mirror of
https://github.com/sreedevk/deduplicator.git
synced 2026-08-26 02:04:39 +00:00
v0.3 (#61)
- [x] parallelization
- [x] (scanning) + (processing sw & processing hw & formatting & printing)
- [x] reduce cloning values on the heap
- [x] add a partial hashing mode (--strict)
- [x] add unit tests
- [x] add silent mode
- [x] update documentation
- [x] remove color output
- [x] progress bar improvements
- [x] use progress bar groups
- [x] remove broken json rendering
- [x] add benchmarks
This commit is contained in:
Sreedev Kodichath
committed by
GitHub
GitHub
parent
f9e86f8522
commit
58e33cc8da
212
Cargo.lock
generated
212
Cargo.lock
generated
@@ -274,7 +274,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "deduplicator"
|
||||
version = "0.2.2"
|
||||
version = "0.3.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bytesize",
|
||||
@@ -284,14 +284,13 @@ dependencies = [
|
||||
"globwalk",
|
||||
"gxhash",
|
||||
"indicatif",
|
||||
"itertools",
|
||||
"memmap2",
|
||||
"pathdiff",
|
||||
"prettytable-rs",
|
||||
"rand",
|
||||
"rayon",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"unicode-segmentation",
|
||||
"tempfile",
|
||||
"threadpool",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -333,6 +332,22 @@ version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
|
||||
|
||||
[[package]]
|
||||
name = "errno"
|
||||
version = "0.3.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fastrand"
|
||||
version = "2.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.15"
|
||||
@@ -341,7 +356,19 @@ checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"wasi",
|
||||
"wasi 0.11.0+wasi-snapshot-preview1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"r-efi",
|
||||
"wasi 0.14.2+wasi-0.2.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -395,6 +422,12 @@ version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"
|
||||
|
||||
[[package]]
|
||||
name = "hermit-abi"
|
||||
version = "0.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone"
|
||||
version = "0.1.60"
|
||||
@@ -463,7 +496,7 @@ version = "0.4.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"hermit-abi 0.3.9",
|
||||
"libc",
|
||||
"windows-sys",
|
||||
]
|
||||
@@ -474,15 +507,6 @@ version = "1.70.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800"
|
||||
|
||||
[[package]]
|
||||
name = "itertools"
|
||||
version = "0.10.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
|
||||
dependencies = [
|
||||
"either",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.11"
|
||||
@@ -506,9 +530,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.155"
|
||||
version = "0.2.174"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
|
||||
checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776"
|
||||
|
||||
[[package]]
|
||||
name = "libredox"
|
||||
@@ -520,6 +544,12 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
version = "0.9.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12"
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.4.12"
|
||||
@@ -560,6 +590,16 @@ dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num_cpus"
|
||||
version = "1.17.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b"
|
||||
dependencies = [
|
||||
"hermit-abi 0.5.2",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "number_prefix"
|
||||
version = "0.4.0"
|
||||
@@ -597,6 +637,15 @@ version = "1.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0"
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
|
||||
dependencies = [
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prettytable-rs"
|
||||
version = "0.10.0"
|
||||
@@ -629,6 +678,41 @@ dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "r-efi"
|
||||
version = "5.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97"
|
||||
dependencies = [
|
||||
"rand_chacha",
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_chacha"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
|
||||
dependencies = [
|
||||
"ppv-lite86",
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.9.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"
|
||||
dependencies = [
|
||||
"getrandom 0.3.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "1.10.0"
|
||||
@@ -664,7 +748,7 @@ version = "0.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
"getrandom 0.2.15",
|
||||
"libredox",
|
||||
"thiserror",
|
||||
]
|
||||
@@ -686,6 +770,19 @@ version = "0.8.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "1.0.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266"
|
||||
dependencies = [
|
||||
"bitflags 2.6.0",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustversion"
|
||||
version = "1.0.17"
|
||||
@@ -733,17 +830,6 @@ dependencies = [
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.120"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"ryu",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.13.2"
|
||||
@@ -767,6 +853,19 @@ dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tempfile"
|
||||
version = "3.20.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.3.3",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "term"
|
||||
version = "0.7.0"
|
||||
@@ -798,18 +897,21 @@ dependencies = [
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "threadpool"
|
||||
version = "1.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa"
|
||||
dependencies = [
|
||||
"num_cpus",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-segmentation"
|
||||
version = "1.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-width"
|
||||
version = "0.1.13"
|
||||
@@ -838,6 +940,15 @@ version = "0.11.0+wasi-snapshot-preview1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.14.2+wasi-0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3"
|
||||
dependencies = [
|
||||
"wit-bindgen-rt",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen"
|
||||
version = "0.2.92"
|
||||
@@ -1004,3 +1115,32 @@ name = "windows_x86_64_msvc"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
|
||||
|
||||
[[package]]
|
||||
name = "wit-bindgen-rt"
|
||||
version = "0.39.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
|
||||
dependencies = [
|
||||
"bitflags 2.6.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.8.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f"
|
||||
dependencies = [
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.8.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
34
Cargo.toml
34
Cargo.toml
@@ -1,8 +1,8 @@
|
||||
[package]
|
||||
name = "deduplicator"
|
||||
version = "0.2.2"
|
||||
version = "0.3.0"
|
||||
edition = "2021"
|
||||
description = "find,filter,delete Duplicates"
|
||||
description = "find,filter and delete duplicate files"
|
||||
repository = "https://github.com/sreedevk/deduplicator"
|
||||
license = "MIT"
|
||||
authors = [
|
||||
@@ -11,8 +11,11 @@ authors = [
|
||||
"Dhruva Sagar <dhruva.sagar@gmail.com>",
|
||||
]
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
[[bin]]
|
||||
name = "deduplicator"
|
||||
path = "src/main.rs"
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
[dependencies]
|
||||
anyhow = "1.0.68"
|
||||
bytesize = "1.1.0"
|
||||
@@ -20,27 +23,36 @@ chrono = "0.4.23"
|
||||
clap = { version = "4.0.32", features = ["derive"] }
|
||||
dashmap = { version = "5.4.0", features = ["rayon"] }
|
||||
globwalk = "0.8.1"
|
||||
gxhash = "3.4.1"
|
||||
gxhash = { version = "3.4.1", default-features = false }
|
||||
indicatif = { version = "0.17.2", features = ["rayon"] }
|
||||
itertools = "0.10.5"
|
||||
memmap2 = "0.5.8"
|
||||
pathdiff = "0.2.1"
|
||||
prettytable-rs = "0.10.0"
|
||||
rand = "0.9.1"
|
||||
rayon = "1.6.1"
|
||||
serde = { version = "1.0.192", features = ["derive"] }
|
||||
serde_json = "1.0.108"
|
||||
unicode-segmentation = "1.10.0"
|
||||
threadpool = "1.8.1"
|
||||
|
||||
[profile.release]
|
||||
strip = true
|
||||
opt-level = 3
|
||||
lto = "thin"
|
||||
debug = false
|
||||
codegen-units = 1
|
||||
|
||||
# generated by 'cargo dist init'
|
||||
[profile.dist]
|
||||
inherits = "release"
|
||||
lto = "thin"
|
||||
|
||||
[workspace.metadata.dist]
|
||||
rust-toolchain-version = "1.78.0"
|
||||
rust-toolchain-version = "1.87.0"
|
||||
ci = ["github"]
|
||||
targets = ["x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc", "aarch64-apple-darwin"]
|
||||
targets = [
|
||||
"x86_64-unknown-linux-gnu",
|
||||
"x86_64-apple-darwin",
|
||||
"x86_64-pc-windows-msvc",
|
||||
"aarch64-apple-darwin",
|
||||
]
|
||||
cargo-dist-version = "0.0.7"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3.20.0"
|
||||
|
||||
184
README.md
184
README.md
@@ -7,6 +7,8 @@
|
||||
## Usage
|
||||
|
||||
```bash
|
||||
find,filter and delete duplicate files
|
||||
|
||||
Usage: deduplicator [OPTIONS] [scan_dir_path]
|
||||
|
||||
Arguments:
|
||||
@@ -15,13 +17,14 @@ Arguments:
|
||||
Options:
|
||||
-t, --types <TYPES> Filetypes to deduplicate [default = all]
|
||||
-i, --interactive Delete files interactively
|
||||
-s, --min-size <MIN_SIZE> Minimum filesize of duplicates to scan (e.g., 100B/1K/2M/3G/4T) [default: 1b]
|
||||
-d, --max-depth <MAX_DEPTH> Max Depth to scan while looking for duplicates
|
||||
--min-depth <MIN_DEPTH> Min Depth to scan while looking for duplicates
|
||||
-m, --min-size <MIN_SIZE> Minimum filesize of duplicates to scan (e.g., 100B/1K/2M/3G/4T) [default: 1b]
|
||||
-D, --max-depth <MAX_DEPTH> Max Depth to scan while looking for duplicates
|
||||
-d, --min-depth <MIN_DEPTH> Min Depth to scan while looking for duplicates
|
||||
-f, --follow-links Follow links while scanning directories
|
||||
-h, --help Print help information
|
||||
-V, --version Print version information
|
||||
--json
|
||||
-s, --strict Guarantees that two files are duplicate (performs a full hash)
|
||||
-p, --progress Show Progress spinners & metrics
|
||||
-h, --help Print help
|
||||
-V, --version Print version
|
||||
```
|
||||
### Examples
|
||||
|
||||
@@ -42,93 +45,118 @@ deduplicator ~/.config --follow-links
|
||||
deduplicator ~/Media --min-size 100mb
|
||||
```
|
||||
|
||||
## Demo
|
||||
|
||||
|
||||
|
||||
|
||||
## Installation
|
||||
Currently, you can only install deduplicator using cargo package manager.
|
||||
|
||||
### Cargo Install
|
||||
|
||||
#### Stable
|
||||
|
||||
> [!WARNING] Note from GxHash: GxHash relies on aes hardware acceleration, you must make sure the aes feature is enabled when building (otherwise it won't build). This can be done by setting the RUSTFLAGS environment variable to -C target-feature=+aes or -C target-cpu=native (the latter should work if your CPU is properly recognized by rustc, which is the case most of the time).
|
||||
> please install version `0.2.1` if you are unable to install `0.2.2`
|
||||
### Cargo
|
||||
> GxHash relies on aes hardware acceleration, so please set `RUSTFLAGS` to `"-C target-feature=+aes"` or `"-C target-cpu=native"` before
|
||||
> installing.
|
||||
|
||||
#### install from crates.io
|
||||
```bash
|
||||
$ RUSTFLAGS="-C target-cpu=native" cargo install deduplicator
|
||||
```
|
||||
|
||||
> [!]
|
||||
|
||||
#### Nightly
|
||||
|
||||
if you'd like to install with nightly features, you can use
|
||||
|
||||
#### install from git
|
||||
```bash
|
||||
$ cargo install --git https://github.com/sreedevk/deduplicator
|
||||
$ RUSTFLAGS="-C target-cpu=native" cargo install deduplicator --git https://github.com/sreedevk/deduplicator
|
||||
```
|
||||
Please note that if you use a version manager to install rust (like asdf), you need to reshim (`asdf reshim rust`).
|
||||
|
||||
### Linux (Pre-built Binary)
|
||||
|
||||
you can download the pre-built binary from the [Releases](https://github.com/sreedevk/deduplicator/releases) page.
|
||||
download the `deduplicator-x86_64-unknown-linux-gnu.tar.gz` for linux. Once you have the tarball file with the executable,
|
||||
you can follow these steps to install:
|
||||
|
||||
```bash
|
||||
$ tar -zxvf deduplicator-x86_64-unknown-linux-gnu.tar.gz
|
||||
$ sudo mv deduplicator /usr/bin/
|
||||
```
|
||||
|
||||
### Mac OS (Pre-built Binary)
|
||||
|
||||
you can download the pre-build binary from the [Releases](https://github.com/sreedevk/deduplicator/releases) page.
|
||||
download the `deduplicator-x86_64-apple-darwin.tar.gz` tarball for mac os. Once you have the tarball file with the executable, you can follow these steps to install:
|
||||
|
||||
```bash
|
||||
$ tar -zxvf deduplicator-x86_64-unknown-linux-gnu.tar.gz
|
||||
$ sudo mv deduplicator /usr/bin/
|
||||
```
|
||||
|
||||
### Windows (Pre-built Binary)
|
||||
|
||||
you can download the pre-build binary from the [Releases](https://github.com/sreedevk/deduplicator/releases) page.
|
||||
download the `deduplicator-x86_64-pc-windows-msvc.zip` zip file for windows. unzip the `zip` file & move the `deduplicator.exe` to a location in the PATH system environment variable.
|
||||
|
||||
Note: If you Run into an msvc error, please install MSCV from [here](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist?view=msvc-170)
|
||||
|
||||
## Performance
|
||||
Deduplicator uses size comparison and [GxHash](https://docs.rs/gxhash/latest/gxhash/) to quickly check a large number of files to find duplicates. its also heavily parallelized. The default behavior of deduplicator is to only hash the first page (4K) of the file. This is to ensure that performance is the default priority. You can modify this behavior by using the `--strict` flag which will hash the whole file and ensure that 2 files are indeed duplicates. I'll add benchmarks in future versions.
|
||||
|
||||
Deduplicator uses size comparison and fxhash (a non non-cryptographic hashing algo) to quickly scan through large number of files to find duplicates. its also highly parallel (uses rayon and dashmap). I was able to scan through 120GB of files (Videos, PDFs, Images) in ~300ms. checkout the benchmarks
|
||||
|
||||
## benchmarks
|
||||
|
||||
| Command | Dirsize | Filecount | Mean [ms] | Min [ms] | Max [ms] | Relative |
|
||||
|:---|:---|---:|---:|---:|---:|---:|
|
||||
| `deduplicator ~/Data/tmp` | (~120G) | 721 files | 33.5 ± 28.6 | 25.3 | 151.5 | 1.87 ± 1.60 |
|
||||
| `deduplicator ~/Data/books` | (~8.6G) | 1419 files | 24.5 ± 1.0 | 22.9 | 28.1 | 1.37 ± 0.08 |
|
||||
| `deduplicator ~/Data/books --min-size 10M` | (~8.6G) | 1419 files | 17.9 ± 0.7 | 16.8 | 20.0 | 1.00 |
|
||||
| `deduplicator ~/Data/ --types pdf,jpg,png,jpeg` | (~290G) | 104222 files | 1207.2 ± 37.0 | 1172.2 | 1287.7 | 67.27 ± 3.33 |
|
||||
|
||||
* The last entry is lower because of the number of files deduplicator had to go through (~660895 Files). The average size of the files rarely affect the performance of deduplicator.
|
||||
|
||||
These benchmarks were run using [hyperfine](https://github.com/sharkdp/hyperfine). Here are the specs of the machine used to benchmark deduplicator:
|
||||
### Benchmarks
|
||||
I've used hyperfine to run deduplicator on files generated by the rake file at `rakelib/benchmark.rake`. The Benchmarking accuracy can further be improved by isolating runs inside restricted docker containers. I'll include that in the future. For now, here's the hyperfine output on my i7-12800H laptop with 32G of RAM.
|
||||
|
||||
#### Fewer Large Files
|
||||
```
|
||||
OS: Arch Linux x86_64
|
||||
Host: Precision 5540
|
||||
Kernel: 5.15.89-1-lts
|
||||
Uptime: 4 hours, 44 mins
|
||||
Shell: zsh 5.9
|
||||
Terminal: kitty
|
||||
CPU: Intel i9-9880H (16) @ 4.800GHz
|
||||
GPU: NVIDIA Quadro T2000 Mobile / Max-Q
|
||||
GPU: Intel CoffeeLake-H GT2 [UHD Graphics 630]
|
||||
Memory: 31731MiB (~32GiB)
|
||||
# hyperfine -N --warmup 80 './target/release/deduplicator bench_artifacts'
|
||||
Benchmark 1: ./target/release/deduplicator bench_artifacts
|
||||
Time (mean ± σ): 2.5 ms ± 0.4 ms [User: 2.2 ms, System: 4.8 ms]
|
||||
Range (min … max): 1.9 ms … 6.8 ms 1322 runs
|
||||
|
||||
# dust 'bench_artifacts'
|
||||
37M ┌── file_1_fwds.bin │██ │ 1%
|
||||
201M ├── file_0_fwds.bin │██████████ │ 8%
|
||||
390M ├── file_0_fwdcbss.bin│████████████████████ │ 15%
|
||||
390M ├── file_0_fwscas.bin │████████████████████ │ 15%
|
||||
390M ├── file_0_fwss.bin │████████████████████ │ 15%
|
||||
390M ├── file_1_fwdcbss.bin│████████████████████ │ 15%
|
||||
390M ├── file_1_fwscas.bin │████████████████████ │ 15%
|
||||
390M ├── file_1_fwss.bin │████████████████████ │ 15%
|
||||
2.5G ┌─┴ bench_artifacts │███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ │ 100%
|
||||
```
|
||||
|
||||
## Screenshots
|
||||
#### Many Small Files
|
||||
```
|
||||
# hyperfine --warmup 20 './target/release/deduplicator bench_artifacts'
|
||||
Benchmark 1: ./target/release/deduplicator bench_artifacts
|
||||
Time (mean ± σ): 22.3 ms ± 1.7 ms [User: 35.8 ms, System: 46.3 ms]
|
||||
Range (min … max): 18.9 ms … 27.2 ms 112 runs
|
||||
|
||||
|
||||
# dust 'bench_artifacts'
|
||||
3.9M ┌── file_992_fwss.bin │█ │ 0%
|
||||
3.9M ├── file_993_fwdcbss.bin│█ │ 0%
|
||||
3.9M ├── file_993_fwscas.bin │█ │ 0%
|
||||
3.9M ├── file_993_fwss.bin │█ │ 0%
|
||||
3.9M ├── file_994_fwdcbss.bin│█ │ 0%
|
||||
3.9M ├── file_994_fwscas.bin │█ │ 0%
|
||||
3.9M ├── file_994_fwss.bin │█ │ 0%
|
||||
3.9M ├── file_995_fwdcbss.bin│█ │ 0%
|
||||
3.9M ├── file_995_fwscas.bin │█ │ 0%
|
||||
3.9M ├── file_995_fwss.bin │█ │ 0%
|
||||
3.9M ├── file_996_fwdcbss.bin│█ │ 0%
|
||||
3.9M ├── file_996_fwscas.bin │█ │ 0%
|
||||
3.9M ├── file_996_fwss.bin │█ │ 0%
|
||||
3.9M ├── file_997_fwdcbss.bin│█ │ 0%
|
||||
3.9M ├── file_997_fwscas.bin │█ │ 0%
|
||||
3.9M ├── file_997_fwss.bin │█ │ 0%
|
||||
3.9M ├── file_998_fwdcbss.bin│█ │ 0%
|
||||
3.9M ├── file_998_fwscas.bin │█ │ 0%
|
||||
3.9M ├── file_998_fwss.bin │█ │ 0%
|
||||
3.9M ├── file_999_fwdcbss.bin│█ │ 0%
|
||||
3.9M ├── file_999_fwscas.bin │█ │ 0%
|
||||
3.9M ├── file_999_fwss.bin │█ │ 0%
|
||||
3.9M ├── file_99_fwdcbss.bin │█ │ 0%
|
||||
3.9M ├── file_99_fwscas.bin │█ │ 0%
|
||||
3.9M ├── file_99_fwss.bin │█ │ 0%
|
||||
3.9M ├── file_9_fwdcbss.bin │█ │ 0%
|
||||
3.9M ├── file_9_fwscas.bin │█ │ 0%
|
||||
3.9M ├── file_9_fwss.bin │█ │ 0%
|
||||
11G ┌─┴ bench_artifacts │█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ │ 100%
|
||||
```
|
||||
|
||||
## Roadmap
|
||||
- Tree format output for duplicate file listing
|
||||
- GUI
|
||||
- Packages for different operating system repositories (currently only installable via cargo)
|
||||
## proposed
|
||||
- [ ] parallelization
|
||||
- [ ] (scanning + processing sw + processing hw) & formatting & printing
|
||||
- [ ] scanning + processing sw + processing hw + formatting + printing
|
||||
- [ ] max file path size should use the last set of duplicates
|
||||
- [ ] add more unit tests
|
||||
- [ ] test against different filesystems
|
||||
- [ ] test against different file name encodings
|
||||
- [ ] restore json output (was removed in 0.3)
|
||||
- [ ] fix memory leak on very large filesystems
|
||||
- [ ] maybe use a bloom filter
|
||||
- [ ] reduce FileInfo size
|
||||
- [ ] output in a tree format
|
||||
- [ ] tui
|
||||
- [ ] change the default hashing method to include the first & last page of a file (8K)
|
||||
|
||||
## v0.3
|
||||
- [x] parallelization
|
||||
- [x] (scanning) + (processing sw & processing hw & formatting & printing)
|
||||
- [x] reduce cloning values on the heap
|
||||
- [x] add a partial hashing mode (--strict)
|
||||
- [x] add unit tests
|
||||
- [x] add silent mode
|
||||
- [x] update documentation
|
||||
- [x] remove color output
|
||||
- [x] progress bar improvements
|
||||
- [x] use progress bar groups
|
||||
- [x] remove broken json rendering
|
||||
- [x] add benchmarks
|
||||
|
||||
93
rakelib/benchmark.rake
Normal file
93
rakelib/benchmark.rake
Normal file
@@ -0,0 +1,93 @@
|
||||
require 'tempfile'
|
||||
require 'fileutils'
|
||||
require 'securerandom'
|
||||
|
||||
namespace :benchmark do
|
||||
file 'target/release/deduplicator' do
|
||||
sh "cargo build --release"
|
||||
end
|
||||
|
||||
task :few_large_files => 'target/release/deduplicator' do
|
||||
# Benchmark 1: ./target/release/deduplicator bench_artifacts
|
||||
# Time (mean ± σ): 2.5 ms ± 0.6 ms [User: 2.4 ms, System: 4.8 ms]
|
||||
# Range (min … max): 1.8 ms … 9.7 ms 1474 runs
|
||||
|
||||
root = "bench_artifacts"
|
||||
Dir.mkdir(root)
|
||||
|
||||
# files with same size
|
||||
2.times.map do |i|
|
||||
File.open(File.join(root, "file_#{i}_fwss.bin"), 'wb') do |f|
|
||||
f.write(SecureRandom.bytes(4096 * 100_000))
|
||||
end
|
||||
end
|
||||
|
||||
# files with different sizes
|
||||
2.times.map do |i|
|
||||
File.open(File.join(root, "file_#{i}_fwds.bin"), 'wb') do |f|
|
||||
f.write(SecureRandom.bytes(4096 * (rand * 100_000).ceil))
|
||||
end
|
||||
end
|
||||
|
||||
# files with same content & size
|
||||
2.times.each do |i|
|
||||
File.open(File.join(root, "file_#{i}_fwscas.bin"), 'wb') do |f|
|
||||
f.write("\0" * (4096 * 100_000))
|
||||
end
|
||||
end
|
||||
|
||||
# files with different content but same size
|
||||
2.times.each do |i|
|
||||
File.open(File.join(root, "file_#{i}_fwdcbss.bin"), 'wb') do |f|
|
||||
f.write(SecureRandom.bytes(4096 * 100_000))
|
||||
end
|
||||
end
|
||||
|
||||
sh("hyperfine -N --warmup 80 './target/release/deduplicator #{root}'")
|
||||
sh("dust '#{root}'")
|
||||
|
||||
FileUtils.rm_rf(root)
|
||||
end
|
||||
|
||||
task :many_small_files => 'target/release/deduplicator' do
|
||||
# Benchmark 1: ./target/release/deduplicator bench_artifacts
|
||||
# Time (mean ± σ): 10.6 ms ± 1.0 ms [User: 20.0 ms, System: 22.5 ms]
|
||||
# Range (min … max): 8.4 ms … 14.2 ms 235 runs
|
||||
|
||||
root = "bench_artifacts"
|
||||
Dir.mkdir(root)
|
||||
|
||||
# files with same size
|
||||
1000.times.each do |i|
|
||||
File.open(File.join(root, "file_#{i}_fwss.bin"), 'wb') do |f|
|
||||
f.write(SecureRandom.bytes(4096 * 1000))
|
||||
end
|
||||
end
|
||||
|
||||
# files with different sizes
|
||||
1000.times.each do |i|
|
||||
File.open(File.join(root, "file_#{i}_fwds.bin"), 'wb') do |f|
|
||||
f.write(SecureRandom.bytes(4096 * (rand * 100).ceil))
|
||||
end
|
||||
end
|
||||
|
||||
# files with same content & size
|
||||
1000.times.each do |i|
|
||||
File.open(File.join(root, "file_#{i}_fwscas.bin"), 'wb') do |f|
|
||||
f.write("\0" * (4096 * 1000))
|
||||
end
|
||||
end
|
||||
|
||||
# files with different content but same size
|
||||
1000.times.each do |i|
|
||||
File.open(File.join(root, "file_#{i}_fwdcbss.bin"), 'wb') do |f|
|
||||
f.write(SecureRandom.bytes(4096 * 1000))
|
||||
end
|
||||
end
|
||||
|
||||
sh("hyperfine --warmup 20 './target/release/deduplicator #{root}'")
|
||||
sh("dust '#{root}'")
|
||||
|
||||
FileUtils.rm_rf(root)
|
||||
end
|
||||
end
|
||||
@@ -1,43 +1,45 @@
|
||||
use anyhow::Result;
|
||||
use gxhash::GxHasher;
|
||||
use gxhash::gxhash128;
|
||||
use memmap2::Mmap;
|
||||
use serde::Serialize;
|
||||
use std::fs;
|
||||
use std::hash::Hasher;
|
||||
use std::{fs::Metadata, path::PathBuf};
|
||||
use std::{
|
||||
fs,
|
||||
io::Read,
|
||||
path::{Path, PathBuf},
|
||||
time::SystemTime,
|
||||
};
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FileInfo {
|
||||
pub path: PathBuf,
|
||||
pub hash: Option<String>,
|
||||
pub path: Box<Path>,
|
||||
pub size: u64,
|
||||
#[serde(skip)]
|
||||
pub filemeta: Metadata,
|
||||
pub modified: SystemTime,
|
||||
}
|
||||
|
||||
impl FileInfo {
|
||||
pub fn hash(&self) -> Result<Self> {
|
||||
let file = fs::File::open(self.path.clone())?;
|
||||
pub fn hash(&self, seed: i64) -> Result<u128> {
|
||||
let file = fs::File::open(&self.path)?;
|
||||
let mapper = unsafe { Mmap::map(&file)? };
|
||||
let mut primhasher = GxHasher::default();
|
||||
let final_hash = mapper.chunks(4096).fold(0u128, |acc, chunk: &[u8]| {
|
||||
acc.wrapping_add(gxhash128(chunk, seed))
|
||||
});
|
||||
|
||||
mapper
|
||||
.chunks(1_000_000)
|
||||
.for_each(|chunk| primhasher.write(chunk));
|
||||
Ok(final_hash)
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
hash: Some(primhasher.finish().to_string()),
|
||||
..self.clone()
|
||||
})
|
||||
pub fn initial_page_hash(&self, seed: i64) -> Result<u128> {
|
||||
let mut file = fs::File::open(&self.path)?;
|
||||
let mut buffer = [0; 4096];
|
||||
let bytes_read = file.read(&mut buffer)?;
|
||||
|
||||
Ok(gxhash128(&buffer[..bytes_read], seed))
|
||||
}
|
||||
|
||||
pub fn new(path: PathBuf) -> Result<Self> {
|
||||
let filemeta = std::fs::metadata(path.clone())?;
|
||||
let filemeta = std::fs::metadata(&path)?;
|
||||
Ok(Self {
|
||||
path,
|
||||
filemeta: filemeta.clone(),
|
||||
hash: None,
|
||||
path: path.into_boxed_path(),
|
||||
size: filemeta.len(),
|
||||
modified: filemeta.modified()?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,16 +1,14 @@
|
||||
pub struct Formatter;
|
||||
use crate::fileinfo::FileInfo;
|
||||
use crate::params::Params;
|
||||
use crate::{fileinfo::FileInfo, params::Params};
|
||||
use anyhow::Result;
|
||||
use chrono::{DateTime, Utc};
|
||||
use dashmap::DashMap;
|
||||
use indicatif::{ProgressBar, ProgressFinish, ProgressIterator, ProgressStyle};
|
||||
use indicatif::{ParallelProgressIterator, ProgressBar, ProgressFinish, ProgressStyle};
|
||||
use pathdiff::diff_paths;
|
||||
use prettytable::{format, row, Table};
|
||||
use std::borrow::Cow;
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
use prettytable::{format, row, Row, Table};
|
||||
use rayon::prelude::*;
|
||||
use std::{borrow::Cow, path::PathBuf, sync::Arc, time::Duration};
|
||||
|
||||
pub struct Formatter;
|
||||
impl Formatter {
|
||||
pub fn human_path(
|
||||
file: &FileInfo,
|
||||
@@ -18,7 +16,7 @@ impl Formatter {
|
||||
min_path_length: usize,
|
||||
) -> Result<String> {
|
||||
let base_directory: PathBuf = app_args.get_directory()?;
|
||||
let relative_path = diff_paths(file.path.clone(), base_directory).unwrap_or_default();
|
||||
let relative_path = diff_paths(&file.path, base_directory).unwrap_or_default();
|
||||
|
||||
let formatted_path = format!(
|
||||
"{:<0width$}",
|
||||
@@ -34,44 +32,62 @@ impl Formatter {
|
||||
}
|
||||
|
||||
pub fn human_mtime(file: &FileInfo) -> Result<String> {
|
||||
let modified_time: DateTime<Utc> = file.filemeta.modified()?.into();
|
||||
let modified_time: DateTime<Utc> = file.modified.into();
|
||||
Ok(modified_time.format("%Y-%m-%d %H:%M:%S").to_string())
|
||||
}
|
||||
|
||||
pub fn generate_table(raw: DashMap<String, Vec<FileInfo>>, max_path_len: usize, app_args: &Params) -> Result<Table> {
|
||||
let mut output_table = Table::new();
|
||||
output_table.set_titles(row!["hash", "duplicates"]);
|
||||
|
||||
let progress_style = ProgressStyle::with_template(
|
||||
"[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}",
|
||||
)?;
|
||||
|
||||
let progress_bar = ProgressBar::new(raw.len() as u64);
|
||||
progress_bar.set_style(progress_style);
|
||||
progress_bar.enable_steady_tick(Duration::from_millis(50));
|
||||
progress_bar.set_message("generating output");
|
||||
|
||||
raw.into_iter()
|
||||
.progress_with(progress_bar)
|
||||
.with_finish(ProgressFinish::WithMessage(Cow::from("output generated")))
|
||||
.for_each(|(hash, group)| {
|
||||
pub fn gen_sub_tbl(items: Vec<FileInfo>, app_args: &Params, max_path_len: u64) -> Table {
|
||||
let mut inner_table = Table::new();
|
||||
inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
|
||||
group.iter().for_each(|file| {
|
||||
items.iter().for_each(|file| {
|
||||
inner_table.add_row(row![
|
||||
Self::human_path(file, app_args, max_path_len).unwrap_or_default(),
|
||||
Self::human_path(file, app_args, max_path_len as usize).unwrap_or_default(),
|
||||
Self::human_filesize(file).unwrap_or_default(),
|
||||
Self::human_mtime(file).unwrap_or_default()
|
||||
]);
|
||||
});
|
||||
inner_table
|
||||
}
|
||||
|
||||
output_table.add_row(row![hash, inner_table]);
|
||||
});
|
||||
pub fn generate_table(
|
||||
raw: Arc<DashMap<u128, Vec<FileInfo>>>,
|
||||
mpath_len: u64,
|
||||
args: &Params,
|
||||
) -> Result<Table> {
|
||||
let progress_bar = match args.progress {
|
||||
true => ProgressBar::new_spinner(),
|
||||
false => ProgressBar::hidden(),
|
||||
};
|
||||
|
||||
let progress_style = ProgressStyle::with_template("[{elapsed_precise}] {pos:>7} {msg}")?;
|
||||
progress_bar.set_style(progress_style);
|
||||
progress_bar.enable_steady_tick(Duration::from_millis(50));
|
||||
progress_bar.set_message("generating output");
|
||||
|
||||
let rows = raw
|
||||
.par_iter_mut()
|
||||
.progress_with(progress_bar)
|
||||
.with_finish(ProgressFinish::WithMessage(Cow::from("output generated")))
|
||||
.filter(|i| i.value().len() > 1)
|
||||
.map(|i| {
|
||||
row![
|
||||
i.key(),
|
||||
Self::gen_sub_tbl(i.value().to_vec(), args, mpath_len)
|
||||
]
|
||||
})
|
||||
.collect::<Vec<Row>>();
|
||||
|
||||
let mut output_table = Table::new();
|
||||
output_table.set_titles(row!["hash", "duplicates"]);
|
||||
output_table.extend(rows);
|
||||
Ok(output_table)
|
||||
}
|
||||
|
||||
pub fn print(raw: DashMap<String, Vec<FileInfo>>, max_path_len: usize, app_args: &Params) -> Result<()> {
|
||||
pub fn print(
|
||||
raw: Arc<DashMap<u128, Vec<FileInfo>>>,
|
||||
max_path_len: u64,
|
||||
app_args: &Params,
|
||||
) -> Result<()> {
|
||||
if raw.is_empty() {
|
||||
println!("\n\nNo duplicates found matching your search criteria.\n");
|
||||
return Ok(());
|
||||
|
||||
@@ -1,46 +1,30 @@
|
||||
use crate::formatter::Formatter;
|
||||
use crate::{fileinfo::FileInfo, params::Params};
|
||||
use crate::{fileinfo::FileInfo, formatter::Formatter, params::Params};
|
||||
use anyhow::Result;
|
||||
use dashmap::DashMap;
|
||||
use prettytable::{format, row, Table};
|
||||
use std::io::{self, Write};
|
||||
use std::{
|
||||
io::{self, Write},
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
pub fn scan_group_confirmation() -> Result<bool> {
|
||||
print!("\nconfirm? [y/N]: ");
|
||||
std::io::stdout().flush()?;
|
||||
let mut user_input = String::new();
|
||||
io::stdin().read_line(&mut user_input)?;
|
||||
pub struct Interactive;
|
||||
|
||||
match user_input.trim() {
|
||||
"Y" | "y" => Ok(true),
|
||||
_ => Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn scan_group_instruction() -> Result<String> {
|
||||
println!("\nEnter the indices of the files you want to delete.");
|
||||
println!("You can enter multiple files using commas to seperate file indices.");
|
||||
println!("example: 1,2");
|
||||
print!("\n> ");
|
||||
std::io::stdout().flush()?;
|
||||
let mut user_input = String::new();
|
||||
io::stdin().read_line(&mut user_input)?;
|
||||
|
||||
Ok(user_input)
|
||||
}
|
||||
|
||||
pub fn init(result: DashMap<String, Vec<FileInfo>>, app_args: &Params) -> Result<()> {
|
||||
impl Interactive {
|
||||
pub fn init(result: Arc<DashMap<u128, Vec<FileInfo>>>, app_args: &Params) -> Result<()> {
|
||||
result
|
||||
.clone()
|
||||
.into_iter()
|
||||
.iter()
|
||||
.filter(|i| i.value().len() > 1)
|
||||
.enumerate()
|
||||
.for_each(|(gindex, (_, group))| {
|
||||
.for_each(|(gindex, i)| {
|
||||
let group = i.value();
|
||||
let mut itable = Table::new();
|
||||
itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
|
||||
itable.set_titles(row!["index", "filename", "size", "updated_at"]);
|
||||
|
||||
let max_path_size = group
|
||||
.iter()
|
||||
.map(|f| f.path.clone().into_os_string().len())
|
||||
.map(|f| f.path.iter().count())
|
||||
.max()
|
||||
.unwrap_or_default();
|
||||
|
||||
@@ -53,21 +37,45 @@ pub fn init(result: DashMap<String, Vec<FileInfo>>, app_args: &Params) -> Result
|
||||
]);
|
||||
});
|
||||
|
||||
process_group_action(&group, gindex, result.len(), itable);
|
||||
Self::process_group_action(group, gindex, result.len(), itable);
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub fn process_group_action(
|
||||
pub fn scan_group_confirmation() -> Result<bool> {
|
||||
print!("\nconfirm? [y/N]: ");
|
||||
std::io::stdout().flush()?;
|
||||
let mut user_input = String::new();
|
||||
io::stdin().read_line(&mut user_input)?;
|
||||
|
||||
match user_input.trim() {
|
||||
"Y" | "y" => Ok(true),
|
||||
_ => Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn scan_group_instruction() -> Result<String> {
|
||||
println!("\nEnter the indices of the files you want to delete.");
|
||||
println!("You can enter multiple files using commas to seperate file indices.");
|
||||
println!("example: 1,2");
|
||||
print!("\n> ");
|
||||
std::io::stdout().flush()?;
|
||||
let mut user_input = String::new();
|
||||
io::stdin().read_line(&mut user_input)?;
|
||||
|
||||
Ok(user_input)
|
||||
}
|
||||
|
||||
pub fn process_group_action(
|
||||
duplicates: &Vec<FileInfo>,
|
||||
dup_index: usize,
|
||||
dup_size: usize,
|
||||
table: Table,
|
||||
) {
|
||||
) {
|
||||
println!("\nDuplicate Set {} of {}\n", dup_index + 1, dup_size);
|
||||
table.printstd();
|
||||
let files_to_delete = scan_group_instruction().unwrap_or_default();
|
||||
let files_to_delete = Self::scan_group_instruction().unwrap_or_default();
|
||||
let parsed_file_indices = files_to_delete
|
||||
.trim()
|
||||
.split(',')
|
||||
@@ -81,7 +89,7 @@ pub fn process_group_action(
|
||||
.any(|index| index > (duplicates.len() - 1))
|
||||
{
|
||||
println!("Err: File Index Out of Bounds!");
|
||||
return process_group_action(duplicates, dup_index, dup_size, table);
|
||||
return Self::process_group_action(duplicates, dup_index, dup_size, table);
|
||||
}
|
||||
|
||||
print!("{esc}[2J{esc}[1;1H", esc = 27 as char);
|
||||
@@ -102,7 +110,7 @@ pub fn process_group_action(
|
||||
println!("{}: {}", index, file.path.display());
|
||||
});
|
||||
|
||||
match scan_group_confirmation().unwrap() {
|
||||
match Self::scan_group_confirmation().unwrap() {
|
||||
true => {
|
||||
files_to_delete.into_iter().for_each(|file| {
|
||||
match std::fs::remove_file(file.path.clone()) {
|
||||
@@ -113,4 +121,5 @@ pub fn process_group_action(
|
||||
}
|
||||
false => println!("\nCancelled Delete Operation."),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
26
src/main.rs
26
src/main.rs
@@ -4,28 +4,32 @@ mod interactive;
|
||||
mod params;
|
||||
mod processor;
|
||||
mod scanner;
|
||||
mod server;
|
||||
|
||||
use self::{formatter::Formatter, interactive::Interactive, server::Server};
|
||||
use anyhow::Result;
|
||||
use clap::Parser;
|
||||
use formatter::Formatter;
|
||||
use params::Params;
|
||||
use processor::Processor;
|
||||
use scanner::Scanner;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let app_args = Params::parse();
|
||||
let scan_results = Scanner::build(&app_args)?.scan()?;
|
||||
let mut processor = Processor::new(scan_results);
|
||||
let server = Server::new(app_args.clone());
|
||||
|
||||
processor.sizewise()?;
|
||||
processor.hashwise()?;
|
||||
|
||||
let results = processor.hashwise_results;
|
||||
server.start()?;
|
||||
|
||||
match app_args.interactive {
|
||||
false => Formatter::print(results, processor.max_path_len, &app_args)?,
|
||||
true => interactive::init(results, &app_args)?,
|
||||
false => {
|
||||
Formatter::print(
|
||||
server.hw_duplicate_set,
|
||||
server.max_file_path_len.load(Ordering::Acquire),
|
||||
&app_args,
|
||||
)?;
|
||||
}
|
||||
true => {
|
||||
Interactive::init(server.hw_duplicate_set, &app_args)?;
|
||||
}
|
||||
};
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ use std::{fs, path::PathBuf};
|
||||
use anyhow::Result;
|
||||
use clap::{Parser, ValueHint};
|
||||
|
||||
#[derive(Parser, Debug, Clone)]
|
||||
#[derive(Parser, Debug, Default, Clone)]
|
||||
#[command(author, version, about, long_about = None)]
|
||||
pub struct Params {
|
||||
/// Filetypes to deduplicate [default = all]
|
||||
@@ -16,20 +16,23 @@ pub struct Params {
|
||||
#[arg(long, short)]
|
||||
pub interactive: bool,
|
||||
/// Minimum filesize of duplicates to scan (e.g., 100B/1K/2M/3G/4T).
|
||||
#[arg(long, short = 's', default_value = "1b")]
|
||||
#[arg(long, short = 'm', default_value = "1b")]
|
||||
pub min_size: Option<String>,
|
||||
/// Max Depth to scan while looking for duplicates
|
||||
#[arg(long, short = 'd')]
|
||||
#[arg(long, short = 'D')]
|
||||
pub max_depth: Option<usize>,
|
||||
/// Min Depth to scan while looking for duplicates
|
||||
#[arg(long)]
|
||||
#[arg(long, short = 'd')]
|
||||
pub min_depth: Option<usize>,
|
||||
/// Follow links while scanning directories
|
||||
#[arg(long, short)]
|
||||
pub follow_links: bool,
|
||||
/// print json output
|
||||
#[arg(long)]
|
||||
pub json: bool,
|
||||
/// Guarantees that two files are duplicate (performs a full hash)
|
||||
#[arg(long, short = 's', default_value = "false")]
|
||||
pub strict: bool,
|
||||
/// Show Progress spinners & metrics
|
||||
#[arg(long, short = 'p', default_value = "false")]
|
||||
pub progress: bool,
|
||||
}
|
||||
|
||||
impl Params {
|
||||
|
||||
402
src/processor.rs
402
src/processor.rs
@@ -1,102 +1,368 @@
|
||||
use anyhow::Result;
|
||||
use dashmap::DashMap;
|
||||
use indicatif::{ParallelProgressIterator, ProgressBar, ProgressFinish, ProgressStyle};
|
||||
use indicatif::{
|
||||
MultiProgress, ParallelProgressIterator, ProgressBar, ProgressFinish, ProgressStyle,
|
||||
};
|
||||
use rayon::prelude::{IntoParallelIterator, ParallelIterator};
|
||||
use std::{borrow::Cow, time::Duration};
|
||||
use std::borrow::Cow;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex, TryLockError, TryLockResult};
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::fileinfo::FileInfo;
|
||||
use crate::params::Params;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Processor {
|
||||
pub files: Vec<FileInfo>,
|
||||
pub hashwise_results: DashMap<String, Vec<FileInfo>>,
|
||||
pub sizewise_results: DashMap<u64, Vec<FileInfo>>,
|
||||
pub max_path_len: usize,
|
||||
}
|
||||
pub struct Processor {}
|
||||
|
||||
impl Processor {
|
||||
pub fn new(files: Vec<FileInfo>) -> Self {
|
||||
Self {
|
||||
files,
|
||||
hashwise_results: DashMap::new(),
|
||||
sizewise_results: DashMap::new(),
|
||||
max_path_len: 0,
|
||||
}
|
||||
}
|
||||
pub fn hashwise(
|
||||
app_args: Arc<Params>,
|
||||
sw_store: Arc<DashMap<u64, Vec<FileInfo>>>,
|
||||
hw_store: Arc<DashMap<u128, Vec<FileInfo>>>,
|
||||
progress_bar_box: Arc<MultiProgress>,
|
||||
max_file_size: Arc<AtomicU64>,
|
||||
seed: i64,
|
||||
) -> Result<()> {
|
||||
let progress_bar = match app_args.progress {
|
||||
true => progress_bar_box.add(ProgressBar::new_spinner()),
|
||||
false => ProgressBar::hidden(),
|
||||
};
|
||||
|
||||
pub fn hashwise(&mut self) -> Result<()> {
|
||||
if self.sizewise_results.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let keys: Vec<u64> = sw_store.clone().iter().map(|i| *i.key()).collect();
|
||||
|
||||
let progress_style = ProgressStyle::with_template(
|
||||
"[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}",
|
||||
)?;
|
||||
let progress_bar = ProgressBar::new(self.files.len() as u64);
|
||||
let progress_style = ProgressStyle::with_template("[{elapsed_precise}] {pos:>7} {msg}")?;
|
||||
progress_bar.set_style(progress_style);
|
||||
progress_bar.enable_steady_tick(Duration::from_millis(50));
|
||||
progress_bar.set_message("indexing file hashes");
|
||||
progress_bar.set_message("files grouped by hash.");
|
||||
|
||||
let filelist = self
|
||||
.sizewise_results
|
||||
.clone()
|
||||
.into_read_only()
|
||||
.values()
|
||||
.filter(|&subfiles| subfiles.len() > 1)
|
||||
.flatten()
|
||||
.cloned()
|
||||
.collect::<Vec<FileInfo>>();
|
||||
|
||||
self.max_path_len = filelist
|
||||
.iter()
|
||||
.map(|x| x.path.clone().into_os_string().len())
|
||||
.max()
|
||||
.unwrap_or_default();
|
||||
|
||||
filelist
|
||||
.into_par_iter()
|
||||
keys.into_par_iter()
|
||||
.progress_with(progress_bar)
|
||||
.with_finish(ProgressFinish::WithMessage(Cow::from(
|
||||
"indexed files hashes",
|
||||
"files grouped by hash.",
|
||||
)))
|
||||
.map(|file| file.hash())
|
||||
.filter_map(Result::ok)
|
||||
.for_each(move |file| {
|
||||
self.hashwise_results
|
||||
.entry(file.hash.clone().unwrap_or_default())
|
||||
.for_each(|key| {
|
||||
let group: Vec<FileInfo> = sw_store.get(&key).unwrap().to_vec();
|
||||
if group.len() > 1 {
|
||||
group.into_par_iter().for_each(|file| {
|
||||
let fhash = if app_args.strict {
|
||||
file.hash(seed).expect("hashing file failed.")
|
||||
} else {
|
||||
file.initial_page_hash(seed).expect("hashing file failed.")
|
||||
};
|
||||
|
||||
Self::compare_and_update_max_path_len(
|
||||
max_file_size.clone(),
|
||||
file.path.to_string_lossy().len() as u64,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
hw_store
|
||||
.entry(fhash)
|
||||
.and_modify(|fileset| fileset.push(file.clone()))
|
||||
.or_insert_with(|| vec![file]);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn sizewise(&mut self) -> Result<()> {
|
||||
if self.files.is_empty() {
|
||||
return Ok(());
|
||||
pub fn compare_and_update_max_path_len(current: Arc<AtomicU64>, next: u64) -> Result<()> {
|
||||
if current.load(Ordering::Relaxed) < next {
|
||||
current.store(next, Ordering::Release);
|
||||
}
|
||||
|
||||
let progress_style = ProgressStyle::with_template(
|
||||
"[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}",
|
||||
)?;
|
||||
let progress_bar = ProgressBar::new(self.files.len() as u64);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn sizewise(
|
||||
app_args: Arc<Params>,
|
||||
scanner_finished: Arc<AtomicBool>,
|
||||
store: Arc<DashMap<u64, Vec<FileInfo>>>,
|
||||
files: Arc<Mutex<Vec<FileInfo>>>,
|
||||
progress_bar_box: Arc<MultiProgress>,
|
||||
) -> Result<()> {
|
||||
let progress_bar = match app_args.progress {
|
||||
true => progress_bar_box.add(ProgressBar::new_spinner()),
|
||||
false => ProgressBar::hidden(),
|
||||
};
|
||||
|
||||
let progress_style = ProgressStyle::with_template("[{elapsed_precise}] {pos:>7} {msg}")?;
|
||||
progress_bar.set_style(progress_style);
|
||||
progress_bar.enable_steady_tick(Duration::from_millis(50));
|
||||
progress_bar.set_message("indexing file sizes");
|
||||
progress_bar.set_message("files grouped by size");
|
||||
|
||||
self.files
|
||||
.clone()
|
||||
.into_par_iter()
|
||||
.progress_with(progress_bar)
|
||||
.with_finish(ProgressFinish::WithMessage(Cow::from(
|
||||
"indexed files sizes",
|
||||
)))
|
||||
.for_each(|file| {
|
||||
self.sizewise_results
|
||||
loop {
|
||||
let fileopt: Option<FileInfo> = {
|
||||
match files.try_lock() {
|
||||
Ok(mut flist) => flist.pop(),
|
||||
TryLockResult::Err(TryLockError::WouldBlock) => None,
|
||||
_ => None,
|
||||
}
|
||||
};
|
||||
|
||||
match fileopt {
|
||||
Some(file) => {
|
||||
progress_bar.inc(1);
|
||||
store
|
||||
.entry(file.size)
|
||||
.and_modify(|fileset| fileset.push(file.clone()))
|
||||
.or_insert_with(|| vec![file]);
|
||||
});
|
||||
continue;
|
||||
}
|
||||
None => match scanner_finished.load(std::sync::atomic::Ordering::Relaxed) {
|
||||
true => {
|
||||
progress_bar.finish_with_message("files grouped by size");
|
||||
break Ok(());
|
||||
}
|
||||
false => continue,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use anyhow::Result;
|
||||
use dashmap::DashMap;
|
||||
use indicatif::MultiProgress;
|
||||
use rand::Rng;
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tempfile::TempDir;
|
||||
|
||||
use crate::{fileinfo::FileInfo, params::Params};
|
||||
|
||||
use super::Processor;
|
||||
|
||||
fn generate_bytes(size: usize) -> Vec<u8> {
|
||||
let mut rng = rand::rng();
|
||||
(0..size).map(|_| rng.random::<u8>()).collect::<Vec<u8>>()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hashwise_sorting_two_files_with_identical_init_page_only_strict_mode() -> Result<()> {
|
||||
let root = TempDir::new()?;
|
||||
let content = generate_bytes(4096);
|
||||
|
||||
let mut content_x = content.clone();
|
||||
let mut content_y = content.clone();
|
||||
|
||||
content_x.extend(generate_bytes(1720320));
|
||||
content_y.extend(generate_bytes(1720320));
|
||||
|
||||
let files = [
|
||||
(root.path().join("fileone.bin"), content_x),
|
||||
(root.path().join("filetwo.bin"), content_y),
|
||||
];
|
||||
|
||||
for (fpath, content) in files.iter() {
|
||||
let mut f = File::create_new(fpath)?;
|
||||
f.write_all(content)?;
|
||||
}
|
||||
|
||||
let dupstore = Arc::new(DashMap::new());
|
||||
let file_queue = Arc::new(Mutex::new(
|
||||
files
|
||||
.iter()
|
||||
.map(|f| FileInfo::new(f.0.clone()).unwrap())
|
||||
.collect::<Vec<FileInfo>>(),
|
||||
));
|
||||
|
||||
let hw_dupstore = Arc::new(DashMap::new());
|
||||
Processor::sizewise(
|
||||
Arc::new(Params::default()),
|
||||
Arc::new(AtomicBool::new(true)),
|
||||
dupstore.clone(),
|
||||
file_queue,
|
||||
Arc::new(MultiProgress::new()),
|
||||
)?;
|
||||
|
||||
let args = Params {
|
||||
strict: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
Processor::hashwise(
|
||||
Arc::new(args),
|
||||
dupstore.clone(),
|
||||
hw_dupstore.clone(),
|
||||
Arc::new(MultiProgress::new()),
|
||||
Arc::new(AtomicU64::new(32)),
|
||||
300,
|
||||
)?;
|
||||
|
||||
assert_eq!(hw_dupstore.len(), 2);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hashwise_sorting_two_files_with_identical_init_page_only_fast_mode() -> Result<()> {
|
||||
let root = TempDir::new()?;
|
||||
let content = generate_bytes(4096);
|
||||
|
||||
let mut content_x = content.clone();
|
||||
let mut content_y = content.clone();
|
||||
|
||||
content_x.extend(generate_bytes(1720320));
|
||||
content_y.extend(generate_bytes(1720320));
|
||||
|
||||
let files = [
|
||||
(root.path().join("fileone.bin"), content_x),
|
||||
(root.path().join("filetwo.bin"), content_y),
|
||||
];
|
||||
|
||||
for (fpath, content) in files.iter() {
|
||||
let mut f = File::create_new(fpath)?;
|
||||
f.write_all(content)?;
|
||||
}
|
||||
|
||||
let dupstore = Arc::new(DashMap::new());
|
||||
let file_queue = Arc::new(Mutex::new(
|
||||
files
|
||||
.iter()
|
||||
.map(|f| FileInfo::new(f.0.clone()).unwrap())
|
||||
.collect::<Vec<FileInfo>>(),
|
||||
));
|
||||
|
||||
let hw_dupstore = Arc::new(DashMap::new());
|
||||
Processor::sizewise(
|
||||
Arc::new(Params::default()),
|
||||
Arc::new(AtomicBool::new(true)),
|
||||
dupstore.clone(),
|
||||
file_queue,
|
||||
Arc::new(MultiProgress::new()),
|
||||
)?;
|
||||
|
||||
Processor::hashwise(
|
||||
Arc::new(Params::default()),
|
||||
dupstore.clone(),
|
||||
hw_dupstore.clone(),
|
||||
Arc::new(MultiProgress::new()),
|
||||
Arc::new(AtomicU64::new(32)),
|
||||
300,
|
||||
)?;
|
||||
|
||||
assert_eq!(hw_dupstore.len(), 1);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hashwise_sorting_two_files_with_identical_data() -> Result<()> {
|
||||
let root = TempDir::new()?;
|
||||
let content = generate_bytes(282624);
|
||||
let files = [
|
||||
(root.path().join("fileone.bin"), content.clone()),
|
||||
(root.path().join("filetwo.bin"), content.clone()),
|
||||
];
|
||||
|
||||
for (fpath, content) in files.iter() {
|
||||
let mut f = File::create_new(fpath)?;
|
||||
f.write_all(content)?;
|
||||
}
|
||||
|
||||
let dupstore = Arc::new(DashMap::new());
|
||||
let file_queue = Arc::new(Mutex::new(
|
||||
files
|
||||
.iter()
|
||||
.map(|f| FileInfo::new(f.0.clone()).unwrap())
|
||||
.collect::<Vec<FileInfo>>(),
|
||||
));
|
||||
|
||||
let hw_dupstore = Arc::new(DashMap::new());
|
||||
Processor::sizewise(
|
||||
Arc::new(Params::default()),
|
||||
Arc::new(AtomicBool::new(true)),
|
||||
dupstore.clone(),
|
||||
file_queue,
|
||||
Arc::new(MultiProgress::new()),
|
||||
)?;
|
||||
|
||||
Processor::hashwise(
|
||||
Arc::new(Params::default()),
|
||||
dupstore.clone(),
|
||||
hw_dupstore.clone(),
|
||||
Arc::new(MultiProgress::new()),
|
||||
Arc::new(AtomicU64::new(32)),
|
||||
300,
|
||||
)?;
|
||||
|
||||
assert_eq!(hw_dupstore.len(), 1);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sizewise_sorting_two_files_of_different_sizes() -> Result<()> {
|
||||
let root = TempDir::new()?;
|
||||
let files = [
|
||||
(root.path().join("fileone.bin"), generate_bytes(282624)),
|
||||
(root.path().join("filetwo.bin"), generate_bytes(1720320)),
|
||||
];
|
||||
|
||||
for (fpath, content) in files.iter() {
|
||||
let mut f = File::create_new(fpath)?;
|
||||
f.write_all(content)?;
|
||||
}
|
||||
|
||||
let file_queue = Arc::new(Mutex::new(
|
||||
files
|
||||
.iter()
|
||||
.map(|f| FileInfo::new(f.0.clone()).unwrap())
|
||||
.collect::<Vec<FileInfo>>(),
|
||||
));
|
||||
|
||||
let dupstore = Arc::new(DashMap::new());
|
||||
|
||||
Processor::sizewise(
|
||||
Arc::new(Params::default()),
|
||||
Arc::new(AtomicBool::new(true)),
|
||||
dupstore.clone(),
|
||||
file_queue,
|
||||
Arc::new(MultiProgress::new()),
|
||||
)?;
|
||||
|
||||
assert_eq!(dupstore.len(), 2);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sizewise_sorting_two_files_of_same_size() -> Result<()> {
|
||||
let root = TempDir::new()?;
|
||||
let files = [
|
||||
(root.path().join("fileone.bin"), generate_bytes(282624)),
|
||||
(root.path().join("filetwo.bin"), generate_bytes(282624)),
|
||||
];
|
||||
|
||||
for (fpath, content) in files.iter() {
|
||||
let mut f = File::create_new(fpath)?;
|
||||
f.write_all(content)?;
|
||||
}
|
||||
|
||||
let file_queue = Arc::new(Mutex::new(
|
||||
files
|
||||
.iter()
|
||||
.map(|f| FileInfo::new(f.0.clone()).unwrap())
|
||||
.collect::<Vec<FileInfo>>(),
|
||||
));
|
||||
|
||||
let dupstore = Arc::new(DashMap::new());
|
||||
|
||||
Processor::sizewise(
|
||||
Arc::new(Params::default()),
|
||||
Arc::new(AtomicBool::new(true)),
|
||||
dupstore.clone(),
|
||||
file_queue,
|
||||
Arc::new(MultiProgress::new()),
|
||||
)?;
|
||||
|
||||
assert_eq!(dupstore.len(), 1);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
134
src/scanner.rs
134
src/scanner.rs
@@ -1,120 +1,41 @@
|
||||
#![allow(unused)]
|
||||
use crate::{fileinfo::FileInfo, params::Params};
|
||||
use anyhow::Result;
|
||||
use indicatif::{ProgressBar, ProgressStyle};
|
||||
use std::{fs, path::PathBuf, time::Duration};
|
||||
use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::{path::Path, time::Duration};
|
||||
|
||||
use globwalk::{GlobWalker, GlobWalkerBuilder};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Scanner {
|
||||
pub directory: Option<PathBuf>,
|
||||
pub directory: Box<Path>,
|
||||
pub filetypes: Option<String>,
|
||||
pub min_depth: Option<usize>,
|
||||
pub max_depth: Option<usize>,
|
||||
pub min_size: Option<u64>,
|
||||
pub follow_links: bool,
|
||||
pub progress: bool,
|
||||
}
|
||||
|
||||
impl Scanner {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
directory: None,
|
||||
filetypes: None,
|
||||
min_depth: None,
|
||||
max_depth: None,
|
||||
min_size: None,
|
||||
follow_links: true,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn build(app_args: &Params) -> Result<Self> {
|
||||
let scan_directory = app_args.get_directory()?;
|
||||
Ok(Scanner::new())
|
||||
.map(|scanner| scanner.directory(scan_directory))
|
||||
.map(|scanner| match app_args.get_min_size() {
|
||||
Some(min_size) => scanner.min_size(min_size),
|
||||
None => scanner,
|
||||
pub fn new(app_args: Arc<Params>) -> Result<Self> {
|
||||
Ok(Self {
|
||||
directory: app_args.get_directory()?.into_boxed_path(),
|
||||
filetypes: app_args.get_types(),
|
||||
min_depth: app_args.min_depth,
|
||||
max_depth: app_args.max_depth,
|
||||
min_size: app_args.get_min_size(),
|
||||
follow_links: app_args.follow_links,
|
||||
progress: app_args.progress,
|
||||
})
|
||||
.map(|scanner| match app_args.get_types() {
|
||||
Some(ftypes) => scanner.filetypes(ftypes),
|
||||
None => scanner,
|
||||
})
|
||||
.map(|scanner| match app_args.min_depth {
|
||||
Some(min_depth) => scanner.min_depth(min_depth),
|
||||
None => scanner,
|
||||
})
|
||||
.map(|scanner| match app_args.max_depth {
|
||||
Some(max_depth) => scanner.max_depth(max_depth),
|
||||
None => scanner,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn min_size(&self, min_size: u64) -> Self {
|
||||
Self {
|
||||
min_size: Some(min_size),
|
||||
..self.clone()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn min_depth(&self, min_depth: usize) -> Self {
|
||||
Self {
|
||||
min_depth: Some(min_depth),
|
||||
..self.clone()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn max_depth(&self, max_depth: usize) -> Self {
|
||||
Self {
|
||||
max_depth: Some(max_depth),
|
||||
..self.clone()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn directory(&self, dir: PathBuf) -> Self {
|
||||
Self {
|
||||
directory: Some(dir),
|
||||
..self.clone()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn filetypes(&self, patterns: String) -> Self {
|
||||
Self {
|
||||
filetypes: Some(patterns),
|
||||
..self.clone()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn ignore_links(&self) -> Self {
|
||||
Self {
|
||||
follow_links: false,
|
||||
..self.clone()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn follow_links(&self) -> Self {
|
||||
Self {
|
||||
follow_links: true,
|
||||
..self.clone()
|
||||
}
|
||||
}
|
||||
|
||||
fn scan_patterns(&self) -> Result<String> {
|
||||
Ok(match self.filetypes.clone() {
|
||||
Ok(match &self.filetypes {
|
||||
Some(ftypes) => format!("**/*{{{ftypes}}}"),
|
||||
None => "**/*".to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
fn scan_dir(&self) -> Result<PathBuf> {
|
||||
let scan_dir = match self.directory.clone() {
|
||||
Some(path) => path,
|
||||
None => std::env::current_dir()?,
|
||||
};
|
||||
|
||||
Ok(fs::canonicalize(scan_dir)?)
|
||||
}
|
||||
|
||||
fn attach_link_opts(&self, walker: GlobWalkerBuilder) -> Result<GlobWalkerBuilder> {
|
||||
Ok(walker.follow_links(self.follow_links))
|
||||
}
|
||||
@@ -134,7 +55,7 @@ impl Scanner {
|
||||
}
|
||||
fn build_walker(&self) -> Result<GlobWalker> {
|
||||
let walker = Ok(GlobWalkerBuilder::from_patterns(
|
||||
self.scan_dir()?,
|
||||
self.directory.clone(),
|
||||
&[self.scan_patterns()?],
|
||||
))
|
||||
.and_then(|walker| self.attach_walker_min_depth(walker))
|
||||
@@ -144,16 +65,23 @@ impl Scanner {
|
||||
Ok(walker.build()?)
|
||||
}
|
||||
|
||||
pub fn scan(&self) -> Result<Vec<FileInfo>> {
|
||||
pub fn scan(
|
||||
&self,
|
||||
files: Arc<Mutex<Vec<FileInfo>>>,
|
||||
progress_bar_box: Arc<MultiProgress>,
|
||||
) -> Result<()> {
|
||||
let progress_bar = match self.progress {
|
||||
true => progress_bar_box.add(ProgressBar::new_spinner()),
|
||||
false => ProgressBar::hidden(),
|
||||
};
|
||||
|
||||
let progress_style = ProgressStyle::with_template("[{elapsed_precise}] {pos:>7} {msg}")?;
|
||||
let progress_bar = ProgressBar::new_spinner();
|
||||
progress_bar.set_style(progress_style);
|
||||
progress_bar.enable_steady_tick(Duration::from_millis(50));
|
||||
progress_bar.set_message("paths mapped");
|
||||
let min_size = self.min_size.unwrap_or_default();
|
||||
|
||||
let results = self
|
||||
.build_walker()?
|
||||
self.build_walker()?
|
||||
.filter_map(Result::ok)
|
||||
.map(|entity| entity.into_path())
|
||||
.inspect(|_path| progress_bar.inc(1))
|
||||
@@ -161,10 +89,12 @@ impl Scanner {
|
||||
.map(FileInfo::new)
|
||||
.filter_map(Result::ok)
|
||||
.filter(|file| file.size > min_size)
|
||||
.collect::<Vec<FileInfo>>();
|
||||
.for_each(|file| {
|
||||
let mut flock = files.lock().unwrap();
|
||||
flock.push(file);
|
||||
});
|
||||
|
||||
progress_bar.finish_with_message("paths mapped");
|
||||
|
||||
Ok(results)
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
99
src/server.rs
Normal file
99
src/server.rs
Normal file
@@ -0,0 +1,99 @@
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use crate::processor::Processor;
|
||||
use crate::scanner::Scanner;
|
||||
use anyhow::Result;
|
||||
use dashmap::DashMap;
|
||||
use indicatif::{MultiProgress, ProgressDrawTarget};
|
||||
use rand::Rng;
|
||||
use threadpool::ThreadPool;
|
||||
|
||||
use crate::fileinfo::FileInfo;
|
||||
use crate::params::Params;
|
||||
|
||||
pub struct Server {
|
||||
filequeue: Arc<Mutex<Vec<FileInfo>>>,
|
||||
sw_duplicate_set: Arc<DashMap<u64, Vec<FileInfo>>>,
|
||||
pub hw_duplicate_set: Arc<DashMap<u128, Vec<FileInfo>>>,
|
||||
threadpool: ThreadPool,
|
||||
app_args: Arc<Params>,
|
||||
pub max_file_path_len: Arc<AtomicU64>,
|
||||
}
|
||||
|
||||
impl Server {
|
||||
pub fn new(opts: Params) -> Self {
|
||||
Self {
|
||||
filequeue: Arc::new(Mutex::new(Vec::new())),
|
||||
sw_duplicate_set: Arc::new(DashMap::new()),
|
||||
hw_duplicate_set: Arc::new(DashMap::new()),
|
||||
threadpool: ThreadPool::new(4),
|
||||
app_args: Arc::new(opts),
|
||||
max_file_path_len: Arc::new(AtomicU64::new(0)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn start(&self) -> Result<()> {
|
||||
let progbarbox = Arc::new(MultiProgress::new());
|
||||
let mut rng = rand::rng();
|
||||
let seed: i64 = rng.random();
|
||||
|
||||
if !self.app_args.progress {
|
||||
progbarbox.set_draw_target(ProgressDrawTarget::hidden());
|
||||
}
|
||||
|
||||
let app_args_clone_for_sc = self.app_args.clone();
|
||||
let app_args_clone_for_pr = self.app_args.clone();
|
||||
let file_queue_clone_sc = self.filequeue.clone();
|
||||
let file_queue_clone_pr = self.filequeue.clone();
|
||||
let scanner_finished = Arc::new(AtomicBool::new(false));
|
||||
|
||||
let sfin_sc_tr_cl = scanner_finished.clone();
|
||||
let sfin_pr_tr_cl = scanner_finished.clone();
|
||||
|
||||
let store_dupl_sw_for_sw = self.sw_duplicate_set.clone();
|
||||
let store_dupl_sw_for_hw = self.sw_duplicate_set.clone();
|
||||
let store_dupl_hw = self.hw_duplicate_set.clone();
|
||||
let max_file_path_len_clone = self.max_file_path_len.clone();
|
||||
|
||||
let progbarbox_sc_clone = progbarbox.clone();
|
||||
|
||||
self.threadpool.execute(move || {
|
||||
Scanner::new(app_args_clone_for_sc)
|
||||
.unwrap()
|
||||
.scan(file_queue_clone_sc, progbarbox_sc_clone)
|
||||
.unwrap();
|
||||
|
||||
sfin_sc_tr_cl.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||
});
|
||||
|
||||
let progbarbox_pr_clone = progbarbox.clone();
|
||||
|
||||
self.threadpool.execute(move || {
|
||||
Processor::sizewise(
|
||||
app_args_clone_for_pr.clone(),
|
||||
sfin_pr_tr_cl,
|
||||
store_dupl_sw_for_sw,
|
||||
file_queue_clone_pr,
|
||||
progbarbox_pr_clone.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
Processor::hashwise(
|
||||
app_args_clone_for_pr,
|
||||
store_dupl_sw_for_hw,
|
||||
store_dupl_hw,
|
||||
progbarbox_pr_clone,
|
||||
max_file_path_len_clone,
|
||||
seed,
|
||||
)
|
||||
.unwrap();
|
||||
});
|
||||
|
||||
progbarbox.clear()?;
|
||||
|
||||
self.threadpool.join();
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user