From 58e33cc8da2a37535d7a4013b7c2f8c8926bf37f Mon Sep 17 00:00:00 2001 From: Sreedev Kodichath Date: Sun, 13 Jul 2025 19:03:14 -0400 Subject: [PATCH] v0.3 (#61) - [x] parallelization - [x] (scanning) + (processing sw & processing hw & formatting & printing) - [x] reduce cloning values on the heap - [x] add a partial hashing mode (--strict) - [x] add unit tests - [x] add silent mode - [x] update documentation - [x] remove color output - [x] progress bar improvements - [x] use progress bar groups - [x] remove broken json rendering - [x] add benchmarks --- Cargo.lock | 212 +++++++++++++++++---- Cargo.toml | 40 ++-- README.md | 184 ++++++++++-------- Rakefile | 1 + rakelib/benchmark.rake | 93 ++++++++++ src/fileinfo.rs | 50 ++--- src/formatter.rs | 80 ++++---- src/interactive.rs | 209 +++++++++++---------- src/main.rs | 28 +-- src/params.rs | 17 +- src/processor.rs | 412 +++++++++++++++++++++++++++++++++-------- src/scanner.rs | 142 ++++---------- src/server.rs | 99 ++++++++++ 13 files changed, 1085 insertions(+), 482 deletions(-) create mode 100644 Rakefile create mode 100644 rakelib/benchmark.rake create mode 100644 src/server.rs diff --git a/Cargo.lock b/Cargo.lock index 2551cc9..c7a2c07 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -274,7 +274,7 @@ dependencies = [ [[package]] name = "deduplicator" -version = "0.2.2" +version = "0.3.0" dependencies = [ "anyhow", "bytesize", @@ -284,14 +284,13 @@ dependencies = [ "globwalk", "gxhash", "indicatif", - "itertools", "memmap2", "pathdiff", "prettytable-rs", + "rand", "rayon", - "serde", - "serde_json", - "unicode-segmentation", + "tempfile", + "threadpool", ] [[package]] @@ -333,6 +332,22 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" +[[package]] +name = "errno" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + [[package]] name = "getrandom" version = "0.2.15" @@ -341,7 +356,19 @@ checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", ] [[package]] @@ -395,6 +422,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + [[package]] name = "iana-time-zone" version = "0.1.60" @@ -463,7 +496,7 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", "windows-sys", ] @@ -474,15 +507,6 @@ version = "1.70.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.11" @@ -506,9 +530,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.155" +version = "0.2.174" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" [[package]] name = "libredox" @@ -520,6 +544,12 @@ dependencies = [ "libc", ] +[[package]] +name = "linux-raw-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" + [[package]] name = "lock_api" version = "0.4.12" @@ -560,6 +590,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi 0.5.2", + "libc", +] + [[package]] name = "number_prefix" version = "0.4.0" @@ -597,6 +637,15 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + [[package]] name = "prettytable-rs" version = "0.10.0" @@ -629,6 +678,41 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.3", +] + [[package]] name = "rayon" version = "1.10.0" @@ -664,7 +748,7 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" dependencies = [ - "getrandom", + "getrandom 0.2.15", "libredox", "thiserror", ] @@ -686,6 +770,19 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +[[package]] +name = "rustix" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" +dependencies = [ + "bitflags 2.6.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + [[package]] name = "rustversion" version = "1.0.17" @@ -733,17 +830,6 @@ dependencies = [ "syn", ] -[[package]] -name = "serde_json" -version = "1.0.120" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" -dependencies = [ - "itoa", - "ryu", - "serde", -] - [[package]] name = "smallvec" version = "1.13.2" @@ -767,6 +853,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tempfile" +version = "3.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" +dependencies = [ + "fastrand", + "getrandom 0.3.3", + "once_cell", + "rustix", + "windows-sys", +] + [[package]] name = "term" version = "0.7.0" @@ -798,18 +897,21 @@ dependencies = [ "syn", ] +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + [[package]] name = "unicode-ident" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" -[[package]] -name = "unicode-segmentation" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" - [[package]] name = "unicode-width" version = "0.1.13" @@ -838,6 +940,15 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "wasm-bindgen" version = "0.2.92" @@ -1004,3 +1115,32 @@ name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags 2.6.0", +] + +[[package]] +name = "zerocopy" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml index 17b0944..b6b5453 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,18 +1,21 @@ [package] name = "deduplicator" -version = "0.2.2" +version = "0.3.0" edition = "2021" -description = "find,filter,delete Duplicates" +description = "find,filter and delete duplicate files" repository = "https://github.com/sreedevk/deduplicator" license = "MIT" authors = [ - "Sreedev Kodichath ", - "Valentin Bersier ", - "Dhruva Sagar ", + "Sreedev Kodichath ", + "Valentin Bersier ", + "Dhruva Sagar ", ] -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[[bin]] +name = "deduplicator" +path = "src/main.rs" +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] anyhow = "1.0.68" bytesize = "1.1.0" @@ -20,27 +23,36 @@ chrono = "0.4.23" clap = { version = "4.0.32", features = ["derive"] } dashmap = { version = "5.4.0", features = ["rayon"] } globwalk = "0.8.1" -gxhash = "3.4.1" +gxhash = { version = "3.4.1", default-features = false } indicatif = { version = "0.17.2", features = ["rayon"] } -itertools = "0.10.5" memmap2 = "0.5.8" pathdiff = "0.2.1" prettytable-rs = "0.10.0" +rand = "0.9.1" rayon = "1.6.1" -serde = { version = "1.0.192", features = ["derive"] } -serde_json = "1.0.108" -unicode-segmentation = "1.10.0" +threadpool = "1.8.1" [profile.release] strip = true +opt-level = 3 +lto = "thin" +debug = false +codegen-units = 1 # generated by 'cargo dist init' [profile.dist] inherits = "release" -lto = "thin" [workspace.metadata.dist] -rust-toolchain-version = "1.78.0" +rust-toolchain-version = "1.87.0" ci = ["github"] -targets = ["x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc", "aarch64-apple-darwin"] +targets = [ + "x86_64-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", + "aarch64-apple-darwin", +] cargo-dist-version = "0.0.7" + +[dev-dependencies] +tempfile = "3.20.0" diff --git a/README.md b/README.md index 8c07d80..1e7a0e7 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,8 @@ ## Usage ```bash +find,filter and delete duplicate files + Usage: deduplicator [OPTIONS] [scan_dir_path] Arguments: @@ -15,13 +17,14 @@ Arguments: Options: -t, --types Filetypes to deduplicate [default = all] -i, --interactive Delete files interactively - -s, --min-size Minimum filesize of duplicates to scan (e.g., 100B/1K/2M/3G/4T) [default: 1b] - -d, --max-depth Max Depth to scan while looking for duplicates - --min-depth Min Depth to scan while looking for duplicates + -m, --min-size Minimum filesize of duplicates to scan (e.g., 100B/1K/2M/3G/4T) [default: 1b] + -D, --max-depth Max Depth to scan while looking for duplicates + -d, --min-depth Min Depth to scan while looking for duplicates -f, --follow-links Follow links while scanning directories - -h, --help Print help information - -V, --version Print version information - --json + -s, --strict Guarantees that two files are duplicate (performs a full hash) + -p, --progress Show Progress spinners & metrics + -h, --help Print help + -V, --version Print version ``` ### Examples @@ -42,93 +45,118 @@ deduplicator ~/.config --follow-links deduplicator ~/Media --min-size 100mb ``` +## Demo +![record](https://github.com/user-attachments/assets/fcfdd9bf-4d05-41b6-a82e-367634eeaa73) + + + ## Installation +Currently, you can only install deduplicator using cargo package manager. -### Cargo Install - -#### Stable - -> [!WARNING] Note from GxHash: GxHash relies on aes hardware acceleration, you must make sure the aes feature is enabled when building (otherwise it won't build). This can be done by setting the RUSTFLAGS environment variable to -C target-feature=+aes or -C target-cpu=native (the latter should work if your CPU is properly recognized by rustc, which is the case most of the time). -> please install version `0.2.1` if you are unable to install `0.2.2` +### Cargo +> GxHash relies on aes hardware acceleration, so please set `RUSTFLAGS` to `"-C target-feature=+aes"` or `"-C target-cpu=native"` before +> installing. +#### install from crates.io ```bash $ RUSTFLAGS="-C target-cpu=native" cargo install deduplicator ``` -> [!] - -#### Nightly - -if you'd like to install with nightly features, you can use - +#### install from git ```bash -$ cargo install --git https://github.com/sreedevk/deduplicator +$ RUSTFLAGS="-C target-cpu=native" cargo install deduplicator --git https://github.com/sreedevk/deduplicator ``` -Please note that if you use a version manager to install rust (like asdf), you need to reshim (`asdf reshim rust`). - -### Linux (Pre-built Binary) - -you can download the pre-built binary from the [Releases](https://github.com/sreedevk/deduplicator/releases) page. -download the `deduplicator-x86_64-unknown-linux-gnu.tar.gz` for linux. Once you have the tarball file with the executable, -you can follow these steps to install: - -```bash -$ tar -zxvf deduplicator-x86_64-unknown-linux-gnu.tar.gz -$ sudo mv deduplicator /usr/bin/ -``` - -### Mac OS (Pre-built Binary) - -you can download the pre-build binary from the [Releases](https://github.com/sreedevk/deduplicator/releases) page. -download the `deduplicator-x86_64-apple-darwin.tar.gz` tarball for mac os. Once you have the tarball file with the executable, you can follow these steps to install: - -```bash -$ tar -zxvf deduplicator-x86_64-unknown-linux-gnu.tar.gz -$ sudo mv deduplicator /usr/bin/ -``` - -### Windows (Pre-built Binary) - -you can download the pre-build binary from the [Releases](https://github.com/sreedevk/deduplicator/releases) page. -download the `deduplicator-x86_64-pc-windows-msvc.zip` zip file for windows. unzip the `zip` file & move the `deduplicator.exe` to a location in the PATH system environment variable. - -Note: If you Run into an msvc error, please install MSCV from [here](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist?view=msvc-170) ## Performance +Deduplicator uses size comparison and [GxHash](https://docs.rs/gxhash/latest/gxhash/) to quickly check a large number of files to find duplicates. its also heavily parallelized. The default behavior of deduplicator is to only hash the first page (4K) of the file. This is to ensure that performance is the default priority. You can modify this behavior by using the `--strict` flag which will hash the whole file and ensure that 2 files are indeed duplicates. I'll add benchmarks in future versions. -Deduplicator uses size comparison and fxhash (a non non-cryptographic hashing algo) to quickly scan through large number of files to find duplicates. its also highly parallel (uses rayon and dashmap). I was able to scan through 120GB of files (Videos, PDFs, Images) in ~300ms. checkout the benchmarks - -## benchmarks - -| Command | Dirsize | Filecount | Mean [ms] | Min [ms] | Max [ms] | Relative | -|:---|:---|---:|---:|---:|---:|---:| -| `deduplicator ~/Data/tmp` | (~120G) | 721 files | 33.5 ± 28.6 | 25.3 | 151.5 | 1.87 ± 1.60 | -| `deduplicator ~/Data/books` | (~8.6G) | 1419 files | 24.5 ± 1.0 | 22.9 | 28.1 | 1.37 ± 0.08 | -| `deduplicator ~/Data/books --min-size 10M` | (~8.6G) | 1419 files | 17.9 ± 0.7 | 16.8 | 20.0 | 1.00 | -| `deduplicator ~/Data/ --types pdf,jpg,png,jpeg` | (~290G) | 104222 files | 1207.2 ± 37.0 | 1172.2 | 1287.7 | 67.27 ± 3.33 | - -* The last entry is lower because of the number of files deduplicator had to go through (~660895 Files). The average size of the files rarely affect the performance of deduplicator. - -These benchmarks were run using [hyperfine](https://github.com/sharkdp/hyperfine). Here are the specs of the machine used to benchmark deduplicator: +### Benchmarks +I've used hyperfine to run deduplicator on files generated by the rake file at `rakelib/benchmark.rake`. The Benchmarking accuracy can further be improved by isolating runs inside restricted docker containers. I'll include that in the future. For now, here's the hyperfine output on my i7-12800H laptop with 32G of RAM. +#### Fewer Large Files ``` -OS: Arch Linux x86_64 -Host: Precision 5540 -Kernel: 5.15.89-1-lts -Uptime: 4 hours, 44 mins -Shell: zsh 5.9 -Terminal: kitty -CPU: Intel i9-9880H (16) @ 4.800GHz -GPU: NVIDIA Quadro T2000 Mobile / Max-Q -GPU: Intel CoffeeLake-H GT2 [UHD Graphics 630] -Memory: 31731MiB (~32GiB) +# hyperfine -N --warmup 80 './target/release/deduplicator bench_artifacts' +Benchmark 1: ./target/release/deduplicator bench_artifacts + Time (mean ± σ): 2.5 ms ± 0.4 ms [User: 2.2 ms, System: 4.8 ms] + Range (min … max): 1.9 ms … 6.8 ms 1322 runs + +# dust 'bench_artifacts' + 37M ┌── file_1_fwds.bin │██ │ 1% +201M ├── file_0_fwds.bin │██████████ │ 8% +390M ├── file_0_fwdcbss.bin│████████████████████ │ 15% +390M ├── file_0_fwscas.bin │████████████████████ │ 15% +390M ├── file_0_fwss.bin │████████████████████ │ 15% +390M ├── file_1_fwdcbss.bin│████████████████████ │ 15% +390M ├── file_1_fwscas.bin │████████████████████ │ 15% +390M ├── file_1_fwss.bin │████████████████████ │ 15% +2.5G ┌─┴ bench_artifacts │███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ │ 100% ``` -## Screenshots +#### Many Small Files +``` +# hyperfine --warmup 20 './target/release/deduplicator bench_artifacts' +Benchmark 1: ./target/release/deduplicator bench_artifacts + Time (mean ± σ): 22.3 ms ± 1.7 ms [User: 35.8 ms, System: 46.3 ms] + Range (min … max): 18.9 ms … 27.2 ms 112 runs -![](https://user-images.githubusercontent.com/36154121/213618143-e5182e39-731e-4817-87dd-1a6a0f38a449.gif) +# dust 'bench_artifacts' +3.9M ┌── file_992_fwss.bin │█ │ 0% +3.9M ├── file_993_fwdcbss.bin│█ │ 0% +3.9M ├── file_993_fwscas.bin │█ │ 0% +3.9M ├── file_993_fwss.bin │█ │ 0% +3.9M ├── file_994_fwdcbss.bin│█ │ 0% +3.9M ├── file_994_fwscas.bin │█ │ 0% +3.9M ├── file_994_fwss.bin │█ │ 0% +3.9M ├── file_995_fwdcbss.bin│█ │ 0% +3.9M ├── file_995_fwscas.bin │█ │ 0% +3.9M ├── file_995_fwss.bin │█ │ 0% +3.9M ├── file_996_fwdcbss.bin│█ │ 0% +3.9M ├── file_996_fwscas.bin │█ │ 0% +3.9M ├── file_996_fwss.bin │█ │ 0% +3.9M ├── file_997_fwdcbss.bin│█ │ 0% +3.9M ├── file_997_fwscas.bin │█ │ 0% +3.9M ├── file_997_fwss.bin │█ │ 0% +3.9M ├── file_998_fwdcbss.bin│█ │ 0% +3.9M ├── file_998_fwscas.bin │█ │ 0% +3.9M ├── file_998_fwss.bin │█ │ 0% +3.9M ├── file_999_fwdcbss.bin│█ │ 0% +3.9M ├── file_999_fwscas.bin │█ │ 0% +3.9M ├── file_999_fwss.bin │█ │ 0% +3.9M ├── file_99_fwdcbss.bin │█ │ 0% +3.9M ├── file_99_fwscas.bin │█ │ 0% +3.9M ├── file_99_fwss.bin │█ │ 0% +3.9M ├── file_9_fwdcbss.bin │█ │ 0% +3.9M ├── file_9_fwscas.bin │█ │ 0% +3.9M ├── file_9_fwss.bin │█ │ 0% + 11G ┌─┴ bench_artifacts │█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ │ 100% +``` -## Roadmap - - Tree format output for duplicate file listing - - GUI - - Packages for different operating system repositories (currently only installable via cargo) +## proposed +- [ ] parallelization + - [ ] (scanning + processing sw + processing hw) & formatting & printing + - [ ] scanning + processing sw + processing hw + formatting + printing +- [ ] max file path size should use the last set of duplicates +- [ ] add more unit tests + - [ ] test against different filesystems + - [ ] test against different file name encodings +- [ ] restore json output (was removed in 0.3) +- [ ] fix memory leak on very large filesystems + - [ ] maybe use a bloom filter + - [ ] reduce FileInfo size +- [ ] output in a tree format +- [ ] tui +- [ ] change the default hashing method to include the first & last page of a file (8K) + +## v0.3 +- [x] parallelization + - [x] (scanning) + (processing sw & processing hw & formatting & printing) +- [x] reduce cloning values on the heap +- [x] add a partial hashing mode (--strict) +- [x] add unit tests +- [x] add silent mode +- [x] update documentation +- [x] remove color output +- [x] progress bar improvements + - [x] use progress bar groups +- [x] remove broken json rendering +- [x] add benchmarks diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..784b7fb --- /dev/null +++ b/Rakefile @@ -0,0 +1 @@ +# tasks are contained in the `rakelib` directory diff --git a/rakelib/benchmark.rake b/rakelib/benchmark.rake new file mode 100644 index 0000000..0ce28c2 --- /dev/null +++ b/rakelib/benchmark.rake @@ -0,0 +1,93 @@ +require 'tempfile' +require 'fileutils' +require 'securerandom' + +namespace :benchmark do + file 'target/release/deduplicator' do + sh "cargo build --release" + end + + task :few_large_files => 'target/release/deduplicator' do + # Benchmark 1: ./target/release/deduplicator bench_artifacts + # Time (mean ± σ): 2.5 ms ± 0.6 ms [User: 2.4 ms, System: 4.8 ms] + # Range (min … max): 1.8 ms … 9.7 ms 1474 runs + + root = "bench_artifacts" + Dir.mkdir(root) + + # files with same size + 2.times.map do |i| + File.open(File.join(root, "file_#{i}_fwss.bin"), 'wb') do |f| + f.write(SecureRandom.bytes(4096 * 100_000)) + end + end + + # files with different sizes + 2.times.map do |i| + File.open(File.join(root, "file_#{i}_fwds.bin"), 'wb') do |f| + f.write(SecureRandom.bytes(4096 * (rand * 100_000).ceil)) + end + end + + # files with same content & size + 2.times.each do |i| + File.open(File.join(root, "file_#{i}_fwscas.bin"), 'wb') do |f| + f.write("\0" * (4096 * 100_000)) + end + end + + # files with different content but same size + 2.times.each do |i| + File.open(File.join(root, "file_#{i}_fwdcbss.bin"), 'wb') do |f| + f.write(SecureRandom.bytes(4096 * 100_000)) + end + end + + sh("hyperfine -N --warmup 80 './target/release/deduplicator #{root}'") + sh("dust '#{root}'") + + FileUtils.rm_rf(root) + end + + task :many_small_files => 'target/release/deduplicator' do + # Benchmark 1: ./target/release/deduplicator bench_artifacts + # Time (mean ± σ): 10.6 ms ± 1.0 ms [User: 20.0 ms, System: 22.5 ms] + # Range (min … max): 8.4 ms … 14.2 ms 235 runs + + root = "bench_artifacts" + Dir.mkdir(root) + + # files with same size + 1000.times.each do |i| + File.open(File.join(root, "file_#{i}_fwss.bin"), 'wb') do |f| + f.write(SecureRandom.bytes(4096 * 1000)) + end + end + + # files with different sizes + 1000.times.each do |i| + File.open(File.join(root, "file_#{i}_fwds.bin"), 'wb') do |f| + f.write(SecureRandom.bytes(4096 * (rand * 100).ceil)) + end + end + + # files with same content & size + 1000.times.each do |i| + File.open(File.join(root, "file_#{i}_fwscas.bin"), 'wb') do |f| + f.write("\0" * (4096 * 1000)) + end + end + + # files with different content but same size + 1000.times.each do |i| + File.open(File.join(root, "file_#{i}_fwdcbss.bin"), 'wb') do |f| + f.write(SecureRandom.bytes(4096 * 1000)) + end + end + + sh("hyperfine --warmup 20 './target/release/deduplicator #{root}'") + sh("dust '#{root}'") + + FileUtils.rm_rf(root) + end +end diff --git a/src/fileinfo.rs b/src/fileinfo.rs index bb74f06..df907f3 100644 --- a/src/fileinfo.rs +++ b/src/fileinfo.rs @@ -1,43 +1,45 @@ use anyhow::Result; -use gxhash::GxHasher; +use gxhash::gxhash128; use memmap2::Mmap; -use serde::Serialize; -use std::fs; -use std::hash::Hasher; -use std::{fs::Metadata, path::PathBuf}; +use std::{ + fs, + io::Read, + path::{Path, PathBuf}, + time::SystemTime, +}; -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone)] pub struct FileInfo { - pub path: PathBuf, - pub hash: Option, + pub path: Box, pub size: u64, - #[serde(skip)] - pub filemeta: Metadata, + pub modified: SystemTime, } impl FileInfo { - pub fn hash(&self) -> Result { - let file = fs::File::open(self.path.clone())?; + pub fn hash(&self, seed: i64) -> Result { + let file = fs::File::open(&self.path)?; let mapper = unsafe { Mmap::map(&file)? }; - let mut primhasher = GxHasher::default(); + let final_hash = mapper.chunks(4096).fold(0u128, |acc, chunk: &[u8]| { + acc.wrapping_add(gxhash128(chunk, seed)) + }); - mapper - .chunks(1_000_000) - .for_each(|chunk| primhasher.write(chunk)); + Ok(final_hash) + } - Ok(Self { - hash: Some(primhasher.finish().to_string()), - ..self.clone() - }) + pub fn initial_page_hash(&self, seed: i64) -> Result { + let mut file = fs::File::open(&self.path)?; + let mut buffer = [0; 4096]; + let bytes_read = file.read(&mut buffer)?; + + Ok(gxhash128(&buffer[..bytes_read], seed)) } pub fn new(path: PathBuf) -> Result { - let filemeta = std::fs::metadata(path.clone())?; + let filemeta = std::fs::metadata(&path)?; Ok(Self { - path, - filemeta: filemeta.clone(), - hash: None, + path: path.into_boxed_path(), size: filemeta.len(), + modified: filemeta.modified()?, }) } } diff --git a/src/formatter.rs b/src/formatter.rs index a34ce11..8da9ec0 100644 --- a/src/formatter.rs +++ b/src/formatter.rs @@ -1,16 +1,14 @@ -pub struct Formatter; -use crate::fileinfo::FileInfo; -use crate::params::Params; +use crate::{fileinfo::FileInfo, params::Params}; use anyhow::Result; use chrono::{DateTime, Utc}; use dashmap::DashMap; -use indicatif::{ProgressBar, ProgressFinish, ProgressIterator, ProgressStyle}; +use indicatif::{ParallelProgressIterator, ProgressBar, ProgressFinish, ProgressStyle}; use pathdiff::diff_paths; -use prettytable::{format, row, Table}; -use std::borrow::Cow; -use std::path::PathBuf; -use std::time::Duration; +use prettytable::{format, row, Row, Table}; +use rayon::prelude::*; +use std::{borrow::Cow, path::PathBuf, sync::Arc, time::Duration}; +pub struct Formatter; impl Formatter { pub fn human_path( file: &FileInfo, @@ -18,7 +16,7 @@ impl Formatter { min_path_length: usize, ) -> Result { let base_directory: PathBuf = app_args.get_directory()?; - let relative_path = diff_paths(file.path.clone(), base_directory).unwrap_or_default(); + let relative_path = diff_paths(&file.path, base_directory).unwrap_or_default(); let formatted_path = format!( "{:<0width$}", @@ -34,44 +32,62 @@ impl Formatter { } pub fn human_mtime(file: &FileInfo) -> Result { - let modified_time: DateTime = file.filemeta.modified()?.into(); + let modified_time: DateTime = file.modified.into(); Ok(modified_time.format("%Y-%m-%d %H:%M:%S").to_string()) } - pub fn generate_table(raw: DashMap>, max_path_len: usize, app_args: &Params) -> Result { - let mut output_table = Table::new(); - output_table.set_titles(row!["hash", "duplicates"]); + pub fn gen_sub_tbl(items: Vec, app_args: &Params, max_path_len: u64) -> Table { + let mut inner_table = Table::new(); + inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR); + items.iter().for_each(|file| { + inner_table.add_row(row![ + Self::human_path(file, app_args, max_path_len as usize).unwrap_or_default(), + Self::human_filesize(file).unwrap_or_default(), + Self::human_mtime(file).unwrap_or_default() + ]); + }); + inner_table + } - let progress_style = ProgressStyle::with_template( - "[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}", - )?; + pub fn generate_table( + raw: Arc>>, + mpath_len: u64, + args: &Params, + ) -> Result
{ + let progress_bar = match args.progress { + true => ProgressBar::new_spinner(), + false => ProgressBar::hidden(), + }; - let progress_bar = ProgressBar::new(raw.len() as u64); + let progress_style = ProgressStyle::with_template("[{elapsed_precise}] {pos:>7} {msg}")?; progress_bar.set_style(progress_style); progress_bar.enable_steady_tick(Duration::from_millis(50)); progress_bar.set_message("generating output"); - raw.into_iter() + let rows = raw + .par_iter_mut() .progress_with(progress_bar) .with_finish(ProgressFinish::WithMessage(Cow::from("output generated"))) - .for_each(|(hash, group)| { - let mut inner_table = Table::new(); - inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR); - group.iter().for_each(|file| { - inner_table.add_row(row![ - Self::human_path(file, app_args, max_path_len).unwrap_or_default(), - Self::human_filesize(file).unwrap_or_default(), - Self::human_mtime(file).unwrap_or_default() - ]); - }); - - output_table.add_row(row![hash, inner_table]); - }); + .filter(|i| i.value().len() > 1) + .map(|i| { + row![ + i.key(), + Self::gen_sub_tbl(i.value().to_vec(), args, mpath_len) + ] + }) + .collect::>(); + let mut output_table = Table::new(); + output_table.set_titles(row!["hash", "duplicates"]); + output_table.extend(rows); Ok(output_table) } - pub fn print(raw: DashMap>, max_path_len: usize, app_args: &Params) -> Result<()> { + pub fn print( + raw: Arc>>, + max_path_len: u64, + app_args: &Params, + ) -> Result<()> { if raw.is_empty() { println!("\n\nNo duplicates found matching your search criteria.\n"); return Ok(()); diff --git a/src/interactive.rs b/src/interactive.rs index 5b36282..51dbf6c 100644 --- a/src/interactive.rs +++ b/src/interactive.rs @@ -1,116 +1,125 @@ -use crate::formatter::Formatter; -use crate::{fileinfo::FileInfo, params::Params}; +use crate::{fileinfo::FileInfo, formatter::Formatter, params::Params}; use anyhow::Result; use dashmap::DashMap; use prettytable::{format, row, Table}; -use std::io::{self, Write}; +use std::{ + io::{self, Write}, + sync::Arc, +}; -pub fn scan_group_confirmation() -> Result { - print!("\nconfirm? [y/N]: "); - std::io::stdout().flush()?; - let mut user_input = String::new(); - io::stdin().read_line(&mut user_input)?; +pub struct Interactive; - match user_input.trim() { - "Y" | "y" => Ok(true), - _ => Ok(false), - } -} +impl Interactive { + pub fn init(result: Arc>>, app_args: &Params) -> Result<()> { + result + .clone() + .iter() + .filter(|i| i.value().len() > 1) + .enumerate() + .for_each(|(gindex, i)| { + let group = i.value(); + let mut itable = Table::new(); + itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR); + itable.set_titles(row!["index", "filename", "size", "updated_at"]); -pub fn scan_group_instruction() -> Result { - println!("\nEnter the indices of the files you want to delete."); - println!("You can enter multiple files using commas to seperate file indices."); - println!("example: 1,2"); - print!("\n> "); - std::io::stdout().flush()?; - let mut user_input = String::new(); - io::stdin().read_line(&mut user_input)?; + let max_path_size = group + .iter() + .map(|f| f.path.iter().count()) + .max() + .unwrap_or_default(); - Ok(user_input) -} + group.iter().enumerate().for_each(|(index, file)| { + itable.add_row(row![ + index, + Formatter::human_path(file, app_args, max_path_size).unwrap_or_default(), + Formatter::human_filesize(file).unwrap_or_default(), + Formatter::human_mtime(file).unwrap_or_default() + ]); + }); -pub fn init(result: DashMap>, app_args: &Params) -> Result<()> { - result - .clone() - .into_iter() - .enumerate() - .for_each(|(gindex, (_, group))| { - let mut itable = Table::new(); - itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR); - itable.set_titles(row!["index", "filename", "size", "updated_at"]); - let max_path_size = group - .iter() - .map(|f| f.path.clone().into_os_string().len()) - .max() - .unwrap_or_default(); - - group.iter().enumerate().for_each(|(index, file)| { - itable.add_row(row![ - index, - Formatter::human_path(file, app_args, max_path_size).unwrap_or_default(), - Formatter::human_filesize(file).unwrap_or_default(), - Formatter::human_mtime(file).unwrap_or_default() - ]); + Self::process_group_action(group, gindex, result.len(), itable); }); - process_group_action(&group, gindex, result.len(), itable); - }); - - Ok(()) -} - -pub fn process_group_action( - duplicates: &Vec, - dup_index: usize, - dup_size: usize, - table: Table, -) { - println!("\nDuplicate Set {} of {}\n", dup_index + 1, dup_size); - table.printstd(); - let files_to_delete = scan_group_instruction().unwrap_or_default(); - let parsed_file_indices = files_to_delete - .trim() - .split(',') - .filter(|element| !element.is_empty()) - .map(|index| index.parse::().unwrap_or_default()) - .collect::>(); - - if parsed_file_indices - .clone() - .into_iter() - .any(|index| index > (duplicates.len() - 1)) - { - println!("Err: File Index Out of Bounds!"); - return process_group_action(duplicates, dup_index, dup_size, table); + Ok(()) } - print!("{esc}[2J{esc}[1;1H", esc = 27 as char); + pub fn scan_group_confirmation() -> Result { + print!("\nconfirm? [y/N]: "); + std::io::stdout().flush()?; + let mut user_input = String::new(); + io::stdin().read_line(&mut user_input)?; - if parsed_file_indices.is_empty() { - return; - } - - let files_to_delete = parsed_file_indices - .into_iter() - .map(|index| duplicates[index].clone()); - - println!("\nThe following files will be deleted:"); - files_to_delete - .clone() - .enumerate() - .for_each(|(index, file)| { - println!("{}: {}", index, file.path.display()); - }); - - match scan_group_confirmation().unwrap() { - true => { - files_to_delete.into_iter().for_each(|file| { - match std::fs::remove_file(file.path.clone()) { - Ok(_) => println!("DELETED: {}", file.path.display()), - Err(_) => println!("FAILED: {}", file.path.display()), - } - }); + match user_input.trim() { + "Y" | "y" => Ok(true), + _ => Ok(false), + } + } + + pub fn scan_group_instruction() -> Result { + println!("\nEnter the indices of the files you want to delete."); + println!("You can enter multiple files using commas to seperate file indices."); + println!("example: 1,2"); + print!("\n> "); + std::io::stdout().flush()?; + let mut user_input = String::new(); + io::stdin().read_line(&mut user_input)?; + + Ok(user_input) + } + + pub fn process_group_action( + duplicates: &Vec, + dup_index: usize, + dup_size: usize, + table: Table, + ) { + println!("\nDuplicate Set {} of {}\n", dup_index + 1, dup_size); + table.printstd(); + let files_to_delete = Self::scan_group_instruction().unwrap_or_default(); + let parsed_file_indices = files_to_delete + .trim() + .split(',') + .filter(|element| !element.is_empty()) + .map(|index| index.parse::().unwrap_or_default()) + .collect::>(); + + if parsed_file_indices + .clone() + .into_iter() + .any(|index| index > (duplicates.len() - 1)) + { + println!("Err: File Index Out of Bounds!"); + return Self::process_group_action(duplicates, dup_index, dup_size, table); + } + + print!("{esc}[2J{esc}[1;1H", esc = 27 as char); + + if parsed_file_indices.is_empty() { + return; + } + + let files_to_delete = parsed_file_indices + .into_iter() + .map(|index| duplicates[index].clone()); + + println!("\nThe following files will be deleted:"); + files_to_delete + .clone() + .enumerate() + .for_each(|(index, file)| { + println!("{}: {}", index, file.path.display()); + }); + + match Self::scan_group_confirmation().unwrap() { + true => { + files_to_delete.into_iter().for_each(|file| { + match std::fs::remove_file(file.path.clone()) { + Ok(_) => println!("DELETED: {}", file.path.display()), + Err(_) => println!("FAILED: {}", file.path.display()), + } + }); + } + false => println!("\nCancelled Delete Operation."), } - false => println!("\nCancelled Delete Operation."), } } diff --git a/src/main.rs b/src/main.rs index 7208f3d..9d5a56d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,28 +4,32 @@ mod interactive; mod params; mod processor; mod scanner; +mod server; +use self::{formatter::Formatter, interactive::Interactive, server::Server}; use anyhow::Result; use clap::Parser; -use formatter::Formatter; use params::Params; -use processor::Processor; -use scanner::Scanner; +use std::sync::atomic::Ordering; fn main() -> Result<()> { let app_args = Params::parse(); - let scan_results = Scanner::build(&app_args)?.scan()?; - let mut processor = Processor::new(scan_results); + let server = Server::new(app_args.clone()); - processor.sizewise()?; - processor.hashwise()?; - - let results = processor.hashwise_results; + server.start()?; match app_args.interactive { - false => Formatter::print(results, processor.max_path_len, &app_args)?, - true => interactive::init(results, &app_args)?, - } + false => { + Formatter::print( + server.hw_duplicate_set, + server.max_file_path_len.load(Ordering::Acquire), + &app_args, + )?; + } + true => { + Interactive::init(server.hw_duplicate_set, &app_args)?; + } + }; Ok(()) } diff --git a/src/params.rs b/src/params.rs index 2e302e3..c74013e 100644 --- a/src/params.rs +++ b/src/params.rs @@ -3,7 +3,7 @@ use std::{fs, path::PathBuf}; use anyhow::Result; use clap::{Parser, ValueHint}; -#[derive(Parser, Debug, Clone)] +#[derive(Parser, Debug, Default, Clone)] #[command(author, version, about, long_about = None)] pub struct Params { /// Filetypes to deduplicate [default = all] @@ -16,20 +16,23 @@ pub struct Params { #[arg(long, short)] pub interactive: bool, /// Minimum filesize of duplicates to scan (e.g., 100B/1K/2M/3G/4T). - #[arg(long, short = 's', default_value = "1b")] + #[arg(long, short = 'm', default_value = "1b")] pub min_size: Option, /// Max Depth to scan while looking for duplicates - #[arg(long, short = 'd')] + #[arg(long, short = 'D')] pub max_depth: Option, /// Min Depth to scan while looking for duplicates - #[arg(long)] + #[arg(long, short = 'd')] pub min_depth: Option, /// Follow links while scanning directories #[arg(long, short)] pub follow_links: bool, - /// print json output - #[arg(long)] - pub json: bool, + /// Guarantees that two files are duplicate (performs a full hash) + #[arg(long, short = 's', default_value = "false")] + pub strict: bool, + /// Show Progress spinners & metrics + #[arg(long, short = 'p', default_value = "false")] + pub progress: bool, } impl Params { diff --git a/src/processor.rs b/src/processor.rs index c99fd35..b7ac840 100644 --- a/src/processor.rs +++ b/src/processor.rs @@ -1,102 +1,368 @@ use anyhow::Result; use dashmap::DashMap; -use indicatif::{ParallelProgressIterator, ProgressBar, ProgressFinish, ProgressStyle}; +use indicatif::{ + MultiProgress, ParallelProgressIterator, ProgressBar, ProgressFinish, ProgressStyle, +}; use rayon::prelude::{IntoParallelIterator, ParallelIterator}; -use std::{borrow::Cow, time::Duration}; +use std::borrow::Cow; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, TryLockError, TryLockResult}; +use std::time::Duration; use crate::fileinfo::FileInfo; +use crate::params::Params; -#[derive(Debug, Clone)] -pub struct Processor { - pub files: Vec, - pub hashwise_results: DashMap>, - pub sizewise_results: DashMap>, - pub max_path_len: usize, -} +pub struct Processor {} impl Processor { - pub fn new(files: Vec) -> Self { - Self { - files, - hashwise_results: DashMap::new(), - sizewise_results: DashMap::new(), - max_path_len: 0, - } - } + pub fn hashwise( + app_args: Arc, + sw_store: Arc>>, + hw_store: Arc>>, + progress_bar_box: Arc, + max_file_size: Arc, + seed: i64, + ) -> Result<()> { + let progress_bar = match app_args.progress { + true => progress_bar_box.add(ProgressBar::new_spinner()), + false => ProgressBar::hidden(), + }; - pub fn hashwise(&mut self) -> Result<()> { - if self.sizewise_results.is_empty() { - return Ok(()); - } + let keys: Vec = sw_store.clone().iter().map(|i| *i.key()).collect(); - let progress_style = ProgressStyle::with_template( - "[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}", - )?; - let progress_bar = ProgressBar::new(self.files.len() as u64); + let progress_style = ProgressStyle::with_template("[{elapsed_precise}] {pos:>7} {msg}")?; progress_bar.set_style(progress_style); progress_bar.enable_steady_tick(Duration::from_millis(50)); - progress_bar.set_message("indexing file hashes"); + progress_bar.set_message("files grouped by hash."); - let filelist = self - .sizewise_results - .clone() - .into_read_only() - .values() - .filter(|&subfiles| subfiles.len() > 1) - .flatten() - .cloned() - .collect::>(); - - self.max_path_len = filelist - .iter() - .map(|x| x.path.clone().into_os_string().len()) - .max() - .unwrap_or_default(); - - filelist - .into_par_iter() + keys.into_par_iter() .progress_with(progress_bar) .with_finish(ProgressFinish::WithMessage(Cow::from( - "indexed files hashes", + "files grouped by hash.", ))) - .map(|file| file.hash()) - .filter_map(Result::ok) - .for_each(move |file| { - self.hashwise_results - .entry(file.hash.clone().unwrap_or_default()) - .and_modify(|fileset| fileset.push(file.clone())) - .or_insert_with(|| vec![file]); + .for_each(|key| { + let group: Vec = sw_store.get(&key).unwrap().to_vec(); + if group.len() > 1 { + group.into_par_iter().for_each(|file| { + let fhash = if app_args.strict { + file.hash(seed).expect("hashing file failed.") + } else { + file.initial_page_hash(seed).expect("hashing file failed.") + }; + + Self::compare_and_update_max_path_len( + max_file_size.clone(), + file.path.to_string_lossy().len() as u64, + ) + .unwrap(); + + hw_store + .entry(fhash) + .and_modify(|fileset| fileset.push(file.clone())) + .or_insert_with(|| vec![file]); + }); + } }); Ok(()) } - pub fn sizewise(&mut self) -> Result<()> { - if self.files.is_empty() { - return Ok(()); + pub fn compare_and_update_max_path_len(current: Arc, next: u64) -> Result<()> { + if current.load(Ordering::Relaxed) < next { + current.store(next, Ordering::Release); } - let progress_style = ProgressStyle::with_template( - "[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}", - )?; - let progress_bar = ProgressBar::new(self.files.len() as u64); + Ok(()) + } + + pub fn sizewise( + app_args: Arc, + scanner_finished: Arc, + store: Arc>>, + files: Arc>>, + progress_bar_box: Arc, + ) -> Result<()> { + let progress_bar = match app_args.progress { + true => progress_bar_box.add(ProgressBar::new_spinner()), + false => ProgressBar::hidden(), + }; + + let progress_style = ProgressStyle::with_template("[{elapsed_precise}] {pos:>7} {msg}")?; progress_bar.set_style(progress_style); progress_bar.enable_steady_tick(Duration::from_millis(50)); - progress_bar.set_message("indexing file sizes"); + progress_bar.set_message("files grouped by size"); - self.files - .clone() - .into_par_iter() - .progress_with(progress_bar) - .with_finish(ProgressFinish::WithMessage(Cow::from( - "indexed files sizes", - ))) - .for_each(|file| { - self.sizewise_results - .entry(file.size) - .and_modify(|fileset| fileset.push(file.clone())) - .or_insert_with(|| vec![file]); - }); + loop { + let fileopt: Option = { + match files.try_lock() { + Ok(mut flist) => flist.pop(), + TryLockResult::Err(TryLockError::WouldBlock) => None, + _ => None, + } + }; + + match fileopt { + Some(file) => { + progress_bar.inc(1); + store + .entry(file.size) + .and_modify(|fileset| fileset.push(file.clone())) + .or_insert_with(|| vec![file]); + continue; + } + None => match scanner_finished.load(std::sync::atomic::Ordering::Relaxed) { + true => { + progress_bar.finish_with_message("files grouped by size"); + break Ok(()); + } + false => continue, + }, + } + } + } +} + +#[cfg(test)] +mod tests { + use anyhow::Result; + use dashmap::DashMap; + use indicatif::MultiProgress; + use rand::Rng; + use std::fs::File; + use std::io::Write; + use std::sync::atomic::{AtomicBool, AtomicU64}; + use std::sync::{Arc, Mutex}; + use tempfile::TempDir; + + use crate::{fileinfo::FileInfo, params::Params}; + + use super::Processor; + + fn generate_bytes(size: usize) -> Vec { + let mut rng = rand::rng(); + (0..size).map(|_| rng.random::()).collect::>() + } + + #[test] + fn hashwise_sorting_two_files_with_identical_init_page_only_strict_mode() -> Result<()> { + let root = TempDir::new()?; + let content = generate_bytes(4096); + + let mut content_x = content.clone(); + let mut content_y = content.clone(); + + content_x.extend(generate_bytes(1720320)); + content_y.extend(generate_bytes(1720320)); + + let files = [ + (root.path().join("fileone.bin"), content_x), + (root.path().join("filetwo.bin"), content_y), + ]; + + for (fpath, content) in files.iter() { + let mut f = File::create_new(fpath)?; + f.write_all(content)?; + } + + let dupstore = Arc::new(DashMap::new()); + let file_queue = Arc::new(Mutex::new( + files + .iter() + .map(|f| FileInfo::new(f.0.clone()).unwrap()) + .collect::>(), + )); + + let hw_dupstore = Arc::new(DashMap::new()); + Processor::sizewise( + Arc::new(Params::default()), + Arc::new(AtomicBool::new(true)), + dupstore.clone(), + file_queue, + Arc::new(MultiProgress::new()), + )?; + + let args = Params { + strict: true, + ..Default::default() + }; + + Processor::hashwise( + Arc::new(args), + dupstore.clone(), + hw_dupstore.clone(), + Arc::new(MultiProgress::new()), + Arc::new(AtomicU64::new(32)), + 300, + )?; + + assert_eq!(hw_dupstore.len(), 2); + + Ok(()) + } + + #[test] + fn hashwise_sorting_two_files_with_identical_init_page_only_fast_mode() -> Result<()> { + let root = TempDir::new()?; + let content = generate_bytes(4096); + + let mut content_x = content.clone(); + let mut content_y = content.clone(); + + content_x.extend(generate_bytes(1720320)); + content_y.extend(generate_bytes(1720320)); + + let files = [ + (root.path().join("fileone.bin"), content_x), + (root.path().join("filetwo.bin"), content_y), + ]; + + for (fpath, content) in files.iter() { + let mut f = File::create_new(fpath)?; + f.write_all(content)?; + } + + let dupstore = Arc::new(DashMap::new()); + let file_queue = Arc::new(Mutex::new( + files + .iter() + .map(|f| FileInfo::new(f.0.clone()).unwrap()) + .collect::>(), + )); + + let hw_dupstore = Arc::new(DashMap::new()); + Processor::sizewise( + Arc::new(Params::default()), + Arc::new(AtomicBool::new(true)), + dupstore.clone(), + file_queue, + Arc::new(MultiProgress::new()), + )?; + + Processor::hashwise( + Arc::new(Params::default()), + dupstore.clone(), + hw_dupstore.clone(), + Arc::new(MultiProgress::new()), + Arc::new(AtomicU64::new(32)), + 300, + )?; + + assert_eq!(hw_dupstore.len(), 1); + + Ok(()) + } + + #[test] + fn hashwise_sorting_two_files_with_identical_data() -> Result<()> { + let root = TempDir::new()?; + let content = generate_bytes(282624); + let files = [ + (root.path().join("fileone.bin"), content.clone()), + (root.path().join("filetwo.bin"), content.clone()), + ]; + + for (fpath, content) in files.iter() { + let mut f = File::create_new(fpath)?; + f.write_all(content)?; + } + + let dupstore = Arc::new(DashMap::new()); + let file_queue = Arc::new(Mutex::new( + files + .iter() + .map(|f| FileInfo::new(f.0.clone()).unwrap()) + .collect::>(), + )); + + let hw_dupstore = Arc::new(DashMap::new()); + Processor::sizewise( + Arc::new(Params::default()), + Arc::new(AtomicBool::new(true)), + dupstore.clone(), + file_queue, + Arc::new(MultiProgress::new()), + )?; + + Processor::hashwise( + Arc::new(Params::default()), + dupstore.clone(), + hw_dupstore.clone(), + Arc::new(MultiProgress::new()), + Arc::new(AtomicU64::new(32)), + 300, + )?; + + assert_eq!(hw_dupstore.len(), 1); + + Ok(()) + } + + #[test] + fn sizewise_sorting_two_files_of_different_sizes() -> Result<()> { + let root = TempDir::new()?; + let files = [ + (root.path().join("fileone.bin"), generate_bytes(282624)), + (root.path().join("filetwo.bin"), generate_bytes(1720320)), + ]; + + for (fpath, content) in files.iter() { + let mut f = File::create_new(fpath)?; + f.write_all(content)?; + } + + let file_queue = Arc::new(Mutex::new( + files + .iter() + .map(|f| FileInfo::new(f.0.clone()).unwrap()) + .collect::>(), + )); + + let dupstore = Arc::new(DashMap::new()); + + Processor::sizewise( + Arc::new(Params::default()), + Arc::new(AtomicBool::new(true)), + dupstore.clone(), + file_queue, + Arc::new(MultiProgress::new()), + )?; + + assert_eq!(dupstore.len(), 2); + + Ok(()) + } + + #[test] + fn sizewise_sorting_two_files_of_same_size() -> Result<()> { + let root = TempDir::new()?; + let files = [ + (root.path().join("fileone.bin"), generate_bytes(282624)), + (root.path().join("filetwo.bin"), generate_bytes(282624)), + ]; + + for (fpath, content) in files.iter() { + let mut f = File::create_new(fpath)?; + f.write_all(content)?; + } + + let file_queue = Arc::new(Mutex::new( + files + .iter() + .map(|f| FileInfo::new(f.0.clone()).unwrap()) + .collect::>(), + )); + + let dupstore = Arc::new(DashMap::new()); + + Processor::sizewise( + Arc::new(Params::default()), + Arc::new(AtomicBool::new(true)), + dupstore.clone(), + file_queue, + Arc::new(MultiProgress::new()), + )?; + + assert_eq!(dupstore.len(), 1); Ok(()) } diff --git a/src/scanner.rs b/src/scanner.rs index 78cb95a..0071e0a 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -1,118 +1,39 @@ -#![allow(unused)] use crate::{fileinfo::FileInfo, params::Params}; use anyhow::Result; -use indicatif::{ProgressBar, ProgressStyle}; -use std::{fs, path::PathBuf, time::Duration}; +use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; +use std::sync::{Arc, Mutex}; +use std::{path::Path, time::Duration}; use globwalk::{GlobWalker, GlobWalkerBuilder}; -#[derive(Debug, Clone)] pub struct Scanner { - pub directory: Option, + pub directory: Box, pub filetypes: Option, pub min_depth: Option, pub max_depth: Option, pub min_size: Option, pub follow_links: bool, + pub progress: bool, } impl Scanner { - pub fn new() -> Self { - Self { - directory: None, - filetypes: None, - min_depth: None, - max_depth: None, - min_size: None, - follow_links: true, - } - } - - pub fn build(app_args: &Params) -> Result { - let scan_directory = app_args.get_directory()?; - Ok(Scanner::new()) - .map(|scanner| scanner.directory(scan_directory)) - .map(|scanner| match app_args.get_min_size() { - Some(min_size) => scanner.min_size(min_size), - None => scanner, - }) - .map(|scanner| match app_args.get_types() { - Some(ftypes) => scanner.filetypes(ftypes), - None => scanner, - }) - .map(|scanner| match app_args.min_depth { - Some(min_depth) => scanner.min_depth(min_depth), - None => scanner, - }) - .map(|scanner| match app_args.max_depth { - Some(max_depth) => scanner.max_depth(max_depth), - None => scanner, - }) - } - - pub fn min_size(&self, min_size: u64) -> Self { - Self { - min_size: Some(min_size), - ..self.clone() - } - } - - pub fn min_depth(&self, min_depth: usize) -> Self { - Self { - min_depth: Some(min_depth), - ..self.clone() - } - } - - pub fn max_depth(&self, max_depth: usize) -> Self { - Self { - max_depth: Some(max_depth), - ..self.clone() - } - } - - pub fn directory(&self, dir: PathBuf) -> Self { - Self { - directory: Some(dir), - ..self.clone() - } - } - - pub fn filetypes(&self, patterns: String) -> Self { - Self { - filetypes: Some(patterns), - ..self.clone() - } - } - - pub fn ignore_links(&self) -> Self { - Self { - follow_links: false, - ..self.clone() - } - } - - pub fn follow_links(&self) -> Self { - Self { - follow_links: true, - ..self.clone() - } - } - - fn scan_patterns(&self) -> Result { - Ok(match self.filetypes.clone() { - Some(ftypes) => format!("**/*{{{ftypes}}}"), - None => "**/*".to_string(), + pub fn new(app_args: Arc) -> Result { + Ok(Self { + directory: app_args.get_directory()?.into_boxed_path(), + filetypes: app_args.get_types(), + min_depth: app_args.min_depth, + max_depth: app_args.max_depth, + min_size: app_args.get_min_size(), + follow_links: app_args.follow_links, + progress: app_args.progress, }) } - fn scan_dir(&self) -> Result { - let scan_dir = match self.directory.clone() { - Some(path) => path, - None => std::env::current_dir()?, - }; - - Ok(fs::canonicalize(scan_dir)?) + fn scan_patterns(&self) -> Result { + Ok(match &self.filetypes { + Some(ftypes) => format!("**/*{{{ftypes}}}"), + None => "**/*".to_string(), + }) } fn attach_link_opts(&self, walker: GlobWalkerBuilder) -> Result { @@ -134,7 +55,7 @@ impl Scanner { } fn build_walker(&self) -> Result { let walker = Ok(GlobWalkerBuilder::from_patterns( - self.scan_dir()?, + self.directory.clone(), &[self.scan_patterns()?], )) .and_then(|walker| self.attach_walker_min_depth(walker)) @@ -144,16 +65,23 @@ impl Scanner { Ok(walker.build()?) } - pub fn scan(&self) -> Result> { + pub fn scan( + &self, + files: Arc>>, + progress_bar_box: Arc, + ) -> Result<()> { + let progress_bar = match self.progress { + true => progress_bar_box.add(ProgressBar::new_spinner()), + false => ProgressBar::hidden(), + }; + let progress_style = ProgressStyle::with_template("[{elapsed_precise}] {pos:>7} {msg}")?; - let progress_bar = ProgressBar::new_spinner(); progress_bar.set_style(progress_style); progress_bar.enable_steady_tick(Duration::from_millis(50)); progress_bar.set_message("paths mapped"); let min_size = self.min_size.unwrap_or_default(); - let results = self - .build_walker()? + self.build_walker()? .filter_map(Result::ok) .map(|entity| entity.into_path()) .inspect(|_path| progress_bar.inc(1)) @@ -161,10 +89,12 @@ impl Scanner { .map(FileInfo::new) .filter_map(Result::ok) .filter(|file| file.size > min_size) - .collect::>(); + .for_each(|file| { + let mut flock = files.lock().unwrap(); + flock.push(file); + }); progress_bar.finish_with_message("paths mapped"); - - Ok(results) + Ok(()) } } diff --git a/src/server.rs b/src/server.rs new file mode 100644 index 0000000..6dcb1ac --- /dev/null +++ b/src/server.rs @@ -0,0 +1,99 @@ +use std::sync::atomic::{AtomicBool, AtomicU64}; +use std::sync::{Arc, Mutex}; + +use crate::processor::Processor; +use crate::scanner::Scanner; +use anyhow::Result; +use dashmap::DashMap; +use indicatif::{MultiProgress, ProgressDrawTarget}; +use rand::Rng; +use threadpool::ThreadPool; + +use crate::fileinfo::FileInfo; +use crate::params::Params; + +pub struct Server { + filequeue: Arc>>, + sw_duplicate_set: Arc>>, + pub hw_duplicate_set: Arc>>, + threadpool: ThreadPool, + app_args: Arc, + pub max_file_path_len: Arc, +} + +impl Server { + pub fn new(opts: Params) -> Self { + Self { + filequeue: Arc::new(Mutex::new(Vec::new())), + sw_duplicate_set: Arc::new(DashMap::new()), + hw_duplicate_set: Arc::new(DashMap::new()), + threadpool: ThreadPool::new(4), + app_args: Arc::new(opts), + max_file_path_len: Arc::new(AtomicU64::new(0)), + } + } + + pub fn start(&self) -> Result<()> { + let progbarbox = Arc::new(MultiProgress::new()); + let mut rng = rand::rng(); + let seed: i64 = rng.random(); + + if !self.app_args.progress { + progbarbox.set_draw_target(ProgressDrawTarget::hidden()); + } + + let app_args_clone_for_sc = self.app_args.clone(); + let app_args_clone_for_pr = self.app_args.clone(); + let file_queue_clone_sc = self.filequeue.clone(); + let file_queue_clone_pr = self.filequeue.clone(); + let scanner_finished = Arc::new(AtomicBool::new(false)); + + let sfin_sc_tr_cl = scanner_finished.clone(); + let sfin_pr_tr_cl = scanner_finished.clone(); + + let store_dupl_sw_for_sw = self.sw_duplicate_set.clone(); + let store_dupl_sw_for_hw = self.sw_duplicate_set.clone(); + let store_dupl_hw = self.hw_duplicate_set.clone(); + let max_file_path_len_clone = self.max_file_path_len.clone(); + + let progbarbox_sc_clone = progbarbox.clone(); + + self.threadpool.execute(move || { + Scanner::new(app_args_clone_for_sc) + .unwrap() + .scan(file_queue_clone_sc, progbarbox_sc_clone) + .unwrap(); + + sfin_sc_tr_cl.store(true, std::sync::atomic::Ordering::Relaxed); + }); + + let progbarbox_pr_clone = progbarbox.clone(); + + self.threadpool.execute(move || { + Processor::sizewise( + app_args_clone_for_pr.clone(), + sfin_pr_tr_cl, + store_dupl_sw_for_sw, + file_queue_clone_pr, + progbarbox_pr_clone.clone(), + ) + .unwrap(); + + Processor::hashwise( + app_args_clone_for_pr, + store_dupl_sw_for_hw, + store_dupl_hw, + progbarbox_pr_clone, + max_file_path_len_clone, + seed, + ) + .unwrap(); + }); + + progbarbox.clear()?; + + self.threadpool.join(); + + Ok(()) + } +}