diff --git a/Cargo.lock b/Cargo.lock index aae92e7..c7a2c07 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -81,22 +81,6 @@ version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" -[[package]] -name = "assert_cmd" -version = "2.0.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bd389a4b2970a01282ee455294913c0a43724daedcd1a24c3eb0ec1c1320b66" -dependencies = [ - "anstyle", - "bstr", - "doc-comment", - "libc", - "predicates", - "predicates-core", - "predicates-tree", - "wait-timeout", -] - [[package]] name = "autocfg" version = "1.3.0" @@ -122,7 +106,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" dependencies = [ "memchr", - "regex-automata", "serde", ] @@ -294,7 +277,6 @@ name = "deduplicator" version = "0.3.0" dependencies = [ "anyhow", - "assert_cmd", "bytesize", "chrono", "clap", @@ -311,12 +293,6 @@ dependencies = [ "threadpool", ] -[[package]] -name = "difflib" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" - [[package]] name = "dirs-next" version = "2.0.0" @@ -338,12 +314,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "doc-comment" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" - [[package]] name = "either" version = "1.13.0" @@ -676,33 +646,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "predicates" -version = "3.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5d19ee57562043d37e82899fade9a22ebab7be9cef5026b07fda9cdd4293573" -dependencies = [ - "anstyle", - "difflib", - "predicates-core", -] - -[[package]] -name = "predicates-core" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "727e462b119fe9c93fd0eb1429a5f7647394014cf3c04ab2c0350eeb09095ffa" - -[[package]] -name = "predicates-tree" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72dd2d6d381dfb73a193c7fca536518d7caee39fc8503f74e7dc0be0531b425c" -dependencies = [ - "predicates-core", - "termtree", -] - [[package]] name = "prettytable-rs" version = "0.10.0" @@ -934,12 +877,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "termtree" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" - [[package]] name = "thiserror" version = "1.0.61" @@ -987,15 +924,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" -[[package]] -name = "wait-timeout" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" -dependencies = [ - "libc", -] - [[package]] name = "walkdir" version = "2.5.0" diff --git a/Cargo.toml b/Cargo.toml index 808a1bb..b6b5453 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,6 @@ pathdiff = "0.2.1" prettytable-rs = "0.10.0" rand = "0.9.1" rayon = "1.6.1" -tempfile = "3.20.0" threadpool = "1.8.1" [profile.release] @@ -56,4 +55,4 @@ targets = [ cargo-dist-version = "0.0.7" [dev-dependencies] -assert_cmd = "2.0.0" +tempfile = "3.20.0" diff --git a/README.md b/README.md index 89431c0..2c8aa13 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,67 @@ $ RUSTFLAGS="-C target-cpu=native" cargo install deduplicator --git https://gith ## Performance Deduplicator uses size comparison and [GxHash](https://docs.rs/gxhash/latest/gxhash/) to quickly check a large number of files to find duplicates. its also heavily parallelized. The default behavior of deduplicator is to only hash the first page (4K) of the file. This is to ensure that performance is the default priority. You can modify this behavior by using the `--strict` flag which will hash the whole file and ensure that 2 files are indeed duplicates. I'll add benchmarks in future versions. +### Benchmarks +I've used hyperfine to run deduplicator on files generated by the rake file at `rakelib/benchmark.rake`. The Benchmarking accuracy can further be improved by isolating runs inside restricted docker containers. I'll include that in the future. For now, here's the hyperfine output on my i7-12800H laptop with 32G of RAM. + +#### Fewer Large Files +``` +# hyperfine -N --warmup 80 './target/release/deduplicator bench_artifacts' +Benchmark 1: ./target/release/deduplicator bench_artifacts + Time (mean ± σ): 2.5 ms ± 0.4 ms [User: 2.2 ms, System: 4.8 ms] + Range (min … max): 1.9 ms … 6.8 ms 1322 runs + +# dust 'bench_artifacts' + 37M ┌── file_1_fwds.bin │██ │ 1% +201M ├── file_0_fwds.bin │██████████ │ 8% +390M ├── file_0_fwdcbss.bin│████████████████████ │ 15% +390M ├── file_0_fwscas.bin │████████████████████ │ 15% +390M ├── file_0_fwss.bin │████████████████████ │ 15% +390M ├── file_1_fwdcbss.bin│████████████████████ │ 15% +390M ├── file_1_fwscas.bin │████████████████████ │ 15% +390M ├── file_1_fwss.bin │████████████████████ │ 15% +2.5G ┌─┴ bench_artifacts │███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ │ 100% +``` + +#### Many Small Files +``` +# hyperfine --warmup 20 './target/release/deduplicator bench_artifacts' +Benchmark 1: ./target/release/deduplicator bench_artifacts + Time (mean ± σ): 22.3 ms ± 1.7 ms [User: 35.8 ms, System: 46.3 ms] + Range (min … max): 18.9 ms … 27.2 ms 112 runs + +# dust 'bench_artifacts' +3.9M ┌── file_992_fwss.bin │█ │ 0% +3.9M ├── file_993_fwdcbss.bin│█ │ 0% +3.9M ├── file_993_fwscas.bin │█ │ 0% +3.9M ├── file_993_fwss.bin │█ │ 0% +3.9M ├── file_994_fwdcbss.bin│█ │ 0% +3.9M ├── file_994_fwscas.bin │█ │ 0% +3.9M ├── file_994_fwss.bin │█ │ 0% +3.9M ├── file_995_fwdcbss.bin│█ │ 0% +3.9M ├── file_995_fwscas.bin │█ │ 0% +3.9M ├── file_995_fwss.bin │█ │ 0% +3.9M ├── file_996_fwdcbss.bin│█ │ 0% +3.9M ├── file_996_fwscas.bin │█ │ 0% +3.9M ├── file_996_fwss.bin │█ │ 0% +3.9M ├── file_997_fwdcbss.bin│█ │ 0% +3.9M ├── file_997_fwscas.bin │█ │ 0% +3.9M ├── file_997_fwss.bin │█ │ 0% +3.9M ├── file_998_fwdcbss.bin│█ │ 0% +3.9M ├── file_998_fwscas.bin │█ │ 0% +3.9M ├── file_998_fwss.bin │█ │ 0% +3.9M ├── file_999_fwdcbss.bin│█ │ 0% +3.9M ├── file_999_fwscas.bin │█ │ 0% +3.9M ├── file_999_fwss.bin │█ │ 0% +3.9M ├── file_99_fwdcbss.bin │█ │ 0% +3.9M ├── file_99_fwscas.bin │█ │ 0% +3.9M ├── file_99_fwss.bin │█ │ 0% +3.9M ├── file_9_fwdcbss.bin │█ │ 0% +3.9M ├── file_9_fwscas.bin │█ │ 0% +3.9M ├── file_9_fwss.bin │█ │ 0% + 11G ┌─┴ bench_artifacts │█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ │ 100% +``` + ## proposed - [ ] parallelization - [ ] (scanning + processing sw + processing hw) & formatting & printing @@ -79,7 +140,6 @@ Deduplicator uses size comparison and [GxHash](https://docs.rs/gxhash/latest/gxh - [ ] reduce FileInfo size - [ ] output in a tree format - [ ] tui -- [ ] add benchmarks - [ ] change the default hashing method to include the first & last page of a file (8K) ## v0.3 @@ -94,3 +154,4 @@ Deduplicator uses size comparison and [GxHash](https://docs.rs/gxhash/latest/gxh - [x] progress bar improvements - [x] use progress bar groups - [x] remove broken json rendering +- [x] add benchmarks diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..784b7fb --- /dev/null +++ b/Rakefile @@ -0,0 +1 @@ +# tasks are contained in the `rakelib` directory diff --git a/rakelib/benchmark.rake b/rakelib/benchmark.rake new file mode 100644 index 0000000..0ce28c2 --- /dev/null +++ b/rakelib/benchmark.rake @@ -0,0 +1,93 @@ +require 'tempfile' +require 'fileutils' +require 'securerandom' + +namespace :benchmark do + file 'target/release/deduplicator' do + sh "cargo build --release" + end + + task :few_large_files => 'target/release/deduplicator' do + # Benchmark 1: ./target/release/deduplicator bench_artifacts + # Time (mean ± σ): 2.5 ms ± 0.6 ms [User: 2.4 ms, System: 4.8 ms] + # Range (min … max): 1.8 ms … 9.7 ms 1474 runs + + root = "bench_artifacts" + Dir.mkdir(root) + + # files with same size + 2.times.map do |i| + File.open(File.join(root, "file_#{i}_fwss.bin"), 'wb') do |f| + f.write(SecureRandom.bytes(4096 * 100_000)) + end + end + + # files with different sizes + 2.times.map do |i| + File.open(File.join(root, "file_#{i}_fwds.bin"), 'wb') do |f| + f.write(SecureRandom.bytes(4096 * (rand * 100_000).ceil)) + end + end + + # files with same content & size + 2.times.each do |i| + File.open(File.join(root, "file_#{i}_fwscas.bin"), 'wb') do |f| + f.write("\0" * (4096 * 100_000)) + end + end + + # files with different content but same size + 2.times.each do |i| + File.open(File.join(root, "file_#{i}_fwdcbss.bin"), 'wb') do |f| + f.write(SecureRandom.bytes(4096 * 100_000)) + end + end + + sh("hyperfine -N --warmup 80 './target/release/deduplicator #{root}'") + sh("dust '#{root}'") + + FileUtils.rm_rf(root) + end + + task :many_small_files => 'target/release/deduplicator' do + # Benchmark 1: ./target/release/deduplicator bench_artifacts + # Time (mean ± σ): 10.6 ms ± 1.0 ms [User: 20.0 ms, System: 22.5 ms] + # Range (min … max): 8.4 ms … 14.2 ms 235 runs + + root = "bench_artifacts" + Dir.mkdir(root) + + # files with same size + 1000.times.each do |i| + File.open(File.join(root, "file_#{i}_fwss.bin"), 'wb') do |f| + f.write(SecureRandom.bytes(4096 * 1000)) + end + end + + # files with different sizes + 1000.times.each do |i| + File.open(File.join(root, "file_#{i}_fwds.bin"), 'wb') do |f| + f.write(SecureRandom.bytes(4096 * (rand * 100).ceil)) + end + end + + # files with same content & size + 1000.times.each do |i| + File.open(File.join(root, "file_#{i}_fwscas.bin"), 'wb') do |f| + f.write("\0" * (4096 * 1000)) + end + end + + # files with different content but same size + 1000.times.each do |i| + File.open(File.join(root, "file_#{i}_fwdcbss.bin"), 'wb') do |f| + f.write(SecureRandom.bytes(4096 * 1000)) + end + end + + sh("hyperfine --warmup 20 './target/release/deduplicator #{root}'") + sh("dust '#{root}'") + + FileUtils.rm_rf(root) + end +end