From 66f49a9aeef91a01f2a6cd9ad4096b22ecb378c2 Mon Sep 17 00:00:00 2001 From: sreedevk Date: Sat, 19 Jul 2025 01:35:24 +0000 Subject: [PATCH] benchmarks updated --- README.md | 41 +++++++++++++++++------------------------ rakelib/benchmark.rake | 8 ++++++++ 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 343848a..302a8b5 100644 --- a/README.md +++ b/README.md @@ -81,38 +81,31 @@ I've used hyperfine to run deduplicator on files generated by the rake file at ` ``` # hyperfine -N --warmup 80 './target/release/deduplicator bench_artifacts' Benchmark 1: ./target/release/deduplicator bench_artifacts - Time (mean ± σ): 2.3 ms ± 0.5 ms [User: 2.4 ms, System: 4.6 ms] - Range (min … max): 1.4 ms … 6.7 ms 1457 runs + Time (mean ± σ): 2.2 ms ± 0.4 ms [User: 2.2 ms, System: 4.4 ms] + Range (min … max): 1.3 ms … 7.1 ms 1522 runs dust 'bench_artifacts' - 40M ┌── file_0_fwds.bin │████ │ 2% -113M ├── file_1_fwds.bin │███████████ │ 5% -390M ├── file_0_fwdcbss.bin│█████████████████████████████████████ │ 16% -390M ├── file_0_fwscas.bin │█████████████████████████████████████ │ 16% -390M ├── file_0_fwss.bin │█████████████████████████████████████ │ 16% -390M ├── file_1_fwdcbss.bin│█████████████████████████████████████ │ 16% -390M ├── file_1_fwscas.bin │█████████████████████████████████████ │ 16% -390M ├── file_1_fwss.bin │█████████████████████████████████████ │ 16% -2.4G ┌─┴ bench_artifacts │██████████████████████████████████████████████████████████████████ │ 100% + + 54M ┌── file_0_fwds.bin │████ │ 2% +122M ├── file_1_fwds.bin │████████ │ 5% +390M ├── file_0_fwdcbss.bin│██████████████████████████ │ 15% +390M ├── file_0_fwscas.bin │██████████████████████████ │ 15% +390M ├── file_0_fwss.bin │██████████████████████████ │ 15% +390M ├── file_1_fwdcbss.bin│██████████████████████████ │ 15% +390M ├── file_1_fwscas.bin │██████████████████████████ │ 15% +390M ├── file_1_fwss.bin │██████████████████████████ │ 15% +2.5G ┌─┴ bench_artifacts │██████████████████████████████████████████████████████████████████ │ 100% ``` #### Many Small Files ``` # hyperfine --warmup 20 './target/release/deduplicator bench_artifacts' Benchmark 1: ./target/release/deduplicator bench_artifacts - Time (mean ± σ): 88.6 ms ± 10.3 ms [User: 275.3 ms, System: 232.7 ms] - Range (min … max): 66.8 ms … 108.0 ms 41 runs + Time (mean ± σ): 40.1 ms ± 2.3 ms [User: 251.0 ms, System: 277.3 ms] + Range (min … max): 35.0 ms … 45.9 ms 72 runs dust 'bench_artifacts' -3.9M ┌── file_98_fwss.bin │█ │ 0% -3.9M ├── file_990_fwdcbss.bin│█ │ 0% -3.9M ├── file_990_fwscas.bin │█ │ 0% -3.9M ├── file_990_fwss.bin │█ │ 0% -3.9M ├── file_991_fwdcbss.bin│█ │ 0% -3.9M ├── file_991_fwscas.bin │█ │ 0% -3.9M ├── file_991_fwss.bin │█ │ 0% -3.9M ├── file_992_fwdcbss.bin│█ │ 0% -3.9M ├── file_992_fwscas.bin │█ │ 0% +3.9M ┌── file_992_fwscas.bin │█ │ 0% 3.9M ├── file_992_fwss.bin │█ │ 0% 3.9M ├── file_993_fwdcbss.bin│█ │ 0% 3.9M ├── file_993_fwscas.bin │█ │ 0% @@ -155,7 +148,6 @@ dust 'bench_artifacts' - [ ] fix memory leak on very large filesystems - [ ] maybe use a bloom filter - [ ] reduce FileInfo size -- [ ] output a duplicate tree - [ ] tui - [ ] change the default hashing method to include the first & last page of a file (8K) - [ ] provide option to localize duplicate detection to arbitrary levels relative to current directory @@ -168,7 +160,8 @@ dust 'bench_artifacts' ## v0.3.1 - [x] parallelization - [x] (scanning + processing sw + processing hw) & formatting & printing -- [ ] remove formatting step and write directly to stdout (tree mode) +- [x] remove formatting step and write directly to stdout +- [x] simplify output to improve performance - [x] Increase the number of pages hashed in partial hashing ## v0.3.0 diff --git a/rakelib/benchmark.rake b/rakelib/benchmark.rake index 0ce28c2..6b4eb6d 100644 --- a/rakelib/benchmark.rake +++ b/rakelib/benchmark.rake @@ -16,6 +16,7 @@ namespace :benchmark do Dir.mkdir(root) # files with same size + puts "generating files of same size ..." 2.times.map do |i| File.open(File.join(root, "file_#{i}_fwss.bin"), 'wb') do |f| f.write(SecureRandom.bytes(4096 * 100_000)) @@ -23,6 +24,7 @@ namespace :benchmark do end # files with different sizes + puts "generating files of different sizes ..." 2.times.map do |i| File.open(File.join(root, "file_#{i}_fwds.bin"), 'wb') do |f| f.write(SecureRandom.bytes(4096 * (rand * 100_000).ceil)) @@ -30,6 +32,7 @@ namespace :benchmark do end # files with same content & size + puts "generating files of same content and sizes ..." 2.times.each do |i| File.open(File.join(root, "file_#{i}_fwscas.bin"), 'wb') do |f| f.write("\0" * (4096 * 100_000)) @@ -37,6 +40,7 @@ namespace :benchmark do end # files with different content but same size + puts "generating files of different content but same sizes ..." 2.times.each do |i| File.open(File.join(root, "file_#{i}_fwdcbss.bin"), 'wb') do |f| f.write(SecureRandom.bytes(4096 * 100_000)) @@ -58,6 +62,7 @@ namespace :benchmark do Dir.mkdir(root) # files with same size + puts "generating 1000 files of the same size ... " 1000.times.each do |i| File.open(File.join(root, "file_#{i}_fwss.bin"), 'wb') do |f| f.write(SecureRandom.bytes(4096 * 1000)) @@ -65,6 +70,7 @@ namespace :benchmark do end # files with different sizes + puts "generating 1000 files of different sizes ... " 1000.times.each do |i| File.open(File.join(root, "file_#{i}_fwds.bin"), 'wb') do |f| f.write(SecureRandom.bytes(4096 * (rand * 100).ceil)) @@ -72,6 +78,7 @@ namespace :benchmark do end # files with same content & size + puts "generating files of same content and sizes ..." 1000.times.each do |i| File.open(File.join(root, "file_#{i}_fwscas.bin"), 'wb') do |f| f.write("\0" * (4096 * 1000)) @@ -79,6 +86,7 @@ namespace :benchmark do end # files with different content but same size + puts "generating files of different content but same sizes ..." 1000.times.each do |i| File.open(File.join(root, "file_#{i}_fwdcbss.bin"), 'wb') do |f| f.write(SecureRandom.bytes(4096 * 1000))