From 130a8f99ba4d45fe25cefbda58eeafd49b02e7a6 Mon Sep 17 00:00:00 2001 From: sreedevk Date: Fri, 18 Jul 2025 11:25:37 +0000 Subject: [PATCH] partial hash page count increased --- Cargo.lock | 2 +- README.md | 2 ++ src/fileinfo.rs | 4 ++-- src/processor.rs | 10 +++++----- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c7a2c07..6b1d6dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -274,7 +274,7 @@ dependencies = [ [[package]] name = "deduplicator" -version = "0.3.0" +version = "0.3.1" dependencies = [ "anyhow", "bytesize", diff --git a/README.md b/README.md index 6e481b2..343848a 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,8 @@ dust 'bench_artifacts' ## v0.3.1 - [x] parallelization - [x] (scanning + processing sw + processing hw) & formatting & printing +- [ ] remove formatting step and write directly to stdout (tree mode) +- [x] Increase the number of pages hashed in partial hashing ## v0.3.0 - [x] parallelization diff --git a/src/fileinfo.rs b/src/fileinfo.rs index d82fa9c..f13191f 100644 --- a/src/fileinfo.rs +++ b/src/fileinfo.rs @@ -33,9 +33,9 @@ impl FileInfo { .fold(0u128, |acc, chunk: &[u8]| acc ^ gxhash128(chunk, seed))) } - pub fn initpage_hash(&self, seed: i64) -> Result { + pub fn initpages_hash(&self, seed: i64) -> Result { let mut file = fs::File::open(&self.path)?; - let mut buffer = [0; 4096]; + let mut buffer = [0; 16384]; let bytes_read = file.read(&mut buffer)?; Ok(gxhash128(&buffer[..bytes_read], seed)) diff --git a/src/processor.rs b/src/processor.rs index 88ac3f2..533cda8 100644 --- a/src/processor.rs +++ b/src/processor.rs @@ -59,7 +59,7 @@ impl Processor { let fhash = match app_args.strict { true => file.hash(seed).expect("hashing file failed."), - false => file.initpage_hash(seed).expect("hashing file failed."), + false => file.initpages_hash(seed).expect("hashing file failed."), }; Self::compare_and_update_max_path_len( @@ -153,9 +153,9 @@ mod tests { } #[test] - fn hashwise_sorting_two_files_with_identical_init_page_only_strict_mode() -> Result<()> { + fn hashwise_sorting_two_files_with_identical_init_pages_only_strict_mode() -> Result<()> { let root = TempDir::new()?; - let content = generate_bytes(4096); + let content = generate_bytes(16384); let mut content_x = content.clone(); let mut content_y = content.clone(); @@ -211,9 +211,9 @@ mod tests { } #[test] - fn hashwise_sorting_two_files_with_identical_init_page_only_fast_mode() -> Result<()> { + fn hashwise_sorting_two_files_with_identical_init_pages_only_fast_mode() -> Result<()> { let root = TempDir::new()?; - let content = generate_bytes(4096); + let content = generate_bytes(16384); let mut content_x = content.clone(); let mut content_y = content.clone();