smolkik_adm/deduplicator
1
0
Fork 0
mirror of https://github.com/sreedevk/deduplicator.git synced 2026-09-14 01:26:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

partial hash page count increased

Browse Source
This commit is contained in:
sreedevk
2025-07-18 11:25:37 +00:00
parent d97af6b8f6
commit 130a8f99ba
4 changed files with 10 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.lock generated
View File

@@ -274,7 +274,7 @@ dependencies = [
[[package]] [[package]]
name = "deduplicator" name = "deduplicator"
version = "0.3.0" version = "0.3.1"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"bytesize", "bytesize",

2
README.md
View File

@@ -168,6 +168,8 @@ dust 'bench_artifacts'
## v0.3.1 ## v0.3.1
- [x] parallelization - [x] parallelization
- [x] (scanning + processing sw + processing hw) & formatting & printing - [x] (scanning + processing sw + processing hw) & formatting & printing
- [ ] remove formatting step and write directly to stdout (tree mode)
- [x] Increase the number of pages hashed in partial hashing
## v0.3.0 ## v0.3.0
- [x] parallelization - [x] parallelization

4
src/fileinfo.rs
View File

@@ -33,9 +33,9 @@ impl FileInfo {
.fold(0u128, |acc, chunk: &[u8]| acc ^ gxhash128(chunk, seed))) .fold(0u128, |acc, chunk: &[u8]| acc ^ gxhash128(chunk, seed)))
} }
pub fn initpage_hash(&self, seed: i64) -> Result<u128> { pub fn initpages_hash(&self, seed: i64) -> Result<u128> {
let mut file = fs::File::open(&self.path)?; let mut file = fs::File::open(&self.path)?;
let mut buffer = [0; 4096]; let mut buffer = [0; 16384];
let bytes_read = file.read(&mut buffer)?; let bytes_read = file.read(&mut buffer)?;
Ok(gxhash128(&buffer[..bytes_read], seed)) Ok(gxhash128(&buffer[..bytes_read], seed))

10
src/processor.rs
View File

@@ -59,7 +59,7 @@ impl Processor {
let fhash = match app_args.strict { let fhash = match app_args.strict {
true => file.hash(seed).expect("hashing file failed."), true => file.hash(seed).expect("hashing file failed."),
false => file.initpage_hash(seed).expect("hashing file failed."), false => file.initpages_hash(seed).expect("hashing file failed."),
}; };
Self::compare_and_update_max_path_len( Self::compare_and_update_max_path_len(
@@ -153,9 +153,9 @@ mod tests {
} }
#[test] #[test]
fn hashwise_sorting_two_files_with_identical_init_page_only_strict_mode() -> Result<()> { fn hashwise_sorting_two_files_with_identical_init_pages_only_strict_mode() -> Result<()> {
let root = TempDir::new()?; let root = TempDir::new()?;
let content = generate_bytes(4096); let content = generate_bytes(16384);
let mut content_x = content.clone(); let mut content_x = content.clone();
let mut content_y = content.clone(); let mut content_y = content.clone();
@@ -211,9 +211,9 @@ mod tests {
} }
#[test] #[test]
fn hashwise_sorting_two_files_with_identical_init_page_only_fast_mode() -> Result<()> { fn hashwise_sorting_two_files_with_identical_init_pages_only_fast_mode() -> Result<()> {
let root = TempDir::new()?; let root = TempDir::new()?;
let content = generate_bytes(4096); let content = generate_bytes(16384);
let mut content_x = content.clone(); let mut content_x = content.clone();
let mut content_y = content.clone(); let mut content_y = content.clone();