mirror of
https://github.com/sreedevk/deduplicator.git
synced 2026-08-26 02:04:39 +00:00
partial hash page count increased
This commit is contained in:
2
Cargo.lock
generated
2
Cargo.lock
generated
@@ -274,7 +274,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "deduplicator"
|
||||
version = "0.3.0"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bytesize",
|
||||
|
||||
@@ -168,6 +168,8 @@ dust 'bench_artifacts'
|
||||
## v0.3.1
|
||||
- [x] parallelization
|
||||
- [x] (scanning + processing sw + processing hw) & formatting & printing
|
||||
- [ ] remove formatting step and write directly to stdout (tree mode)
|
||||
- [x] Increase the number of pages hashed in partial hashing
|
||||
|
||||
## v0.3.0
|
||||
- [x] parallelization
|
||||
|
||||
@@ -33,9 +33,9 @@ impl FileInfo {
|
||||
.fold(0u128, |acc, chunk: &[u8]| acc ^ gxhash128(chunk, seed)))
|
||||
}
|
||||
|
||||
pub fn initpage_hash(&self, seed: i64) -> Result<u128> {
|
||||
pub fn initpages_hash(&self, seed: i64) -> Result<u128> {
|
||||
let mut file = fs::File::open(&self.path)?;
|
||||
let mut buffer = [0; 4096];
|
||||
let mut buffer = [0; 16384];
|
||||
let bytes_read = file.read(&mut buffer)?;
|
||||
|
||||
Ok(gxhash128(&buffer[..bytes_read], seed))
|
||||
|
||||
@@ -59,7 +59,7 @@ impl Processor {
|
||||
|
||||
let fhash = match app_args.strict {
|
||||
true => file.hash(seed).expect("hashing file failed."),
|
||||
false => file.initpage_hash(seed).expect("hashing file failed."),
|
||||
false => file.initpages_hash(seed).expect("hashing file failed."),
|
||||
};
|
||||
|
||||
Self::compare_and_update_max_path_len(
|
||||
@@ -153,9 +153,9 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hashwise_sorting_two_files_with_identical_init_page_only_strict_mode() -> Result<()> {
|
||||
fn hashwise_sorting_two_files_with_identical_init_pages_only_strict_mode() -> Result<()> {
|
||||
let root = TempDir::new()?;
|
||||
let content = generate_bytes(4096);
|
||||
let content = generate_bytes(16384);
|
||||
|
||||
let mut content_x = content.clone();
|
||||
let mut content_y = content.clone();
|
||||
@@ -211,9 +211,9 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hashwise_sorting_two_files_with_identical_init_page_only_fast_mode() -> Result<()> {
|
||||
fn hashwise_sorting_two_files_with_identical_init_pages_only_fast_mode() -> Result<()> {
|
||||
let root = TempDir::new()?;
|
||||
let content = generate_bytes(4096);
|
||||
let content = generate_bytes(16384);
|
||||
|
||||
let mut content_x = content.clone();
|
||||
let mut content_y = content.clone();
|
||||
|
||||
Reference in New Issue
Block a user