diff --git a/README.md b/README.md index 8c07d80..0b74407 100644 --- a/README.md +++ b/README.md @@ -132,3 +132,10 @@ Memory: 31731MiB (~32GiB) - Tree format output for duplicate file listing - GUI - Packages for different operating system repositories (currently only installable via cargo) + +## v0.3 checklist +- [x] parallelization of scanning, processing and formatting +- [x] reduce cloning values on the heap +- [ ] add a partial hashing mode +- [ ] add an option to use a bloom filter for very large filesystems +- [ ] max file path size should use the last set of duplicates diff --git a/src/fileinfo.rs b/src/fileinfo.rs index bb74f06..409b6e4 100644 --- a/src/fileinfo.rs +++ b/src/fileinfo.rs @@ -4,19 +4,19 @@ use memmap2::Mmap; use serde::Serialize; use std::fs; use std::hash::Hasher; +use std::io::Read; use std::{fs::Metadata, path::PathBuf}; #[derive(Debug, Clone, Serialize)] pub struct FileInfo { pub path: PathBuf, - pub hash: Option, pub size: u64, #[serde(skip)] pub filemeta: Metadata, } impl FileInfo { - pub fn hash(&self) -> Result { + pub fn hash(&self) -> Result { let file = fs::File::open(self.path.clone())?; let mapper = unsafe { Mmap::map(&file)? }; let mut primhasher = GxHasher::default(); @@ -25,10 +25,16 @@ impl FileInfo { .chunks(1_000_000) .for_each(|chunk| primhasher.write(chunk)); - Ok(Self { - hash: Some(primhasher.finish().to_string()), - ..self.clone() - }) + Ok(primhasher.finish().to_string()) + } + + pub fn initial_page_hash(&self) -> Result { + let file = fs::File::open(self.path.clone())?; + let mapper = unsafe { Mmap::map(&file)? }; + let mut primhasher = GxHasher::default(); + primhasher.write(mapper.take(4096).into_inner()); + + Ok(primhasher.finish().to_string()) } pub fn new(path: PathBuf) -> Result { @@ -36,7 +42,6 @@ impl FileInfo { Ok(Self { path, filemeta: filemeta.clone(), - hash: None, size: filemeta.len(), }) } diff --git a/src/processor.rs b/src/processor.rs index 9fc0db5..b2346de 100644 --- a/src/processor.rs +++ b/src/processor.rs @@ -33,7 +33,7 @@ impl Processor { if group.len() > 1 { group.into_par_iter().for_each(|file| { hw_store - .entry(file.hash.clone().unwrap_or_default()) + .entry(file.hash().expect("hashing file failed.")) .and_modify(|fileset| fileset.push(file.clone())) .or_insert_with(|| vec![file]); });