diff --git a/Cargo.toml b/Cargo.toml index db4afd5..96b071b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,9 +6,9 @@ description = "find,filter and delete duplicate files" repository = "https://github.com/sreedevk/deduplicator" license = "MIT" authors = [ - "Sreedev Kodichath ", - "Valentin Bersier ", - "Dhruva Sagar ", + "Sreedev Kodichath ", + "Valentin Bersier ", + "Dhruva Sagar ", ] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -19,7 +19,7 @@ chrono = "0.4.23" clap = { version = "4.0.32", features = ["derive"] } dashmap = { version = "5.4.0", features = ["rayon"] } globwalk = "0.8.1" -gxhash = "3.4.1" +gxhash = { version = "3.4.1", default-features = false } indicatif = { version = "0.17.2", features = ["rayon"] } itertools = "0.10.5" memmap2 = "0.5.8" @@ -40,7 +40,12 @@ inherits = "release" lto = "thin" [workspace.metadata.dist] -rust-toolchain-version = "1.78.0" +rust-toolchain-version = "1.87.0" ci = ["github"] -targets = ["x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc", "aarch64-apple-darwin"] +targets = [ + "x86_64-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", + "aarch64-apple-darwin", +] cargo-dist-version = "0.0.7" diff --git a/README.md b/README.md index 1ebf3e7..775a88b 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,6 @@ deduplicator ~/Media --min-size 100mb ## Installation ### Cargo Install - #### Stable > [!WARNING] Note from GxHash: GxHash relies on aes hardware acceleration, you must make sure the aes feature is enabled when building (otherwise it won't build). This can be done by setting the RUSTFLAGS environment variable to -C target-feature=+aes or -C target-cpu=native (the latter should work if your CPU is properly recognized by rustc, which is the case most of the time). > please install version `0.2.1` if you are unable to install `0.3.0` diff --git a/src/fileinfo.rs b/src/fileinfo.rs index b480490..2a23800 100644 --- a/src/fileinfo.rs +++ b/src/fileinfo.rs @@ -1,9 +1,8 @@ use anyhow::Result; -use gxhash::GxHasher; +use gxhash::gxhash128; use memmap2::Mmap; use std::{ fs, - hash::Hasher, io::Read, path::{Path, PathBuf}, time::SystemTime, @@ -17,25 +16,22 @@ pub struct FileInfo { } impl FileInfo { - pub fn hash(&self) -> Result { + pub fn hash(&self, seed: i64) -> Result { let file = fs::File::open(&self.path)?; let mapper = unsafe { Mmap::map(&file)? }; - let mut primhasher = GxHasher::default(); + let final_hash = mapper + .chunks(4096) + .fold(0, |acc, chunk: &[u8]| acc + gxhash128(chunk, seed)); - mapper - .chunks(1_000_000) - .for_each(|chunk| primhasher.write(chunk)); - - Ok(primhasher.finish_u128()) + Ok(final_hash) } - pub fn initial_page_hash(&self) -> Result { - let file = fs::File::open(&self.path)?; - let mapper = unsafe { Mmap::map(&file)? }; - let mut primhasher = GxHasher::default(); - primhasher.write(mapper.take(4096).into_inner()); + pub fn initial_page_hash(&self, seed: i64) -> Result { + let mut file = fs::File::open(&self.path)?; + let mut buffer = [0; 4096]; + let bytes_read = file.read(&mut buffer)?; - Ok(primhasher.finish_u128()) + Ok(gxhash128(&buffer[..bytes_read], seed)) } pub fn new(path: PathBuf) -> Result { diff --git a/src/processor.rs b/src/processor.rs index ee4b75d..04cee53 100644 --- a/src/processor.rs +++ b/src/processor.rs @@ -21,6 +21,7 @@ impl Processor { hw_store: Arc>>, progress_bar_box: Arc, max_file_size: Arc, + seed: i64, ) -> Result<()> { let progress_bar = match app_args.progress { true => progress_bar_box.add(ProgressBar::new_spinner()), @@ -44,9 +45,9 @@ impl Processor { if group.len() > 1 { group.into_par_iter().for_each(|file| { let fhash = if app_args.strict { - file.hash().expect("hashing file failed.") + file.hash(seed).expect("hashing file failed.") } else { - file.initial_page_hash().expect("hashing file failed.") + file.initial_page_hash(seed).expect("hashing file failed.") }; Self::compare_and_update_max_path_len( diff --git a/src/server.rs b/src/server.rs index 44ec19d..6dcb1ac 100644 --- a/src/server.rs +++ b/src/server.rs @@ -6,6 +6,7 @@ use crate::scanner::Scanner; use anyhow::Result; use dashmap::DashMap; use indicatif::{MultiProgress, ProgressDrawTarget}; +use rand::Rng; use threadpool::ThreadPool; use crate::fileinfo::FileInfo; @@ -34,6 +35,8 @@ impl Server { pub fn start(&self) -> Result<()> { let progbarbox = Arc::new(MultiProgress::new()); + let mut rng = rand::rng(); + let seed: i64 = rng.random(); if !self.app_args.progress { progbarbox.set_draw_target(ProgressDrawTarget::hidden()); @@ -82,6 +85,7 @@ impl Server { store_dupl_hw, progbarbox_pr_clone, max_file_path_len_clone, + seed, ) .unwrap(); });