speeeeeeeeeeed

Cargo.toml

@@ -6,9 +6,9 @@ description = "find,filter and delete duplicate files"
 repository = "https://github.com/sreedevk/deduplicator"
 license = "MIT"
 authors = [
-    "Sreedev Kodichath <sreedevpadmakumar@gmail.com>",
-    "Valentin Bersier <vbersier@gmail.com>",
-    "Dhruva Sagar <dhruva.sagar@gmail.com>",
+  "Sreedev Kodichath <sreedevpadmakumar@gmail.com>",
+  "Valentin Bersier <vbersier@gmail.com>",
+  "Dhruva Sagar <dhruva.sagar@gmail.com>",
 ]
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -19,7 +19,7 @@ chrono = "0.4.23"
 clap = { version = "4.0.32", features = ["derive"] }
 dashmap = { version = "5.4.0", features = ["rayon"] }
 globwalk = "0.8.1"
-gxhash = "3.4.1"
+gxhash = { version = "3.4.1", default-features = false }
 indicatif = { version = "0.17.2", features = ["rayon"] }
 itertools = "0.10.5"
 memmap2 = "0.5.8"
@@ -40,7 +40,12 @@ inherits = "release"
 lto = "thin"
 
 [workspace.metadata.dist]
-rust-toolchain-version = "1.78.0"
+rust-toolchain-version = "1.87.0"
 ci = ["github"]
-targets = ["x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc", "aarch64-apple-darwin"]
+targets = [
+  "x86_64-unknown-linux-gnu",
+  "x86_64-apple-darwin",
+  "x86_64-pc-windows-msvc",
+  "aarch64-apple-darwin",
+]
 cargo-dist-version = "0.0.7"

README.md

@@ -48,7 +48,6 @@ deduplicator ~/Media --min-size 100mb
 ## Installation
 
 ### Cargo Install
-
 #### Stable
 > [!WARNING] Note from GxHash: GxHash relies on aes hardware acceleration, you must make sure the aes feature is enabled when building (otherwise it won't build). This can be done by setting the RUSTFLAGS environment variable to -C target-feature=+aes or -C target-cpu=native (the latter should work if your CPU is properly recognized by rustc, which is the case most of the time).
 > please install version `0.2.1`  if you are unable to install `0.3.0`

src/fileinfo.rs

@@ -1,9 +1,8 @@
 use anyhow::Result;
-use gxhash::GxHasher;
+use gxhash::gxhash128;
 use memmap2::Mmap;
 use std::{
     fs,
-    hash::Hasher,
     io::Read,
     path::{Path, PathBuf},
     time::SystemTime,
@@ -17,25 +16,22 @@ pub struct FileInfo {
 }
 
 impl FileInfo {
-    pub fn hash(&self) -> Result<u128> {
+    pub fn hash(&self, seed: i64) -> Result<u128> {
         let file = fs::File::open(&self.path)?;
         let mapper = unsafe { Mmap::map(&file)? };
-        let mut primhasher = GxHasher::default();
+        let final_hash = mapper
+            .chunks(4096)
+            .fold(0, |acc, chunk: &[u8]| acc + gxhash128(chunk, seed));
 
-        mapper
-            .chunks(1_000_000)
-            .for_each(|chunk| primhasher.write(chunk));
-
-        Ok(primhasher.finish_u128())
+        Ok(final_hash)
     }
 
-    pub fn initial_page_hash(&self) -> Result<u128> {
-        let file = fs::File::open(&self.path)?;
-        let mapper = unsafe { Mmap::map(&file)? };
-        let mut primhasher = GxHasher::default();
-        primhasher.write(mapper.take(4096).into_inner());
+    pub fn initial_page_hash(&self, seed: i64) -> Result<u128> {
+        let mut file = fs::File::open(&self.path)?;
+        let mut buffer = [0; 4096];
+        let bytes_read = file.read(&mut buffer)?;
 
-        Ok(primhasher.finish_u128())
+        Ok(gxhash128(&buffer[..bytes_read], seed))
     }
 
     pub fn new(path: PathBuf) -> Result<Self> {

src/processor.rs

@@ -21,6 +21,7 @@ impl Processor {
         hw_store: Arc<DashMap<u128, Vec<FileInfo>>>,
         progress_bar_box: Arc<MultiProgress>,
         max_file_size: Arc<AtomicU64>,
+        seed: i64,
     ) -> Result<()> {
         let progress_bar = match app_args.progress {
             true => progress_bar_box.add(ProgressBar::new_spinner()),
@@ -44,9 +45,9 @@ impl Processor {
                 if group.len() > 1 {
                     group.into_par_iter().for_each(|file| {
                         let fhash = if app_args.strict {
-                            file.hash().expect("hashing file failed.")
+                            file.hash(seed).expect("hashing file failed.")
                         } else {
-                            file.initial_page_hash().expect("hashing file failed.")
+                            file.initial_page_hash(seed).expect("hashing file failed.")
                         };
 
                         Self::compare_and_update_max_path_len(

src/server.rs

@@ -6,6 +6,7 @@ use crate::scanner::Scanner;
 use anyhow::Result;
 use dashmap::DashMap;
 use indicatif::{MultiProgress, ProgressDrawTarget};
+use rand::Rng;
 use threadpool::ThreadPool;
 
 use crate::fileinfo::FileInfo;
@@ -34,6 +35,8 @@ impl Server {
 
     pub fn start(&self) -> Result<()> {
         let progbarbox = Arc::new(MultiProgress::new());
+        let mut rng = rand::rng();
+        let seed: i64 = rng.random();
 
         if !self.app_args.progress {
             progbarbox.set_draw_target(ProgressDrawTarget::hidden());
@@ -82,6 +85,7 @@ impl Server {
                 store_dupl_hw,
                 progbarbox_pr_clone,
                 max_file_path_len_clone,
+                seed,
             )
             .unwrap();
         });