added strict mode with partial hashing default

src/params.rs

@@ -30,6 +30,9 @@ pub struct Params {
     /// print json output
     #[arg(long)]
     pub json: bool,
+    /// Guarantees that two files are duplicate (performs a full hash)
+    #[arg(long, short = 'f', default_value = "false")]
+    pub strict: bool,
 }
 
 impl Params {

src/processor.rs

@@ -8,11 +8,13 @@ use std::sync::{Arc, Mutex, TryLockError, TryLockResult};
 use std::time::Duration;
 
 use crate::fileinfo::FileInfo;
+use crate::params::Params;
 
 pub struct Processor {}
 
 impl Processor {
     pub fn hashwise(
+        app_args: Arc<Params>,
         sw_store: Arc<DashMap<u64, Vec<FileInfo>>>,
         hw_store: Arc<DashMap<String, Vec<FileInfo>>>,
     ) -> Result<()> {
@@ -32,8 +34,14 @@ impl Processor {
                 let group: Vec<FileInfo> = sw_store.get(&key).unwrap().to_vec();
                 if group.len() > 1 {
                     group.into_par_iter().for_each(|file| {
+                        let fhash = if app_args.strict {
+                            file.hash().expect("hashing file failed.")
+                        } else {
+                            file.initial_page_hash().expect("hashing file failed.")
+                        };
+
                         hw_store
-                            .entry(file.hash().expect("hashing file failed."))
+                            .entry(fhash)
                             .and_modify(|fileset| fileset.push(file.clone()))
                             .or_insert_with(|| vec![file]);
                     });
@@ -51,7 +59,6 @@ impl Processor {
         Ok(())
     }
 
-    // TODO: reduce the amount of time files remain locked for
     pub fn sizewise(
         scanner_finished: Arc<AtomicBool>,
         store: Arc<DashMap<u64, Vec<FileInfo>>>,
@@ -65,32 +72,31 @@ impl Processor {
         progress_bar.set_message("files grouped by size");
 
         loop {
-            match files.try_lock() {
-                Ok(mut flist) => match flist.pop() {
-                    Some(file) => {
-                        progress_bar.inc(1);
-                        Self::compare_and_update_max_path_len(
-                            max_file_size.clone(),
-                            file.path.to_string_lossy().len() as u64,
-                        )?;
-                        store
-                            .entry(file.size)
-                            .and_modify(|fileset| fileset.push(file.clone()))
-                            .or_insert_with(|| vec![file]);
-                        continue;
-                    }
-                    None => match scanner_finished.load(std::sync::atomic::Ordering::Relaxed) {
-                        true => break Ok(()),
-                        false => continue,
-                    },
-                },
-                TryLockResult::Err(TryLockError::WouldBlock) => continue,
-                _ => {
-                    break {
-                        progress_bar.finish_with_message("files grouped by size.");
-                        Ok(())
-                    }
+            let fileopt: Option<FileInfo> = {
+                match files.try_lock() {
+                    Ok(mut flist) => flist.pop(),
+                    TryLockResult::Err(TryLockError::WouldBlock) => None,
+                    _ => None,
                 }
+            };
+
+            match fileopt {
+                Some(file) => {
+                    progress_bar.inc(1);
+                    Self::compare_and_update_max_path_len(
+                        max_file_size.clone(),
+                        file.path.to_string_lossy().len() as u64,
+                    )?;
+                    store
+                        .entry(file.size)
+                        .and_modify(|fileset| fileset.push(file.clone()))
+                        .or_insert_with(|| vec![file]);
+                    continue;
+                }
+                None => match scanner_finished.load(std::sync::atomic::Ordering::Relaxed) {
+                    true => break Ok(()),
+                    false => continue,
+                },
             }
         }
     }

src/server.rs

@@ -25,14 +25,15 @@ impl Server {
             filequeue: Arc::new(Mutex::new(Vec::new())),
             sw_duplicate_set: Arc::new(DashMap::new()),
             hw_duplicate_set: Arc::new(DashMap::new()),
-            threadpool: ThreadPool::new(8),
+            threadpool: ThreadPool::new(4),
             app_args: Arc::new(opts),
             max_file_path_len: Arc::new(AtomicU64::new(0)),
         }
     }
 
     pub fn start(&self) -> Result<()> {
-        let app_args_clone = self.app_args.clone();
+        let app_args_clone_for_sc = self.app_args.clone();
+        let app_args_clone_for_pr = self.app_args.clone();
         let file_queue_clone_sc = self.filequeue.clone();
         let file_queue_clone_pr = self.filequeue.clone();
         let scanner_finished = Arc::new(AtomicBool::new(false));
@@ -46,7 +47,7 @@ impl Server {
         let max_file_path_len_clone = self.max_file_path_len.clone();
 
         self.threadpool.execute(move || {
-            Scanner::build(app_args_clone)
+            Scanner::build(app_args_clone_for_sc)
                 .unwrap()
                 .scan(file_queue_clone_sc)
                 .unwrap();
@@ -63,7 +64,8 @@ impl Server {
             )
             .unwrap();
 
-            Processor::hashwise(store_dupl_sw_for_hw, store_dupl_hw).unwrap();
+            Processor::hashwise(app_args_clone_for_pr, store_dupl_sw_for_hw, store_dupl_hw)
+                .unwrap();
         });
 
         self.threadpool.join();