From 21f0eb3cb081287e6a8745cbd6699e9dcb37c9f9 Mon Sep 17 00:00:00 2001 From: sreedevk Date: Sat, 19 Jul 2025 16:26:12 +0000 Subject: [PATCH] all file type related filtering issues fixed --- README.md | 2 +- src/params.rs | 45 ------------- src/scanner.rs | 180 ++++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 171 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index e9f0ae5..d934489 100644 --- a/README.md +++ b/README.md @@ -177,10 +177,10 @@ dust 'bench_artifacts' - [ ] fix: partial hash collision - a file full of null bytes ("\0") and an empty file. This is a known trade off in gxhash. - [ ] include initial pages and final pages of the file - [ ] append the offset between the last initial page hashed and the first final page hashed in the content passed to the hasher. -- [ ] fix: --exclude-types and --types flag behave identically. ## v0.3.2 - [x] fix: single file groups are printed to screen +- [x] fix: --exclude-types and --types flag behave identically. ## v0.3.1 - [x] parallelization diff --git a/src/params.rs b/src/params.rs index 9d55316..3a1918b 100644 --- a/src/params.rs +++ b/src/params.rs @@ -2,7 +2,6 @@ use std::{fs, path::PathBuf}; use anyhow::Result; use clap::{Parser, ValueHint}; -use std::collections::HashSet; #[derive(Parser, Debug, Default, Clone)] #[command(author, version, about, long_about = None)] @@ -56,48 +55,4 @@ impl Params { let dir = fs::canonicalize(dir_path)?; Ok(dir) } - - pub fn types_intersection(itypes: &str, xtypes: &str) -> String { - let iset = itypes - .split(",") - .map(String::from) - .collect::>(); - let xset = xtypes - .split(",") - .map(String::from) - .collect::>(); - - iset.difference(&xset) - .cloned() - .collect::>() - .join(",") - } - - pub fn get_types(&self) -> Option { - match &self.types { - Some(itypes) => match &self.exclude_types { - Some(xtypes) => Some(Self::types_intersection(itypes, xtypes)), - None => Some(itypes.to_string()), - }, - None => self.exclude_types.as_ref().map(|xtypes| xtypes.to_string()), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn mixing_include_and_exclude_types_works_as_expected() { - let params = Params { - types: Some(String::from("js,xml,ts,pdf,tiff")), - exclude_types: Some(String::from("js,ts,xml")), - ..Default::default() - }; - - assert!(params - .get_types() - .is_some_and(|x| x == "pdf,tiff" || x == "tiff,pdf")) - } } diff --git a/src/scanner.rs b/src/scanner.rs index 0071e0a..f2e690b 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -8,9 +8,10 @@ use globwalk::{GlobWalker, GlobWalkerBuilder}; pub struct Scanner { pub directory: Box, - pub filetypes: Option, pub min_depth: Option, pub max_depth: Option, + pub include_types: Option, + pub exclude_types: Option, pub min_size: Option, pub follow_links: bool, pub progress: bool, @@ -20,7 +21,8 @@ impl Scanner { pub fn new(app_args: Arc) -> Result { Ok(Self { directory: app_args.get_directory()?.into_boxed_path(), - filetypes: app_args.get_types(), + include_types: app_args.types.clone(), + exclude_types: app_args.exclude_types.clone(), min_depth: app_args.min_depth, max_depth: app_args.max_depth, min_size: app_args.get_min_size(), @@ -29,11 +31,21 @@ impl Scanner { }) } - fn scan_patterns(&self) -> Result { - Ok(match &self.filetypes { - Some(ftypes) => format!("**/*{{{ftypes}}}"), - None => "**/*".to_string(), - }) + fn scan_patterns(&self) -> Result> { + let include_types = match &self.include_types { + Some(ftypes) => Some(format!("**/*.{{{ftypes}}}")), + None => Some("**/*".to_string()), + }; + + let exclude_types = self + .exclude_types + .as_ref() + .map(|ftypes| format!("!**/*.{{{ftypes}}}")); + + Ok(vec![include_types, exclude_types] + .into_iter() + .flatten() + .collect()) } fn attach_link_opts(&self, walker: GlobWalkerBuilder) -> Result { @@ -56,7 +68,7 @@ impl Scanner { fn build_walker(&self) -> Result { let walker = Ok(GlobWalkerBuilder::from_patterns( self.directory.clone(), - &[self.scan_patterns()?], + &self.scan_patterns()?, )) .and_then(|walker| self.attach_walker_min_depth(walker)) .and_then(|walker| self.attach_walker_max_depth(walker)) @@ -79,7 +91,7 @@ impl Scanner { progress_bar.set_style(progress_style); progress_bar.enable_steady_tick(Duration::from_millis(50)); progress_bar.set_message("paths mapped"); - let min_size = self.min_size.unwrap_or_default(); + let min_size = self.min_size.unwrap_or(0); self.build_walker()? .filter_map(Result::ok) @@ -88,7 +100,7 @@ impl Scanner { .filter(|path| path.is_file()) .map(FileInfo::new) .filter_map(Result::ok) - .filter(|file| file.size > min_size) + .filter(|file| file.size >= min_size) .for_each(|file| { let mut flock = files.lock().unwrap(); flock.push(file); @@ -98,3 +110,151 @@ impl Scanner { Ok(()) } } + +#[cfg(test)] +mod tests { + use crate::fileinfo::FileInfo; + use crate::params::Params; + use std::fs::File; + use std::sync::{Arc, Mutex}; + + use super::Scanner; + use indicatif::MultiProgress; + use tempfile::TempDir; + + #[test] + fn ensure_file_include_type_filter_includes_expected_file_types() { + let root = + TempDir::with_prefix("deduplicator_test_root").expect("unable to create tempdir"); + [ + "this-is-a-js-file.js", + "this-is-a-css-file.css", + "this-is-a-csv-file.csv", + "this-is-a-rust-file.rs", + ] + .iter() + .for_each(|path| { + File::create_new(root.path().join(path)).unwrap_or_else(|_| { + panic!("unable to create file {path}"); + }); + }); + + let params = Params { + types: Some(String::from("js,csv")), + dir: Some(root.path().into()), + ..Default::default() + }; + + let progress = Arc::new(MultiProgress::new()); + let scanlist = Arc::new(Mutex::>::new(vec![])); + let scanner = Scanner::new(Arc::new(params)).expect("scanner initialization failed"); + + scanner + .scan(scanlist.clone(), progress) + .expect("scanning failed."); + + let scan_list_mg = scanlist.lock().unwrap(); + + assert!(scan_list_mg.iter().any(|f| f.path.to_str().unwrap() + == root.path().join("this-is-a-js-file.js").to_str().unwrap())); + + assert!(scan_list_mg.iter().any(|f| f.path.to_str().unwrap() + == root.path().join("this-is-a-csv-file.csv").to_str().unwrap())); + + assert!(scan_list_mg.iter().all(|f| f.path.to_str().unwrap() + != root.path().join("this-is-a-css-file.css").to_str().unwrap())); + + assert!(scan_list_mg.iter().all(|f| f.path.to_str().unwrap() + != root.path().join("this-is-a-rust-file.rs").to_str().unwrap())); + } + + #[test] + fn ensure_file_exclude_type_filter_excludes_expected_file_types() { + let root = + TempDir::with_prefix("deduplicator_test_root").expect("unable to create tempdir"); + [ + "this-is-a-js-file.js", + "this-is-a-css-file.css", + "this-is-a-csv-file.csv", + "this-is-a-rust-file.rs", + ] + .iter() + .for_each(|path| { + File::create_new(root.path().join(path)).unwrap_or_else(|_| { + panic!("unable to create file {path}"); + }); + }); + + let params = Params { + exclude_types: Some(String::from("js,csv")), + dir: Some(root.path().into()), + ..Default::default() + }; + + let progress = Arc::new(MultiProgress::new()); + let scanlist = Arc::new(Mutex::>::new(vec![])); + let scanner = Scanner::new(Arc::new(params)).expect("scanner initialization failed"); + + scanner + .scan(scanlist.clone(), progress) + .expect("scanning failed."); + + let scan_list_mg = scanlist.lock().unwrap(); + + assert!(scan_list_mg.iter().all(|f| f.path.to_str().unwrap() + != root.path().join("this-is-a-js-file.js").to_str().unwrap())); + + assert!(scan_list_mg.iter().all(|f| f.path.to_str().unwrap() + != root.path().join("this-is-a-csv-file.csv").to_str().unwrap())); + + assert!(scan_list_mg.iter().any(|f| f.path.to_str().unwrap() + == root.path().join("this-is-a-css-file.css").to_str().unwrap())); + + assert!(scan_list_mg.iter().any(|f| f.path.to_str().unwrap() + == root.path().join("this-is-a-rust-file.rs").to_str().unwrap())); + } + + #[test] + fn complex_file_type_params() { + let root = + TempDir::with_prefix("deduplicator_test_root").expect("unable to create tempdir"); + [ + "this-is-a-js-file.js", + "this-is-a-css-file.css", + "this-is-a-csv-file.csv", + "this-is-a-rust-file.rs", + ] + .iter() + .for_each(|path| { + File::create_new(root.path().join(path)).unwrap_or_else(|_| { + panic!("unable to create file {path}"); + }); + }); + + let params = Params { + types: Some(String::from("js,csv,rs")), + exclude_types: Some(String::from("csv")), + dir: Some(root.path().into()), + ..Default::default() + }; + + let progress = Arc::new(MultiProgress::new()); + let scanlist = Arc::new(Mutex::>::new(vec![])); + let scanner = Scanner::new(Arc::new(params)).expect("scanner initialization failed"); + + scanner + .scan(scanlist.clone(), progress) + .expect("scanning failed."); + + let scan_list_mg = scanlist.lock().unwrap(); + + assert!(scan_list_mg.iter().any(|f| f.path.to_str().unwrap() + == root.path().join("this-is-a-js-file.js").to_str().unwrap())); + + assert!(scan_list_mg.iter().all(|f| f.path.to_str().unwrap() + != root.path().join("this-is-a-csv-file.csv").to_str().unwrap())); + + assert!(scan_list_mg.iter().any(|f| f.path.to_str().unwrap() + == root.path().join("this-is-a-rust-file.rs").to_str().unwrap())); + } +}