smolkik_adm/deduplicator
1
0
Fork 0
mirror of https://github.com/sreedevk/deduplicator.git synced 2026-08-31 04:25:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

all file type related filtering issues fixed

Browse Source
This commit is contained in:
sreedevk
2025-07-19 16:26:12 +00:00
parent f5d2d4e22c
commit 21f0eb3cb0
3 changed files with 171 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@@ -177,10 +177,10 @@ dust 'bench_artifacts'
- [ ] fix: partial hash collision - a file full of null bytes ("\0") and an empty file. This is a known trade off in gxhash.
- [ ] include initial pages and final pages of the file
- [ ] append the offset between the last initial page hashed and the first final page hashed in the content passed to the hasher.
- [ ] fix: --exclude-types and --types flag behave identically.
## v0.3.2
- [x] fix: single file groups are printed to screen
- [x] fix: --exclude-types and --types flag behave identically.
## v0.3.1
- [x] parallelization

45
src/params.rs
View File

@@ -2,7 +2,6 @@ use std::{fs, path::PathBuf};
use anyhow::Result;
use clap::{Parser, ValueHint};
use std::collections::HashSet;
#[derive(Parser, Debug, Default, Clone)]
#[command(author, version, about, long_about = None)]
@@ -56,48 +55,4 @@ impl Params {
let dir = fs::canonicalize(dir_path)?;
Ok(dir)
}
pub fn types_intersection(itypes: &str, xtypes: &str) -> String {
let iset = itypes
.split(",")
.map(String::from)
.collect::<HashSet<String>>();
let xset = xtypes
.split(",")
.map(String::from)
.collect::<HashSet<String>>();
iset.difference(&xset)
.cloned()
.collect::<Vec<String>>()
.join(",")
}
pub fn get_types(&self) -> Option<String> {
match &self.types {
Some(itypes) => match &self.exclude_types {
Some(xtypes) => Some(Self::types_intersection(itypes, xtypes)),
None => Some(itypes.to_string()),
},
None => self.exclude_types.as_ref().map(|xtypes| xtypes.to_string()),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn mixing_include_and_exclude_types_works_as_expected() {
let params = Params {
types: Some(String::from("js,xml,ts,pdf,tiff")),
exclude_types: Some(String::from("js,ts,xml")),
..Default::default()
};
assert!(params
.get_types()
.is_some_and(|x| x == "pdf,tiff" || x == "tiff,pdf"))
}
}

180
src/scanner.rs
View File

@@ -8,9 +8,10 @@ use globwalk::{GlobWalker, GlobWalkerBuilder};
pub struct Scanner {
pub directory: Box<Path>,
pub filetypes: Option<String>,
pub min_depth: Option<usize>,
pub max_depth: Option<usize>,
pub include_types: Option<String>,
pub exclude_types: Option<String>,
pub min_size: Option<u64>,
pub follow_links: bool,
pub progress: bool,
@@ -20,7 +21,8 @@ impl Scanner {
pub fn new(app_args: Arc<Params>) -> Result<Self> {
Ok(Self {
directory: app_args.get_directory()?.into_boxed_path(),
filetypes: app_args.get_types(),
include_types: app_args.types.clone(),
exclude_types: app_args.exclude_types.clone(),
min_depth: app_args.min_depth,
max_depth: app_args.max_depth,
min_size: app_args.get_min_size(),
@@ -29,11 +31,21 @@ impl Scanner {
})
}
fn scan_patterns(&self) -> Result<String> {
Ok(match &self.filetypes {
Some(ftypes) => format!("**/*{{{ftypes}}}"),
None => "**/*".to_string(),
})
fn scan_patterns(&self) -> Result<Vec<String>> {
let include_types = match &self.include_types {
Some(ftypes) => Some(format!("**/*.{{{ftypes}}}")),
None => Some("**/*".to_string()),
};
let exclude_types = self
.exclude_types
.as_ref()
.map(|ftypes| format!("!**/*.{{{ftypes}}}"));
Ok(vec![include_types, exclude_types]
.into_iter()
.flatten()
.collect())
}
fn attach_link_opts(&self, walker: GlobWalkerBuilder) -> Result<GlobWalkerBuilder> {
@@ -56,7 +68,7 @@ impl Scanner {
fn build_walker(&self) -> Result<GlobWalker> {
let walker = Ok(GlobWalkerBuilder::from_patterns(
self.directory.clone(),
&[self.scan_patterns()?],
&self.scan_patterns()?,
))
.and_then(|walker| self.attach_walker_min_depth(walker))
.and_then(|walker| self.attach_walker_max_depth(walker))
@@ -79,7 +91,7 @@ impl Scanner {
progress_bar.set_style(progress_style);
progress_bar.enable_steady_tick(Duration::from_millis(50));
progress_bar.set_message("paths mapped");
let min_size = self.min_size.unwrap_or_default();
let min_size = self.min_size.unwrap_or(0);
self.build_walker()?
.filter_map(Result::ok)
@@ -88,7 +100,7 @@ impl Scanner {
.filter(|path| path.is_file())
.map(FileInfo::new)
.filter_map(Result::ok)
.filter(|file| file.size > min_size)
.filter(|file| file.size >= min_size)
.for_each(|file| {
let mut flock = files.lock().unwrap();
flock.push(file);
@@ -98,3 +110,151 @@ impl Scanner {
Ok(())
}
}
#[cfg(test)]
mod tests {
use crate::fileinfo::FileInfo;
use crate::params::Params;
use std::fs::File;
use std::sync::{Arc, Mutex};
use super::Scanner;
use indicatif::MultiProgress;
use tempfile::TempDir;
#[test]
fn ensure_file_include_type_filter_includes_expected_file_types() {
let root =
TempDir::with_prefix("deduplicator_test_root").expect("unable to create tempdir");
[
"this-is-a-js-file.js",
"this-is-a-css-file.css",
"this-is-a-csv-file.csv",
"this-is-a-rust-file.rs",
]
.iter()
.for_each(|path| {
File::create_new(root.path().join(path)).unwrap_or_else(|_| {
panic!("unable to create file {path}");
});
});
let params = Params {
types: Some(String::from("js,csv")),
dir: Some(root.path().into()),
..Default::default()
};
let progress = Arc::new(MultiProgress::new());
let scanlist = Arc::new(Mutex::<Vec<FileInfo>>::new(vec![]));
let scanner = Scanner::new(Arc::new(params)).expect("scanner initialization failed");
scanner
.scan(scanlist.clone(), progress)
.expect("scanning failed.");
let scan_list_mg = scanlist.lock().unwrap();
assert!(scan_list_mg.iter().any(|f| f.path.to_str().unwrap()
== root.path().join("this-is-a-js-file.js").to_str().unwrap()));
assert!(scan_list_mg.iter().any(|f| f.path.to_str().unwrap()
== root.path().join("this-is-a-csv-file.csv").to_str().unwrap()));
assert!(scan_list_mg.iter().all(|f| f.path.to_str().unwrap()
!= root.path().join("this-is-a-css-file.css").to_str().unwrap()));
assert!(scan_list_mg.iter().all(|f| f.path.to_str().unwrap()
!= root.path().join("this-is-a-rust-file.rs").to_str().unwrap()));
}
#[test]
fn ensure_file_exclude_type_filter_excludes_expected_file_types() {
let root =
TempDir::with_prefix("deduplicator_test_root").expect("unable to create tempdir");
[
"this-is-a-js-file.js",
"this-is-a-css-file.css",
"this-is-a-csv-file.csv",
"this-is-a-rust-file.rs",
]
.iter()
.for_each(|path| {
File::create_new(root.path().join(path)).unwrap_or_else(|_| {
panic!("unable to create file {path}");
});
});
let params = Params {
exclude_types: Some(String::from("js,csv")),
dir: Some(root.path().into()),
..Default::default()
};
let progress = Arc::new(MultiProgress::new());
let scanlist = Arc::new(Mutex::<Vec<FileInfo>>::new(vec![]));
let scanner = Scanner::new(Arc::new(params)).expect("scanner initialization failed");
scanner
.scan(scanlist.clone(), progress)
.expect("scanning failed.");
let scan_list_mg = scanlist.lock().unwrap();
assert!(scan_list_mg.iter().all(|f| f.path.to_str().unwrap()
!= root.path().join("this-is-a-js-file.js").to_str().unwrap()));
assert!(scan_list_mg.iter().all(|f| f.path.to_str().unwrap()
!= root.path().join("this-is-a-csv-file.csv").to_str().unwrap()));
assert!(scan_list_mg.iter().any(|f| f.path.to_str().unwrap()
== root.path().join("this-is-a-css-file.css").to_str().unwrap()));
assert!(scan_list_mg.iter().any(|f| f.path.to_str().unwrap()
== root.path().join("this-is-a-rust-file.rs").to_str().unwrap()));
}
#[test]
fn complex_file_type_params() {
let root =
TempDir::with_prefix("deduplicator_test_root").expect("unable to create tempdir");
[
"this-is-a-js-file.js",
"this-is-a-css-file.css",
"this-is-a-csv-file.csv",
"this-is-a-rust-file.rs",
]
.iter()
.for_each(|path| {
File::create_new(root.path().join(path)).unwrap_or_else(|_| {
panic!("unable to create file {path}");
});
});
let params = Params {
types: Some(String::from("js,csv,rs")),
exclude_types: Some(String::from("csv")),
dir: Some(root.path().into()),
..Default::default()
};
let progress = Arc::new(MultiProgress::new());
let scanlist = Arc::new(Mutex::<Vec<FileInfo>>::new(vec![]));
let scanner = Scanner::new(Arc::new(params)).expect("scanner initialization failed");
scanner
.scan(scanlist.clone(), progress)
.expect("scanning failed.");
let scan_list_mg = scanlist.lock().unwrap();
assert!(scan_list_mg.iter().any(|f| f.path.to_str().unwrap()
== root.path().join("this-is-a-js-file.js").to_str().unwrap()));
assert!(scan_list_mg.iter().all(|f| f.path.to_str().unwrap()
!= root.path().join("this-is-a-csv-file.csv").to_str().unwrap()));
assert!(scan_list_mg.iter().any(|f| f.path.to_str().unwrap()
== root.path().join("this-is-a-rust-file.rs").to_str().unwrap()));
}
}