diff --git a/Cargo.lock b/Cargo.lock index 868bac2..7f27993 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -274,7 +274,7 @@ dependencies = [ [[package]] name = "deduplicator" -version = "0.2.2" +version = "0.3.0" dependencies = [ "anyhow", "bytesize", diff --git a/Cargo.toml b/Cargo.toml index 4b42588..db4afd5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "deduplicator" -version = "0.2.2" +version = "0.3.0" edition = "2021" -description = "find,filter,delete Duplicates" +description = "find,filter and delete duplicate files" repository = "https://github.com/sreedevk/deduplicator" license = "MIT" authors = [ @@ -12,7 +12,6 @@ authors = [ ] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] anyhow = "1.0.68" bytesize = "1.1.0" diff --git a/README.md b/README.md index f635c1e..1ebf3e7 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,8 @@ ## Usage ```bash +find,filter,delete duplicate files + Usage: deduplicator [OPTIONS] [scan_dir_path] Arguments: @@ -15,12 +17,14 @@ Arguments: Options: -t, --types Filetypes to deduplicate [default = all] -i, --interactive Delete files interactively - -s, --min-size Minimum filesize of duplicates to scan (e.g., 100B/1K/2M/3G/4T) [default: 1b] - -d, --max-depth Max Depth to scan while looking for duplicates - --min-depth Min Depth to scan while looking for duplicates + -m, --min-size Minimum filesize of duplicates to scan (e.g., 100B/1K/2M/3G/4T) [default: 1b] + -D, --max-depth Max Depth to scan while looking for duplicates + -d, --min-depth Min Depth to scan while looking for duplicates -f, --follow-links Follow links while scanning directories - -h, --help Print help information - -V, --version Print version information + -s, --strict Guarantees that two files are duplicate (performs a full hash) + -p, --progress Show Progress spinners & metrics + -h, --help Print help + -V, --version Print version ``` ### Examples @@ -46,16 +50,13 @@ deduplicator ~/Media --min-size 100mb ### Cargo Install #### Stable - > [!WARNING] Note from GxHash: GxHash relies on aes hardware acceleration, you must make sure the aes feature is enabled when building (otherwise it won't build). This can be done by setting the RUSTFLAGS environment variable to -C target-feature=+aes or -C target-cpu=native (the latter should work if your CPU is properly recognized by rustc, which is the case most of the time). -> please install version `0.2.1` if you are unable to install `0.2.2` +> please install version `0.2.1` if you are unable to install `0.3.0` ```bash $ RUSTFLAGS="-C target-cpu=native" cargo install deduplicator ``` -> [!] - #### Nightly if you'd like to install with nightly features, you can use @@ -95,23 +96,9 @@ Note: If you Run into an msvc error, please install MSCV from [here](https://lea ## Performance -Deduplicator uses size comparison and fxhash (a non non-cryptographic hashing algo) to quickly scan through large number of files to find duplicates. its also highly parallel (uses rayon and dashmap). I was able to scan through 120GB of files (Videos, PDFs, Images) in ~300ms. checkout the benchmarks - -## benchmarks - -| Command | Dirsize | Filecount | Mean [ms] | Min [ms] | Max [ms] | Relative | -|:---|:---|---:|---:|---:|---:|---:| -| `deduplicator ~/Data/tmp` | (~120G) | 721 files | 33.5 ± 28.6 | 25.3 | 151.5 | 1.87 ± 1.60 | -| `deduplicator ~/Data/books` | (~8.6G) | 1419 files | 24.5 ± 1.0 | 22.9 | 28.1 | 1.37 ± 0.08 | -| `deduplicator ~/Data/books --min-size 10M` | (~8.6G) | 1419 files | 17.9 ± 0.7 | 16.8 | 20.0 | 1.00 | -| `deduplicator ~/Data/ --types pdf,jpg,png,jpeg` | (~290G) | 104222 files | 1207.2 ± 37.0 | 1172.2 | 1287.7 | 67.27 ± 3.33 | - -* The last entry is lower because of the number of files deduplicator had to go through (~660895 Files). The average size of the files rarely affect the performance of deduplicator. - -These benchmarks were run using [hyperfine](https://github.com/sharkdp/hyperfine). Here are the specs of the machine used to benchmark deduplicator: +Deduplicator uses size comparison and gxhash (a non non-cryptographic hashing algorithm) to quickly scan through large number of files to find duplicates. its also highly parallel (uses rayon and dashmap). ## Screenshots - ![](https://user-images.githubusercontent.com/36154121/213618143-e5182e39-731e-4817-87dd-1a6a0f38a449.gif) ## Roadmap @@ -132,8 +119,8 @@ These benchmarks were run using [hyperfine](https://github.com/sharkdp/hyperfine - [ ] restore json output - [ ] update documentation - [x] remove color output -- [ ] progress bar improvements - - [ ] use progress bar groups +- [x] progress bar improvements + - [x] use progress bar groups - [ ] fix memory leak on very large filesystems - [ ] maybe use a bloom filter - [ ] reduce FileInfo size diff --git a/src/fileinfo.rs b/src/fileinfo.rs index 9b20015..b480490 100644 --- a/src/fileinfo.rs +++ b/src/fileinfo.rs @@ -17,7 +17,7 @@ pub struct FileInfo { } impl FileInfo { - pub fn hash(&self) -> Result { + pub fn hash(&self) -> Result { let file = fs::File::open(&self.path)?; let mapper = unsafe { Mmap::map(&file)? }; let mut primhasher = GxHasher::default(); @@ -26,16 +26,16 @@ impl FileInfo { .chunks(1_000_000) .for_each(|chunk| primhasher.write(chunk)); - Ok(primhasher.finish().to_string()) + Ok(primhasher.finish_u128()) } - pub fn initial_page_hash(&self) -> Result { + pub fn initial_page_hash(&self) -> Result { let file = fs::File::open(&self.path)?; let mapper = unsafe { Mmap::map(&file)? }; let mut primhasher = GxHasher::default(); primhasher.write(mapper.take(4096).into_inner()); - Ok(primhasher.finish().to_string()) + Ok(primhasher.finish_u128()) } pub fn new(path: PathBuf) -> Result { diff --git a/src/formatter.rs b/src/formatter.rs index 5f5aba2..4842358 100644 --- a/src/formatter.rs +++ b/src/formatter.rs @@ -36,16 +36,25 @@ impl Formatter { Ok(modified_time.format("%Y-%m-%d %H:%M:%S").to_string()) } + pub fn gen_sub_tbl(items: Vec, app_args: &Params, max_path_len: u64) -> Table { + let mut inner_table = Table::new(); + inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR); + items.iter().for_each(|file| { + inner_table.add_row(row![ + Self::human_path(file, app_args, max_path_len as usize).unwrap_or_default(), + Self::human_filesize(file).unwrap_or_default(), + Self::human_mtime(file).unwrap_or_default() + ]); + }); + inner_table + } + pub fn generate_table( - raw: Arc>>, - max_path_len: u64, - app_args: &Params, + raw: Arc>>, + mpath_len: u64, + args: &Params, ) -> Result { - let mut output_table = Table::new(); - - output_table.set_titles(row!["hash", "duplicates"]); - - let progress_bar = match app_args.progress { + let progress_bar = match args.progress { true => ProgressBar::new_spinner(), false => ProgressBar::hidden(), }; @@ -60,26 +69,21 @@ impl Formatter { .progress_with(progress_bar) .with_finish(ProgressFinish::WithMessage(Cow::from("output generated"))) .map(|i| { - let mut inner_table = Table::new(); - inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR); - i.value().iter().for_each(|file| { - inner_table.add_row(row![ - Self::human_path(file, app_args, max_path_len as usize).unwrap_or_default(), - Self::human_filesize(file).unwrap_or_default(), - Self::human_mtime(file).unwrap_or_default() - ]); - }); - - row![i.key(), inner_table] + row![ + i.key(), + Self::gen_sub_tbl(i.value().to_vec(), args, mpath_len) + ] }) .collect::>(); + let mut output_table = Table::new(); + output_table.set_titles(row!["hash", "duplicates"]); output_table.extend(rows); Ok(output_table) } pub fn print( - raw: Arc>>, + raw: Arc>>, max_path_len: u64, app_args: &Params, ) -> Result<()> { diff --git a/src/interactive.rs b/src/interactive.rs index d8dfd9f..2bf2bdd 100644 --- a/src/interactive.rs +++ b/src/interactive.rs @@ -10,7 +10,7 @@ use std::{ pub struct Interactive; impl Interactive { - pub fn init(result: Arc>>, app_args: &Params) -> Result<()> { + pub fn init(result: Arc>>, app_args: &Params) -> Result<()> { result.clone().iter().enumerate().for_each(|(gindex, i)| { let group = i.value(); let mut itable = Table::new(); diff --git a/src/params.rs b/src/params.rs index 9d31f81..c74013e 100644 --- a/src/params.rs +++ b/src/params.rs @@ -16,19 +16,19 @@ pub struct Params { #[arg(long, short)] pub interactive: bool, /// Minimum filesize of duplicates to scan (e.g., 100B/1K/2M/3G/4T). - #[arg(long, short = 's', default_value = "1b")] + #[arg(long, short = 'm', default_value = "1b")] pub min_size: Option, /// Max Depth to scan while looking for duplicates - #[arg(long, short = 'd')] + #[arg(long, short = 'D')] pub max_depth: Option, /// Min Depth to scan while looking for duplicates - #[arg(long)] + #[arg(long, short = 'd')] pub min_depth: Option, /// Follow links while scanning directories #[arg(long, short)] pub follow_links: bool, /// Guarantees that two files are duplicate (performs a full hash) - #[arg(long, short = 'f', default_value = "false")] + #[arg(long, short = 's', default_value = "false")] pub strict: bool, /// Show Progress spinners & metrics #[arg(long, short = 'p', default_value = "false")] diff --git a/src/processor.rs b/src/processor.rs index ba0ef86..ee4b75d 100644 --- a/src/processor.rs +++ b/src/processor.rs @@ -18,7 +18,7 @@ impl Processor { pub fn hashwise( app_args: Arc, sw_store: Arc>>, - hw_store: Arc>>, + hw_store: Arc>>, progress_bar_box: Arc, max_file_size: Arc, ) -> Result<()> { @@ -129,7 +129,7 @@ mod tests { use rand::Rng; use std::fs::File; use std::io::Write; - use std::sync::atomic::{AtomicBool, AtomicU64}; + use std::sync::atomic::AtomicBool; use std::sync::{Arc, Mutex}; use tempfile::TempDir; @@ -169,7 +169,6 @@ mod tests { Arc::new(AtomicBool::new(true)), dupstore.clone(), file_queue, - Arc::new(AtomicU64::new(0)), Arc::new(MultiProgress::new()), )?; diff --git a/src/server.rs b/src/server.rs index 49301a9..44ec19d 100644 --- a/src/server.rs +++ b/src/server.rs @@ -14,7 +14,7 @@ use crate::params::Params; pub struct Server { filequeue: Arc>>, sw_duplicate_set: Arc>>, - pub hw_duplicate_set: Arc>>, + pub hw_duplicate_set: Arc>>, threadpool: ThreadPool, app_args: Arc, pub max_file_path_len: Arc,