diff --git a/Cargo.lock b/Cargo.lock index 249bc8f..739bdf3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -70,6 +70,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" +[[package]] +name = "bytesize" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c58ec36aac5066d5ca17df51b3e70279f5670a72102f5752cb7e7c856adfc70" + [[package]] name = "cc" version = "1.0.78" @@ -302,13 +308,13 @@ name = "deduplicator" version = "0.1.1" dependencies = [ "anyhow", + "bytesize", "chrono", "clap", "colored", "dashmap", "fxhash", "glob", - "humansize", "indicatif", "itertools", "memmap2", @@ -435,15 +441,6 @@ dependencies = [ "libc", ] -[[package]] -name = "humansize" -version = "2.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e682e2bd70ecbcce5209f11a992a4ba001fea8e60acf7860ce007629e6d2756" -dependencies = [ - "libm", -] - [[package]] name = "iana-time-zone" version = "0.1.53" @@ -540,12 +537,6 @@ version = "0.2.139" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" -[[package]] -name = "libm" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" - [[package]] name = "link-cplusplus" version = "1.0.8" diff --git a/Cargo.toml b/Cargo.toml index 5abc561..d7c0c94 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,13 +10,13 @@ authors = ["Sreedev Kodichath ", "Valentin Bersier [dependencies] anyhow = "1.0.68" +bytesize = "1.1.0" chrono = "0.4.23" clap = { version = "4.0.32", features = ["derive"] } colored = "2.0.0" dashmap = { version = "5.4.0", features = ["rayon"] } fxhash = "0.2.1" glob = "0.3.0" -humansize = "2.1.2" indicatif = { version = "0.17.2", features = ["rayon", "tokio"] } itertools = "0.10.5" memmap2 = "0.5.8" diff --git a/README.md b/README.md index 75b1638..908b807 100644 --- a/README.md +++ b/README.md @@ -14,11 +14,12 @@ NOTE: This project is still being developed. At the moment, as shown in the scre Usage: deduplicator [OPTIONS] Options: - -t, --types Filetypes to deduplicate (default = all) - --dir Run Deduplicator on dir different from pwd - -i, --interactive Delete files interactively - -h, --help Print help information - -V, --version Print version information + -t, --types Filetypes to deduplicate (default = all) + --dir Run Deduplicator on dir different from pwd + -i, --interactive Delete files interactively + -m, --minsize Minimum filesize of duplicates to scan (e.g., 100B/1K/2M/3G/4T). [default = 0] + -h, --help Print help information + -V, --version Print version information ```

Installation

diff --git a/src/filters.rs b/src/filters.rs new file mode 100644 index 0000000..db95efc --- /dev/null +++ b/src/filters.rs @@ -0,0 +1,12 @@ +use crate::file_manager::File; +use crate::params::Params; + +pub fn is_file_gt_minsize(app_opts: &Params, file: &File) -> bool { + match app_opts.get_minsize() { + Some(msize) => match file.size { + Some(fsize) => fsize >= msize, + None => true, + }, + None => true, + } +} diff --git a/src/main.rs b/src/main.rs index 4a82824..132bae8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,6 +3,7 @@ mod file_manager; mod output; mod params; mod scanner; +mod filters; use anyhow::Result; use app::App; diff --git a/src/output.rs b/src/output.rs index 3bd0e8a..3ed023f 100644 --- a/src/output.rs +++ b/src/output.rs @@ -5,7 +5,6 @@ use chrono::offset::Utc; use chrono::DateTime; use colored::Colorize; use dashmap::DashMap; -use humansize::{format_size, DECIMAL}; use itertools::Itertools; use prettytable::{format, row, Table}; use std::io::Write; @@ -30,10 +29,8 @@ fn format_path(path: &str, opts: &Params) -> Result { Ok(format!("...{:<32}", display_range)) } -fn file_size(path: &String) -> Result { - let mdata = fs::metadata(path)?; - let formatted_size = format!("{:>12}", format_size(mdata.len(), DECIMAL)); - Ok(formatted_size) +fn file_size(file: &File) -> Result { + Ok(format!("{:>12}", bytesize::ByteSize::b(file.size.unwrap()))) } fn modified_time(path: &String) -> Result { @@ -119,6 +116,15 @@ fn process_group_action(duplicates: &Vec, dup_index: usize, dup_size: usiz pub fn interactive(duplicates: DashMap>, opts: &Params) { print_meta_info(); + + if duplicates.is_empty() { + println!( + "\n{}", + "No duplicates found matching your search criteria.".green() + ); + return; + } + duplicates .clone() .into_iter() @@ -131,7 +137,7 @@ pub fn interactive(duplicates: DashMap>, opts: &Params) { itable.add_row(row![ index, format_path(&file.path, opts).unwrap_or_default().blue(), - file_size(&file.path).unwrap_or_default().red(), + file_size(&file).unwrap_or_default().red(), modified_time(&file.path).unwrap_or_default().yellow() ]); }); @@ -143,33 +149,31 @@ pub fn interactive(duplicates: DashMap>, opts: &Params) { pub fn print(duplicates: DashMap>, opts: &Params) { print_meta_info(); + if duplicates.is_empty() { + println!( + "\n{}", + "No duplicates found matching your search criteria.".green() + ); + return; + } + let mut output_table = Table::new(); output_table.set_titles(row!["hash", "duplicates"]); duplicates .into_iter() - .map(|f| { - // we extract the file size of the first file in the group - let size = - f.1.first() - .and_then(|ff| fs::metadata(&ff.path).ok()) - .and_then(|m| Some(m.len())) - .unwrap_or_default(); - (f.0, f.1, size) - }) - .sorted_unstable_by_key(|f| f.2) // sort by ascending size - .for_each(|(hash, group, size)| { + .sorted_unstable_by_key(|f| f.1.first().and_then(|ff| ff.size).unwrap_or_default()) // sort by ascending size + .for_each(|(hash, group)| { let mut inner_table = Table::new(); inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR); group.iter().for_each(|file| { inner_table.add_row(row![ format_path(&file.path, opts).unwrap_or_default().blue(), - // since all files should logically have the same size, - // we can print the pre-computed size of the first element in the group - format!("{:>12}", format_size(size, DECIMAL)).red(), + file_size(&file).unwrap_or_default().red(), modified_time(&file.path).unwrap_or_default().yellow() ]); }); output_table.add_row(row![hash.green(), inner_table]); }); + output_table.printstd(); } diff --git a/src/params.rs b/src/params.rs index 0ab59b8..db21ab7 100644 --- a/src/params.rs +++ b/src/params.rs @@ -14,9 +14,24 @@ pub struct Params { /// Delete files interactively #[arg(long, short)] pub interactive: bool, + /// Minimum filesize of duplicates to scan (e.g., 100B/1K/2M/3G/4T). [default = 0] + #[arg(long, short)] + pub minsize: Option, } impl Params { + pub fn get_minsize(&self) -> Option { + match &self.minsize { + Some(msize) => { + match msize.parse::() { + Ok(units) => Some(units.0), + Err(_) => None + } + }, + None => None + } + } + pub fn get_directory(&self) -> Result { let dir_pathbuf: PathBuf = self .dir diff --git a/src/scanner.rs b/src/scanner.rs index c5d1086..5383267 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -1,3 +1,4 @@ +use crate::{file_manager::File, filters, params::Params}; use anyhow::Result; use dashmap::DashMap; use fxhash::hash64 as hasher; @@ -8,8 +9,6 @@ use rayon::prelude::*; use std::hash::Hasher; use std::{fs, path::PathBuf}; -use crate::{file_manager::File, params::Params}; - #[derive(Clone, Copy)] enum IndexCritera { Size, @@ -28,12 +27,7 @@ pub fn duplicates(app_opts: &Params) -> Result>> { .collect::>(); if sizewize_duplicate_files.len() > 1 { - let size_wise_duplicate_paths = sizewize_duplicate_files - .into_par_iter() - .map(|file| file.path) - .collect::>(); - - let hash_index_store = index_files(size_wise_duplicate_paths, IndexCritera::Hash)?; + let hash_index_store = index_files(sizewize_duplicate_files, IndexCritera::Hash)?; let duplicate_files = hash_index_store .into_par_iter() .filter(|(_, files)| files.len() > 1) @@ -45,9 +39,9 @@ pub fn duplicates(app_opts: &Params) -> Result>> { } } -fn scan(app_opts: &Params) -> Result> { +fn scan(app_opts: &Params) -> Result> { let glob_patterns: Vec = app_opts.get_glob_patterns(); - let files: Vec = glob_patterns + let files: Vec = glob_patterns .par_iter() .progress_with_style(ProgressStyle::with_template( "{spinner:.green} [scanning files] [{wide_bar:.cyan/blue}] {pos}/{len} files", @@ -63,42 +57,39 @@ fn scan(app_opts: &Params) -> Result> { }) .collect::>() }) + .map(|file_path| File { + path: file_path.clone(), + hash: None, + size: Some(fs::metadata(file_path).unwrap().len()), + }) + .filter(|file| filters::is_file_gt_minsize(app_opts, file)) .collect(); Ok(files) } -fn process_file_size_index(fpath: String) -> Result { +fn process_file_hash_index(file: &File) -> Result { Ok(File { - path: fpath.clone(), - size: Some(fs::metadata(fpath)?.len()), - hash: None, - }) -} - -fn process_file_hash_index(fpath: String) -> Result { - Ok(File { - path: fpath.clone(), - size: None, - hash: Some(hash_file(&fpath).unwrap_or_default()), + path: file.path.clone(), + size: file.size, + hash: Some(hash_file(&file.path).unwrap_or_default()), }) } fn process_file_index( - fpath: String, + file: File, store: &DashMap>, index_criteria: IndexCritera, ) { match index_criteria { IndexCritera::Size => { - let processed_file = process_file_size_index(fpath).unwrap(); store - .entry(processed_file.size.unwrap_or_default().to_string()) - .and_modify(|fileset| fileset.push(processed_file.clone())) - .or_insert_with(|| vec![processed_file]); + .entry(file.size.unwrap_or_default().to_string()) + .and_modify(|fileset| fileset.push(file.clone())) + .or_insert_with(|| vec![file]); } IndexCritera::Hash => { - let processed_file = process_file_hash_index(fpath).unwrap(); + let processed_file = process_file_hash_index(&file).unwrap(); let indexhash = processed_file.clone().hash.unwrap_or_default(); store @@ -110,7 +101,7 @@ fn process_file_index( } fn index_files( - files: Vec, + files: Vec, index_criteria: IndexCritera, ) -> Result>> { let store: DashMap> = DashMap::new(); @@ -124,7 +115,7 @@ fn index_files( Ok(store) } -pub fn incremental_hashing(filepath: &str) -> Result { +fn incremental_hashing(filepath: &str) -> Result { let file = fs::File::open(filepath)?; let fmap = unsafe { Mmap::map(&file)? }; let mut inchasher = fxhash::FxHasher::default(); @@ -135,12 +126,12 @@ pub fn incremental_hashing(filepath: &str) -> Result { Ok(format!("{}", inchasher.finish())) } -pub fn standard_hashing(filepath: &str) -> Result { +fn standard_hashing(filepath: &str) -> Result { let file = fs::read(filepath)?; Ok(hasher(&*file).to_string()) } -pub fn hash_file(filepath: &str) -> Result { +fn hash_file(filepath: &str) -> Result { let filemeta = fs::metadata(filepath)?; // NOTE: USE INCREMENTAL HASHING ONLY FOR FILES > 100MB