From 0e624b042da0448455441ae9486931b55d63315a Mon Sep 17 00:00:00 2001 From: sreedev Date: Wed, 18 Jan 2023 02:30:48 -0500 Subject: [PATCH] added minsize filter --- Cargo.lock | 23 +++++++---------------- Cargo.toml | 2 +- src/filters.rs | 12 ++++++++++++ src/main.rs | 1 + src/output.rs | 11 ++++------- src/params.rs | 15 +++++++++++++++ src/scanner.rs | 20 ++++++++++++-------- 7 files changed, 52 insertions(+), 32 deletions(-) create mode 100644 src/filters.rs diff --git a/Cargo.lock b/Cargo.lock index 249bc8f..739bdf3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -70,6 +70,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" +[[package]] +name = "bytesize" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c58ec36aac5066d5ca17df51b3e70279f5670a72102f5752cb7e7c856adfc70" + [[package]] name = "cc" version = "1.0.78" @@ -302,13 +308,13 @@ name = "deduplicator" version = "0.1.1" dependencies = [ "anyhow", + "bytesize", "chrono", "clap", "colored", "dashmap", "fxhash", "glob", - "humansize", "indicatif", "itertools", "memmap2", @@ -435,15 +441,6 @@ dependencies = [ "libc", ] -[[package]] -name = "humansize" -version = "2.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e682e2bd70ecbcce5209f11a992a4ba001fea8e60acf7860ce007629e6d2756" -dependencies = [ - "libm", -] - [[package]] name = "iana-time-zone" version = "0.1.53" @@ -540,12 +537,6 @@ version = "0.2.139" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" -[[package]] -name = "libm" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" - [[package]] name = "link-cplusplus" version = "1.0.8" diff --git a/Cargo.toml b/Cargo.toml index 5abc561..d7c0c94 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,13 +10,13 @@ authors = ["Sreedev Kodichath ", "Valentin Bersier [dependencies] anyhow = "1.0.68" +bytesize = "1.1.0" chrono = "0.4.23" clap = { version = "4.0.32", features = ["derive"] } colored = "2.0.0" dashmap = { version = "5.4.0", features = ["rayon"] } fxhash = "0.2.1" glob = "0.3.0" -humansize = "2.1.2" indicatif = { version = "0.17.2", features = ["rayon", "tokio"] } itertools = "0.10.5" memmap2 = "0.5.8" diff --git a/src/filters.rs b/src/filters.rs new file mode 100644 index 0000000..db95efc --- /dev/null +++ b/src/filters.rs @@ -0,0 +1,12 @@ +use crate::file_manager::File; +use crate::params::Params; + +pub fn is_file_gt_minsize(app_opts: &Params, file: &File) -> bool { + match app_opts.get_minsize() { + Some(msize) => match file.size { + Some(fsize) => fsize >= msize, + None => true, + }, + None => true, + } +} diff --git a/src/main.rs b/src/main.rs index 4a82824..132bae8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,6 +3,7 @@ mod file_manager; mod output; mod params; mod scanner; +mod filters; use anyhow::Result; use app::App; diff --git a/src/output.rs b/src/output.rs index 4767d6f..b16d111 100644 --- a/src/output.rs +++ b/src/output.rs @@ -5,7 +5,6 @@ use chrono::offset::Utc; use chrono::DateTime; use colored::Colorize; use dashmap::DashMap; -use humansize::{format_size, DECIMAL}; use itertools::Itertools; use prettytable::{format, row, Table}; use std::io::Write; @@ -30,10 +29,8 @@ fn format_path(path: &str, opts: &Params) -> Result { Ok(format!("...{:<32}", display_range)) } -fn file_size(path: &String) -> Result { - let mdata = fs::metadata(path)?; - let formatted_size = format!("{:>12}", format_size(mdata.len(), DECIMAL)); - Ok(formatted_size) +fn file_size(file: &File) -> Result { + Ok(format!("{:>12}", bytesize::ByteSize::b(file.size.unwrap()))) } fn modified_time(path: &String) -> Result { @@ -131,7 +128,7 @@ pub fn interactive(duplicates: DashMap>, opts: &Params) { itable.add_row(row![ index, format_path(&file.path, opts).unwrap_or_default().blue(), - file_size(&file.path).unwrap_or_default().red(), + file_size(&file).unwrap_or_default().red(), modified_time(&file.path).unwrap_or_default().yellow() ]); }); @@ -151,7 +148,7 @@ pub fn print(duplicates: DashMap>, opts: &Params) { group.iter().for_each(|file| { inner_table.add_row(row![ format_path(&file.path, opts).unwrap_or_default().blue(), - file_size(&file.path).unwrap_or_default().red(), + file_size(&file).unwrap_or_default().red(), modified_time(&file.path).unwrap_or_default().yellow() ]); }); diff --git a/src/params.rs b/src/params.rs index 0ab59b8..db21ab7 100644 --- a/src/params.rs +++ b/src/params.rs @@ -14,9 +14,24 @@ pub struct Params { /// Delete files interactively #[arg(long, short)] pub interactive: bool, + /// Minimum filesize of duplicates to scan (e.g., 100B/1K/2M/3G/4T). [default = 0] + #[arg(long, short)] + pub minsize: Option, } impl Params { + pub fn get_minsize(&self) -> Option { + match &self.minsize { + Some(msize) => { + match msize.parse::() { + Ok(units) => Some(units.0), + Err(_) => None + } + }, + None => None + } + } + pub fn get_directory(&self) -> Result { let dir_pathbuf: PathBuf = self .dir diff --git a/src/scanner.rs b/src/scanner.rs index ad2269c..5383267 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -1,3 +1,4 @@ +use crate::{file_manager::File, filters, params::Params}; use anyhow::Result; use dashmap::DashMap; use fxhash::hash64 as hasher; @@ -8,8 +9,6 @@ use rayon::prelude::*; use std::hash::Hasher; use std::{fs, path::PathBuf}; -use crate::{file_manager::File, params::Params}; - #[derive(Clone, Copy)] enum IndexCritera { Size, @@ -58,17 +57,22 @@ fn scan(app_opts: &Params) -> Result> { }) .collect::>() }) - .map(|file_path| File { path: file_path.clone(), hash: None, size: Some(fs::metadata(file_path).unwrap().len()) }) + .map(|file_path| File { + path: file_path.clone(), + hash: None, + size: Some(fs::metadata(file_path).unwrap().len()), + }) + .filter(|file| filters::is_file_gt_minsize(app_opts, file)) .collect(); Ok(files) } -fn process_file_hash_index(fpath: String) -> Result { +fn process_file_hash_index(file: &File) -> Result { Ok(File { - path: fpath.clone(), - size: None, - hash: Some(hash_file(&fpath).unwrap_or_default()), + path: file.path.clone(), + size: file.size, + hash: Some(hash_file(&file.path).unwrap_or_default()), }) } @@ -85,7 +89,7 @@ fn process_file_index( .or_insert_with(|| vec![file]); } IndexCritera::Hash => { - let processed_file = process_file_hash_index(file.path).unwrap(); + let processed_file = process_file_hash_index(&file).unwrap(); let indexhash = processed_file.clone().hash.unwrap_or_default(); store