From d99d7833270a2787618217e7fff96735955fd423 Mon Sep 17 00:00:00 2001 From: sreedevk Date: Sun, 13 Jul 2025 21:24:31 +0000 Subject: [PATCH] add more tests --- Cargo.lock | 89 ++++++++++++++++---- Cargo.toml | 14 +++- README.md | 38 --------- src/fileinfo.rs | 6 +- src/formatter.rs | 1 + src/interactive.rs | 45 ++++++----- src/processor.rs | 197 ++++++++++++++++++++++++++++++++++++++++++++- src/scanner.rs | 5 +- 8 files changed, 306 insertions(+), 89 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7f27993..aae92e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -81,6 +81,22 @@ version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +[[package]] +name = "assert_cmd" +version = "2.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bd389a4b2970a01282ee455294913c0a43724daedcd1a24c3eb0ec1c1320b66" +dependencies = [ + "anstyle", + "bstr", + "doc-comment", + "libc", + "predicates", + "predicates-core", + "predicates-tree", + "wait-timeout", +] + [[package]] name = "autocfg" version = "1.3.0" @@ -106,6 +122,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" dependencies = [ "memchr", + "regex-automata", "serde", ] @@ -277,6 +294,7 @@ name = "deduplicator" version = "0.3.0" dependencies = [ "anyhow", + "assert_cmd", "bytesize", "chrono", "clap", @@ -284,7 +302,6 @@ dependencies = [ "globwalk", "gxhash", "indicatif", - "itertools", "memmap2", "pathdiff", "prettytable-rs", @@ -292,9 +309,14 @@ dependencies = [ "rayon", "tempfile", "threadpool", - "unicode-segmentation", ] +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + [[package]] name = "dirs-next" version = "2.0.0" @@ -316,6 +338,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "doc-comment" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + [[package]] name = "either" version = "1.13.0" @@ -509,15 +537,6 @@ version = "1.70.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.11" @@ -657,6 +676,33 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "predicates" +version = "3.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d19ee57562043d37e82899fade9a22ebab7be9cef5026b07fda9cdd4293573" +dependencies = [ + "anstyle", + "difflib", + "predicates-core", +] + +[[package]] +name = "predicates-core" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "727e462b119fe9c93fd0eb1429a5f7647394014cf3c04ab2c0350eeb09095ffa" + +[[package]] +name = "predicates-tree" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72dd2d6d381dfb73a193c7fca536518d7caee39fc8503f74e7dc0be0531b425c" +dependencies = [ + "predicates-core", + "termtree", +] + [[package]] name = "prettytable-rs" version = "0.10.0" @@ -888,6 +934,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "termtree" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" + [[package]] name = "thiserror" version = "1.0.61" @@ -923,12 +975,6 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" -[[package]] -name = "unicode-segmentation" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" - [[package]] name = "unicode-width" version = "0.1.13" @@ -941,6 +987,15 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + [[package]] name = "walkdir" version = "2.5.0" diff --git a/Cargo.toml b/Cargo.toml index 96b071b..808a1bb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,10 @@ authors = [ "Dhruva Sagar ", ] +[[bin]] +name = "deduplicator" +path = "src/main.rs" + # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] anyhow = "1.0.68" @@ -21,7 +25,6 @@ dashmap = { version = "5.4.0", features = ["rayon"] } globwalk = "0.8.1" gxhash = { version = "3.4.1", default-features = false } indicatif = { version = "0.17.2", features = ["rayon"] } -itertools = "0.10.5" memmap2 = "0.5.8" pathdiff = "0.2.1" prettytable-rs = "0.10.0" @@ -29,15 +32,17 @@ rand = "0.9.1" rayon = "1.6.1" tempfile = "3.20.0" threadpool = "1.8.1" -unicode-segmentation = "1.10.0" [profile.release] strip = true +opt-level = 3 +lto = "thin" +debug = false +codegen-units = 1 # generated by 'cargo dist init' [profile.dist] inherits = "release" -lto = "thin" [workspace.metadata.dist] rust-toolchain-version = "1.87.0" @@ -49,3 +54,6 @@ targets = [ "aarch64-apple-darwin", ] cargo-dist-version = "0.0.7" + +[dev-dependencies] +assert_cmd = "2.0.0" diff --git a/README.md b/README.md index 775a88b..bcfe4fc 100644 --- a/README.md +++ b/README.md @@ -56,45 +56,7 @@ deduplicator ~/Media --min-size 100mb $ RUSTFLAGS="-C target-cpu=native" cargo install deduplicator ``` -#### Nightly - -if you'd like to install with nightly features, you can use - -```bash -$ cargo install --git https://github.com/sreedevk/deduplicator -``` -Please note that if you use a version manager to install rust (like asdf), you need to reshim (`asdf reshim rust`). - -### Linux (Pre-built Binary) - -you can download the pre-built binary from the [Releases](https://github.com/sreedevk/deduplicator/releases) page. -download the `deduplicator-x86_64-unknown-linux-gnu.tar.gz` for linux. Once you have the tarball file with the executable, -you can follow these steps to install: - -```bash -$ tar -zxvf deduplicator-x86_64-unknown-linux-gnu.tar.gz -$ sudo mv deduplicator /usr/bin/ -``` - -### Mac OS (Pre-built Binary) - -you can download the pre-build binary from the [Releases](https://github.com/sreedevk/deduplicator/releases) page. -download the `deduplicator-x86_64-apple-darwin.tar.gz` tarball for mac os. Once you have the tarball file with the executable, you can follow these steps to install: - -```bash -$ tar -zxvf deduplicator-x86_64-unknown-linux-gnu.tar.gz -$ sudo mv deduplicator /usr/bin/ -``` - -### Windows (Pre-built Binary) - -you can download the pre-build binary from the [Releases](https://github.com/sreedevk/deduplicator/releases) page. -download the `deduplicator-x86_64-pc-windows-msvc.zip` zip file for windows. unzip the `zip` file & move the `deduplicator.exe` to a location in the PATH system environment variable. - -Note: If you Run into an msvc error, please install MSCV from [here](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist?view=msvc-170) - ## Performance - Deduplicator uses size comparison and gxhash (a non non-cryptographic hashing algorithm) to quickly scan through large number of files to find duplicates. its also highly parallel (uses rayon and dashmap). ## Screenshots diff --git a/src/fileinfo.rs b/src/fileinfo.rs index 2a23800..df907f3 100644 --- a/src/fileinfo.rs +++ b/src/fileinfo.rs @@ -19,9 +19,9 @@ impl FileInfo { pub fn hash(&self, seed: i64) -> Result { let file = fs::File::open(&self.path)?; let mapper = unsafe { Mmap::map(&file)? }; - let final_hash = mapper - .chunks(4096) - .fold(0, |acc, chunk: &[u8]| acc + gxhash128(chunk, seed)); + let final_hash = mapper.chunks(4096).fold(0u128, |acc, chunk: &[u8]| { + acc.wrapping_add(gxhash128(chunk, seed)) + }); Ok(final_hash) } diff --git a/src/formatter.rs b/src/formatter.rs index 4842358..8da9ec0 100644 --- a/src/formatter.rs +++ b/src/formatter.rs @@ -68,6 +68,7 @@ impl Formatter { .par_iter_mut() .progress_with(progress_bar) .with_finish(ProgressFinish::WithMessage(Cow::from("output generated"))) + .filter(|i| i.value().len() > 1) .map(|i| { row![ i.key(), diff --git a/src/interactive.rs b/src/interactive.rs index 2bf2bdd..51dbf6c 100644 --- a/src/interactive.rs +++ b/src/interactive.rs @@ -11,30 +11,35 @@ pub struct Interactive; impl Interactive { pub fn init(result: Arc>>, app_args: &Params) -> Result<()> { - result.clone().iter().enumerate().for_each(|(gindex, i)| { - let group = i.value(); - let mut itable = Table::new(); - itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR); - itable.set_titles(row!["index", "filename", "size", "updated_at"]); + result + .clone() + .iter() + .filter(|i| i.value().len() > 1) + .enumerate() + .for_each(|(gindex, i)| { + let group = i.value(); + let mut itable = Table::new(); + itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR); + itable.set_titles(row!["index", "filename", "size", "updated_at"]); - let max_path_size = group - .iter() - .map(|f| f.path.iter().count()) - .max() - .unwrap_or_default(); + let max_path_size = group + .iter() + .map(|f| f.path.iter().count()) + .max() + .unwrap_or_default(); - group.iter().enumerate().for_each(|(index, file)| { - itable.add_row(row![ - index, - Formatter::human_path(file, app_args, max_path_size).unwrap_or_default(), - Formatter::human_filesize(file).unwrap_or_default(), - Formatter::human_mtime(file).unwrap_or_default() - ]); + group.iter().enumerate().for_each(|(index, file)| { + itable.add_row(row![ + index, + Formatter::human_path(file, app_args, max_path_size).unwrap_or_default(), + Formatter::human_filesize(file).unwrap_or_default(), + Formatter::human_mtime(file).unwrap_or_default() + ]); + }); + + Self::process_group_action(group, gindex, result.len(), itable); }); - Self::process_group_action(group, gindex, result.len(), itable); - }); - Ok(()) } diff --git a/src/processor.rs b/src/processor.rs index 04cee53..b7ac840 100644 --- a/src/processor.rs +++ b/src/processor.rs @@ -130,7 +130,7 @@ mod tests { use rand::Rng; use std::fs::File; use std::io::Write; - use std::sync::atomic::AtomicBool; + use std::sync::atomic::{AtomicBool, AtomicU64}; use std::sync::{Arc, Mutex}; use tempfile::TempDir; @@ -144,11 +144,165 @@ mod tests { } #[test] - fn sizewise_sorting_normal() -> Result<()> { + fn hashwise_sorting_two_files_with_identical_init_page_only_strict_mode() -> Result<()> { + let root = TempDir::new()?; + let content = generate_bytes(4096); + + let mut content_x = content.clone(); + let mut content_y = content.clone(); + + content_x.extend(generate_bytes(1720320)); + content_y.extend(generate_bytes(1720320)); + + let files = [ + (root.path().join("fileone.bin"), content_x), + (root.path().join("filetwo.bin"), content_y), + ]; + + for (fpath, content) in files.iter() { + let mut f = File::create_new(fpath)?; + f.write_all(content)?; + } + + let dupstore = Arc::new(DashMap::new()); + let file_queue = Arc::new(Mutex::new( + files + .iter() + .map(|f| FileInfo::new(f.0.clone()).unwrap()) + .collect::>(), + )); + + let hw_dupstore = Arc::new(DashMap::new()); + Processor::sizewise( + Arc::new(Params::default()), + Arc::new(AtomicBool::new(true)), + dupstore.clone(), + file_queue, + Arc::new(MultiProgress::new()), + )?; + + let args = Params { + strict: true, + ..Default::default() + }; + + Processor::hashwise( + Arc::new(args), + dupstore.clone(), + hw_dupstore.clone(), + Arc::new(MultiProgress::new()), + Arc::new(AtomicU64::new(32)), + 300, + )?; + + assert_eq!(hw_dupstore.len(), 2); + + Ok(()) + } + + #[test] + fn hashwise_sorting_two_files_with_identical_init_page_only_fast_mode() -> Result<()> { + let root = TempDir::new()?; + let content = generate_bytes(4096); + + let mut content_x = content.clone(); + let mut content_y = content.clone(); + + content_x.extend(generate_bytes(1720320)); + content_y.extend(generate_bytes(1720320)); + + let files = [ + (root.path().join("fileone.bin"), content_x), + (root.path().join("filetwo.bin"), content_y), + ]; + + for (fpath, content) in files.iter() { + let mut f = File::create_new(fpath)?; + f.write_all(content)?; + } + + let dupstore = Arc::new(DashMap::new()); + let file_queue = Arc::new(Mutex::new( + files + .iter() + .map(|f| FileInfo::new(f.0.clone()).unwrap()) + .collect::>(), + )); + + let hw_dupstore = Arc::new(DashMap::new()); + Processor::sizewise( + Arc::new(Params::default()), + Arc::new(AtomicBool::new(true)), + dupstore.clone(), + file_queue, + Arc::new(MultiProgress::new()), + )?; + + Processor::hashwise( + Arc::new(Params::default()), + dupstore.clone(), + hw_dupstore.clone(), + Arc::new(MultiProgress::new()), + Arc::new(AtomicU64::new(32)), + 300, + )?; + + assert_eq!(hw_dupstore.len(), 1); + + Ok(()) + } + + #[test] + fn hashwise_sorting_two_files_with_identical_data() -> Result<()> { + let root = TempDir::new()?; + let content = generate_bytes(282624); + let files = [ + (root.path().join("fileone.bin"), content.clone()), + (root.path().join("filetwo.bin"), content.clone()), + ]; + + for (fpath, content) in files.iter() { + let mut f = File::create_new(fpath)?; + f.write_all(content)?; + } + + let dupstore = Arc::new(DashMap::new()); + let file_queue = Arc::new(Mutex::new( + files + .iter() + .map(|f| FileInfo::new(f.0.clone()).unwrap()) + .collect::>(), + )); + + let hw_dupstore = Arc::new(DashMap::new()); + Processor::sizewise( + Arc::new(Params::default()), + Arc::new(AtomicBool::new(true)), + dupstore.clone(), + file_queue, + Arc::new(MultiProgress::new()), + )?; + + Processor::hashwise( + Arc::new(Params::default()), + dupstore.clone(), + hw_dupstore.clone(), + Arc::new(MultiProgress::new()), + Arc::new(AtomicU64::new(32)), + 300, + )?; + + assert_eq!(hw_dupstore.len(), 1); + + Ok(()) + } + + #[test] + fn sizewise_sorting_two_files_of_different_sizes() -> Result<()> { let root = TempDir::new()?; let files = [ - (root.path().join("fileone.bin"), generate_bytes(80)), - (root.path().join("filetwo.bin"), generate_bytes(120)), + (root.path().join("fileone.bin"), generate_bytes(282624)), + (root.path().join("filetwo.bin"), generate_bytes(1720320)), ]; for (fpath, content) in files.iter() { @@ -177,4 +331,39 @@ mod tests { Ok(()) } + + #[test] + fn sizewise_sorting_two_files_of_same_size() -> Result<()> { + let root = TempDir::new()?; + let files = [ + (root.path().join("fileone.bin"), generate_bytes(282624)), + (root.path().join("filetwo.bin"), generate_bytes(282624)), + ]; + + for (fpath, content) in files.iter() { + let mut f = File::create_new(fpath)?; + f.write_all(content)?; + } + + let file_queue = Arc::new(Mutex::new( + files + .iter() + .map(|f| FileInfo::new(f.0.clone()).unwrap()) + .collect::>(), + )); + + let dupstore = Arc::new(DashMap::new()); + + Processor::sizewise( + Arc::new(Params::default()), + Arc::new(AtomicBool::new(true)), + dupstore.clone(), + file_queue, + Arc::new(MultiProgress::new()), + )?; + + assert_eq!(dupstore.len(), 1); + + Ok(()) + } } diff --git a/src/scanner.rs b/src/scanner.rs index 9419bcf..0071e0a 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -1,15 +1,13 @@ -#![allow(unused)] use crate::{fileinfo::FileInfo, params::Params}; use anyhow::Result; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use std::sync::{Arc, Mutex}; -use std::{fs, path::Path, time::Duration}; +use std::{path::Path, time::Duration}; use globwalk::{GlobWalker, GlobWalkerBuilder}; pub struct Scanner { pub directory: Box, - pub app_args: Arc, pub filetypes: Option, pub min_depth: Option, pub max_depth: Option, @@ -28,7 +26,6 @@ impl Scanner { min_size: app_args.get_min_size(), follow_links: app_args.follow_links, progress: app_args.progress, - app_args, }) }