add more tests

2026-09-05 23:10:45 +00:00 · 2025-07-13 21:24:31 +00:00
parent ed55aaa005
commit d99d783327
8 changed files with 306 additions and 89 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -81,6 +81,22 @@ version = "1.0.86"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da"

+[[package]]
+name = "assert_cmd"
+version = "2.0.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2bd389a4b2970a01282ee455294913c0a43724daedcd1a24c3eb0ec1c1320b66"
+dependencies = [
+ "anstyle",
+ "bstr",
+ "doc-comment",
+ "libc",
+ "predicates",
+ "predicates-core",
+ "predicates-tree",
+ "wait-timeout",
+]
+
 [[package]]
 name = "autocfg"
 version = "1.3.0"
@@ -106,6 +122,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706"
 dependencies = [
 "memchr",
+ "regex-automata",
 "serde",
 ]

@@ -277,6 +294,7 @@ name = "deduplicator"
 version = "0.3.0"
 dependencies = [
 "anyhow",
+ "assert_cmd",
 "bytesize",
 "chrono",
 "clap",
@@ -284,7 +302,6 @@ dependencies = [
 "globwalk",
 "gxhash",
 "indicatif",
- "itertools",
 "memmap2",
 "pathdiff",
 "prettytable-rs",
@@ -292,9 +309,14 @@ dependencies = [
 "rayon",
 "tempfile",
 "threadpool",
- "unicode-segmentation",
 ]

+[[package]]
+name = "difflib"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8"
+
 [[package]]
 name = "dirs-next"
 version = "2.0.0"
@@ -316,6 +338,12 @@ dependencies = [
 "winapi",
 ]

+[[package]]
+name = "doc-comment"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
+
 [[package]]
 name = "either"
 version = "1.13.0"
@@ -509,15 +537,6 @@ version = "1.70.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800"

-[[package]]
-name = "itertools"
-version = "0.10.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
-dependencies = [
- "either",
-]
-
 [[package]]
 name = "itoa"
 version = "1.0.11"
@@ -657,6 +676,33 @@ dependencies = [
 "zerocopy",
 ]

+[[package]]
+name = "predicates"
+version = "3.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a5d19ee57562043d37e82899fade9a22ebab7be9cef5026b07fda9cdd4293573"
+dependencies = [
+ "anstyle",
+ "difflib",
+ "predicates-core",
+]
+
+[[package]]
+name = "predicates-core"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "727e462b119fe9c93fd0eb1429a5f7647394014cf3c04ab2c0350eeb09095ffa"
+
+[[package]]
+name = "predicates-tree"
+version = "1.0.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72dd2d6d381dfb73a193c7fca536518d7caee39fc8503f74e7dc0be0531b425c"
+dependencies = [
+ "predicates-core",
+ "termtree",
+]
+
 [[package]]
 name = "prettytable-rs"
 version = "0.10.0"
@@ -888,6 +934,12 @@ dependencies = [
 "winapi",
 ]

+[[package]]
+name = "termtree"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683"
+
 [[package]]
 name = "thiserror"
 version = "1.0.61"
@@ -923,12 +975,6 @@ version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"

-[[package]]
-name = "unicode-segmentation"
-version = "1.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202"
-
 [[package]]
 name = "unicode-width"
 version = "0.1.13"
@@ -941,6 +987,15 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"

+[[package]]
+name = "wait-timeout"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "walkdir"
 version = "2.5.0"
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -11,6 +11,10 @@ authors = [
  "Dhruva Sagar <dhruva.sagar@gmail.com>",
 ]

+[[bin]]
+name = "deduplicator"
+path = "src/main.rs"
+
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 [dependencies]
 anyhow = "1.0.68"
@@ -21,7 +25,6 @@ dashmap = { version = "5.4.0", features = ["rayon"] }
 globwalk = "0.8.1"
 gxhash = { version = "3.4.1", default-features = false }
 indicatif = { version = "0.17.2", features = ["rayon"] }
-itertools = "0.10.5"
 memmap2 = "0.5.8"
 pathdiff = "0.2.1"
 prettytable-rs = "0.10.0"
@@ -29,15 +32,17 @@ rand = "0.9.1"
 rayon = "1.6.1"
 tempfile = "3.20.0"
 threadpool = "1.8.1"
-unicode-segmentation = "1.10.0"

 [profile.release]
 strip = true
+opt-level = 3
+lto = "thin"
+debug = false
+codegen-units = 1

 # generated by 'cargo dist init'
 [profile.dist]
 inherits = "release"
-lto = "thin"

 [workspace.metadata.dist]
 rust-toolchain-version = "1.87.0"
@@ -49,3 +54,6 @@ targets = [
  "aarch64-apple-darwin",
 ]
 cargo-dist-version = "0.0.7"
+
+[dev-dependencies]
+assert_cmd = "2.0.0"
--- a/README.md
+++ b/README.md
@@ -56,45 +56,7 @@ deduplicator ~/Media --min-size 100mb
 $ RUSTFLAGS="-C target-cpu=native" cargo install deduplicator
 ```

-#### Nightly
-
-if you'd like to install with nightly features, you can use
-
-```bash
-$ cargo install --git https://github.com/sreedevk/deduplicator
-```
-Please note that if you use a version manager to install rust (like asdf), you need to reshim (`asdf reshim rust`).
-
-### Linux (Pre-built Binary)
-
-you can download the pre-built binary from the [Releases](https://github.com/sreedevk/deduplicator/releases) page.
-download the `deduplicator-x86_64-unknown-linux-gnu.tar.gz` for linux. Once you have the tarball file with the executable,
-you can follow these steps to install:
-
-```bash
-$ tar -zxvf deduplicator-x86_64-unknown-linux-gnu.tar.gz
-$ sudo mv deduplicator /usr/bin/
-```
-
-### Mac OS (Pre-built Binary)
-
-you can download the pre-build binary from the [Releases](https://github.com/sreedevk/deduplicator/releases) page.
-download the `deduplicator-x86_64-apple-darwin.tar.gz` tarball for mac os. Once you have the tarball file with the executable, you can follow these steps to install:
-
-```bash
-$ tar -zxvf deduplicator-x86_64-unknown-linux-gnu.tar.gz
-$ sudo mv deduplicator /usr/bin/
-```
-
-### Windows (Pre-built Binary)
-
-you can download the pre-build binary from the [Releases](https://github.com/sreedevk/deduplicator/releases) page.
-download the `deduplicator-x86_64-pc-windows-msvc.zip` zip file for windows. unzip the `zip`  file & move the `deduplicator.exe` to a location in the PATH system environment variable.
-
-Note: If you Run into an msvc error, please install MSCV from [here](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist?view=msvc-170)
-
 ## Performance
-
 Deduplicator uses size comparison and gxhash (a non non-cryptographic hashing algorithm) to quickly scan through large number of files to find duplicates. its also highly parallel (uses rayon and dashmap). 

 ## Screenshots
--- a/src/fileinfo.rs
+++ b/src/fileinfo.rs
@@ -19,9 +19,9 @@ impl FileInfo {
    pub fn hash(&self, seed: i64) -> Result<u128> {
        let file = fs::File::open(&self.path)?;
        let mapper = unsafe { Mmap::map(&file)? };
-        let final_hash = mapper
-            .chunks(4096)
-            .fold(0, |acc, chunk: &[u8]| acc + gxhash128(chunk, seed));
+        let final_hash = mapper.chunks(4096).fold(0u128, |acc, chunk: &[u8]| {
+            acc.wrapping_add(gxhash128(chunk, seed))
+        });

        Ok(final_hash)
    }
--- a/src/formatter.rs
+++ b/src/formatter.rs
@@ -68,6 +68,7 @@ impl Formatter {
            .par_iter_mut()
            .progress_with(progress_bar)
            .with_finish(ProgressFinish::WithMessage(Cow::from("output generated")))
+            .filter(|i| i.value().len() > 1)
            .map(|i| {
                row![
                    i.key(),
--- a/src/interactive.rs
+++ b/src/interactive.rs
@@ -11,30 +11,35 @@ pub struct Interactive;

 impl Interactive {
    pub fn init(result: Arc<DashMap<u128, Vec<FileInfo>>>, app_args: &Params) -> Result<()> {
-        result.clone().iter().enumerate().for_each(|(gindex, i)| {
-            let group = i.value();
-            let mut itable = Table::new();
-            itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
-            itable.set_titles(row!["index", "filename", "size", "updated_at"]);
+        result
+            .clone()
+            .iter()
+            .filter(|i| i.value().len() > 1)
+            .enumerate()
+            .for_each(|(gindex, i)| {
+                let group = i.value();
+                let mut itable = Table::new();
+                itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
+                itable.set_titles(row!["index", "filename", "size", "updated_at"]);

-            let max_path_size = group
-                .iter()
-                .map(|f| f.path.iter().count())
-                .max()
-                .unwrap_or_default();
+                let max_path_size = group
+                    .iter()
+                    .map(|f| f.path.iter().count())
+                    .max()
+                    .unwrap_or_default();

-            group.iter().enumerate().for_each(|(index, file)| {
-                itable.add_row(row![
-                    index,
-                    Formatter::human_path(file, app_args, max_path_size).unwrap_or_default(),
-                    Formatter::human_filesize(file).unwrap_or_default(),
-                    Formatter::human_mtime(file).unwrap_or_default()
-                ]);
+                group.iter().enumerate().for_each(|(index, file)| {
+                    itable.add_row(row![
+                        index,
+                        Formatter::human_path(file, app_args, max_path_size).unwrap_or_default(),
+                        Formatter::human_filesize(file).unwrap_or_default(),
+                        Formatter::human_mtime(file).unwrap_or_default()
+                    ]);
+                });
+
+                Self::process_group_action(group, gindex, result.len(), itable);
            });

-            Self::process_group_action(group, gindex, result.len(), itable);
-        });
-
        Ok(())
    }

--- a/src/processor.rs
+++ b/src/processor.rs
@@ -130,7 +130,7 @@ mod tests {
    use rand::Rng;
    use std::fs::File;
    use std::io::Write;
-    use std::sync::atomic::AtomicBool;
+    use std::sync::atomic::{AtomicBool, AtomicU64};
    use std::sync::{Arc, Mutex};
    use tempfile::TempDir;

@@ -144,11 +144,165 @@ mod tests {
    }

    #[test]
-    fn sizewise_sorting_normal() -> Result<()> {
+    fn hashwise_sorting_two_files_with_identical_init_page_only_strict_mode() -> Result<()> {
+        let root = TempDir::new()?;
+        let content = generate_bytes(4096);
+
+        let mut content_x = content.clone();
+        let mut content_y = content.clone();
+
+        content_x.extend(generate_bytes(1720320));
+        content_y.extend(generate_bytes(1720320));
+
+        let files = [
+            (root.path().join("fileone.bin"), content_x),
+            (root.path().join("filetwo.bin"), content_y),
+        ];
+
+        for (fpath, content) in files.iter() {
+            let mut f = File::create_new(fpath)?;
+            f.write_all(content)?;
+        }
+
+        let dupstore = Arc::new(DashMap::new());
+        let file_queue = Arc::new(Mutex::new(
+            files
+                .iter()
+                .map(|f| FileInfo::new(f.0.clone()).unwrap())
+                .collect::<Vec<FileInfo>>(),
+        ));
+
+        let hw_dupstore = Arc::new(DashMap::new());
+        Processor::sizewise(
+            Arc::new(Params::default()),
+            Arc::new(AtomicBool::new(true)),
+            dupstore.clone(),
+            file_queue,
+            Arc::new(MultiProgress::new()),
+        )?;
+
+        let args = Params {
+            strict: true,
+            ..Default::default()
+        };
+
+        Processor::hashwise(
+            Arc::new(args),
+            dupstore.clone(),
+            hw_dupstore.clone(),
+            Arc::new(MultiProgress::new()),
+            Arc::new(AtomicU64::new(32)),
+            300,
+        )?;
+
+        assert_eq!(hw_dupstore.len(), 2);
+
+        Ok(())
+    }
+
+    #[test]
+    fn hashwise_sorting_two_files_with_identical_init_page_only_fast_mode() -> Result<()> {
+        let root = TempDir::new()?;
+        let content = generate_bytes(4096);
+
+        let mut content_x = content.clone();
+        let mut content_y = content.clone();
+
+        content_x.extend(generate_bytes(1720320));
+        content_y.extend(generate_bytes(1720320));
+
+        let files = [
+            (root.path().join("fileone.bin"), content_x),
+            (root.path().join("filetwo.bin"), content_y),
+        ];
+
+        for (fpath, content) in files.iter() {
+            let mut f = File::create_new(fpath)?;
+            f.write_all(content)?;
+        }
+
+        let dupstore = Arc::new(DashMap::new());
+        let file_queue = Arc::new(Mutex::new(
+            files
+                .iter()
+                .map(|f| FileInfo::new(f.0.clone()).unwrap())
+                .collect::<Vec<FileInfo>>(),
+        ));
+
+        let hw_dupstore = Arc::new(DashMap::new());
+        Processor::sizewise(
+            Arc::new(Params::default()),
+            Arc::new(AtomicBool::new(true)),
+            dupstore.clone(),
+            file_queue,
+            Arc::new(MultiProgress::new()),
+        )?;
+
+        Processor::hashwise(
+            Arc::new(Params::default()),
+            dupstore.clone(),
+            hw_dupstore.clone(),
+            Arc::new(MultiProgress::new()),
+            Arc::new(AtomicU64::new(32)),
+            300,
+        )?;
+
+        assert_eq!(hw_dupstore.len(), 1);
+
+        Ok(())
+    }
+
+    #[test]
+    fn hashwise_sorting_two_files_with_identical_data() -> Result<()> {
+        let root = TempDir::new()?;
+        let content = generate_bytes(282624);
+        let files = [
+            (root.path().join("fileone.bin"), content.clone()),
+            (root.path().join("filetwo.bin"), content.clone()),
+        ];
+
+        for (fpath, content) in files.iter() {
+            let mut f = File::create_new(fpath)?;
+            f.write_all(content)?;
+        }
+
+        let dupstore = Arc::new(DashMap::new());
+        let file_queue = Arc::new(Mutex::new(
+            files
+                .iter()
+                .map(|f| FileInfo::new(f.0.clone()).unwrap())
+                .collect::<Vec<FileInfo>>(),
+        ));
+
+        let hw_dupstore = Arc::new(DashMap::new());
+        Processor::sizewise(
+            Arc::new(Params::default()),
+            Arc::new(AtomicBool::new(true)),
+            dupstore.clone(),
+            file_queue,
+            Arc::new(MultiProgress::new()),
+        )?;
+
+        Processor::hashwise(
+            Arc::new(Params::default()),
+            dupstore.clone(),
+            hw_dupstore.clone(),
+            Arc::new(MultiProgress::new()),
+            Arc::new(AtomicU64::new(32)),
+            300,
+        )?;
+
+        assert_eq!(hw_dupstore.len(), 1);
+
+        Ok(())
+    }
+
+    #[test]
+    fn sizewise_sorting_two_files_of_different_sizes() -> Result<()> {
        let root = TempDir::new()?;
        let files = [
-            (root.path().join("fileone.bin"), generate_bytes(80)),
-            (root.path().join("filetwo.bin"), generate_bytes(120)),
+            (root.path().join("fileone.bin"), generate_bytes(282624)),
+            (root.path().join("filetwo.bin"), generate_bytes(1720320)),
        ];

        for (fpath, content) in files.iter() {
@@ -177,4 +331,39 @@ mod tests {

        Ok(())
    }
+
+    #[test]
+    fn sizewise_sorting_two_files_of_same_size() -> Result<()> {
+        let root = TempDir::new()?;
+        let files = [
+            (root.path().join("fileone.bin"), generate_bytes(282624)),
+            (root.path().join("filetwo.bin"), generate_bytes(282624)),
+        ];
+
+        for (fpath, content) in files.iter() {
+            let mut f = File::create_new(fpath)?;
+            f.write_all(content)?;
+        }
+
+        let file_queue = Arc::new(Mutex::new(
+            files
+                .iter()
+                .map(|f| FileInfo::new(f.0.clone()).unwrap())
+                .collect::<Vec<FileInfo>>(),
+        ));
+
+        let dupstore = Arc::new(DashMap::new());
+
+        Processor::sizewise(
+            Arc::new(Params::default()),
+            Arc::new(AtomicBool::new(true)),
+            dupstore.clone(),
+            file_queue,
+            Arc::new(MultiProgress::new()),
+        )?;
+
+        assert_eq!(dupstore.len(), 1);
+
+        Ok(())
+    }
 }
--- a/src/scanner.rs
+++ b/src/scanner.rs
@@ -1,15 +1,13 @@
-#![allow(unused)]
 use crate::{fileinfo::FileInfo, params::Params};
 use anyhow::Result;
 use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
 use std::sync::{Arc, Mutex};
-use std::{fs, path::Path, time::Duration};
+use std::{path::Path, time::Duration};

 use globwalk::{GlobWalker, GlobWalkerBuilder};

 pub struct Scanner {
    pub directory: Box<Path>,
-    pub app_args: Arc<Params>,
    pub filetypes: Option<String>,
    pub min_depth: Option<usize>,
    pub max_depth: Option<usize>,
@@ -28,7 +26,6 @@ impl Scanner {
            min_size: app_args.get_min_size(),
            follow_links: app_args.follow_links,
            progress: app_args.progress,
-            app_args,
        })
    }