11 Commits
v0.3.1 ... main

Author SHA1 Message Date
Sreedev Kodichath
245e910c39 reorder AI disclaimer on the readme 2026-07-26 15:18:46 -04:00
Sreedev Kodichath
312d66a5e1 add AI use disclaimer 2026-07-26 15:17:12 -04:00
sreedevk
d102d92bfa cleanup clone littering in server.rs 2025-08-07 23:18:02 +00:00
sreedevk
68c2491fd0 added file store partitioning idea to readme. 2025-07-30 13:08:54 +00:00
sreedevk
da24efbb36 count graphemes instead of chars in path length check 2025-07-30 10:47:58 +00:00
sreedevk
2191763b4d added potential optimizations to readme 2025-07-28 14:22:13 +00:00
sreedevk
b84510f18e tree rendering ui improvements 2025-07-19 16:38:17 +00:00
Sreedev Kodichath
bfb89edd16 Update README.md 2025-07-19 12:32:09 -04:00
sreedevk
21f0eb3cb0 all file type related filtering issues fixed 2025-07-19 16:26:12 +00:00
sreedevk
f5d2d4e22c fix: single file hash groups printed to screen 2025-07-19 14:22:43 +00:00
sreedevk
360af25318 updated readme 2025-07-19 13:57:26 +00:00
10 changed files with 297 additions and 115 deletions

2
.cargo/config.toml Normal file
View File

@@ -0,0 +1,2 @@
[build]
rustflags = ["-C", "target-feature=+aes,+sse2"]

9
Cargo.lock generated
View File

@@ -273,7 +273,7 @@ dependencies = [
[[package]]
name = "deduplicator"
version = "0.3.1"
version = "0.3.2"
dependencies = [
"anyhow",
"bytesize",
@@ -290,6 +290,7 @@ dependencies = [
"rayon",
"tempfile",
"threadpool",
"unicode-segmentation",
]
[[package]]
@@ -898,6 +899,12 @@ version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"
[[package]]
name = "unicode-segmentation"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
[[package]]
name = "unicode-width"
version = "0.1.14"

View File

@@ -1,6 +1,6 @@
[package]
name = "deduplicator"
version = "0.3.1"
version = "0.3.2"
edition = "2021"
description = "find,filter and delete duplicate files"
repository = "https://github.com/sreedevk/deduplicator"
@@ -31,6 +31,7 @@ prettytable-rs = "0.10.0"
rand = "0.9.1"
rayon = "1.6.1"
threadpool = "1.8.1"
unicode-segmentation = "1.12.0"
[profile.release]
strip = true

View File

@@ -4,6 +4,9 @@
Find, Sort, Filter & Delete duplicate files
</p>
> [!NOTE]
> This project is maintained with the assistance of AI tools. All changes are subject to manual review and a comprehensive test suite to ensure stability and quality.
## Usage
```bash
@@ -50,7 +53,8 @@ deduplicator ~/Media --min-size 100mb
```
## Demo
![record](https://github.com/user-attachments/assets/fcfdd9bf-4d05-41b6-a82e-367634eeaa73)
![demo](https://github.com/user-attachments/assets/bdb95831-542d-4902-a458-4e0f5d171a33)
@@ -80,6 +84,12 @@ $ RUSTFLAGS="-C target-cpu=native" cargo install deduplicator --git https://gith
$ RUSTFLAGS="-C target-feature=+aes,+sse2" cargo install --git https://github.com/sreedevk/deduplicator
```
### Manual Installation
- Download the right pre-compiled binary archive for your platform from [github release page](https://github.com/sreedevk/deduplicator/releases/tag/latest).
- Decompress it using `tar -zxvf <archive>.tar.gz`
- Move it to a directory included in `$PATH`.
- ideally `/usr/local/bin/`.
## Performance
Deduplicator uses size comparison and [GxHash](https://docs.rs/gxhash/latest/gxhash/) to quickly check a large number of files to find duplicates. its also heavily parallelized. The default behavior of deduplicator is to only hash the first page (4K) of the file. This is to ensure that performance is the default priority. You can modify this behavior by using the `--strict` flag which will hash the whole file and ensure that 2 files are indeed duplicates. I'll add benchmarks in future versions.
@@ -149,6 +159,8 @@ dust 'bench_artifacts'
## proposed
- [ ] parallelization
- [ ] scanning + processing sw + processing hw + formatting + printing
- [ ] user supplied cache file path for faster re-runs
- [ ] hardlinks / symlinks support
- [ ] max file path size should use the last set of duplicates
- [ ] add more unit tests
- [ ] test against different filesystems
@@ -160,6 +172,7 @@ dust 'bench_artifacts'
- [ ] tui
- [ ] change the default hashing method to include the first & last page of a file (8K)
- [ ] provide option to localize duplicate detection to arbitrary levels relative to current directory
- [ ] localize file meta store locks to sub path levels to avoid global lock contention from multiple threads.
- [ ] bulk operations
- [ ] --keep-latest
- [ ] --keep-oldest
@@ -169,6 +182,13 @@ dust 'bench_artifacts'
- [ ] fix: partial hash collision - a file full of null bytes ("\0") and an empty file. This is a known trade off in gxhash.
- [ ] include initial pages and final pages of the file
- [ ] append the offset between the last initial page hashed and the first final page hashed in the content passed to the hasher.
- [ ] potential optimizations
- [ ] lookup the memory efficiency gains if instead of directly inserting into a hashmap, deduplicator looksup the file in a bloom filter. this way, the duplicate store hashmap does
not require to be locked for every single file. The bloom filter can be stored in an atomically updateable type to improve performance as well.
## v0.3.2
- [x] fix: single file groups are printed to screen
- [x] fix: --exclude-types and --types flag behave identically.
## v0.3.1
- [x] parallelization

View File

@@ -4,6 +4,7 @@ use chrono::{DateTime, Utc};
use dashmap::DashMap;
use pathdiff::diff_paths;
use rayon::prelude::*;
use std::sync::atomic::AtomicU64;
use std::{path::PathBuf, sync::Arc};
const YELLOW: &str = "\x1b[33m";
@@ -11,14 +12,14 @@ const RESET: &str = "\x1b[0m";
pub struct Formatter;
impl Formatter {
pub fn human_path(file: &FileInfo, aargs: &Params, min_path_length: usize) -> Result<String> {
pub fn human_path(file: &FileInfo, aargs: &Params, max_path_length: usize) -> Result<String> {
let base_directory: PathBuf = aargs.get_directory()?;
let relative_path = diff_paths(&file.path, base_directory).unwrap_or_default();
let formatted_path = format!(
"{:<0width$}",
relative_path.to_str().unwrap_or_default().to_string(),
width = min_path_length
width = max_path_length
);
Ok(formatted_path)
@@ -39,26 +40,41 @@ impl Formatter {
if raw.is_empty() {
println!("No duplicates found matching your search criteria.");
} else {
let printed_count: AtomicU64 = AtomicU64::new(0);
raw.par_iter().for_each(|sref| {
let mut ostring = format!("{}{:32x}{}\n", YELLOW, sref.key(), RESET);
let subfields = sref
.value()
.par_iter()
.map(|finfo| {
format!(
"├─ {}\t{}\t{}\n",
Self::human_path(finfo, aargs, max_path_len as usize)
.expect("path formatting failed."),
Self::human_filesize(finfo).expect("filesize formatting failed."),
Self::human_mtime(finfo).expect("modified time formatting failed.")
)
})
.collect::<String>();
if sref.value().len() > 1 {
printed_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let mut ostring = format!("{}{:32x}{}\n", YELLOW, sref.key(), RESET);
let subfields = sref
.value()
.par_iter()
.enumerate()
.map(|(i, finfo)| {
let nodechar = if i == sref.value().len() - 1 {
"└─"
} else {
"├─"
};
format!(
"{}\t{}\t{}\t{}\n",
nodechar,
Self::human_path(finfo, aargs, max_path_len as usize)
.expect("path formatting failed."),
Self::human_filesize(finfo).expect("filesize formatting failed."),
Self::human_mtime(finfo).expect("modified time formatting failed.")
)
})
.collect::<String>();
ostring.push_str(&subfields);
println!("{ostring}");
ostring.push_str(&subfields);
println!("{ostring}");
}
});
if printed_count.load(std::sync::atomic::Ordering::Relaxed) < 1 {
println!("No duplicates found matching your search criteria.");
}
}
}
}

View File

@@ -2,6 +2,7 @@ use crate::{fileinfo::FileInfo, formatter::Formatter, params::Params};
use anyhow::Result;
use dashmap::DashMap;
use prettytable::{format, row, Table};
use std::sync::atomic::AtomicU64;
use std::{
io::{self, Write},
sync::Arc,
@@ -11,12 +12,19 @@ pub struct Interactive;
impl Interactive {
pub fn init(result: Arc<DashMap<u128, Vec<FileInfo>>>, app_args: &Params) -> Result<()> {
result
.clone()
let store = result.clone();
if store.is_empty() {
println!("No duplicates found matching your search criteria.");
}
let printed_count: AtomicU64 = AtomicU64::new(0);
store
.iter()
.filter(|i| i.value().len() > 1)
.enumerate()
.for_each(|(gindex, i)| {
printed_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let group = i.value();
let mut itable = Table::new();
itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
@@ -40,6 +48,10 @@ impl Interactive {
Self::process_group_action(group, gindex, result.len(), itable);
});
if printed_count.load(std::sync::atomic::Ordering::Relaxed) < 1 {
println!("No duplicates found matching your search criteria.");
}
Ok(())
}

View File

@@ -2,7 +2,6 @@ use std::{fs, path::PathBuf};
use anyhow::Result;
use clap::{Parser, ValueHint};
use std::collections::HashSet;
#[derive(Parser, Debug, Default, Clone)]
#[command(author, version, about, long_about = None)]
@@ -56,48 +55,4 @@ impl Params {
let dir = fs::canonicalize(dir_path)?;
Ok(dir)
}
pub fn types_intersection(itypes: &str, xtypes: &str) -> String {
let iset = itypes
.split(",")
.map(String::from)
.collect::<HashSet<String>>();
let xset = xtypes
.split(",")
.map(String::from)
.collect::<HashSet<String>>();
iset.difference(&xset)
.cloned()
.collect::<Vec<String>>()
.join(",")
}
pub fn get_types(&self) -> Option<String> {
match &self.types {
Some(itypes) => match &self.exclude_types {
Some(xtypes) => Some(Self::types_intersection(itypes, xtypes)),
None => Some(itypes.to_string()),
},
None => self.exclude_types.as_ref().map(|xtypes| xtypes.to_string()),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn mixing_include_and_exclude_types_works_as_expected() {
let params = Params {
types: Some(String::from("js,xml,ts,pdf,tiff")),
exclude_types: Some(String::from("js,ts,xml")),
..Default::default()
};
assert!(params
.get_types()
.is_some_and(|x| x == "pdf,tiff" || x == "tiff,pdf"))
}
}

View File

@@ -6,6 +6,7 @@ use rayon::prelude::{IntoParallelIterator, ParallelIterator};
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Mutex, TryLockError, TryLockResult};
use std::time::Duration;
use unicode_segmentation::UnicodeSegmentation;
use crate::fileinfo::FileInfo;
use crate::params::Params;
@@ -64,7 +65,7 @@ impl Processor {
Self::compare_and_update_max_path_len(
max_file_size.clone(),
file.path.to_string_lossy().len() as u64,
file.path.to_string_lossy().graphemes(true).count() as u64,
);
hw_store

View File

@@ -8,9 +8,10 @@ use globwalk::{GlobWalker, GlobWalkerBuilder};
pub struct Scanner {
pub directory: Box<Path>,
pub filetypes: Option<String>,
pub min_depth: Option<usize>,
pub max_depth: Option<usize>,
pub include_types: Option<String>,
pub exclude_types: Option<String>,
pub min_size: Option<u64>,
pub follow_links: bool,
pub progress: bool,
@@ -20,7 +21,8 @@ impl Scanner {
pub fn new(app_args: Arc<Params>) -> Result<Self> {
Ok(Self {
directory: app_args.get_directory()?.into_boxed_path(),
filetypes: app_args.get_types(),
include_types: app_args.types.clone(),
exclude_types: app_args.exclude_types.clone(),
min_depth: app_args.min_depth,
max_depth: app_args.max_depth,
min_size: app_args.get_min_size(),
@@ -29,11 +31,21 @@ impl Scanner {
})
}
fn scan_patterns(&self) -> Result<String> {
Ok(match &self.filetypes {
Some(ftypes) => format!("**/*{{{ftypes}}}"),
None => "**/*".to_string(),
})
fn scan_patterns(&self) -> Result<Vec<String>> {
let include_types = match &self.include_types {
Some(ftypes) => Some(format!("**/*.{{{ftypes}}}")),
None => Some("**/*".to_string()),
};
let exclude_types = self
.exclude_types
.as_ref()
.map(|ftypes| format!("!**/*.{{{ftypes}}}"));
Ok(vec![include_types, exclude_types]
.into_iter()
.flatten()
.collect())
}
fn attach_link_opts(&self, walker: GlobWalkerBuilder) -> Result<GlobWalkerBuilder> {
@@ -56,7 +68,7 @@ impl Scanner {
fn build_walker(&self) -> Result<GlobWalker> {
let walker = Ok(GlobWalkerBuilder::from_patterns(
self.directory.clone(),
&[self.scan_patterns()?],
&self.scan_patterns()?,
))
.and_then(|walker| self.attach_walker_min_depth(walker))
.and_then(|walker| self.attach_walker_max_depth(walker))
@@ -79,7 +91,7 @@ impl Scanner {
progress_bar.set_style(progress_style);
progress_bar.enable_steady_tick(Duration::from_millis(50));
progress_bar.set_message("paths mapped");
let min_size = self.min_size.unwrap_or_default();
let min_size = self.min_size.unwrap_or(0);
self.build_walker()?
.filter_map(Result::ok)
@@ -88,7 +100,7 @@ impl Scanner {
.filter(|path| path.is_file())
.map(FileInfo::new)
.filter_map(Result::ok)
.filter(|file| file.size > min_size)
.filter(|file| file.size >= min_size)
.for_each(|file| {
let mut flock = files.lock().unwrap();
flock.push(file);
@@ -98,3 +110,151 @@ impl Scanner {
Ok(())
}
}
#[cfg(test)]
mod tests {
use crate::fileinfo::FileInfo;
use crate::params::Params;
use std::fs::File;
use std::sync::{Arc, Mutex};
use super::Scanner;
use indicatif::MultiProgress;
use tempfile::TempDir;
#[test]
fn ensure_file_include_type_filter_includes_expected_file_types() {
let root =
TempDir::with_prefix("deduplicator_test_root").expect("unable to create tempdir");
[
"this-is-a-js-file.js",
"this-is-a-css-file.css",
"this-is-a-csv-file.csv",
"this-is-a-rust-file.rs",
]
.iter()
.for_each(|path| {
File::create_new(root.path().join(path)).unwrap_or_else(|_| {
panic!("unable to create file {path}");
});
});
let params = Params {
types: Some(String::from("js,csv")),
dir: Some(root.path().into()),
..Default::default()
};
let progress = Arc::new(MultiProgress::new());
let scanlist = Arc::new(Mutex::<Vec<FileInfo>>::new(vec![]));
let scanner = Scanner::new(Arc::new(params)).expect("scanner initialization failed");
scanner
.scan(scanlist.clone(), progress)
.expect("scanning failed.");
let scan_list_mg = scanlist.lock().unwrap();
assert!(scan_list_mg.iter().any(|f| f.path.to_str().unwrap()
== root.path().join("this-is-a-js-file.js").to_str().unwrap()));
assert!(scan_list_mg.iter().any(|f| f.path.to_str().unwrap()
== root.path().join("this-is-a-csv-file.csv").to_str().unwrap()));
assert!(scan_list_mg.iter().all(|f| f.path.to_str().unwrap()
!= root.path().join("this-is-a-css-file.css").to_str().unwrap()));
assert!(scan_list_mg.iter().all(|f| f.path.to_str().unwrap()
!= root.path().join("this-is-a-rust-file.rs").to_str().unwrap()));
}
#[test]
fn ensure_file_exclude_type_filter_excludes_expected_file_types() {
let root =
TempDir::with_prefix("deduplicator_test_root").expect("unable to create tempdir");
[
"this-is-a-js-file.js",
"this-is-a-css-file.css",
"this-is-a-csv-file.csv",
"this-is-a-rust-file.rs",
]
.iter()
.for_each(|path| {
File::create_new(root.path().join(path)).unwrap_or_else(|_| {
panic!("unable to create file {path}");
});
});
let params = Params {
exclude_types: Some(String::from("js,csv")),
dir: Some(root.path().into()),
..Default::default()
};
let progress = Arc::new(MultiProgress::new());
let scanlist = Arc::new(Mutex::<Vec<FileInfo>>::new(vec![]));
let scanner = Scanner::new(Arc::new(params)).expect("scanner initialization failed");
scanner
.scan(scanlist.clone(), progress)
.expect("scanning failed.");
let scan_list_mg = scanlist.lock().unwrap();
assert!(scan_list_mg.iter().all(|f| f.path.to_str().unwrap()
!= root.path().join("this-is-a-js-file.js").to_str().unwrap()));
assert!(scan_list_mg.iter().all(|f| f.path.to_str().unwrap()
!= root.path().join("this-is-a-csv-file.csv").to_str().unwrap()));
assert!(scan_list_mg.iter().any(|f| f.path.to_str().unwrap()
== root.path().join("this-is-a-css-file.css").to_str().unwrap()));
assert!(scan_list_mg.iter().any(|f| f.path.to_str().unwrap()
== root.path().join("this-is-a-rust-file.rs").to_str().unwrap()));
}
#[test]
fn complex_file_type_params() {
let root =
TempDir::with_prefix("deduplicator_test_root").expect("unable to create tempdir");
[
"this-is-a-js-file.js",
"this-is-a-css-file.css",
"this-is-a-csv-file.csv",
"this-is-a-rust-file.rs",
]
.iter()
.for_each(|path| {
File::create_new(root.path().join(path)).unwrap_or_else(|_| {
panic!("unable to create file {path}");
});
});
let params = Params {
types: Some(String::from("js,csv,rs")),
exclude_types: Some(String::from("csv")),
dir: Some(root.path().into()),
..Default::default()
};
let progress = Arc::new(MultiProgress::new());
let scanlist = Arc::new(Mutex::<Vec<FileInfo>>::new(vec![]));
let scanner = Scanner::new(Arc::new(params)).expect("scanner initialization failed");
scanner
.scan(scanlist.clone(), progress)
.expect("scanning failed.");
let scan_list_mg = scanlist.lock().unwrap();
assert!(scan_list_mg.iter().any(|f| f.path.to_str().unwrap()
== root.path().join("this-is-a-js-file.js").to_str().unwrap()));
assert!(scan_list_mg.iter().all(|f| f.path.to_str().unwrap()
!= root.path().join("this-is-a-csv-file.csv").to_str().unwrap()));
assert!(scan_list_mg.iter().any(|f| f.path.to_str().unwrap()
== root.path().join("this-is-a-rust-file.rs").to_str().unwrap()));
}
}

View File

@@ -42,60 +42,68 @@ impl Server {
progbarbox.set_draw_target(ProgressDrawTarget::hidden());
}
let app_args_clone_for_sc = self.app_args.clone();
let app_args_clone_for_sw = self.app_args.clone();
let app_args_clone_for_hw = self.app_args.clone();
let file_queue_clone_sc = self.filequeue.clone();
let file_queue_clone_pr = self.filequeue.clone();
let (app_args_sc, app_args_sw, app_args_hw) = (
Arc::clone(&self.app_args),
Arc::clone(&self.app_args),
Arc::clone(&self.app_args),
);
let (file_queue_sc, file_queue_pr) = (
Arc::clone(&self.filequeue),
Arc::clone(&self.filequeue),
);
let scanner_finished = Arc::new(AtomicBool::new(false));
let sw_sort_finished = Arc::new(AtomicBool::new(false));
let sfin_sc_tr_cl = scanner_finished.clone();
let sfin_pr_tr_cl = scanner_finished.clone();
let swfin_pr_tr_sw = sw_sort_finished.clone();
let swfin_pr_tr_hw = sw_sort_finished.clone();
let store_dupl_sw_for_sw = self.sw_duplicate_set.clone();
let store_dupl_sw_for_hw = self.sw_duplicate_set.clone();
let store_dupl_hw = self.hw_duplicate_set.clone();
let max_file_path_len_clone = self.max_file_path_len.clone();
let progbarbox_sc_clone = progbarbox.clone();
let progbarbox_pr_clone_for_sw = progbarbox.clone();
let progbarbox_pr_clone_for_hw = progbarbox.clone();
let (sfin_sc, sfin_pr) = (
Arc::clone(&scanner_finished),
Arc::clone(&scanner_finished),
);
let (swfin_pr_sw, swfin_pr_hw) = (
Arc::clone(&sw_sort_finished),
Arc::clone(&sw_sort_finished),
);
let (store_sw, store_sw2, store_hw) = (
Arc::clone(&self.sw_duplicate_set),
Arc::clone(&self.sw_duplicate_set),
Arc::clone(&self.hw_duplicate_set),
);
let max_file_path_len = Arc::clone(&self.max_file_path_len);
let (prog_sc, prog_sw, prog_hw) = (
Arc::clone(&progbarbox),
Arc::clone(&progbarbox),
Arc::clone(&progbarbox),
);
self.threadpool.execute(move || {
Scanner::new(app_args_clone_for_sc)
Scanner::new(app_args_sc)
.expect("unable to initialize scanner.")
.scan(file_queue_clone_sc, progbarbox_sc_clone)
.scan(file_queue_sc, prog_sc)
.expect("scanner failed.");
sfin_sc_tr_cl.store(true, std::sync::atomic::Ordering::Relaxed);
sfin_sc.store(true, std::sync::atomic::Ordering::Relaxed);
});
self.threadpool.execute(move || {
Processor::sizewise(
app_args_clone_for_sw,
sfin_pr_tr_cl,
store_dupl_sw_for_sw,
file_queue_clone_pr,
progbarbox_pr_clone_for_sw,
app_args_sw,
sfin_pr,
store_sw,
file_queue_pr,
prog_sw,
)
.expect("sizewise scanner failed.");
swfin_pr_tr_sw.store(true, std::sync::atomic::Ordering::Relaxed);
swfin_pr_sw.store(true, std::sync::atomic::Ordering::Relaxed);
});
self.threadpool.execute(move || {
Processor::hashwise(
app_args_clone_for_hw,
store_dupl_sw_for_hw,
store_dupl_hw,
progbarbox_pr_clone_for_hw,
max_file_path_len_clone,
app_args_hw,
store_sw2,
store_hw,
prog_hw,
max_file_path_len,
seed,
swfin_pr_tr_hw,
swfin_pr_hw,
)
.expect("sizewise scanner failed.");
});