smolkik_adm/deduplicator
1
0
Fork 0
mirror of https://github.com/sreedevk/deduplicator.git synced 2026-08-30 12:05:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

changed hash type to u128

Browse Source
This commit is contained in:
sreedevk
2025-07-13 18:11:07 +00:00
parent 47b2fc4a87
commit fe79a07b43
9 changed files with 52 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.lock generated
View File

@@ -274,7 +274,7 @@ dependencies = [
[[package]]
name = "deduplicator"
version = "0.2.2"
version = "0.3.0"
dependencies = [
"anyhow",
"bytesize",

5
Cargo.toml
View File

@@ -1,8 +1,8 @@
[package]
name = "deduplicator"
version = "0.2.2"
version = "0.3.0"
edition = "2021"
description = "find,filter,delete Duplicates"
description = "find,filter and delete duplicate files"
repository = "https://github.com/sreedevk/deduplicator"
license = "MIT"
authors = [
@@ -12,7 +12,6 @@ authors = [
]
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
anyhow = "1.0.68"
bytesize = "1.1.0"

39
README.md
View File

@@ -7,6 +7,8 @@
## Usage
```bash
find,filter,delete duplicate files
Usage: deduplicator [OPTIONS] [scan_dir_path]
Arguments:
@@ -15,12 +17,14 @@ Arguments:
Options:
-t, --types <TYPES> Filetypes to deduplicate [default = all]
-i, --interactive Delete files interactively
-s, --min-size <MIN_SIZE> Minimum filesize of duplicates to scan (e.g., 100B/1K/2M/3G/4T) [default: 1b]
-d, --max-depth <MAX_DEPTH> Max Depth to scan while looking for duplicates
--min-depth <MIN_DEPTH> Min Depth to scan while looking for duplicates
-m, --min-size <MIN_SIZE> Minimum filesize of duplicates to scan (e.g., 100B/1K/2M/3G/4T) [default: 1b]
-D, --max-depth <MAX_DEPTH> Max Depth to scan while looking for duplicates
-d, --min-depth <MIN_DEPTH> Min Depth to scan while looking for duplicates
-f, --follow-links Follow links while scanning directories
-h, --help Print help information
-V, --version Print version information
-s, --strict Guarantees that two files are duplicate (performs a full hash)
-p, --progress Show Progress spinners & metrics
-h, --help Print help
-V, --version Print version
```
### Examples
@@ -46,16 +50,13 @@ deduplicator ~/Media --min-size 100mb
### Cargo Install
#### Stable
> [!WARNING] Note from GxHash: GxHash relies on aes hardware acceleration, you must make sure the aes feature is enabled when building (otherwise it won't build). This can be done by setting the RUSTFLAGS environment variable to -C target-feature=+aes or -C target-cpu=native (the latter should work if your CPU is properly recognized by rustc, which is the case most of the time).
> please install version `0.2.1` if you are unable to install `0.2.2`
> please install version `0.2.1` if you are unable to install `0.3.0`
```bash
$ RUSTFLAGS="-C target-cpu=native" cargo install deduplicator
```
> [!]
#### Nightly
if you'd like to install with nightly features, you can use
@@ -95,23 +96,9 @@ Note: If you Run into an msvc error, please install MSCV from [here](https://lea
## Performance
Deduplicator uses size comparison and fxhash (a non non-cryptographic hashing algo) to quickly scan through large number of files to find duplicates. its also highly parallel (uses rayon and dashmap). I was able to scan through 120GB of files (Videos, PDFs, Images) in ~300ms. checkout the benchmarks
## benchmarks
| Command | Dirsize | Filecount | Mean [ms] | Min [ms] | Max [ms] | Relative |
|:---|:---|---:|---:|---:|---:|---:|
| `deduplicator ~/Data/tmp` | (~120G) | 721 files | 33.5 ± 28.6 | 25.3 | 151.5 | 1.87 ± 1.60 |
| `deduplicator ~/Data/books` | (~8.6G) | 1419 files | 24.5 ± 1.0 | 22.9 | 28.1 | 1.37 ± 0.08 |
| `deduplicator ~/Data/books --min-size 10M` | (~8.6G) | 1419 files | 17.9 ± 0.7 | 16.8 | 20.0 | 1.00 |
| `deduplicator ~/Data/ --types pdf,jpg,png,jpeg` | (~290G) | 104222 files | 1207.2 ± 37.0 | 1172.2 | 1287.7 | 67.27 ± 3.33 |
* The last entry is lower because of the number of files deduplicator had to go through (~660895 Files). The average size of the files rarely affect the performance of deduplicator.
These benchmarks were run using [hyperfine](https://github.com/sharkdp/hyperfine). Here are the specs of the machine used to benchmark deduplicator:
Deduplicator uses size comparison and gxhash (a non non-cryptographic hashing algorithm) to quickly scan through large number of files to find duplicates. its also highly parallel (uses rayon and dashmap).
## Screenshots
![](https://user-images.githubusercontent.com/36154121/213618143-e5182e39-731e-4817-87dd-1a6a0f38a449.gif)
## Roadmap
@@ -132,8 +119,8 @@ These benchmarks were run using [hyperfine](https://github.com/sharkdp/hyperfine
- [ ] restore json output
- [ ] update documentation
- [x] remove color output
- [ ] progress bar improvements
- [ ] use progress bar groups
- [x] progress bar improvements
- [x] use progress bar groups
- [ ] fix memory leak on very large filesystems
- [ ] maybe use a bloom filter
- [ ] reduce FileInfo size

8
src/fileinfo.rs
View File

@@ -17,7 +17,7 @@ pub struct FileInfo {
}
impl FileInfo {
pub fn hash(&self) -> Result<String> {
pub fn hash(&self) -> Result<u128> {
let file = fs::File::open(&self.path)?;
let mapper = unsafe { Mmap::map(&file)? };
let mut primhasher = GxHasher::default();
@@ -26,16 +26,16 @@ impl FileInfo {
.chunks(1_000_000)
.for_each(|chunk| primhasher.write(chunk));
Ok(primhasher.finish().to_string())
Ok(primhasher.finish_u128())
}
pub fn initial_page_hash(&self) -> Result<String> {
pub fn initial_page_hash(&self) -> Result<u128> {
let file = fs::File::open(&self.path)?;
let mapper = unsafe { Mmap::map(&file)? };
let mut primhasher = GxHasher::default();
primhasher.write(mapper.take(4096).into_inner());
Ok(primhasher.finish().to_string())
Ok(primhasher.finish_u128())
}
pub fn new(path: PathBuf) -> Result<Self> {

44
src/formatter.rs
View File

@@ -36,16 +36,25 @@ impl Formatter {
Ok(modified_time.format("%Y-%m-%d %H:%M:%S").to_string())
}
pub fn gen_sub_tbl(items: Vec<FileInfo>, app_args: &Params, max_path_len: u64) -> Table {
let mut inner_table = Table::new();
inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
items.iter().for_each(|file| {
inner_table.add_row(row![
Self::human_path(file, app_args, max_path_len as usize).unwrap_or_default(),
Self::human_filesize(file).unwrap_or_default(),
Self::human_mtime(file).unwrap_or_default()
]);
});
inner_table
}
pub fn generate_table(
raw: Arc<DashMap<String, Vec<FileInfo>>>,
max_path_len: u64,
app_args: &Params,
raw: Arc<DashMap<u128, Vec<FileInfo>>>,
mpath_len: u64,
args: &Params,
) -> Result<Table> {
let mut output_table = Table::new();
output_table.set_titles(row!["hash", "duplicates"]);
let progress_bar = match app_args.progress {
let progress_bar = match args.progress {
true => ProgressBar::new_spinner(),
false => ProgressBar::hidden(),
};
@@ -60,26 +69,21 @@ impl Formatter {
.progress_with(progress_bar)
.with_finish(ProgressFinish::WithMessage(Cow::from("output generated")))
.map(|i| {
let mut inner_table = Table::new();
inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
i.value().iter().for_each(|file| {
inner_table.add_row(row![
Self::human_path(file, app_args, max_path_len as usize).unwrap_or_default(),
Self::human_filesize(file).unwrap_or_default(),
Self::human_mtime(file).unwrap_or_default()
]);
});
row![i.key(), inner_table]
row![
i.key(),
Self::gen_sub_tbl(i.value().to_vec(), args, mpath_len)
]
})
.collect::<Vec<Row>>();
let mut output_table = Table::new();
output_table.set_titles(row!["hash", "duplicates"]);
output_table.extend(rows);
Ok(output_table)
}
pub fn print(
raw: Arc<DashMap<String, Vec<FileInfo>>>,
raw: Arc<DashMap<u128, Vec<FileInfo>>>,
max_path_len: u64,
app_args: &Params,
) -> Result<()> {

2
src/interactive.rs
View File

@@ -10,7 +10,7 @@ use std::{
pub struct Interactive;
impl Interactive {
pub fn init(result: Arc<DashMap<String, Vec<FileInfo>>>, app_args: &Params) -> Result<()> {
pub fn init(result: Arc<DashMap<u128, Vec<FileInfo>>>, app_args: &Params) -> Result<()> {
result.clone().iter().enumerate().for_each(|(gindex, i)| {
let group = i.value();
let mut itable = Table::new();

8
src/params.rs
View File

@@ -16,19 +16,19 @@ pub struct Params {
#[arg(long, short)]
pub interactive: bool,
/// Minimum filesize of duplicates to scan (e.g., 100B/1K/2M/3G/4T).
#[arg(long, short = 's', default_value = "1b")]
#[arg(long, short = 'm', default_value = "1b")]
pub min_size: Option<String>,
/// Max Depth to scan while looking for duplicates
#[arg(long, short = 'd')]
#[arg(long, short = 'D')]
pub max_depth: Option<usize>,
/// Min Depth to scan while looking for duplicates
#[arg(long)]
#[arg(long, short = 'd')]
pub min_depth: Option<usize>,
/// Follow links while scanning directories
#[arg(long, short)]
pub follow_links: bool,
/// Guarantees that two files are duplicate (performs a full hash)
#[arg(long, short = 'f', default_value = "false")]
#[arg(long, short = 's', default_value = "false")]
pub strict: bool,
/// Show Progress spinners & metrics
#[arg(long, short = 'p', default_value = "false")]

5
src/processor.rs
View File

@@ -18,7 +18,7 @@ impl Processor {
pub fn hashwise(
app_args: Arc<Params>,
sw_store: Arc<DashMap<u64, Vec<FileInfo>>>,
hw_store: Arc<DashMap<String, Vec<FileInfo>>>,
hw_store: Arc<DashMap<u128, Vec<FileInfo>>>,
progress_bar_box: Arc<MultiProgress>,
max_file_size: Arc<AtomicU64>,
) -> Result<()> {
@@ -129,7 +129,7 @@ mod tests {
use rand::Rng;
use std::fs::File;
use std::io::Write;
use std::sync::atomic::{AtomicBool, AtomicU64};
use std::sync::atomic::AtomicBool;
use std::sync::{Arc, Mutex};
use tempfile::TempDir;
@@ -169,7 +169,6 @@ mod tests {
Arc::new(AtomicBool::new(true)),
dupstore.clone(),
file_queue,
Arc::new(AtomicU64::new(0)),
Arc::new(MultiProgress::new()),
)?;

2
src/server.rs
View File

@@ -14,7 +14,7 @@ use crate::params::Params;
pub struct Server {
filequeue: Arc<Mutex<Vec<FileInfo>>>,
sw_duplicate_set: Arc<DashMap<u64, Vec<FileInfo>>>,
pub hw_duplicate_set: Arc<DashMap<String, Vec<FileInfo>>>,
pub hw_duplicate_set: Arc<DashMap<u128, Vec<FileInfo>>>,
threadpool: ThreadPool,
app_args: Arc<Params>,
pub max_file_path_len: Arc<AtomicU64>,