mirror of
https://github.com/sreedevk/deduplicator.git
synced 2026-09-01 04:55:33 +00:00
Compare commits
11 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
245e910c39 | ||
|
|
312d66a5e1 | ||
|
|
d102d92bfa | ||
|
|
68c2491fd0 | ||
|
|
da24efbb36 | ||
|
|
2191763b4d | ||
|
|
b84510f18e | ||
|
|
bfb89edd16 | ||
|
|
21f0eb3cb0 | ||
|
|
f5d2d4e22c | ||
|
|
360af25318 |
2
.cargo/config.toml
Normal file
2
.cargo/config.toml
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
[build]
|
||||||
|
rustflags = ["-C", "target-feature=+aes,+sse2"]
|
||||||
9
Cargo.lock
generated
9
Cargo.lock
generated
@@ -273,7 +273,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "deduplicator"
|
name = "deduplicator"
|
||||||
version = "0.3.1"
|
version = "0.3.2"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"bytesize",
|
"bytesize",
|
||||||
@@ -290,6 +290,7 @@ dependencies = [
|
|||||||
"rayon",
|
"rayon",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
"threadpool",
|
"threadpool",
|
||||||
|
"unicode-segmentation",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -898,6 +899,12 @@ version = "1.0.18"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"
|
checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "unicode-segmentation"
|
||||||
|
version = "1.12.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "unicode-width"
|
name = "unicode-width"
|
||||||
version = "0.1.14"
|
version = "0.1.14"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "deduplicator"
|
name = "deduplicator"
|
||||||
version = "0.3.1"
|
version = "0.3.2"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
description = "find,filter and delete duplicate files"
|
description = "find,filter and delete duplicate files"
|
||||||
repository = "https://github.com/sreedevk/deduplicator"
|
repository = "https://github.com/sreedevk/deduplicator"
|
||||||
@@ -31,6 +31,7 @@ prettytable-rs = "0.10.0"
|
|||||||
rand = "0.9.1"
|
rand = "0.9.1"
|
||||||
rayon = "1.6.1"
|
rayon = "1.6.1"
|
||||||
threadpool = "1.8.1"
|
threadpool = "1.8.1"
|
||||||
|
unicode-segmentation = "1.12.0"
|
||||||
|
|
||||||
[profile.release]
|
[profile.release]
|
||||||
strip = true
|
strip = true
|
||||||
|
|||||||
22
README.md
22
README.md
@@ -4,6 +4,9 @@
|
|||||||
Find, Sort, Filter & Delete duplicate files
|
Find, Sort, Filter & Delete duplicate files
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
|
> [!NOTE]
|
||||||
|
> This project is maintained with the assistance of AI tools. All changes are subject to manual review and a comprehensive test suite to ensure stability and quality.
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -50,7 +53,8 @@ deduplicator ~/Media --min-size 100mb
|
|||||||
```
|
```
|
||||||
|
|
||||||
## Demo
|
## Demo
|
||||||

|

|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@@ -80,6 +84,12 @@ $ RUSTFLAGS="-C target-cpu=native" cargo install deduplicator --git https://gith
|
|||||||
$ RUSTFLAGS="-C target-feature=+aes,+sse2" cargo install --git https://github.com/sreedevk/deduplicator
|
$ RUSTFLAGS="-C target-feature=+aes,+sse2" cargo install --git https://github.com/sreedevk/deduplicator
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Manual Installation
|
||||||
|
- Download the right pre-compiled binary archive for your platform from [github release page](https://github.com/sreedevk/deduplicator/releases/tag/latest).
|
||||||
|
- Decompress it using `tar -zxvf <archive>.tar.gz`
|
||||||
|
- Move it to a directory included in `$PATH`.
|
||||||
|
- ideally `/usr/local/bin/`.
|
||||||
|
|
||||||
## Performance
|
## Performance
|
||||||
Deduplicator uses size comparison and [GxHash](https://docs.rs/gxhash/latest/gxhash/) to quickly check a large number of files to find duplicates. its also heavily parallelized. The default behavior of deduplicator is to only hash the first page (4K) of the file. This is to ensure that performance is the default priority. You can modify this behavior by using the `--strict` flag which will hash the whole file and ensure that 2 files are indeed duplicates. I'll add benchmarks in future versions.
|
Deduplicator uses size comparison and [GxHash](https://docs.rs/gxhash/latest/gxhash/) to quickly check a large number of files to find duplicates. its also heavily parallelized. The default behavior of deduplicator is to only hash the first page (4K) of the file. This is to ensure that performance is the default priority. You can modify this behavior by using the `--strict` flag which will hash the whole file and ensure that 2 files are indeed duplicates. I'll add benchmarks in future versions.
|
||||||
|
|
||||||
@@ -149,6 +159,8 @@ dust 'bench_artifacts'
|
|||||||
## proposed
|
## proposed
|
||||||
- [ ] parallelization
|
- [ ] parallelization
|
||||||
- [ ] scanning + processing sw + processing hw + formatting + printing
|
- [ ] scanning + processing sw + processing hw + formatting + printing
|
||||||
|
- [ ] user supplied cache file path for faster re-runs
|
||||||
|
- [ ] hardlinks / symlinks support
|
||||||
- [ ] max file path size should use the last set of duplicates
|
- [ ] max file path size should use the last set of duplicates
|
||||||
- [ ] add more unit tests
|
- [ ] add more unit tests
|
||||||
- [ ] test against different filesystems
|
- [ ] test against different filesystems
|
||||||
@@ -160,6 +172,7 @@ dust 'bench_artifacts'
|
|||||||
- [ ] tui
|
- [ ] tui
|
||||||
- [ ] change the default hashing method to include the first & last page of a file (8K)
|
- [ ] change the default hashing method to include the first & last page of a file (8K)
|
||||||
- [ ] provide option to localize duplicate detection to arbitrary levels relative to current directory
|
- [ ] provide option to localize duplicate detection to arbitrary levels relative to current directory
|
||||||
|
- [ ] localize file meta store locks to sub path levels to avoid global lock contention from multiple threads.
|
||||||
- [ ] bulk operations
|
- [ ] bulk operations
|
||||||
- [ ] --keep-latest
|
- [ ] --keep-latest
|
||||||
- [ ] --keep-oldest
|
- [ ] --keep-oldest
|
||||||
@@ -169,6 +182,13 @@ dust 'bench_artifacts'
|
|||||||
- [ ] fix: partial hash collision - a file full of null bytes ("\0") and an empty file. This is a known trade off in gxhash.
|
- [ ] fix: partial hash collision - a file full of null bytes ("\0") and an empty file. This is a known trade off in gxhash.
|
||||||
- [ ] include initial pages and final pages of the file
|
- [ ] include initial pages and final pages of the file
|
||||||
- [ ] append the offset between the last initial page hashed and the first final page hashed in the content passed to the hasher.
|
- [ ] append the offset between the last initial page hashed and the first final page hashed in the content passed to the hasher.
|
||||||
|
- [ ] potential optimizations
|
||||||
|
- [ ] lookup the memory efficiency gains if instead of directly inserting into a hashmap, deduplicator looksup the file in a bloom filter. this way, the duplicate store hashmap does
|
||||||
|
not require to be locked for every single file. The bloom filter can be stored in an atomically updateable type to improve performance as well.
|
||||||
|
|
||||||
|
## v0.3.2
|
||||||
|
- [x] fix: single file groups are printed to screen
|
||||||
|
- [x] fix: --exclude-types and --types flag behave identically.
|
||||||
|
|
||||||
## v0.3.1
|
## v0.3.1
|
||||||
- [x] parallelization
|
- [x] parallelization
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ use chrono::{DateTime, Utc};
|
|||||||
use dashmap::DashMap;
|
use dashmap::DashMap;
|
||||||
use pathdiff::diff_paths;
|
use pathdiff::diff_paths;
|
||||||
use rayon::prelude::*;
|
use rayon::prelude::*;
|
||||||
|
use std::sync::atomic::AtomicU64;
|
||||||
use std::{path::PathBuf, sync::Arc};
|
use std::{path::PathBuf, sync::Arc};
|
||||||
|
|
||||||
const YELLOW: &str = "\x1b[33m";
|
const YELLOW: &str = "\x1b[33m";
|
||||||
@@ -11,14 +12,14 @@ const RESET: &str = "\x1b[0m";
|
|||||||
|
|
||||||
pub struct Formatter;
|
pub struct Formatter;
|
||||||
impl Formatter {
|
impl Formatter {
|
||||||
pub fn human_path(file: &FileInfo, aargs: &Params, min_path_length: usize) -> Result<String> {
|
pub fn human_path(file: &FileInfo, aargs: &Params, max_path_length: usize) -> Result<String> {
|
||||||
let base_directory: PathBuf = aargs.get_directory()?;
|
let base_directory: PathBuf = aargs.get_directory()?;
|
||||||
let relative_path = diff_paths(&file.path, base_directory).unwrap_or_default();
|
let relative_path = diff_paths(&file.path, base_directory).unwrap_or_default();
|
||||||
|
|
||||||
let formatted_path = format!(
|
let formatted_path = format!(
|
||||||
"{:<0width$}",
|
"{:<0width$}",
|
||||||
relative_path.to_str().unwrap_or_default().to_string(),
|
relative_path.to_str().unwrap_or_default().to_string(),
|
||||||
width = min_path_length
|
width = max_path_length
|
||||||
);
|
);
|
||||||
|
|
||||||
Ok(formatted_path)
|
Ok(formatted_path)
|
||||||
@@ -39,26 +40,41 @@ impl Formatter {
|
|||||||
if raw.is_empty() {
|
if raw.is_empty() {
|
||||||
println!("No duplicates found matching your search criteria.");
|
println!("No duplicates found matching your search criteria.");
|
||||||
} else {
|
} else {
|
||||||
|
let printed_count: AtomicU64 = AtomicU64::new(0);
|
||||||
|
|
||||||
raw.par_iter().for_each(|sref| {
|
raw.par_iter().for_each(|sref| {
|
||||||
let mut ostring = format!("{}{:32x}{}\n", YELLOW, sref.key(), RESET);
|
if sref.value().len() > 1 {
|
||||||
let subfields = sref
|
printed_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||||
.value()
|
let mut ostring = format!("{}{:32x}{}\n", YELLOW, sref.key(), RESET);
|
||||||
.par_iter()
|
let subfields = sref
|
||||||
.map(|finfo| {
|
.value()
|
||||||
format!(
|
.par_iter()
|
||||||
"├─ {}\t{}\t{}\n",
|
.enumerate()
|
||||||
Self::human_path(finfo, aargs, max_path_len as usize)
|
.map(|(i, finfo)| {
|
||||||
.expect("path formatting failed."),
|
let nodechar = if i == sref.value().len() - 1 {
|
||||||
Self::human_filesize(finfo).expect("filesize formatting failed."),
|
"└─"
|
||||||
Self::human_mtime(finfo).expect("modified time formatting failed.")
|
} else {
|
||||||
)
|
"├─"
|
||||||
})
|
};
|
||||||
.collect::<String>();
|
format!(
|
||||||
|
"{}\t{}\t{}\t{}\n",
|
||||||
|
nodechar,
|
||||||
|
Self::human_path(finfo, aargs, max_path_len as usize)
|
||||||
|
.expect("path formatting failed."),
|
||||||
|
Self::human_filesize(finfo).expect("filesize formatting failed."),
|
||||||
|
Self::human_mtime(finfo).expect("modified time formatting failed.")
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect::<String>();
|
||||||
|
|
||||||
ostring.push_str(&subfields);
|
ostring.push_str(&subfields);
|
||||||
|
println!("{ostring}");
|
||||||
println!("{ostring}");
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
if printed_count.load(std::sync::atomic::Ordering::Relaxed) < 1 {
|
||||||
|
println!("No duplicates found matching your search criteria.");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ use crate::{fileinfo::FileInfo, formatter::Formatter, params::Params};
|
|||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use dashmap::DashMap;
|
use dashmap::DashMap;
|
||||||
use prettytable::{format, row, Table};
|
use prettytable::{format, row, Table};
|
||||||
|
use std::sync::atomic::AtomicU64;
|
||||||
use std::{
|
use std::{
|
||||||
io::{self, Write},
|
io::{self, Write},
|
||||||
sync::Arc,
|
sync::Arc,
|
||||||
@@ -11,12 +12,19 @@ pub struct Interactive;
|
|||||||
|
|
||||||
impl Interactive {
|
impl Interactive {
|
||||||
pub fn init(result: Arc<DashMap<u128, Vec<FileInfo>>>, app_args: &Params) -> Result<()> {
|
pub fn init(result: Arc<DashMap<u128, Vec<FileInfo>>>, app_args: &Params) -> Result<()> {
|
||||||
result
|
let store = result.clone();
|
||||||
.clone()
|
if store.is_empty() {
|
||||||
|
println!("No duplicates found matching your search criteria.");
|
||||||
|
}
|
||||||
|
|
||||||
|
let printed_count: AtomicU64 = AtomicU64::new(0);
|
||||||
|
|
||||||
|
store
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|i| i.value().len() > 1)
|
.filter(|i| i.value().len() > 1)
|
||||||
.enumerate()
|
.enumerate()
|
||||||
.for_each(|(gindex, i)| {
|
.for_each(|(gindex, i)| {
|
||||||
|
printed_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||||
let group = i.value();
|
let group = i.value();
|
||||||
let mut itable = Table::new();
|
let mut itable = Table::new();
|
||||||
itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
|
itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
|
||||||
@@ -40,6 +48,10 @@ impl Interactive {
|
|||||||
Self::process_group_action(group, gindex, result.len(), itable);
|
Self::process_group_action(group, gindex, result.len(), itable);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
if printed_count.load(std::sync::atomic::Ordering::Relaxed) < 1 {
|
||||||
|
println!("No duplicates found matching your search criteria.");
|
||||||
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ use std::{fs, path::PathBuf};
|
|||||||
|
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use clap::{Parser, ValueHint};
|
use clap::{Parser, ValueHint};
|
||||||
use std::collections::HashSet;
|
|
||||||
|
|
||||||
#[derive(Parser, Debug, Default, Clone)]
|
#[derive(Parser, Debug, Default, Clone)]
|
||||||
#[command(author, version, about, long_about = None)]
|
#[command(author, version, about, long_about = None)]
|
||||||
@@ -56,48 +55,4 @@ impl Params {
|
|||||||
let dir = fs::canonicalize(dir_path)?;
|
let dir = fs::canonicalize(dir_path)?;
|
||||||
Ok(dir)
|
Ok(dir)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn types_intersection(itypes: &str, xtypes: &str) -> String {
|
|
||||||
let iset = itypes
|
|
||||||
.split(",")
|
|
||||||
.map(String::from)
|
|
||||||
.collect::<HashSet<String>>();
|
|
||||||
let xset = xtypes
|
|
||||||
.split(",")
|
|
||||||
.map(String::from)
|
|
||||||
.collect::<HashSet<String>>();
|
|
||||||
|
|
||||||
iset.difference(&xset)
|
|
||||||
.cloned()
|
|
||||||
.collect::<Vec<String>>()
|
|
||||||
.join(",")
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn get_types(&self) -> Option<String> {
|
|
||||||
match &self.types {
|
|
||||||
Some(itypes) => match &self.exclude_types {
|
|
||||||
Some(xtypes) => Some(Self::types_intersection(itypes, xtypes)),
|
|
||||||
None => Some(itypes.to_string()),
|
|
||||||
},
|
|
||||||
None => self.exclude_types.as_ref().map(|xtypes| xtypes.to_string()),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn mixing_include_and_exclude_types_works_as_expected() {
|
|
||||||
let params = Params {
|
|
||||||
types: Some(String::from("js,xml,ts,pdf,tiff")),
|
|
||||||
exclude_types: Some(String::from("js,ts,xml")),
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
|
|
||||||
assert!(params
|
|
||||||
.get_types()
|
|
||||||
.is_some_and(|x| x == "pdf,tiff" || x == "tiff,pdf"))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ use rayon::prelude::{IntoParallelIterator, ParallelIterator};
|
|||||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||||
use std::sync::{Arc, Mutex, TryLockError, TryLockResult};
|
use std::sync::{Arc, Mutex, TryLockError, TryLockResult};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
use unicode_segmentation::UnicodeSegmentation;
|
||||||
|
|
||||||
use crate::fileinfo::FileInfo;
|
use crate::fileinfo::FileInfo;
|
||||||
use crate::params::Params;
|
use crate::params::Params;
|
||||||
@@ -64,7 +65,7 @@ impl Processor {
|
|||||||
|
|
||||||
Self::compare_and_update_max_path_len(
|
Self::compare_and_update_max_path_len(
|
||||||
max_file_size.clone(),
|
max_file_size.clone(),
|
||||||
file.path.to_string_lossy().len() as u64,
|
file.path.to_string_lossy().graphemes(true).count() as u64,
|
||||||
);
|
);
|
||||||
|
|
||||||
hw_store
|
hw_store
|
||||||
|
|||||||
180
src/scanner.rs
180
src/scanner.rs
@@ -8,9 +8,10 @@ use globwalk::{GlobWalker, GlobWalkerBuilder};
|
|||||||
|
|
||||||
pub struct Scanner {
|
pub struct Scanner {
|
||||||
pub directory: Box<Path>,
|
pub directory: Box<Path>,
|
||||||
pub filetypes: Option<String>,
|
|
||||||
pub min_depth: Option<usize>,
|
pub min_depth: Option<usize>,
|
||||||
pub max_depth: Option<usize>,
|
pub max_depth: Option<usize>,
|
||||||
|
pub include_types: Option<String>,
|
||||||
|
pub exclude_types: Option<String>,
|
||||||
pub min_size: Option<u64>,
|
pub min_size: Option<u64>,
|
||||||
pub follow_links: bool,
|
pub follow_links: bool,
|
||||||
pub progress: bool,
|
pub progress: bool,
|
||||||
@@ -20,7 +21,8 @@ impl Scanner {
|
|||||||
pub fn new(app_args: Arc<Params>) -> Result<Self> {
|
pub fn new(app_args: Arc<Params>) -> Result<Self> {
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
directory: app_args.get_directory()?.into_boxed_path(),
|
directory: app_args.get_directory()?.into_boxed_path(),
|
||||||
filetypes: app_args.get_types(),
|
include_types: app_args.types.clone(),
|
||||||
|
exclude_types: app_args.exclude_types.clone(),
|
||||||
min_depth: app_args.min_depth,
|
min_depth: app_args.min_depth,
|
||||||
max_depth: app_args.max_depth,
|
max_depth: app_args.max_depth,
|
||||||
min_size: app_args.get_min_size(),
|
min_size: app_args.get_min_size(),
|
||||||
@@ -29,11 +31,21 @@ impl Scanner {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn scan_patterns(&self) -> Result<String> {
|
fn scan_patterns(&self) -> Result<Vec<String>> {
|
||||||
Ok(match &self.filetypes {
|
let include_types = match &self.include_types {
|
||||||
Some(ftypes) => format!("**/*{{{ftypes}}}"),
|
Some(ftypes) => Some(format!("**/*.{{{ftypes}}}")),
|
||||||
None => "**/*".to_string(),
|
None => Some("**/*".to_string()),
|
||||||
})
|
};
|
||||||
|
|
||||||
|
let exclude_types = self
|
||||||
|
.exclude_types
|
||||||
|
.as_ref()
|
||||||
|
.map(|ftypes| format!("!**/*.{{{ftypes}}}"));
|
||||||
|
|
||||||
|
Ok(vec![include_types, exclude_types]
|
||||||
|
.into_iter()
|
||||||
|
.flatten()
|
||||||
|
.collect())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn attach_link_opts(&self, walker: GlobWalkerBuilder) -> Result<GlobWalkerBuilder> {
|
fn attach_link_opts(&self, walker: GlobWalkerBuilder) -> Result<GlobWalkerBuilder> {
|
||||||
@@ -56,7 +68,7 @@ impl Scanner {
|
|||||||
fn build_walker(&self) -> Result<GlobWalker> {
|
fn build_walker(&self) -> Result<GlobWalker> {
|
||||||
let walker = Ok(GlobWalkerBuilder::from_patterns(
|
let walker = Ok(GlobWalkerBuilder::from_patterns(
|
||||||
self.directory.clone(),
|
self.directory.clone(),
|
||||||
&[self.scan_patterns()?],
|
&self.scan_patterns()?,
|
||||||
))
|
))
|
||||||
.and_then(|walker| self.attach_walker_min_depth(walker))
|
.and_then(|walker| self.attach_walker_min_depth(walker))
|
||||||
.and_then(|walker| self.attach_walker_max_depth(walker))
|
.and_then(|walker| self.attach_walker_max_depth(walker))
|
||||||
@@ -79,7 +91,7 @@ impl Scanner {
|
|||||||
progress_bar.set_style(progress_style);
|
progress_bar.set_style(progress_style);
|
||||||
progress_bar.enable_steady_tick(Duration::from_millis(50));
|
progress_bar.enable_steady_tick(Duration::from_millis(50));
|
||||||
progress_bar.set_message("paths mapped");
|
progress_bar.set_message("paths mapped");
|
||||||
let min_size = self.min_size.unwrap_or_default();
|
let min_size = self.min_size.unwrap_or(0);
|
||||||
|
|
||||||
self.build_walker()?
|
self.build_walker()?
|
||||||
.filter_map(Result::ok)
|
.filter_map(Result::ok)
|
||||||
@@ -88,7 +100,7 @@ impl Scanner {
|
|||||||
.filter(|path| path.is_file())
|
.filter(|path| path.is_file())
|
||||||
.map(FileInfo::new)
|
.map(FileInfo::new)
|
||||||
.filter_map(Result::ok)
|
.filter_map(Result::ok)
|
||||||
.filter(|file| file.size > min_size)
|
.filter(|file| file.size >= min_size)
|
||||||
.for_each(|file| {
|
.for_each(|file| {
|
||||||
let mut flock = files.lock().unwrap();
|
let mut flock = files.lock().unwrap();
|
||||||
flock.push(file);
|
flock.push(file);
|
||||||
@@ -98,3 +110,151 @@ impl Scanner {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use crate::fileinfo::FileInfo;
|
||||||
|
use crate::params::Params;
|
||||||
|
use std::fs::File;
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
|
use super::Scanner;
|
||||||
|
use indicatif::MultiProgress;
|
||||||
|
use tempfile::TempDir;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ensure_file_include_type_filter_includes_expected_file_types() {
|
||||||
|
let root =
|
||||||
|
TempDir::with_prefix("deduplicator_test_root").expect("unable to create tempdir");
|
||||||
|
[
|
||||||
|
"this-is-a-js-file.js",
|
||||||
|
"this-is-a-css-file.css",
|
||||||
|
"this-is-a-csv-file.csv",
|
||||||
|
"this-is-a-rust-file.rs",
|
||||||
|
]
|
||||||
|
.iter()
|
||||||
|
.for_each(|path| {
|
||||||
|
File::create_new(root.path().join(path)).unwrap_or_else(|_| {
|
||||||
|
panic!("unable to create file {path}");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
let params = Params {
|
||||||
|
types: Some(String::from("js,csv")),
|
||||||
|
dir: Some(root.path().into()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let progress = Arc::new(MultiProgress::new());
|
||||||
|
let scanlist = Arc::new(Mutex::<Vec<FileInfo>>::new(vec![]));
|
||||||
|
let scanner = Scanner::new(Arc::new(params)).expect("scanner initialization failed");
|
||||||
|
|
||||||
|
scanner
|
||||||
|
.scan(scanlist.clone(), progress)
|
||||||
|
.expect("scanning failed.");
|
||||||
|
|
||||||
|
let scan_list_mg = scanlist.lock().unwrap();
|
||||||
|
|
||||||
|
assert!(scan_list_mg.iter().any(|f| f.path.to_str().unwrap()
|
||||||
|
== root.path().join("this-is-a-js-file.js").to_str().unwrap()));
|
||||||
|
|
||||||
|
assert!(scan_list_mg.iter().any(|f| f.path.to_str().unwrap()
|
||||||
|
== root.path().join("this-is-a-csv-file.csv").to_str().unwrap()));
|
||||||
|
|
||||||
|
assert!(scan_list_mg.iter().all(|f| f.path.to_str().unwrap()
|
||||||
|
!= root.path().join("this-is-a-css-file.css").to_str().unwrap()));
|
||||||
|
|
||||||
|
assert!(scan_list_mg.iter().all(|f| f.path.to_str().unwrap()
|
||||||
|
!= root.path().join("this-is-a-rust-file.rs").to_str().unwrap()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ensure_file_exclude_type_filter_excludes_expected_file_types() {
|
||||||
|
let root =
|
||||||
|
TempDir::with_prefix("deduplicator_test_root").expect("unable to create tempdir");
|
||||||
|
[
|
||||||
|
"this-is-a-js-file.js",
|
||||||
|
"this-is-a-css-file.css",
|
||||||
|
"this-is-a-csv-file.csv",
|
||||||
|
"this-is-a-rust-file.rs",
|
||||||
|
]
|
||||||
|
.iter()
|
||||||
|
.for_each(|path| {
|
||||||
|
File::create_new(root.path().join(path)).unwrap_or_else(|_| {
|
||||||
|
panic!("unable to create file {path}");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
let params = Params {
|
||||||
|
exclude_types: Some(String::from("js,csv")),
|
||||||
|
dir: Some(root.path().into()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let progress = Arc::new(MultiProgress::new());
|
||||||
|
let scanlist = Arc::new(Mutex::<Vec<FileInfo>>::new(vec![]));
|
||||||
|
let scanner = Scanner::new(Arc::new(params)).expect("scanner initialization failed");
|
||||||
|
|
||||||
|
scanner
|
||||||
|
.scan(scanlist.clone(), progress)
|
||||||
|
.expect("scanning failed.");
|
||||||
|
|
||||||
|
let scan_list_mg = scanlist.lock().unwrap();
|
||||||
|
|
||||||
|
assert!(scan_list_mg.iter().all(|f| f.path.to_str().unwrap()
|
||||||
|
!= root.path().join("this-is-a-js-file.js").to_str().unwrap()));
|
||||||
|
|
||||||
|
assert!(scan_list_mg.iter().all(|f| f.path.to_str().unwrap()
|
||||||
|
!= root.path().join("this-is-a-csv-file.csv").to_str().unwrap()));
|
||||||
|
|
||||||
|
assert!(scan_list_mg.iter().any(|f| f.path.to_str().unwrap()
|
||||||
|
== root.path().join("this-is-a-css-file.css").to_str().unwrap()));
|
||||||
|
|
||||||
|
assert!(scan_list_mg.iter().any(|f| f.path.to_str().unwrap()
|
||||||
|
== root.path().join("this-is-a-rust-file.rs").to_str().unwrap()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn complex_file_type_params() {
|
||||||
|
let root =
|
||||||
|
TempDir::with_prefix("deduplicator_test_root").expect("unable to create tempdir");
|
||||||
|
[
|
||||||
|
"this-is-a-js-file.js",
|
||||||
|
"this-is-a-css-file.css",
|
||||||
|
"this-is-a-csv-file.csv",
|
||||||
|
"this-is-a-rust-file.rs",
|
||||||
|
]
|
||||||
|
.iter()
|
||||||
|
.for_each(|path| {
|
||||||
|
File::create_new(root.path().join(path)).unwrap_or_else(|_| {
|
||||||
|
panic!("unable to create file {path}");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
let params = Params {
|
||||||
|
types: Some(String::from("js,csv,rs")),
|
||||||
|
exclude_types: Some(String::from("csv")),
|
||||||
|
dir: Some(root.path().into()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let progress = Arc::new(MultiProgress::new());
|
||||||
|
let scanlist = Arc::new(Mutex::<Vec<FileInfo>>::new(vec![]));
|
||||||
|
let scanner = Scanner::new(Arc::new(params)).expect("scanner initialization failed");
|
||||||
|
|
||||||
|
scanner
|
||||||
|
.scan(scanlist.clone(), progress)
|
||||||
|
.expect("scanning failed.");
|
||||||
|
|
||||||
|
let scan_list_mg = scanlist.lock().unwrap();
|
||||||
|
|
||||||
|
assert!(scan_list_mg.iter().any(|f| f.path.to_str().unwrap()
|
||||||
|
== root.path().join("this-is-a-js-file.js").to_str().unwrap()));
|
||||||
|
|
||||||
|
assert!(scan_list_mg.iter().all(|f| f.path.to_str().unwrap()
|
||||||
|
!= root.path().join("this-is-a-csv-file.csv").to_str().unwrap()));
|
||||||
|
|
||||||
|
assert!(scan_list_mg.iter().any(|f| f.path.to_str().unwrap()
|
||||||
|
== root.path().join("this-is-a-rust-file.rs").to_str().unwrap()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -42,60 +42,68 @@ impl Server {
|
|||||||
progbarbox.set_draw_target(ProgressDrawTarget::hidden());
|
progbarbox.set_draw_target(ProgressDrawTarget::hidden());
|
||||||
}
|
}
|
||||||
|
|
||||||
let app_args_clone_for_sc = self.app_args.clone();
|
let (app_args_sc, app_args_sw, app_args_hw) = (
|
||||||
let app_args_clone_for_sw = self.app_args.clone();
|
Arc::clone(&self.app_args),
|
||||||
let app_args_clone_for_hw = self.app_args.clone();
|
Arc::clone(&self.app_args),
|
||||||
let file_queue_clone_sc = self.filequeue.clone();
|
Arc::clone(&self.app_args),
|
||||||
let file_queue_clone_pr = self.filequeue.clone();
|
);
|
||||||
|
let (file_queue_sc, file_queue_pr) = (
|
||||||
|
Arc::clone(&self.filequeue),
|
||||||
|
Arc::clone(&self.filequeue),
|
||||||
|
);
|
||||||
let scanner_finished = Arc::new(AtomicBool::new(false));
|
let scanner_finished = Arc::new(AtomicBool::new(false));
|
||||||
let sw_sort_finished = Arc::new(AtomicBool::new(false));
|
let sw_sort_finished = Arc::new(AtomicBool::new(false));
|
||||||
|
let (sfin_sc, sfin_pr) = (
|
||||||
let sfin_sc_tr_cl = scanner_finished.clone();
|
Arc::clone(&scanner_finished),
|
||||||
let sfin_pr_tr_cl = scanner_finished.clone();
|
Arc::clone(&scanner_finished),
|
||||||
|
);
|
||||||
let swfin_pr_tr_sw = sw_sort_finished.clone();
|
let (swfin_pr_sw, swfin_pr_hw) = (
|
||||||
let swfin_pr_tr_hw = sw_sort_finished.clone();
|
Arc::clone(&sw_sort_finished),
|
||||||
|
Arc::clone(&sw_sort_finished),
|
||||||
let store_dupl_sw_for_sw = self.sw_duplicate_set.clone();
|
);
|
||||||
let store_dupl_sw_for_hw = self.sw_duplicate_set.clone();
|
let (store_sw, store_sw2, store_hw) = (
|
||||||
let store_dupl_hw = self.hw_duplicate_set.clone();
|
Arc::clone(&self.sw_duplicate_set),
|
||||||
let max_file_path_len_clone = self.max_file_path_len.clone();
|
Arc::clone(&self.sw_duplicate_set),
|
||||||
|
Arc::clone(&self.hw_duplicate_set),
|
||||||
let progbarbox_sc_clone = progbarbox.clone();
|
);
|
||||||
let progbarbox_pr_clone_for_sw = progbarbox.clone();
|
let max_file_path_len = Arc::clone(&self.max_file_path_len);
|
||||||
let progbarbox_pr_clone_for_hw = progbarbox.clone();
|
let (prog_sc, prog_sw, prog_hw) = (
|
||||||
|
Arc::clone(&progbarbox),
|
||||||
|
Arc::clone(&progbarbox),
|
||||||
|
Arc::clone(&progbarbox),
|
||||||
|
);
|
||||||
|
|
||||||
self.threadpool.execute(move || {
|
self.threadpool.execute(move || {
|
||||||
Scanner::new(app_args_clone_for_sc)
|
Scanner::new(app_args_sc)
|
||||||
.expect("unable to initialize scanner.")
|
.expect("unable to initialize scanner.")
|
||||||
.scan(file_queue_clone_sc, progbarbox_sc_clone)
|
.scan(file_queue_sc, prog_sc)
|
||||||
.expect("scanner failed.");
|
.expect("scanner failed.");
|
||||||
|
|
||||||
sfin_sc_tr_cl.store(true, std::sync::atomic::Ordering::Relaxed);
|
sfin_sc.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||||
});
|
});
|
||||||
|
|
||||||
self.threadpool.execute(move || {
|
self.threadpool.execute(move || {
|
||||||
Processor::sizewise(
|
Processor::sizewise(
|
||||||
app_args_clone_for_sw,
|
app_args_sw,
|
||||||
sfin_pr_tr_cl,
|
sfin_pr,
|
||||||
store_dupl_sw_for_sw,
|
store_sw,
|
||||||
file_queue_clone_pr,
|
file_queue_pr,
|
||||||
progbarbox_pr_clone_for_sw,
|
prog_sw,
|
||||||
)
|
)
|
||||||
.expect("sizewise scanner failed.");
|
.expect("sizewise scanner failed.");
|
||||||
|
|
||||||
swfin_pr_tr_sw.store(true, std::sync::atomic::Ordering::Relaxed);
|
swfin_pr_sw.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||||
});
|
});
|
||||||
|
|
||||||
self.threadpool.execute(move || {
|
self.threadpool.execute(move || {
|
||||||
Processor::hashwise(
|
Processor::hashwise(
|
||||||
app_args_clone_for_hw,
|
app_args_hw,
|
||||||
store_dupl_sw_for_hw,
|
store_sw2,
|
||||||
store_dupl_hw,
|
store_hw,
|
||||||
progbarbox_pr_clone_for_hw,
|
prog_hw,
|
||||||
max_file_path_len_clone,
|
max_file_path_len,
|
||||||
seed,
|
seed,
|
||||||
swfin_pr_tr_hw,
|
swfin_pr_hw,
|
||||||
)
|
)
|
||||||
.expect("sizewise scanner failed.");
|
.expect("sizewise scanner failed.");
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user