mirror of
https://github.com/sreedevk/deduplicator.git
synced 2026-08-26 02:04:39 +00:00
performance improvements
This commit is contained in:
sreedevk
1
.gitignore
vendored
1
.gitignore
vendored
@@ -1,3 +1,4 @@
|
||||
/target
|
||||
/Cargo.lock
|
||||
/result-bin
|
||||
/.bacon-locations
|
||||
|
||||
111
Cargo.lock
generated
111
Cargo.lock
generated
@@ -1,6 +1,6 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 3
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
@@ -62,7 +62,7 @@ version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ad186efb764318d35165f1758e7dcef3b10628e26d41a44bc5550652e6804391"
|
||||
dependencies = [
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -72,7 +72,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -144,7 +144,7 @@ dependencies = [
|
||||
"js-sys",
|
||||
"num-traits",
|
||||
"wasm-bindgen",
|
||||
"windows-targets 0.52.6",
|
||||
"windows-targets",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -193,16 +193,6 @@ version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422"
|
||||
|
||||
[[package]]
|
||||
name = "colored"
|
||||
version = "2.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "console"
|
||||
version = "0.15.8"
|
||||
@@ -213,7 +203,7 @@ dependencies = [
|
||||
"lazy_static",
|
||||
"libc",
|
||||
"unicode-width",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -290,7 +280,6 @@ dependencies = [
|
||||
"bytesize",
|
||||
"chrono",
|
||||
"clap",
|
||||
"colored",
|
||||
"dashmap",
|
||||
"globwalk",
|
||||
"gxhash",
|
||||
@@ -476,7 +465,7 @@ checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"libc",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -593,7 +582,7 @@ dependencies = [
|
||||
"libc",
|
||||
"redox_syscall",
|
||||
"smallvec",
|
||||
"windows-targets 0.52.6",
|
||||
"windows-targets",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -925,7 +914,7 @@ version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b"
|
||||
dependencies = [
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -940,16 +929,7 @@ version = "0.52.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
|
||||
dependencies = [
|
||||
"windows-targets 0.52.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.48.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
|
||||
dependencies = [
|
||||
"windows-targets 0.48.5",
|
||||
"windows-targets",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -958,22 +938,7 @@ version = "0.52.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
|
||||
dependencies = [
|
||||
"windows-targets 0.52.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-targets"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
|
||||
dependencies = [
|
||||
"windows_aarch64_gnullvm 0.48.5",
|
||||
"windows_aarch64_msvc 0.48.5",
|
||||
"windows_i686_gnu 0.48.5",
|
||||
"windows_i686_msvc 0.48.5",
|
||||
"windows_x86_64_gnu 0.48.5",
|
||||
"windows_x86_64_gnullvm 0.48.5",
|
||||
"windows_x86_64_msvc 0.48.5",
|
||||
"windows-targets",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -982,46 +947,28 @@ version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
|
||||
dependencies = [
|
||||
"windows_aarch64_gnullvm 0.52.6",
|
||||
"windows_aarch64_msvc 0.52.6",
|
||||
"windows_i686_gnu 0.52.6",
|
||||
"windows_aarch64_gnullvm",
|
||||
"windows_aarch64_msvc",
|
||||
"windows_i686_gnu",
|
||||
"windows_i686_gnullvm",
|
||||
"windows_i686_msvc 0.52.6",
|
||||
"windows_x86_64_gnu 0.52.6",
|
||||
"windows_x86_64_gnullvm 0.52.6",
|
||||
"windows_x86_64_msvc 0.52.6",
|
||||
"windows_i686_msvc",
|
||||
"windows_x86_64_gnu",
|
||||
"windows_x86_64_gnullvm",
|
||||
"windows_x86_64_msvc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_gnullvm"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_gnullvm"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_msvc"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_msvc"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
version = "0.52.6"
|
||||
@@ -1034,48 +981,24 @@ version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_msvc"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_msvc"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnu"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnu"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnullvm"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnullvm"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
version = "0.52.6"
|
||||
|
||||
@@ -18,7 +18,6 @@ anyhow = "1.0.68"
|
||||
bytesize = "1.1.0"
|
||||
chrono = "0.4.23"
|
||||
clap = { version = "4.0.32", features = ["derive"] }
|
||||
colored = "2.0.0"
|
||||
dashmap = { version = "5.4.0", features = ["rayon"] }
|
||||
globwalk = "0.8.1"
|
||||
gxhash = "3.4.1"
|
||||
|
||||
@@ -3,14 +3,10 @@ use crate::fileinfo::FileInfo;
|
||||
use crate::params::Params;
|
||||
use anyhow::Result;
|
||||
use chrono::{DateTime, Utc};
|
||||
use colored::Colorize;
|
||||
use dashmap::DashMap;
|
||||
use indicatif::{
|
||||
ParallelProgressIterator, ProgressBar, ProgressFinish, ProgressIterator, ProgressStyle,
|
||||
};
|
||||
use indicatif::{ProgressBar, ProgressFinish, ProgressIterator, ProgressStyle};
|
||||
use pathdiff::diff_paths;
|
||||
use prettytable::{format, row, Table};
|
||||
use rayon::prelude::*;
|
||||
use std::borrow::Cow;
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
@@ -42,41 +38,7 @@ impl Formatter {
|
||||
Ok(modified_time.format("%Y-%m-%d %H:%M:%S").to_string())
|
||||
}
|
||||
|
||||
pub fn generate_table(raw: Vec<FileInfo>, app_args: &Params) -> Result<Table> {
|
||||
let basepath_length = app_args.get_directory()?.to_str().unwrap_or_default().len();
|
||||
let max_filepath_length = raw
|
||||
.iter()
|
||||
.map(|file| file.path.to_str().unwrap_or_default().len())
|
||||
.max()
|
||||
.unwrap_or_default();
|
||||
|
||||
let min_path_length = if max_filepath_length > basepath_length {
|
||||
max_filepath_length - basepath_length
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
let progress_style = ProgressStyle::with_template(
|
||||
"[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}",
|
||||
)?;
|
||||
let progress_bar = ProgressBar::new(raw.len() as u64);
|
||||
progress_bar.set_style(progress_style);
|
||||
progress_bar.enable_steady_tick(Duration::from_millis(50));
|
||||
progress_bar.set_message("reconciling data");
|
||||
|
||||
let duplicates_table: DashMap<String, Vec<FileInfo>> = DashMap::new();
|
||||
raw.into_par_iter()
|
||||
.progress_with(progress_bar)
|
||||
.with_finish(ProgressFinish::WithMessage(Cow::from("data reconciled")))
|
||||
.map(|file| file.hash())
|
||||
.filter_map(Result::ok)
|
||||
.for_each(|file| {
|
||||
duplicates_table
|
||||
.entry(file.hash.clone().unwrap_or_default())
|
||||
.and_modify(|fileset| fileset.push(file.clone()))
|
||||
.or_insert_with(|| vec![file]);
|
||||
});
|
||||
|
||||
pub fn generate_table(raw: DashMap<String, Vec<FileInfo>>, max_path_len: usize, app_args: &Params) -> Result<Table> {
|
||||
let mut output_table = Table::new();
|
||||
output_table.set_titles(row!["hash", "duplicates"]);
|
||||
|
||||
@@ -84,13 +46,12 @@ impl Formatter {
|
||||
"[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}",
|
||||
)?;
|
||||
|
||||
let progress_bar = ProgressBar::new(duplicates_table.len() as u64);
|
||||
let progress_bar = ProgressBar::new(raw.len() as u64);
|
||||
progress_bar.set_style(progress_style);
|
||||
progress_bar.enable_steady_tick(Duration::from_millis(50));
|
||||
progress_bar.set_message("generating output");
|
||||
|
||||
duplicates_table
|
||||
.into_iter()
|
||||
raw.into_iter()
|
||||
.progress_with(progress_bar)
|
||||
.with_finish(ProgressFinish::WithMessage(Cow::from("output generated")))
|
||||
.for_each(|(hash, group)| {
|
||||
@@ -98,36 +59,26 @@ impl Formatter {
|
||||
inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
|
||||
group.iter().for_each(|file| {
|
||||
inner_table.add_row(row![
|
||||
Self::human_path(file, app_args, min_path_length)
|
||||
.unwrap_or_default()
|
||||
.blue(),
|
||||
Self::human_filesize(file).unwrap_or_default().red(),
|
||||
Self::human_mtime(file).unwrap_or_default().yellow()
|
||||
Self::human_path(file, app_args, max_path_len).unwrap_or_default(),
|
||||
Self::human_filesize(file).unwrap_or_default(),
|
||||
Self::human_mtime(file).unwrap_or_default()
|
||||
]);
|
||||
});
|
||||
|
||||
output_table.add_row(row![hash.green(), inner_table]);
|
||||
output_table.add_row(row![hash, inner_table]);
|
||||
});
|
||||
|
||||
Ok(output_table)
|
||||
}
|
||||
|
||||
pub fn print(raw: Vec<FileInfo>, app_args: &Params) -> Result<()> {
|
||||
pub fn print(raw: DashMap<String, Vec<FileInfo>>, max_path_len: usize, app_args: &Params) -> Result<()> {
|
||||
if raw.is_empty() {
|
||||
println!(
|
||||
"\n\n{}\n",
|
||||
"No duplicates found matching your search criteria.".green()
|
||||
);
|
||||
println!("\n\nNo duplicates found matching your search criteria.\n");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if app_args.json {
|
||||
let output_json = serde_json::to_string_pretty(&raw)?;
|
||||
println!("{}", output_json);
|
||||
} else {
|
||||
let output_table = Self::generate_table(raw, app_args)?;
|
||||
output_table.printstd();
|
||||
}
|
||||
let output_table = Self::generate_table(raw, max_path_len, app_args)?;
|
||||
output_table.printstd();
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1,16 +1,9 @@
|
||||
use crate::formatter::Formatter;
|
||||
use crate::{fileinfo::FileInfo, params::Params};
|
||||
use anyhow::Result;
|
||||
use colored::Colorize;
|
||||
use dashmap::DashMap;
|
||||
use indicatif::{ParallelProgressIterator, ProgressBar, ProgressFinish, ProgressStyle};
|
||||
use prettytable::{format, row, Table};
|
||||
use rayon::prelude::*;
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
io::{self, Write},
|
||||
time::Duration,
|
||||
};
|
||||
use std::io::{self, Write};
|
||||
|
||||
pub fn scan_group_confirmation() -> Result<bool> {
|
||||
print!("\nconfirm? [y/N]: ");
|
||||
@@ -36,43 +29,8 @@ pub fn scan_group_instruction() -> Result<String> {
|
||||
Ok(user_input)
|
||||
}
|
||||
|
||||
pub fn init(result: Vec<FileInfo>, app_args: &Params) -> Result<()> {
|
||||
let basepath_length = app_args.get_directory()?.to_str().unwrap_or_default().len();
|
||||
let max_filepath_length = result
|
||||
.iter()
|
||||
.map(|file| file.path.to_str().unwrap_or_default().len())
|
||||
.max()
|
||||
.unwrap_or_default();
|
||||
|
||||
let min_path_length = if max_filepath_length > basepath_length {
|
||||
max_filepath_length - basepath_length
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
let progress_style = ProgressStyle::with_template(
|
||||
"[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}",
|
||||
)?;
|
||||
let progress_bar = ProgressBar::new(result.len() as u64);
|
||||
progress_bar.set_style(progress_style);
|
||||
progress_bar.enable_steady_tick(Duration::from_millis(50));
|
||||
progress_bar.set_message("reconciling data");
|
||||
|
||||
let duplicates: DashMap<String, Vec<FileInfo>> = DashMap::new();
|
||||
pub fn init(result: DashMap<String, Vec<FileInfo>>, app_args: &Params) -> Result<()> {
|
||||
result
|
||||
.into_par_iter()
|
||||
.progress_with(progress_bar)
|
||||
.with_finish(ProgressFinish::WithMessage(Cow::from("data reconciled")))
|
||||
.map(|file| file.hash())
|
||||
.filter_map(Result::ok)
|
||||
.for_each(|file| {
|
||||
duplicates
|
||||
.entry(file.hash.clone().unwrap_or_default())
|
||||
.and_modify(|fileset| fileset.push(file.clone()))
|
||||
.or_insert_with(|| vec![file]);
|
||||
});
|
||||
|
||||
duplicates
|
||||
.clone()
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
@@ -80,18 +38,22 @@ pub fn init(result: Vec<FileInfo>, app_args: &Params) -> Result<()> {
|
||||
let mut itable = Table::new();
|
||||
itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
|
||||
itable.set_titles(row!["index", "filename", "size", "updated_at"]);
|
||||
let max_path_size = group
|
||||
.iter()
|
||||
.map(|f| f.path.clone().into_os_string().len())
|
||||
.max()
|
||||
.unwrap_or_default();
|
||||
|
||||
group.iter().enumerate().for_each(|(index, file)| {
|
||||
itable.add_row(row![
|
||||
index,
|
||||
Formatter::human_path(file, app_args, min_path_length)
|
||||
.unwrap_or_default()
|
||||
.blue(),
|
||||
Formatter::human_filesize(file).unwrap_or_default().red(),
|
||||
Formatter::human_mtime(file).unwrap_or_default().yellow()
|
||||
Formatter::human_path(file, app_args, max_path_size).unwrap_or_default(),
|
||||
Formatter::human_filesize(file).unwrap_or_default(),
|
||||
Formatter::human_mtime(file).unwrap_or_default()
|
||||
]);
|
||||
});
|
||||
|
||||
process_group_action(&group, gindex, duplicates.len(), itable);
|
||||
process_group_action(&group, gindex, result.len(), itable);
|
||||
});
|
||||
|
||||
Ok(())
|
||||
@@ -118,7 +80,7 @@ pub fn process_group_action(
|
||||
.into_iter()
|
||||
.any(|index| index > (duplicates.len() - 1))
|
||||
{
|
||||
println!("{}", "Err: File Index Out of Bounds!".red());
|
||||
println!("Err: File Index Out of Bounds!");
|
||||
return process_group_action(duplicates, dup_index, dup_size, table);
|
||||
}
|
||||
|
||||
@@ -132,23 +94,23 @@ pub fn process_group_action(
|
||||
.into_iter()
|
||||
.map(|index| duplicates[index].clone());
|
||||
|
||||
println!("\n{}", "The following files will be deleted:".red());
|
||||
println!("\nThe following files will be deleted:");
|
||||
files_to_delete
|
||||
.clone()
|
||||
.enumerate()
|
||||
.for_each(|(index, file)| {
|
||||
println!("{}: {}", index.to_string().blue(), file.path.display());
|
||||
println!("{}: {}", index, file.path.display());
|
||||
});
|
||||
|
||||
match scan_group_confirmation().unwrap() {
|
||||
true => {
|
||||
files_to_delete.into_iter().for_each(|file| {
|
||||
match std::fs::remove_file(file.path.clone()) {
|
||||
Ok(_) => println!("{}: {}", "DELETED".green(), file.path.display()),
|
||||
Err(_) => println!("{}: {}", "FAILED".red(), file.path.display()),
|
||||
Ok(_) => println!("DELETED: {}", file.path.display()),
|
||||
Err(_) => println!("FAILED: {}", file.path.display()),
|
||||
}
|
||||
});
|
||||
}
|
||||
false => println!("{}", "\nCancelled Delete Operation.".red()),
|
||||
false => println!("\nCancelled Delete Operation."),
|
||||
}
|
||||
}
|
||||
|
||||
16
src/main.rs
16
src/main.rs
@@ -15,16 +15,16 @@ use scanner::Scanner;
|
||||
fn main() -> Result<()> {
|
||||
let app_args = Params::parse();
|
||||
let scan_results = Scanner::build(&app_args)?.scan()?;
|
||||
let processor = Processor::new(scan_results);
|
||||
let results = processor.sizewise()?.hashwise()?;
|
||||
let mut processor = Processor::new(scan_results);
|
||||
|
||||
processor.sizewise()?;
|
||||
processor.hashwise()?;
|
||||
|
||||
let results = processor.hashwise_results;
|
||||
|
||||
match app_args.interactive {
|
||||
false => {
|
||||
Formatter::print(results.files, &app_args)?;
|
||||
}
|
||||
true => {
|
||||
interactive::init(results.files, &app_args)?;
|
||||
}
|
||||
false => Formatter::print(results, processor.max_path_len, &app_args)?,
|
||||
true => interactive::init(results, &app_args)?,
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -1,107 +1,103 @@
|
||||
use anyhow::Result;
|
||||
use dashmap::DashMap;
|
||||
use indicatif::{ParallelProgressIterator, ProgressBar, ProgressStyle, ProgressFinish};
|
||||
use indicatif::{ParallelProgressIterator, ProgressBar, ProgressFinish, ProgressStyle};
|
||||
use rayon::prelude::{IntoParallelIterator, ParallelIterator};
|
||||
use std::{time::Duration, borrow::Cow};
|
||||
use std::{borrow::Cow, time::Duration};
|
||||
|
||||
use crate::fileinfo::FileInfo;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum State {
|
||||
Initial,
|
||||
SizeWise,
|
||||
HashWise,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Processor {
|
||||
pub files: Vec<FileInfo>,
|
||||
pub state: State,
|
||||
pub hashwise_results: DashMap<String, Vec<FileInfo>>,
|
||||
pub sizewise_results: DashMap<u64, Vec<FileInfo>>,
|
||||
pub max_path_len: usize,
|
||||
}
|
||||
|
||||
impl Processor {
|
||||
pub fn new(files: Vec<FileInfo>) -> Self {
|
||||
Self {
|
||||
files,
|
||||
state: State::Initial,
|
||||
hashwise_results: DashMap::new(),
|
||||
sizewise_results: DashMap::new(),
|
||||
max_path_len: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn hashwise(&self) -> Result<Self> {
|
||||
if self.files.is_empty() {
|
||||
return Ok(self.clone());
|
||||
pub fn hashwise(&mut self) -> Result<()> {
|
||||
if self.sizewise_results.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let progress_style = ProgressStyle::with_template("[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}")?;
|
||||
let progress_style = ProgressStyle::with_template(
|
||||
"[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}",
|
||||
)?;
|
||||
let progress_bar = ProgressBar::new(self.files.len() as u64);
|
||||
progress_bar.set_style(progress_style);
|
||||
progress_bar.enable_steady_tick(Duration::from_millis(50));
|
||||
progress_bar.set_message("indexing file hashes");
|
||||
|
||||
let duplicates_table: DashMap<String, Vec<FileInfo>> = DashMap::new();
|
||||
self.files
|
||||
let filelist = self
|
||||
.sizewise_results
|
||||
.clone()
|
||||
.into_read_only()
|
||||
.values()
|
||||
.filter(|&subfiles| subfiles.len() > 1)
|
||||
.flatten()
|
||||
.cloned()
|
||||
.collect::<Vec<FileInfo>>();
|
||||
|
||||
self.max_path_len = filelist
|
||||
.iter()
|
||||
.map(|x| x.path.clone().into_os_string().len())
|
||||
.max()
|
||||
.unwrap_or_default();
|
||||
|
||||
filelist
|
||||
.into_par_iter()
|
||||
.progress_with(progress_bar)
|
||||
.with_finish(ProgressFinish::WithMessage(Cow::from("indexed files hashes")))
|
||||
.with_finish(ProgressFinish::WithMessage(Cow::from(
|
||||
"indexed files hashes",
|
||||
)))
|
||||
.map(|file| file.hash())
|
||||
.filter_map(Result::ok)
|
||||
.for_each(|file| {
|
||||
duplicates_table
|
||||
.for_each(move |file| {
|
||||
self.hashwise_results
|
||||
.entry(file.hash.clone().unwrap_or_default())
|
||||
.and_modify(|fileset| fileset.push(file.clone()))
|
||||
.or_insert_with(|| vec![file]);
|
||||
});
|
||||
|
||||
let files = duplicates_table
|
||||
.into_read_only()
|
||||
.values()
|
||||
.cloned()
|
||||
.filter(|subfiles| subfiles.len() > 1)
|
||||
.flatten()
|
||||
.collect::<Vec<FileInfo>>();
|
||||
|
||||
Ok(Self {
|
||||
files,
|
||||
state: State::HashWise,
|
||||
})
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn sizewise(&self) -> Result<Self> {
|
||||
pub fn sizewise(&mut self) -> Result<()> {
|
||||
if self.files.is_empty() {
|
||||
return Ok(self.clone());
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let progress_style = ProgressStyle::with_template("[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}")?;
|
||||
let progress_style = ProgressStyle::with_template(
|
||||
"[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}",
|
||||
)?;
|
||||
let progress_bar = ProgressBar::new(self.files.len() as u64);
|
||||
progress_bar.set_style(progress_style);
|
||||
progress_bar.enable_steady_tick(Duration::from_millis(50));
|
||||
progress_bar.set_message("indexing file sizes");
|
||||
|
||||
let duplicates_table: DashMap<u64, Vec<FileInfo>> = DashMap::new();
|
||||
self.files
|
||||
.clone()
|
||||
.into_par_iter()
|
||||
.progress_with(progress_bar)
|
||||
.with_finish(ProgressFinish::WithMessage(Cow::from("indexed files sizes")))
|
||||
.with_finish(ProgressFinish::WithMessage(Cow::from(
|
||||
"indexed files sizes",
|
||||
)))
|
||||
.for_each(|file| {
|
||||
duplicates_table
|
||||
self.sizewise_results
|
||||
.entry(file.size)
|
||||
.and_modify(|fileset| fileset.push(file.clone()))
|
||||
.or_insert_with(|| vec![file]);
|
||||
});
|
||||
|
||||
let files = duplicates_table
|
||||
.into_read_only()
|
||||
.values()
|
||||
.cloned()
|
||||
.filter(|subfiles| subfiles.len() > 1)
|
||||
.flatten()
|
||||
.collect::<Vec<FileInfo>>();
|
||||
|
||||
Ok(Self {
|
||||
files,
|
||||
state: State::SizeWise,
|
||||
})
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -156,10 +156,7 @@ impl Scanner {
|
||||
.build_walker()?
|
||||
.filter_map(Result::ok)
|
||||
.map(|entity| entity.into_path())
|
||||
.map(|path| {
|
||||
progress_bar.inc(1);
|
||||
path
|
||||
})
|
||||
.inspect(|_path| progress_bar.inc(1))
|
||||
.filter(|path| path.is_file())
|
||||
.map(FileInfo::new)
|
||||
.filter_map(Result::ok)
|
||||
|
||||
Reference in New Issue
Block a user