smolkik_adm/deduplicator
1
0
Fork 0
mirror of https://github.com/sreedevk/deduplicator.git synced 2026-08-31 12:35:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

performance improvements

Browse Source
This commit is contained in:
sreedevk
2025-07-12 00:24:25 +00:00
parent b4bf5d4fb3
commit f9e86f8522
8 changed files with 103 additions and 274 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
src/formatter.rs
View File

@@ -3,14 +3,10 @@ use crate::fileinfo::FileInfo;
use crate::params::Params;
use anyhow::Result;
use chrono::{DateTime, Utc};
use colored::Colorize;
use dashmap::DashMap;
use indicatif::{
ParallelProgressIterator, ProgressBar, ProgressFinish, ProgressIterator, ProgressStyle,
};
use indicatif::{ProgressBar, ProgressFinish, ProgressIterator, ProgressStyle};
use pathdiff::diff_paths;
use prettytable::{format, row, Table};
use rayon::prelude::*;
use std::borrow::Cow;
use std::path::PathBuf;
use std::time::Duration;
@@ -42,41 +38,7 @@ impl Formatter {
Ok(modified_time.format("%Y-%m-%d %H:%M:%S").to_string())
}
pub fn generate_table(raw: Vec<FileInfo>, app_args: &Params) -> Result<Table> {
let basepath_length = app_args.get_directory()?.to_str().unwrap_or_default().len();
let max_filepath_length = raw
.iter()
.map(|file| file.path.to_str().unwrap_or_default().len())
.max()
.unwrap_or_default();
let min_path_length = if max_filepath_length > basepath_length {
max_filepath_length - basepath_length
} else {
0
};
let progress_style = ProgressStyle::with_template(
"[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}",
)?;
let progress_bar = ProgressBar::new(raw.len() as u64);
progress_bar.set_style(progress_style);
progress_bar.enable_steady_tick(Duration::from_millis(50));
progress_bar.set_message("reconciling data");
let duplicates_table: DashMap<String, Vec<FileInfo>> = DashMap::new();
raw.into_par_iter()
.progress_with(progress_bar)
.with_finish(ProgressFinish::WithMessage(Cow::from("data reconciled")))
.map(|file| file.hash())
.filter_map(Result::ok)
.for_each(|file| {
duplicates_table
.entry(file.hash.clone().unwrap_or_default())
.and_modify(|fileset| fileset.push(file.clone()))
.or_insert_with(|| vec![file]);
});
pub fn generate_table(raw: DashMap<String, Vec<FileInfo>>, max_path_len: usize, app_args: &Params) -> Result<Table> {
let mut output_table = Table::new();
output_table.set_titles(row!["hash", "duplicates"]);
@@ -84,13 +46,12 @@ impl Formatter {
"[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}",
)?;
let progress_bar = ProgressBar::new(duplicates_table.len() as u64);
let progress_bar = ProgressBar::new(raw.len() as u64);
progress_bar.set_style(progress_style);
progress_bar.enable_steady_tick(Duration::from_millis(50));
progress_bar.set_message("generating output");
duplicates_table
.into_iter()
raw.into_iter()
.progress_with(progress_bar)
.with_finish(ProgressFinish::WithMessage(Cow::from("output generated")))
.for_each(|(hash, group)| {
@@ -98,36 +59,26 @@ impl Formatter {
inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
group.iter().for_each(|file| {
inner_table.add_row(row![
Self::human_path(file, app_args, min_path_length)
.unwrap_or_default()
.blue(),
Self::human_filesize(file).unwrap_or_default().red(),
Self::human_mtime(file).unwrap_or_default().yellow()
Self::human_path(file, app_args, max_path_len).unwrap_or_default(),
Self::human_filesize(file).unwrap_or_default(),
Self::human_mtime(file).unwrap_or_default()
]);
});
output_table.add_row(row![hash.green(), inner_table]);
output_table.add_row(row![hash, inner_table]);
});
Ok(output_table)
}
pub fn print(raw: Vec<FileInfo>, app_args: &Params) -> Result<()> {
pub fn print(raw: DashMap<String, Vec<FileInfo>>, max_path_len: usize, app_args: &Params) -> Result<()> {
if raw.is_empty() {
println!(
"\n\n{}\n",
"No duplicates found matching your search criteria.".green()
);
println!("\n\nNo duplicates found matching your search criteria.\n");
return Ok(());
}
if app_args.json {
let output_json = serde_json::to_string_pretty(&raw)?;
println!("{}", output_json);
} else {
let output_table = Self::generate_table(raw, app_args)?;
output_table.printstd();
}
let output_table = Self::generate_table(raw, max_path_len, app_args)?;
output_table.printstd();
Ok(())
}

74
src/interactive.rs
View File

@@ -1,16 +1,9 @@
use crate::formatter::Formatter;
use crate::{fileinfo::FileInfo, params::Params};
use anyhow::Result;
use colored::Colorize;
use dashmap::DashMap;
use indicatif::{ParallelProgressIterator, ProgressBar, ProgressFinish, ProgressStyle};
use prettytable::{format, row, Table};
use rayon::prelude::*;
use std::{
borrow::Cow,
io::{self, Write},
time::Duration,
};
use std::io::{self, Write};
pub fn scan_group_confirmation() -> Result<bool> {
print!("\nconfirm? [y/N]: ");
@@ -36,43 +29,8 @@ pub fn scan_group_instruction() -> Result<String> {
Ok(user_input)
}
pub fn init(result: Vec<FileInfo>, app_args: &Params) -> Result<()> {
let basepath_length = app_args.get_directory()?.to_str().unwrap_or_default().len();
let max_filepath_length = result
.iter()
.map(|file| file.path.to_str().unwrap_or_default().len())
.max()
.unwrap_or_default();
let min_path_length = if max_filepath_length > basepath_length {
max_filepath_length - basepath_length
} else {
0
};
let progress_style = ProgressStyle::with_template(
"[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}",
)?;
let progress_bar = ProgressBar::new(result.len() as u64);
progress_bar.set_style(progress_style);
progress_bar.enable_steady_tick(Duration::from_millis(50));
progress_bar.set_message("reconciling data");
let duplicates: DashMap<String, Vec<FileInfo>> = DashMap::new();
pub fn init(result: DashMap<String, Vec<FileInfo>>, app_args: &Params) -> Result<()> {
result
.into_par_iter()
.progress_with(progress_bar)
.with_finish(ProgressFinish::WithMessage(Cow::from("data reconciled")))
.map(|file| file.hash())
.filter_map(Result::ok)
.for_each(|file| {
duplicates
.entry(file.hash.clone().unwrap_or_default())
.and_modify(|fileset| fileset.push(file.clone()))
.or_insert_with(|| vec![file]);
});
duplicates
.clone()
.into_iter()
.enumerate()
@@ -80,18 +38,22 @@ pub fn init(result: Vec<FileInfo>, app_args: &Params) -> Result<()> {
let mut itable = Table::new();
itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
itable.set_titles(row!["index", "filename", "size", "updated_at"]);
let max_path_size = group
.iter()
.map(|f| f.path.clone().into_os_string().len())
.max()
.unwrap_or_default();
group.iter().enumerate().for_each(|(index, file)| {
itable.add_row(row![
index,
Formatter::human_path(file, app_args, min_path_length)
.unwrap_or_default()
.blue(),
Formatter::human_filesize(file).unwrap_or_default().red(),
Formatter::human_mtime(file).unwrap_or_default().yellow()
Formatter::human_path(file, app_args, max_path_size).unwrap_or_default(),
Formatter::human_filesize(file).unwrap_or_default(),
Formatter::human_mtime(file).unwrap_or_default()
]);
});
process_group_action(&group, gindex, duplicates.len(), itable);
process_group_action(&group, gindex, result.len(), itable);
});
Ok(())
@@ -118,7 +80,7 @@ pub fn process_group_action(
.into_iter()
.any(|index| index > (duplicates.len() - 1))
{
println!("{}", "Err: File Index Out of Bounds!".red());
println!("Err: File Index Out of Bounds!");
return process_group_action(duplicates, dup_index, dup_size, table);
}
@@ -132,23 +94,23 @@ pub fn process_group_action(
.into_iter()
.map(|index| duplicates[index].clone());
println!("\n{}", "The following files will be deleted:".red());
println!("\nThe following files will be deleted:");
files_to_delete
.clone()
.enumerate()
.for_each(|(index, file)| {
println!("{}: {}", index.to_string().blue(), file.path.display());
println!("{}: {}", index, file.path.display());
});
match scan_group_confirmation().unwrap() {
true => {
files_to_delete.into_iter().for_each(|file| {
match std::fs::remove_file(file.path.clone()) {
Ok(_) => println!("{}: {}", "DELETED".green(), file.path.display()),
Err(_) => println!("{}: {}", "FAILED".red(), file.path.display()),
Ok(_) => println!("DELETED: {}", file.path.display()),
Err(_) => println!("FAILED: {}", file.path.display()),
}
});
}
false => println!("{}", "\nCancelled Delete Operation.".red()),
false => println!("\nCancelled Delete Operation."),
}
}

16
src/main.rs
View File

@@ -15,16 +15,16 @@ use scanner::Scanner;
fn main() -> Result<()> {
let app_args = Params::parse();
let scan_results = Scanner::build(&app_args)?.scan()?;
let processor = Processor::new(scan_results);
let results = processor.sizewise()?.hashwise()?;
let mut processor = Processor::new(scan_results);
processor.sizewise()?;
processor.hashwise()?;
let results = processor.hashwise_results;
match app_args.interactive {
false => {
Formatter::print(results.files, &app_args)?;
}
true => {
interactive::init(results.files, &app_args)?;
}
false => Formatter::print(results, processor.max_path_len, &app_args)?,
true => interactive::init(results, &app_args)?,
}
Ok(())

96
src/processor.rs
View File

@@ -1,107 +1,103 @@
use anyhow::Result;
use dashmap::DashMap;
use indicatif::{ParallelProgressIterator, ProgressBar, ProgressStyle, ProgressFinish};
use indicatif::{ParallelProgressIterator, ProgressBar, ProgressFinish, ProgressStyle};
use rayon::prelude::{IntoParallelIterator, ParallelIterator};
use std::{time::Duration, borrow::Cow};
use std::{borrow::Cow, time::Duration};
use crate::fileinfo::FileInfo;
#[derive(Debug, Clone)]
pub enum State {
Initial,
SizeWise,
HashWise,
}
#[derive(Debug, Clone)]
pub struct Processor {
pub files: Vec<FileInfo>,
pub state: State,
pub hashwise_results: DashMap<String, Vec<FileInfo>>,
pub sizewise_results: DashMap<u64, Vec<FileInfo>>,
pub max_path_len: usize,
}
impl Processor {
pub fn new(files: Vec<FileInfo>) -> Self {
Self {
files,
state: State::Initial,
hashwise_results: DashMap::new(),
sizewise_results: DashMap::new(),
max_path_len: 0,
}
}
pub fn hashwise(&self) -> Result<Self> {
if self.files.is_empty() {
return Ok(self.clone());
pub fn hashwise(&mut self) -> Result<()> {
if self.sizewise_results.is_empty() {
return Ok(());
}
let progress_style = ProgressStyle::with_template("[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}")?;
let progress_style = ProgressStyle::with_template(
"[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}",
)?;
let progress_bar = ProgressBar::new(self.files.len() as u64);
progress_bar.set_style(progress_style);
progress_bar.enable_steady_tick(Duration::from_millis(50));
progress_bar.set_message("indexing file hashes");
let duplicates_table: DashMap<String, Vec<FileInfo>> = DashMap::new();
self.files
let filelist = self
.sizewise_results
.clone()
.into_read_only()
.values()
.filter(|&subfiles| subfiles.len() > 1)
.flatten()
.cloned()
.collect::<Vec<FileInfo>>();
self.max_path_len = filelist
.iter()
.map(|x| x.path.clone().into_os_string().len())
.max()
.unwrap_or_default();
filelist
.into_par_iter()
.progress_with(progress_bar)
.with_finish(ProgressFinish::WithMessage(Cow::from("indexed files hashes")))
.with_finish(ProgressFinish::WithMessage(Cow::from(
"indexed files hashes",
)))
.map(|file| file.hash())
.filter_map(Result::ok)
.for_each(|file| {
duplicates_table
.for_each(move |file| {
self.hashwise_results
.entry(file.hash.clone().unwrap_or_default())
.and_modify(|fileset| fileset.push(file.clone()))
.or_insert_with(|| vec![file]);
});
let files = duplicates_table
.into_read_only()
.values()
.cloned()
.filter(|subfiles| subfiles.len() > 1)
.flatten()
.collect::<Vec<FileInfo>>();
Ok(Self {
files,
state: State::HashWise,
})
Ok(())
}
pub fn sizewise(&self) -> Result<Self> {
pub fn sizewise(&mut self) -> Result<()> {
if self.files.is_empty() {
return Ok(self.clone());
return Ok(());
}
let progress_style = ProgressStyle::with_template("[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}")?;
let progress_style = ProgressStyle::with_template(
"[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}",
)?;
let progress_bar = ProgressBar::new(self.files.len() as u64);
progress_bar.set_style(progress_style);
progress_bar.enable_steady_tick(Duration::from_millis(50));
progress_bar.set_message("indexing file sizes");
let duplicates_table: DashMap<u64, Vec<FileInfo>> = DashMap::new();
self.files
.clone()
.into_par_iter()
.progress_with(progress_bar)
.with_finish(ProgressFinish::WithMessage(Cow::from("indexed files sizes")))
.with_finish(ProgressFinish::WithMessage(Cow::from(
"indexed files sizes",
)))
.for_each(|file| {
duplicates_table
self.sizewise_results
.entry(file.size)
.and_modify(|fileset| fileset.push(file.clone()))
.or_insert_with(|| vec![file]);
});
let files = duplicates_table
.into_read_only()
.values()
.cloned()
.filter(|subfiles| subfiles.len() > 1)
.flatten()
.collect::<Vec<FileInfo>>();
Ok(Self {
files,
state: State::SizeWise,
})
Ok(())
}
}

5
src/scanner.rs
View File

@@ -156,10 +156,7 @@ impl Scanner {
.build_walker()?
.filter_map(Result::ok)
.map(|entity| entity.into_path())
.map(|path| {
progress_bar.inc(1);
path
})
.inspect(|_path| progress_bar.inc(1))
.filter(|path| path.is_file())
.map(FileInfo::new)
.filter_map(Result::ok)