smolkik_adm/deduplicator
1
0
Fork 0
mirror of https://github.com/sreedevk/deduplicator.git synced 2026-09-04 14:35:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

performance tweaks

Browse Source
This commit is contained in:
sreedev
2023-01-18 00:24:50 -05:00
parent 51f5f8e61a
commit ea748c60d8
5 changed files with 107 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/file_manager.rs
View File

@@ -4,14 +4,15 @@ use colored::Colorize;
#[derive(Debug, Clone)]
pub struct File {
pub path: String,
pub hash: String,
pub size: Option<u64>,
pub hash: Option<String>,
}
pub fn delete_files(files: Vec<File>) -> Result<()> {
files.into_iter().for_each(|file| {
match std::fs::remove_file(file.path.clone()) {
Ok(_) => println!("{}: {}", "DELETED".green(), file.path),
Err(e) => println!("{}: {}", "FAILED".red(), file.path)
Err(_) => println!("{}: {}", "FAILED".red(), file.path)
}
});

48
src/output.rs
View File

@@ -1,14 +1,15 @@
use std::io::Write;
use std::{collections::HashMap, fs, io};
use crate::file_manager::{self, File};
use crate::params::Params;
use anyhow::Result;
use chrono::offset::Utc;
use chrono::DateTime;
use colored::Colorize;
use dashmap::DashMap;
use humansize::{format_size, DECIMAL};
use itertools::Itertools;
use crate::file_manager::{self, File};
use crate::params::Params;
use prettytable::{format, row, Table};
use std::io::Write;
use std::{fs, io};
use unicode_segmentation::UnicodeSegmentation;
fn format_path(path: &str, opts: &Params) -> Result<String> {
@@ -42,19 +43,7 @@ fn modified_time(path: &String) -> Result<String> {
Ok(modified_time.format("%Y-%m-%d %H:%M:%S").to_string())
}
fn group_duplicates(duplicates: Vec<File>) -> HashMap<String, Vec<File>> {
let mut duplicate_mapper: HashMap<String, Vec<File>> = HashMap::new();
duplicates.into_iter().for_each(|file| {
duplicate_mapper
.entry(file.hash.clone())
.and_modify(|value| value.push(file.clone()))
.or_insert_with(|| vec![file]);
});
duplicate_mapper
}
fn print_meta_info(duplicates: &Vec<File>, opts: &Params) {
fn print_meta_info() {
println!("Deduplicator v{}", std::env!("CARGO_PKG_VERSION"));
}
@@ -122,20 +111,19 @@ fn process_group_action(duplicates: &Vec<File>, dup_index: usize, dup_size: usiz
match scan_group_confirmation().unwrap() {
true => {
file_manager::delete_files(files_to_delete.collect_vec());
file_manager::delete_files(files_to_delete.collect_vec()).ok();
}
false => println!("{}", "\nCancelled Delete Operation.".red()),
}
}
pub fn interactive(duplicates: Vec<File>, opts: &Params) {
print_meta_info(&duplicates, opts);
let grouped_duplicates = group_duplicates(duplicates);
grouped_duplicates
.iter()
pub fn interactive(duplicates: DashMap<String, Vec<File>>, opts: &Params) {
print_meta_info();
duplicates
.clone()
.into_iter()
.enumerate()
.for_each(|(gindex, (hash, group))| {
.for_each(|(gindex, (_, group))| {
let mut itable = Table::new();
itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
itable.set_titles(row!["index", "filename", "size", "updated_at"]);
@@ -148,18 +136,16 @@ pub fn interactive(duplicates: Vec<File>, opts: &Params) {
]);
});
process_group_action(group, gindex, grouped_duplicates.len(), itable);
process_group_action(&group, gindex, duplicates.len(), itable);
});
}
pub fn print(duplicates: Vec<File>, opts: &Params) {
print_meta_info(&duplicates, opts);
pub fn print(duplicates: DashMap<String, Vec<File>>, opts: &Params) {
print_meta_info();
let mut output_table = Table::new();
let grouped_duplicates: HashMap<String, Vec<File>> = group_duplicates(duplicates);
output_table.set_titles(row!["hash", "duplicates"]);
grouped_duplicates.iter().for_each(|(hash, group)| {
duplicates.into_iter().for_each(|(hash, group)| {
let mut inner_table = Table::new();
inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
group.iter().for_each(|file| {

121
src/scanner.rs
View File

@@ -1,39 +1,57 @@
use anyhow::Result;
use dashmap::DashMap;
use fxhash::hash64 as hasher;
use glob::glob;
use indicatif::{ParallelProgressIterator, ProgressStyle};
use memmap2::Mmap;
use rayon::prelude::*;
use std::hash::Hasher;
use std::{fs, path::PathBuf};
use memmap2::Mmap;
use dashmap::DashMap;
use crate::{file_manager::File, params::Params};
pub fn duplicates(app_opts: &Params) -> Result<Vec<File>> {
let scan_results = scan(app_opts)?;
let index_store = index_files(scan_results)?;
#[derive(Clone, Copy)]
enum IndexCritera {
Size,
Hash,
}
let duplicate_files = index_store
pub fn duplicates(app_opts: &Params) -> Result<DashMap<String, Vec<File>>> {
let scan_results = scan(app_opts)?;
let size_index_store = index_files(scan_results, IndexCritera::Size)?;
let sizewize_duplicate_files = size_index_store
.into_par_iter()
.filter(|(_, files)| files.len() > 1)
.map(|(_, files)| files )
.map(|(_, files)| files)
.flatten()
.collect::<Vec<File>>();
Ok(duplicate_files)
if sizewize_duplicate_files.len() > 1 {
let size_wise_duplicate_paths = sizewize_duplicate_files
.into_par_iter()
.map(|file| file.path)
.collect::<Vec<String>>();
let hash_index_store = index_files(size_wise_duplicate_paths, IndexCritera::Hash)?;
let duplicate_files = hash_index_store
.into_par_iter()
.filter(|(_, files)| files.len() > 1)
.collect();
Ok(duplicate_files)
} else {
Ok(DashMap::new())
}
}
fn scan(app_opts: &Params) -> Result<Vec<String>> {
let glob_patterns: Vec<PathBuf> = app_opts.get_glob_patterns();
let files: Vec<String> = glob_patterns
.par_iter()
.progress_with_style(
ProgressStyle::with_template(
"{spinner:.green} [scanning files] [{wide_bar:.cyan/blue}] {pos}/{len} files",
)
.unwrap(),
)
.progress_with_style(ProgressStyle::with_template(
"{spinner:.green} [scanning files] [{wide_bar:.cyan/blue}] {pos}/{len} files",
)?)
.filter_map(|glob_pattern| glob(glob_pattern.as_os_str().to_str()?).ok())
.flat_map(|file_vec| {
file_vec
@@ -50,25 +68,60 @@ fn scan(app_opts: &Params) -> Result<Vec<String>> {
Ok(files)
}
fn index_files(files: Vec<String>) -> Result<DashMap<String, Vec<File>>> {
fn process_file_size_index(fpath: String) -> Result<File> {
Ok(File {
path: fpath.clone(),
size: Some(fs::metadata(fpath)?.len()),
hash: None,
})
}
fn process_file_hash_index(fpath: String) -> Result<File> {
Ok(File {
path: fpath.clone(),
size: None,
hash: Some(hash_file(&fpath).unwrap_or_default()),
})
}
fn process_file_index(
fpath: String,
store: &DashMap<String, Vec<File>>,
index_criteria: IndexCritera,
) {
match index_criteria {
IndexCritera::Size => {
let processed_file = process_file_size_index(fpath).unwrap();
store
.entry(processed_file.size.unwrap_or_default().to_string())
.and_modify(|fileset| fileset.push(processed_file.clone()))
.or_insert_with(|| vec![processed_file]);
}
IndexCritera::Hash => {
let processed_file = process_file_hash_index(fpath).unwrap();
let indexhash = processed_file.clone().hash.unwrap_or_default();
store
.entry(indexhash)
.and_modify(|fileset| fileset.push(processed_file.clone()))
.or_insert_with(|| vec![processed_file]);
}
}
}
fn index_files(
files: Vec<String>,
index_criteria: IndexCritera,
) -> Result<DashMap<String, Vec<File>>> {
let store: DashMap<String, Vec<File>> = DashMap::new();
files
.into_par_iter()
.progress_with_style(
ProgressStyle::with_template(
"{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files",
)?,
)
.for_each(|file| {
let hash = hash_file(&file).unwrap_or_default();
let fobj = File { path: file, hash: hash.clone() };
store
.entry(hash)
.and_modify(|fileset| fileset.push(fobj.clone()) )
.or_insert_with(|| vec![fobj]);
});
.progress_with_style(ProgressStyle::with_template(
"{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files",
)?)
.for_each(|file| process_file_index(file, &store, index_criteria));
Ok(store)
Ok(store)
}
pub fn incremental_hashing(filepath: &str) -> Result<String> {
@@ -76,9 +129,8 @@ pub fn incremental_hashing(filepath: &str) -> Result<String> {
let fmap = unsafe { Mmap::map(&file)? };
let mut inchasher = fxhash::FxHasher::default();
fmap
.chunks(1_000)
.for_each(|kilo| { inchasher.write(kilo) });
fmap.chunks(1_000_000)
.for_each(|mega| inchasher.write(mega));
Ok(format!("{}", inchasher.finish()))
}
@@ -91,8 +143,9 @@ pub fn standard_hashing(filepath: &str) -> Result<String> {
pub fn hash_file(filepath: &str) -> Result<String> {
let filemeta = fs::metadata(filepath)?;
match filemeta.len() < 1_000_000 {
// NOTE: USE INCREMENTAL HASHING ONLY FOR FILES > 100MB
match filemeta.len() < 100_000_000 {
true => standard_hashing(filepath),
false => incremental_hashing(filepath)
false => incremental_hashing(filepath),
}
}