smolkik_adm/deduplicator
1
0
Fork 0
mirror of https://github.com/sreedevk/deduplicator.git synced 2026-08-26 10:05:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

performance tweaks

Browse Source
This commit is contained in:
sreedev
2023-01-18 00:24:50 -05:00
parent 51f5f8e61a
commit ea748c60d8
5 changed files with 107 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
Cargo.lock generated
View File

@@ -314,7 +314,6 @@ dependencies = [
"memmap2",
"prettytable-rs",
"rayon",
"sqlite",
"thiserror",
"tokio",
"unicode-segmentation",
@@ -693,12 +692,6 @@ version = "0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
[[package]]
name = "pkg-config"
version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
[[package]]
name = "portable-atomic"
version = "0.3.19"
@@ -878,36 +871,6 @@ dependencies = [
"winapi",
]
[[package]]
name = "sqlite"
version = "0.30.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12e072cb5fb89b3fe5e9c9584676348feb503f9fb3ae829d9868171bc5372d48"
dependencies = [
"libc",
"sqlite3-sys",
]
[[package]]
name = "sqlite3-src"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d1815a7a02c996eb8e5c64f61fcb6fd9b12e593ce265c512c5853b2513635691"
dependencies = [
"cc",
"pkg-config",
]
[[package]]
name = "sqlite3-sys"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d47c99824fc55360ba00caf28de0b8a0458369b832e016a64c13af0ad9fbb9ee"
dependencies = [
"libc",
"sqlite3-src",
]
[[package]]
name = "strsim"
version = "0.10.0"

1
Cargo.toml
View File

@@ -22,7 +22,6 @@ itertools = "0.10.5"
memmap2 = "0.5.8"
prettytable-rs = "0.10.0"
rayon = "1.6.1"
sqlite = "0.30.3"
thiserror = "1.0.38"
tokio = { version = "1.23.0", features = ["full"] }
unicode-segmentation = "1.10.0"

5
src/file_manager.rs
View File

@@ -4,14 +4,15 @@ use colored::Colorize;
#[derive(Debug, Clone)]
pub struct File {
pub path: String,
pub hash: String,
pub size: Option<u64>,
pub hash: Option<String>,
}
pub fn delete_files(files: Vec<File>) -> Result<()> {
files.into_iter().for_each(|file| {
match std::fs::remove_file(file.path.clone()) {
Ok(_) => println!("{}: {}", "DELETED".green(), file.path),
Err(e) => println!("{}: {}", "FAILED".red(), file.path)
Err(_) => println!("{}: {}", "FAILED".red(), file.path)
}
});

48
src/output.rs
View File

@@ -1,14 +1,15 @@
use std::io::Write;
use std::{collections::HashMap, fs, io};
use crate::file_manager::{self, File};
use crate::params::Params;
use anyhow::Result;
use chrono::offset::Utc;
use chrono::DateTime;
use colored::Colorize;
use dashmap::DashMap;
use humansize::{format_size, DECIMAL};
use itertools::Itertools;
use crate::file_manager::{self, File};
use crate::params::Params;
use prettytable::{format, row, Table};
use std::io::Write;
use std::{fs, io};
use unicode_segmentation::UnicodeSegmentation;
fn format_path(path: &str, opts: &Params) -> Result<String> {
@@ -42,19 +43,7 @@ fn modified_time(path: &String) -> Result<String> {
Ok(modified_time.format("%Y-%m-%d %H:%M:%S").to_string())
}
fn group_duplicates(duplicates: Vec<File>) -> HashMap<String, Vec<File>> {
let mut duplicate_mapper: HashMap<String, Vec<File>> = HashMap::new();
duplicates.into_iter().for_each(|file| {
duplicate_mapper
.entry(file.hash.clone())
.and_modify(|value| value.push(file.clone()))
.or_insert_with(|| vec![file]);
});
duplicate_mapper
}
fn print_meta_info(duplicates: &Vec<File>, opts: &Params) {
fn print_meta_info() {
println!("Deduplicator v{}", std::env!("CARGO_PKG_VERSION"));
}
@@ -122,20 +111,19 @@ fn process_group_action(duplicates: &Vec<File>, dup_index: usize, dup_size: usiz
match scan_group_confirmation().unwrap() {
true => {
file_manager::delete_files(files_to_delete.collect_vec());
file_manager::delete_files(files_to_delete.collect_vec()).ok();
}
false => println!("{}", "\nCancelled Delete Operation.".red()),
}
}
pub fn interactive(duplicates: Vec<File>, opts: &Params) {
print_meta_info(&duplicates, opts);
let grouped_duplicates = group_duplicates(duplicates);
grouped_duplicates
.iter()
pub fn interactive(duplicates: DashMap<String, Vec<File>>, opts: &Params) {
print_meta_info();
duplicates
.clone()
.into_iter()
.enumerate()
.for_each(|(gindex, (hash, group))| {
.for_each(|(gindex, (_, group))| {
let mut itable = Table::new();
itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
itable.set_titles(row!["index", "filename", "size", "updated_at"]);
@@ -148,18 +136,16 @@ pub fn interactive(duplicates: Vec<File>, opts: &Params) {
]);
});
process_group_action(group, gindex, grouped_duplicates.len(), itable);
process_group_action(&group, gindex, duplicates.len(), itable);
});
}
pub fn print(duplicates: Vec<File>, opts: &Params) {
print_meta_info(&duplicates, opts);
pub fn print(duplicates: DashMap<String, Vec<File>>, opts: &Params) {
print_meta_info();
let mut output_table = Table::new();
let grouped_duplicates: HashMap<String, Vec<File>> = group_duplicates(duplicates);
output_table.set_titles(row!["hash", "duplicates"]);
grouped_duplicates.iter().for_each(|(hash, group)| {
duplicates.into_iter().for_each(|(hash, group)| {
let mut inner_table = Table::new();
inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
group.iter().for_each(|file| {

121
src/scanner.rs
View File

@@ -1,39 +1,57 @@
use anyhow::Result;
use dashmap::DashMap;
use fxhash::hash64 as hasher;
use glob::glob;
use indicatif::{ParallelProgressIterator, ProgressStyle};
use memmap2::Mmap;
use rayon::prelude::*;
use std::hash::Hasher;
use std::{fs, path::PathBuf};
use memmap2::Mmap;
use dashmap::DashMap;
use crate::{file_manager::File, params::Params};
pub fn duplicates(app_opts: &Params) -> Result<Vec<File>> {
let scan_results = scan(app_opts)?;
let index_store = index_files(scan_results)?;
#[derive(Clone, Copy)]
enum IndexCritera {
Size,
Hash,
}
let duplicate_files = index_store
pub fn duplicates(app_opts: &Params) -> Result<DashMap<String, Vec<File>>> {
let scan_results = scan(app_opts)?;
let size_index_store = index_files(scan_results, IndexCritera::Size)?;
let sizewize_duplicate_files = size_index_store
.into_par_iter()
.filter(|(_, files)| files.len() > 1)
.map(|(_, files)| files )
.map(|(_, files)| files)
.flatten()
.collect::<Vec<File>>();
Ok(duplicate_files)
if sizewize_duplicate_files.len() > 1 {
let size_wise_duplicate_paths = sizewize_duplicate_files
.into_par_iter()
.map(|file| file.path)
.collect::<Vec<String>>();
let hash_index_store = index_files(size_wise_duplicate_paths, IndexCritera::Hash)?;
let duplicate_files = hash_index_store
.into_par_iter()
.filter(|(_, files)| files.len() > 1)
.collect();
Ok(duplicate_files)
} else {
Ok(DashMap::new())
}
}
fn scan(app_opts: &Params) -> Result<Vec<String>> {
let glob_patterns: Vec<PathBuf> = app_opts.get_glob_patterns();
let files: Vec<String> = glob_patterns
.par_iter()
.progress_with_style(
ProgressStyle::with_template(
"{spinner:.green} [scanning files] [{wide_bar:.cyan/blue}] {pos}/{len} files",
)
.unwrap(),
)
.progress_with_style(ProgressStyle::with_template(
"{spinner:.green} [scanning files] [{wide_bar:.cyan/blue}] {pos}/{len} files",
)?)
.filter_map(|glob_pattern| glob(glob_pattern.as_os_str().to_str()?).ok())
.flat_map(|file_vec| {
file_vec
@@ -50,25 +68,60 @@ fn scan(app_opts: &Params) -> Result<Vec<String>> {
Ok(files)
}
fn index_files(files: Vec<String>) -> Result<DashMap<String, Vec<File>>> {
fn process_file_size_index(fpath: String) -> Result<File> {
Ok(File {
path: fpath.clone(),
size: Some(fs::metadata(fpath)?.len()),
hash: None,
})
}
fn process_file_hash_index(fpath: String) -> Result<File> {
Ok(File {
path: fpath.clone(),
size: None,
hash: Some(hash_file(&fpath).unwrap_or_default()),
})
}
fn process_file_index(
fpath: String,
store: &DashMap<String, Vec<File>>,
index_criteria: IndexCritera,
) {
match index_criteria {
IndexCritera::Size => {
let processed_file = process_file_size_index(fpath).unwrap();
store
.entry(processed_file.size.unwrap_or_default().to_string())
.and_modify(|fileset| fileset.push(processed_file.clone()))
.or_insert_with(|| vec![processed_file]);
}
IndexCritera::Hash => {
let processed_file = process_file_hash_index(fpath).unwrap();
let indexhash = processed_file.clone().hash.unwrap_or_default();
store
.entry(indexhash)
.and_modify(|fileset| fileset.push(processed_file.clone()))
.or_insert_with(|| vec![processed_file]);
}
}
}
fn index_files(
files: Vec<String>,
index_criteria: IndexCritera,
) -> Result<DashMap<String, Vec<File>>> {
let store: DashMap<String, Vec<File>> = DashMap::new();
files
.into_par_iter()
.progress_with_style(
ProgressStyle::with_template(
"{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files",
)?,
)
.for_each(|file| {
let hash = hash_file(&file).unwrap_or_default();
let fobj = File { path: file, hash: hash.clone() };
store
.entry(hash)
.and_modify(|fileset| fileset.push(fobj.clone()) )
.or_insert_with(|| vec![fobj]);
});
.progress_with_style(ProgressStyle::with_template(
"{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files",
)?)
.for_each(|file| process_file_index(file, &store, index_criteria));
Ok(store)
Ok(store)
}
pub fn incremental_hashing(filepath: &str) -> Result<String> {
@@ -76,9 +129,8 @@ pub fn incremental_hashing(filepath: &str) -> Result<String> {
let fmap = unsafe { Mmap::map(&file)? };
let mut inchasher = fxhash::FxHasher::default();
fmap
.chunks(1_000)
.for_each(|kilo| { inchasher.write(kilo) });
fmap.chunks(1_000_000)
.for_each(|mega| inchasher.write(mega));
Ok(format!("{}", inchasher.finish()))
}
@@ -91,8 +143,9 @@ pub fn standard_hashing(filepath: &str) -> Result<String> {
pub fn hash_file(filepath: &str) -> Result<String> {
let filemeta = fs::metadata(filepath)?;
match filemeta.len() < 1_000_000 {
// NOTE: USE INCREMENTAL HASHING ONLY FOR FILES > 100MB
match filemeta.len() < 100_000_000 {
true => standard_hashing(filepath),
false => incremental_hashing(filepath)
false => incremental_hashing(filepath),
}
}