From 51f5f8e61af28ac839931f16d8f27dc5576d5f48 Mon Sep 17 00:00:00 2001 From: sreedev Date: Tue, 17 Jan 2023 23:40:57 -0500 Subject: [PATCH 1/3] incremental hashing + remove sqlite + mmap file reads --- Cargo.lock | 98 +++++++++++++++------------------------------- Cargo.toml | 4 +- README.md | 1 - src/app.rs | 4 +- src/database.rs | 82 -------------------------------------- src/main.rs | 1 - src/params.rs | 19 +++++++-- src/scanner.rs | 102 +++++++++++++++++++++++++----------------------- 8 files changed, 102 insertions(+), 209 deletions(-) delete mode 100644 src/database.rs diff --git a/Cargo.lock b/Cargo.lock index 7b7a3f2..1240734 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -70,12 +70,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" -[[package]] -name = "cassowary" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" - [[package]] name = "cc" version = "1.0.78" @@ -223,31 +217,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crossterm" -version = "0.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e64e6c0fbe2c17357405f7c758c1ef960fce08bdfb2c03d88d2a18d7e09c4b67" -dependencies = [ - "bitflags", - "crossterm_winapi", - "libc", - "mio", - "parking_lot", - "signal-hook", - "signal-hook-mio", - "winapi", -] - -[[package]] -name = "crossterm_winapi" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ae1b35a484aa10e07fe0638d02301c5ad24de82d310ccbd2f3693da5f09bf1c" -dependencies = [ - "winapi", -] - [[package]] name = "csv" version = "1.1.6" @@ -314,6 +283,20 @@ dependencies = [ "syn", ] +[[package]] +name = "dashmap" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" +dependencies = [ + "cfg-if", + "hashbrown", + "lock_api", + "once_cell", + "parking_lot_core", + "rayon", +] + [[package]] name = "deduplicator" version = "0.0.9" @@ -322,18 +305,18 @@ dependencies = [ "chrono", "clap", "colored", - "crossterm", + "dashmap", "fxhash", "glob", "humansize", "indicatif", "itertools", + "memmap2", "prettytable-rs", "rayon", "sqlite", "thiserror", "tokio", - "tui", "unicode-segmentation", ] @@ -423,6 +406,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "heck" version = "0.4.0" @@ -598,6 +587,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "memmap2" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b182332558b18d807c4ce1ca8ca983b34c3ee32765e47b3f0f69b90355cc1dc" +dependencies = [ + "libc", +] + [[package]] name = "memoffset" version = "0.7.1" @@ -855,27 +853,6 @@ version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" -[[package]] -name = "signal-hook" -version = "0.3.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a253b5e89e2698464fc26b545c9edceb338e18a89effeeecfea192c3025be29d" -dependencies = [ - "libc", - "signal-hook-registry", -] - -[[package]] -name = "signal-hook-mio" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" -dependencies = [ - "libc", - "mio", - "signal-hook", -] - [[package]] name = "signal-hook-registry" version = "1.4.0" @@ -1030,19 +1007,6 @@ dependencies = [ "syn", ] -[[package]] -name = "tui" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccdd26cbd674007e649a272da4475fb666d3aa0ad0531da7136db6fab0e5bad1" -dependencies = [ - "bitflags", - "cassowary", - "crossterm", - "unicode-segmentation", - "unicode-width", -] - [[package]] name = "unicode-ident" version = "1.0.6" diff --git a/Cargo.toml b/Cargo.toml index a38c053..5fd7a11 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,16 +13,16 @@ anyhow = "1.0.68" chrono = "0.4.23" clap = { version = "4.0.32", features = ["derive"] } colored = "2.0.0" -crossterm = "0.25.0" +dashmap = { version = "5.4.0", features = ["rayon"] } fxhash = "0.2.1" glob = "0.3.0" humansize = "2.1.2" indicatif = { version = "0.17.2", features = ["rayon", "tokio"] } itertools = "0.10.5" +memmap2 = "0.5.8" prettytable-rs = "0.10.0" rayon = "1.6.1" sqlite = "0.30.3" thiserror = "1.0.38" tokio = { version = "1.23.0", features = ["full"] } -tui = "0.19.0" unicode-segmentation = "1.10.0" diff --git a/README.md b/README.md index 726c36f..482ac1f 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,6 @@ Usage: deduplicator [OPTIONS] Options: -t, --types Filetypes to deduplicate (default = all) --dir Run Deduplicator on dir different from pwd - -n, --nocache Don't use cache for indexing files (default = false) -i, --interactive Delete files interactively -h, --help Print help information -V, --version Print version information diff --git a/src/app.rs b/src/app.rs index 3356b25..0409738 100644 --- a/src/app.rs +++ b/src/app.rs @@ -1,4 +1,3 @@ -use crate::database; use crate::output; use crate::params::Params; use crate::scanner; @@ -8,8 +7,7 @@ pub struct App; impl App { pub fn init(app_args: &Params) -> Result<()> { - let connection = database::get_connection(app_args)?; - let duplicates = scanner::duplicates(app_args, &connection)?; + let duplicates = scanner::duplicates(app_args)?; match app_args.interactive { true => output::interactive(duplicates, app_args), false => output::print(duplicates, app_args), diff --git a/src/database.rs b/src/database.rs deleted file mode 100644 index 3f8595c..0000000 --- a/src/database.rs +++ /dev/null @@ -1,82 +0,0 @@ -use std::env::temp_dir; -use anyhow::Result; -use crate::params::Params; -use crate::file_manager::File; - -fn db_connection_url(args: &Params) -> String { - match args.nocache { - true => String::from(":memory:"), - false => { - let temp_dir_path = temp_dir(); - format!("{}/deduplicator.db", temp_dir_path.display()) - } - } -} - -pub fn get_connection(args: &Params) -> Result { - sqlite::open(db_connection_url(args)).and_then(|conn| { - setup(&conn).ok(); - Ok(conn) - }) -} - -pub fn setup(connection: &sqlite::Connection) -> Result<()> { - let query = "CREATE TABLE files (file_identifier STRING, hash STRING)"; - connection.execute(query).ok(); - Ok(()) -} - -pub fn put(file: &File, connection: &sqlite::Connection) -> Result<()> { - let query = format!( - "INSERT INTO files (file_identifier, hash) VALUES (\"{}\", \"{}\")", - file.path, file.hash - ); - connection.execute(query)?; - Ok(()) -} - -pub fn indexed_paths(connection: &sqlite::Connection) -> Result> { - let query = "SELECT * FROM files"; - - let result: Vec = connection - .prepare(query)? - .into_iter() - .filter_map(|row_result| row_result.ok()) - .map(|row| { - let path = row.read::<&str, _>("file_identifier").to_string(); - let hash = row.read::("hash").to_string(); - File { path, hash } - }) - .collect(); - - Ok(result) -} - -pub fn duplicate_hashes(connection: &sqlite::Connection, path: &str) -> Result> { - let query = format!( - " - SELECT a.* FROM files a - JOIN (SELECT file_identifier, hash, COUNT(*) - FROM files - GROUP BY hash - HAVING count(*) > 1 ) b - ON a.hash = b.hash - WHERE a.file_identifier LIKE \"{}%\" - ORDER BY a.file_identifier - ", - path - ); - - let result: Vec = connection - .prepare(query)? - .into_iter() - .filter_map(|row_result| row_result.ok()) - .map(|row| { - let path = row.read::<&str, _>("file_identifier").to_string(); - let hash = row.read::("hash").to_string(); - File { path, hash } - }) - .collect(); - - Ok(result) -} diff --git a/src/main.rs b/src/main.rs index 25b703c..4a82824 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,4 @@ mod app; -mod database; mod file_manager; mod output; mod params; diff --git a/src/params.rs b/src/params.rs index 5694eec..0ab59b8 100644 --- a/src/params.rs +++ b/src/params.rs @@ -1,6 +1,6 @@ -use std::{fs, path::PathBuf}; use anyhow::{anyhow, Result}; use clap::Parser; +use std::{fs, path::PathBuf}; #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] @@ -11,9 +11,6 @@ pub struct Params { /// Run Deduplicator on dir different from pwd #[arg(long)] pub dir: Option, - /// Don't use cache for indexing files (default = false) - #[arg(long, short)] - pub nocache: bool, /// Delete files interactively #[arg(long, short)] pub interactive: bool, @@ -36,4 +33,18 @@ impl Params { Ok(dir) } + + pub fn get_glob_patterns(&self) -> Vec { + self.types + .clone() + .unwrap_or_else(|| String::from("*")) + .split(',') + .map(|filetype| format!("*.{}", filetype)) + .map(|filetype| { + vec![self.get_directory().unwrap(), String::from("**"), filetype] + .iter() + .collect() + }) + .collect() + } } diff --git a/src/scanner.rs b/src/scanner.rs index ba0fc6c..e4b8b4d 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -1,46 +1,31 @@ use anyhow::Result; -use fxhash::hash32 as hasher; +use fxhash::hash64 as hasher; use glob::glob; use indicatif::{ParallelProgressIterator, ProgressStyle}; -use itertools::Itertools; use rayon::prelude::*; +use std::hash::Hasher; use std::{fs, path::PathBuf}; +use memmap2::Mmap; +use dashmap::DashMap; -use crate::{database, file_manager::File, params::Params}; +use crate::{file_manager::File, params::Params}; -pub fn duplicates(app_opts: &Params, connection: &sqlite::Connection) -> Result> { - let scan_results = scan(app_opts, connection)?; - let base_path = app_opts.get_directory()?; +pub fn duplicates(app_opts: &Params) -> Result> { + let scan_results = scan(app_opts)?; + let index_store = index_files(scan_results)?; - index_files(scan_results, connection)?; - database::duplicate_hashes(connection, &base_path) + let duplicate_files = index_store + .into_par_iter() + .filter(|(_, files)| files.len() > 1) + .map(|(_, files)| files ) + .flatten() + .collect::>(); + + Ok(duplicate_files) } -fn get_glob_patterns(opts: &Params, directory: &str) -> Vec { - opts.types - .clone() - .unwrap_or_else(|| String::from("*")) - .split(',') - .map(|filetype| format!("*.{}", filetype)) - .map(|filetype| { - vec![directory.to_owned(), String::from("**"), filetype] - .iter() - .collect() - }) - .collect() -} - -fn is_indexed_file(path: impl Into, indexed: &[File]) -> bool { - indexed - .iter() - .map(|file| file.path.clone()) - .contains(&path.into()) -} - -fn scan(app_opts: &Params, connection: &sqlite::Connection) -> Result> { - let directory = app_opts.get_directory()?; - let glob_patterns: Vec = get_glob_patterns(app_opts, &directory); - let indexed_paths = database::indexed_paths(connection)?; +fn scan(app_opts: &Params) -> Result> { + let glob_patterns: Vec = app_opts.get_glob_patterns(); let files: Vec = glob_patterns .par_iter() .progress_with_style( @@ -53,7 +38,6 @@ fn scan(app_opts: &Params, connection: &sqlite::Connection) -> Result Result, connection: &sqlite::Connection) -> Result<()> { - let hashed: Vec = files +fn index_files(files: Vec) -> Result>> { + let store: DashMap> = DashMap::new(); + files .into_par_iter() .progress_with_style( ProgressStyle::with_template( "{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files", - ) - .unwrap(), + )?, ) - .filter_map(|file| { - let hash = hash_file(&file).ok()?; - Some(File { path: file, hash }) - }) - .collect(); + .for_each(|file| { + let hash = hash_file(&file).unwrap_or_default(); + let fobj = File { path: file, hash: hash.clone() }; + store + .entry(hash) + .and_modify(|fileset| fileset.push(fobj.clone()) ) + .or_insert_with(|| vec![fobj]); + }); - hashed - .iter() - .try_for_each(|file| database::put(file, connection)) + Ok(store) +} + +pub fn incremental_hashing(filepath: &str) -> Result { + let file = fs::File::open(filepath)?; + let fmap = unsafe { Mmap::map(&file)? }; + let mut inchasher = fxhash::FxHasher::default(); + + fmap + .chunks(1_000) + .for_each(|kilo| { inchasher.write(kilo) }); + + Ok(format!("{}", inchasher.finish())) +} + +pub fn standard_hashing(filepath: &str) -> Result { + let file = fs::read(filepath)?; + Ok(hasher(&*file).to_string()) } pub fn hash_file(filepath: &str) -> Result { - let file = fs::read(filepath)?; - let hash = hasher(&*file).to_string(); + let filemeta = fs::metadata(filepath)?; - Ok(hash) + match filemeta.len() < 1_000_000 { + true => standard_hashing(filepath), + false => incremental_hashing(filepath) + } } From ea748c60d8fd74a13b8e65214f8c20ba044dbfbe Mon Sep 17 00:00:00 2001 From: sreedev Date: Wed, 18 Jan 2023 00:24:50 -0500 Subject: [PATCH 2/3] performance tweaks --- Cargo.lock | 37 -------------- Cargo.toml | 1 - src/file_manager.rs | 5 +- src/output.rs | 48 +++++++----------- src/scanner.rs | 121 +++++++++++++++++++++++++++++++------------- 5 files changed, 107 insertions(+), 105 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1240734..9260bf6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -314,7 +314,6 @@ dependencies = [ "memmap2", "prettytable-rs", "rayon", - "sqlite", "thiserror", "tokio", "unicode-segmentation", @@ -693,12 +692,6 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" -[[package]] -name = "pkg-config" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" - [[package]] name = "portable-atomic" version = "0.3.19" @@ -878,36 +871,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "sqlite" -version = "0.30.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12e072cb5fb89b3fe5e9c9584676348feb503f9fb3ae829d9868171bc5372d48" -dependencies = [ - "libc", - "sqlite3-sys", -] - -[[package]] -name = "sqlite3-src" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1815a7a02c996eb8e5c64f61fcb6fd9b12e593ce265c512c5853b2513635691" -dependencies = [ - "cc", - "pkg-config", -] - -[[package]] -name = "sqlite3-sys" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d47c99824fc55360ba00caf28de0b8a0458369b832e016a64c13af0ad9fbb9ee" -dependencies = [ - "libc", - "sqlite3-src", -] - [[package]] name = "strsim" version = "0.10.0" diff --git a/Cargo.toml b/Cargo.toml index 5fd7a11..d883b4e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,6 @@ itertools = "0.10.5" memmap2 = "0.5.8" prettytable-rs = "0.10.0" rayon = "1.6.1" -sqlite = "0.30.3" thiserror = "1.0.38" tokio = { version = "1.23.0", features = ["full"] } unicode-segmentation = "1.10.0" diff --git a/src/file_manager.rs b/src/file_manager.rs index 9f97318..25d5b8d 100644 --- a/src/file_manager.rs +++ b/src/file_manager.rs @@ -4,14 +4,15 @@ use colored::Colorize; #[derive(Debug, Clone)] pub struct File { pub path: String, - pub hash: String, + pub size: Option, + pub hash: Option, } pub fn delete_files(files: Vec) -> Result<()> { files.into_iter().for_each(|file| { match std::fs::remove_file(file.path.clone()) { Ok(_) => println!("{}: {}", "DELETED".green(), file.path), - Err(e) => println!("{}: {}", "FAILED".red(), file.path) + Err(_) => println!("{}: {}", "FAILED".red(), file.path) } }); diff --git a/src/output.rs b/src/output.rs index b52eca5..95c4567 100644 --- a/src/output.rs +++ b/src/output.rs @@ -1,14 +1,15 @@ -use std::io::Write; -use std::{collections::HashMap, fs, io}; +use crate::file_manager::{self, File}; +use crate::params::Params; use anyhow::Result; use chrono::offset::Utc; use chrono::DateTime; use colored::Colorize; +use dashmap::DashMap; use humansize::{format_size, DECIMAL}; use itertools::Itertools; -use crate::file_manager::{self, File}; -use crate::params::Params; use prettytable::{format, row, Table}; +use std::io::Write; +use std::{fs, io}; use unicode_segmentation::UnicodeSegmentation; fn format_path(path: &str, opts: &Params) -> Result { @@ -42,19 +43,7 @@ fn modified_time(path: &String) -> Result { Ok(modified_time.format("%Y-%m-%d %H:%M:%S").to_string()) } -fn group_duplicates(duplicates: Vec) -> HashMap> { - let mut duplicate_mapper: HashMap> = HashMap::new(); - duplicates.into_iter().for_each(|file| { - duplicate_mapper - .entry(file.hash.clone()) - .and_modify(|value| value.push(file.clone())) - .or_insert_with(|| vec![file]); - }); - - duplicate_mapper -} - -fn print_meta_info(duplicates: &Vec, opts: &Params) { +fn print_meta_info() { println!("Deduplicator v{}", std::env!("CARGO_PKG_VERSION")); } @@ -122,20 +111,19 @@ fn process_group_action(duplicates: &Vec, dup_index: usize, dup_size: usiz match scan_group_confirmation().unwrap() { true => { - file_manager::delete_files(files_to_delete.collect_vec()); + file_manager::delete_files(files_to_delete.collect_vec()).ok(); } false => println!("{}", "\nCancelled Delete Operation.".red()), } } -pub fn interactive(duplicates: Vec, opts: &Params) { - print_meta_info(&duplicates, opts); - let grouped_duplicates = group_duplicates(duplicates); - - grouped_duplicates - .iter() +pub fn interactive(duplicates: DashMap>, opts: &Params) { + print_meta_info(); + duplicates + .clone() + .into_iter() .enumerate() - .for_each(|(gindex, (hash, group))| { + .for_each(|(gindex, (_, group))| { let mut itable = Table::new(); itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR); itable.set_titles(row!["index", "filename", "size", "updated_at"]); @@ -148,18 +136,16 @@ pub fn interactive(duplicates: Vec, opts: &Params) { ]); }); - process_group_action(group, gindex, grouped_duplicates.len(), itable); + process_group_action(&group, gindex, duplicates.len(), itable); }); } -pub fn print(duplicates: Vec, opts: &Params) { - print_meta_info(&duplicates, opts); +pub fn print(duplicates: DashMap>, opts: &Params) { + print_meta_info(); let mut output_table = Table::new(); - let grouped_duplicates: HashMap> = group_duplicates(duplicates); - output_table.set_titles(row!["hash", "duplicates"]); - grouped_duplicates.iter().for_each(|(hash, group)| { + duplicates.into_iter().for_each(|(hash, group)| { let mut inner_table = Table::new(); inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR); group.iter().for_each(|file| { diff --git a/src/scanner.rs b/src/scanner.rs index e4b8b4d..c5d1086 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -1,39 +1,57 @@ use anyhow::Result; +use dashmap::DashMap; use fxhash::hash64 as hasher; use glob::glob; use indicatif::{ParallelProgressIterator, ProgressStyle}; +use memmap2::Mmap; use rayon::prelude::*; use std::hash::Hasher; use std::{fs, path::PathBuf}; -use memmap2::Mmap; -use dashmap::DashMap; use crate::{file_manager::File, params::Params}; -pub fn duplicates(app_opts: &Params) -> Result> { - let scan_results = scan(app_opts)?; - let index_store = index_files(scan_results)?; +#[derive(Clone, Copy)] +enum IndexCritera { + Size, + Hash, +} - let duplicate_files = index_store +pub fn duplicates(app_opts: &Params) -> Result>> { + let scan_results = scan(app_opts)?; + let size_index_store = index_files(scan_results, IndexCritera::Size)?; + + let sizewize_duplicate_files = size_index_store .into_par_iter() .filter(|(_, files)| files.len() > 1) - .map(|(_, files)| files ) + .map(|(_, files)| files) .flatten() .collect::>(); - Ok(duplicate_files) + if sizewize_duplicate_files.len() > 1 { + let size_wise_duplicate_paths = sizewize_duplicate_files + .into_par_iter() + .map(|file| file.path) + .collect::>(); + + let hash_index_store = index_files(size_wise_duplicate_paths, IndexCritera::Hash)?; + let duplicate_files = hash_index_store + .into_par_iter() + .filter(|(_, files)| files.len() > 1) + .collect(); + + Ok(duplicate_files) + } else { + Ok(DashMap::new()) + } } fn scan(app_opts: &Params) -> Result> { let glob_patterns: Vec = app_opts.get_glob_patterns(); let files: Vec = glob_patterns .par_iter() - .progress_with_style( - ProgressStyle::with_template( - "{spinner:.green} [scanning files] [{wide_bar:.cyan/blue}] {pos}/{len} files", - ) - .unwrap(), - ) + .progress_with_style(ProgressStyle::with_template( + "{spinner:.green} [scanning files] [{wide_bar:.cyan/blue}] {pos}/{len} files", + )?) .filter_map(|glob_pattern| glob(glob_pattern.as_os_str().to_str()?).ok()) .flat_map(|file_vec| { file_vec @@ -50,25 +68,60 @@ fn scan(app_opts: &Params) -> Result> { Ok(files) } -fn index_files(files: Vec) -> Result>> { +fn process_file_size_index(fpath: String) -> Result { + Ok(File { + path: fpath.clone(), + size: Some(fs::metadata(fpath)?.len()), + hash: None, + }) +} + +fn process_file_hash_index(fpath: String) -> Result { + Ok(File { + path: fpath.clone(), + size: None, + hash: Some(hash_file(&fpath).unwrap_or_default()), + }) +} + +fn process_file_index( + fpath: String, + store: &DashMap>, + index_criteria: IndexCritera, +) { + match index_criteria { + IndexCritera::Size => { + let processed_file = process_file_size_index(fpath).unwrap(); + store + .entry(processed_file.size.unwrap_or_default().to_string()) + .and_modify(|fileset| fileset.push(processed_file.clone())) + .or_insert_with(|| vec![processed_file]); + } + IndexCritera::Hash => { + let processed_file = process_file_hash_index(fpath).unwrap(); + let indexhash = processed_file.clone().hash.unwrap_or_default(); + + store + .entry(indexhash) + .and_modify(|fileset| fileset.push(processed_file.clone())) + .or_insert_with(|| vec![processed_file]); + } + } +} + +fn index_files( + files: Vec, + index_criteria: IndexCritera, +) -> Result>> { let store: DashMap> = DashMap::new(); files .into_par_iter() - .progress_with_style( - ProgressStyle::with_template( - "{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files", - )?, - ) - .for_each(|file| { - let hash = hash_file(&file).unwrap_or_default(); - let fobj = File { path: file, hash: hash.clone() }; - store - .entry(hash) - .and_modify(|fileset| fileset.push(fobj.clone()) ) - .or_insert_with(|| vec![fobj]); - }); + .progress_with_style(ProgressStyle::with_template( + "{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files", + )?) + .for_each(|file| process_file_index(file, &store, index_criteria)); - Ok(store) + Ok(store) } pub fn incremental_hashing(filepath: &str) -> Result { @@ -76,9 +129,8 @@ pub fn incremental_hashing(filepath: &str) -> Result { let fmap = unsafe { Mmap::map(&file)? }; let mut inchasher = fxhash::FxHasher::default(); - fmap - .chunks(1_000) - .for_each(|kilo| { inchasher.write(kilo) }); + fmap.chunks(1_000_000) + .for_each(|mega| inchasher.write(mega)); Ok(format!("{}", inchasher.finish())) } @@ -91,8 +143,9 @@ pub fn standard_hashing(filepath: &str) -> Result { pub fn hash_file(filepath: &str) -> Result { let filemeta = fs::metadata(filepath)?; - match filemeta.len() < 1_000_000 { + // NOTE: USE INCREMENTAL HASHING ONLY FOR FILES > 100MB + match filemeta.len() < 100_000_000 { true => standard_hashing(filepath), - false => incremental_hashing(filepath) + false => incremental_hashing(filepath), } } From 1ea57054741fc51e801034d4a156df5942ea3099 Mon Sep 17 00:00:00 2001 From: sreedev Date: Wed, 18 Jan 2023 00:30:39 -0500 Subject: [PATCH 3/3] fix confusing delete confirmation --- src/output.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/output.rs b/src/output.rs index 95c4567..4767d6f 100644 --- a/src/output.rs +++ b/src/output.rs @@ -60,7 +60,7 @@ fn scan_group_instruction() -> Result { } fn scan_group_confirmation() -> Result { - print!("\nconfirm? [Y/n]: "); + print!("\nconfirm? [y/N]: "); std::io::stdout().flush()?; let mut user_input = String::new(); io::stdin().read_line(&mut user_input)?;