diff --git a/Cargo.lock b/Cargo.lock index 1240734..9260bf6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -314,7 +314,6 @@ dependencies = [ "memmap2", "prettytable-rs", "rayon", - "sqlite", "thiserror", "tokio", "unicode-segmentation", @@ -693,12 +692,6 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" -[[package]] -name = "pkg-config" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" - [[package]] name = "portable-atomic" version = "0.3.19" @@ -878,36 +871,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "sqlite" -version = "0.30.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12e072cb5fb89b3fe5e9c9584676348feb503f9fb3ae829d9868171bc5372d48" -dependencies = [ - "libc", - "sqlite3-sys", -] - -[[package]] -name = "sqlite3-src" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1815a7a02c996eb8e5c64f61fcb6fd9b12e593ce265c512c5853b2513635691" -dependencies = [ - "cc", - "pkg-config", -] - -[[package]] -name = "sqlite3-sys" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d47c99824fc55360ba00caf28de0b8a0458369b832e016a64c13af0ad9fbb9ee" -dependencies = [ - "libc", - "sqlite3-src", -] - [[package]] name = "strsim" version = "0.10.0" diff --git a/Cargo.toml b/Cargo.toml index 5fd7a11..d883b4e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,6 @@ itertools = "0.10.5" memmap2 = "0.5.8" prettytable-rs = "0.10.0" rayon = "1.6.1" -sqlite = "0.30.3" thiserror = "1.0.38" tokio = { version = "1.23.0", features = ["full"] } unicode-segmentation = "1.10.0" diff --git a/src/file_manager.rs b/src/file_manager.rs index 9f97318..25d5b8d 100644 --- a/src/file_manager.rs +++ b/src/file_manager.rs @@ -4,14 +4,15 @@ use colored::Colorize; #[derive(Debug, Clone)] pub struct File { pub path: String, - pub hash: String, + pub size: Option, + pub hash: Option, } pub fn delete_files(files: Vec) -> Result<()> { files.into_iter().for_each(|file| { match std::fs::remove_file(file.path.clone()) { Ok(_) => println!("{}: {}", "DELETED".green(), file.path), - Err(e) => println!("{}: {}", "FAILED".red(), file.path) + Err(_) => println!("{}: {}", "FAILED".red(), file.path) } }); diff --git a/src/output.rs b/src/output.rs index b52eca5..95c4567 100644 --- a/src/output.rs +++ b/src/output.rs @@ -1,14 +1,15 @@ -use std::io::Write; -use std::{collections::HashMap, fs, io}; +use crate::file_manager::{self, File}; +use crate::params::Params; use anyhow::Result; use chrono::offset::Utc; use chrono::DateTime; use colored::Colorize; +use dashmap::DashMap; use humansize::{format_size, DECIMAL}; use itertools::Itertools; -use crate::file_manager::{self, File}; -use crate::params::Params; use prettytable::{format, row, Table}; +use std::io::Write; +use std::{fs, io}; use unicode_segmentation::UnicodeSegmentation; fn format_path(path: &str, opts: &Params) -> Result { @@ -42,19 +43,7 @@ fn modified_time(path: &String) -> Result { Ok(modified_time.format("%Y-%m-%d %H:%M:%S").to_string()) } -fn group_duplicates(duplicates: Vec) -> HashMap> { - let mut duplicate_mapper: HashMap> = HashMap::new(); - duplicates.into_iter().for_each(|file| { - duplicate_mapper - .entry(file.hash.clone()) - .and_modify(|value| value.push(file.clone())) - .or_insert_with(|| vec![file]); - }); - - duplicate_mapper -} - -fn print_meta_info(duplicates: &Vec, opts: &Params) { +fn print_meta_info() { println!("Deduplicator v{}", std::env!("CARGO_PKG_VERSION")); } @@ -122,20 +111,19 @@ fn process_group_action(duplicates: &Vec, dup_index: usize, dup_size: usiz match scan_group_confirmation().unwrap() { true => { - file_manager::delete_files(files_to_delete.collect_vec()); + file_manager::delete_files(files_to_delete.collect_vec()).ok(); } false => println!("{}", "\nCancelled Delete Operation.".red()), } } -pub fn interactive(duplicates: Vec, opts: &Params) { - print_meta_info(&duplicates, opts); - let grouped_duplicates = group_duplicates(duplicates); - - grouped_duplicates - .iter() +pub fn interactive(duplicates: DashMap>, opts: &Params) { + print_meta_info(); + duplicates + .clone() + .into_iter() .enumerate() - .for_each(|(gindex, (hash, group))| { + .for_each(|(gindex, (_, group))| { let mut itable = Table::new(); itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR); itable.set_titles(row!["index", "filename", "size", "updated_at"]); @@ -148,18 +136,16 @@ pub fn interactive(duplicates: Vec, opts: &Params) { ]); }); - process_group_action(group, gindex, grouped_duplicates.len(), itable); + process_group_action(&group, gindex, duplicates.len(), itable); }); } -pub fn print(duplicates: Vec, opts: &Params) { - print_meta_info(&duplicates, opts); +pub fn print(duplicates: DashMap>, opts: &Params) { + print_meta_info(); let mut output_table = Table::new(); - let grouped_duplicates: HashMap> = group_duplicates(duplicates); - output_table.set_titles(row!["hash", "duplicates"]); - grouped_duplicates.iter().for_each(|(hash, group)| { + duplicates.into_iter().for_each(|(hash, group)| { let mut inner_table = Table::new(); inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR); group.iter().for_each(|file| { diff --git a/src/scanner.rs b/src/scanner.rs index e4b8b4d..c5d1086 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -1,39 +1,57 @@ use anyhow::Result; +use dashmap::DashMap; use fxhash::hash64 as hasher; use glob::glob; use indicatif::{ParallelProgressIterator, ProgressStyle}; +use memmap2::Mmap; use rayon::prelude::*; use std::hash::Hasher; use std::{fs, path::PathBuf}; -use memmap2::Mmap; -use dashmap::DashMap; use crate::{file_manager::File, params::Params}; -pub fn duplicates(app_opts: &Params) -> Result> { - let scan_results = scan(app_opts)?; - let index_store = index_files(scan_results)?; +#[derive(Clone, Copy)] +enum IndexCritera { + Size, + Hash, +} - let duplicate_files = index_store +pub fn duplicates(app_opts: &Params) -> Result>> { + let scan_results = scan(app_opts)?; + let size_index_store = index_files(scan_results, IndexCritera::Size)?; + + let sizewize_duplicate_files = size_index_store .into_par_iter() .filter(|(_, files)| files.len() > 1) - .map(|(_, files)| files ) + .map(|(_, files)| files) .flatten() .collect::>(); - Ok(duplicate_files) + if sizewize_duplicate_files.len() > 1 { + let size_wise_duplicate_paths = sizewize_duplicate_files + .into_par_iter() + .map(|file| file.path) + .collect::>(); + + let hash_index_store = index_files(size_wise_duplicate_paths, IndexCritera::Hash)?; + let duplicate_files = hash_index_store + .into_par_iter() + .filter(|(_, files)| files.len() > 1) + .collect(); + + Ok(duplicate_files) + } else { + Ok(DashMap::new()) + } } fn scan(app_opts: &Params) -> Result> { let glob_patterns: Vec = app_opts.get_glob_patterns(); let files: Vec = glob_patterns .par_iter() - .progress_with_style( - ProgressStyle::with_template( - "{spinner:.green} [scanning files] [{wide_bar:.cyan/blue}] {pos}/{len} files", - ) - .unwrap(), - ) + .progress_with_style(ProgressStyle::with_template( + "{spinner:.green} [scanning files] [{wide_bar:.cyan/blue}] {pos}/{len} files", + )?) .filter_map(|glob_pattern| glob(glob_pattern.as_os_str().to_str()?).ok()) .flat_map(|file_vec| { file_vec @@ -50,25 +68,60 @@ fn scan(app_opts: &Params) -> Result> { Ok(files) } -fn index_files(files: Vec) -> Result>> { +fn process_file_size_index(fpath: String) -> Result { + Ok(File { + path: fpath.clone(), + size: Some(fs::metadata(fpath)?.len()), + hash: None, + }) +} + +fn process_file_hash_index(fpath: String) -> Result { + Ok(File { + path: fpath.clone(), + size: None, + hash: Some(hash_file(&fpath).unwrap_or_default()), + }) +} + +fn process_file_index( + fpath: String, + store: &DashMap>, + index_criteria: IndexCritera, +) { + match index_criteria { + IndexCritera::Size => { + let processed_file = process_file_size_index(fpath).unwrap(); + store + .entry(processed_file.size.unwrap_or_default().to_string()) + .and_modify(|fileset| fileset.push(processed_file.clone())) + .or_insert_with(|| vec![processed_file]); + } + IndexCritera::Hash => { + let processed_file = process_file_hash_index(fpath).unwrap(); + let indexhash = processed_file.clone().hash.unwrap_or_default(); + + store + .entry(indexhash) + .and_modify(|fileset| fileset.push(processed_file.clone())) + .or_insert_with(|| vec![processed_file]); + } + } +} + +fn index_files( + files: Vec, + index_criteria: IndexCritera, +) -> Result>> { let store: DashMap> = DashMap::new(); files .into_par_iter() - .progress_with_style( - ProgressStyle::with_template( - "{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files", - )?, - ) - .for_each(|file| { - let hash = hash_file(&file).unwrap_or_default(); - let fobj = File { path: file, hash: hash.clone() }; - store - .entry(hash) - .and_modify(|fileset| fileset.push(fobj.clone()) ) - .or_insert_with(|| vec![fobj]); - }); + .progress_with_style(ProgressStyle::with_template( + "{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files", + )?) + .for_each(|file| process_file_index(file, &store, index_criteria)); - Ok(store) + Ok(store) } pub fn incremental_hashing(filepath: &str) -> Result { @@ -76,9 +129,8 @@ pub fn incremental_hashing(filepath: &str) -> Result { let fmap = unsafe { Mmap::map(&file)? }; let mut inchasher = fxhash::FxHasher::default(); - fmap - .chunks(1_000) - .for_each(|kilo| { inchasher.write(kilo) }); + fmap.chunks(1_000_000) + .for_each(|mega| inchasher.write(mega)); Ok(format!("{}", inchasher.finish())) } @@ -91,8 +143,9 @@ pub fn standard_hashing(filepath: &str) -> Result { pub fn hash_file(filepath: &str) -> Result { let filemeta = fs::metadata(filepath)?; - match filemeta.len() < 1_000_000 { + // NOTE: USE INCREMENTAL HASHING ONLY FOR FILES > 100MB + match filemeta.len() < 100_000_000 { true => standard_hashing(filepath), - false => incremental_hashing(filepath) + false => incremental_hashing(filepath), } }