performance tweaks

Cargo.lock

@@ -314,7 +314,6 @@ dependencies = [
  "memmap2",
  "prettytable-rs",
  "rayon",
- "sqlite",
  "thiserror",
  "tokio",
  "unicode-segmentation",
@@ -693,12 +692,6 @@ version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
 
-[[package]]
-name = "pkg-config"
-version = "0.3.26"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
-
 [[package]]
 name = "portable-atomic"
 version = "0.3.19"
@@ -878,36 +871,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "sqlite"
-version = "0.30.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12e072cb5fb89b3fe5e9c9584676348feb503f9fb3ae829d9868171bc5372d48"
-dependencies = [
- "libc",
- "sqlite3-sys",
-]
-
-[[package]]
-name = "sqlite3-src"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d1815a7a02c996eb8e5c64f61fcb6fd9b12e593ce265c512c5853b2513635691"
-dependencies = [
- "cc",
- "pkg-config",
-]
-
-[[package]]
-name = "sqlite3-sys"
-version = "0.14.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d47c99824fc55360ba00caf28de0b8a0458369b832e016a64c13af0ad9fbb9ee"
-dependencies = [
- "libc",
- "sqlite3-src",
-]
-
 [[package]]
 name = "strsim"
 version = "0.10.0"

Cargo.toml

@@ -22,7 +22,6 @@ itertools = "0.10.5"
 memmap2 = "0.5.8"
 prettytable-rs = "0.10.0"
 rayon = "1.6.1"
-sqlite = "0.30.3"
 thiserror = "1.0.38"
 tokio = { version = "1.23.0", features = ["full"] }
 unicode-segmentation = "1.10.0"

src/file_manager.rs

@@ -4,14 +4,15 @@ use colored::Colorize;
 #[derive(Debug, Clone)]
 pub struct File {
     pub path: String,
-    pub hash: String,
+    pub size: Option<u64>,
+    pub hash: Option<String>,
 }
 
 pub fn delete_files(files: Vec<File>) -> Result<()> {
     files.into_iter().for_each(|file| {
         match std::fs::remove_file(file.path.clone()) {
             Ok(_) => println!("{}: {}", "DELETED".green(), file.path),
-            Err(e) => println!("{}: {}", "FAILED".red(), file.path)
+            Err(_) => println!("{}: {}", "FAILED".red(), file.path)
         }
     });
 

src/output.rs

@@ -1,14 +1,15 @@
-use std::io::Write;
+use crate::file_manager::{self, File};
-use std::{collections::HashMap, fs, io};
+use crate::params::Params;
 use anyhow::Result;
 use chrono::offset::Utc;
 use chrono::DateTime;
 use colored::Colorize;
+use dashmap::DashMap;
 use humansize::{format_size, DECIMAL};
 use itertools::Itertools;
-use crate::file_manager::{self, File};
-use crate::params::Params;
 use prettytable::{format, row, Table};
+use std::io::Write;
+use std::{fs, io};
 use unicode_segmentation::UnicodeSegmentation;
 
 fn format_path(path: &str, opts: &Params) -> Result<String> {
@@ -42,19 +43,7 @@ fn modified_time(path: &String) -> Result<String> {
     Ok(modified_time.format("%Y-%m-%d %H:%M:%S").to_string())
 }
 
-fn group_duplicates(duplicates: Vec<File>) -> HashMap<String, Vec<File>> {
+fn print_meta_info() {
-    let mut duplicate_mapper: HashMap<String, Vec<File>> = HashMap::new();
-    duplicates.into_iter().for_each(|file| {
-        duplicate_mapper
-            .entry(file.hash.clone())
-            .and_modify(|value| value.push(file.clone()))
-            .or_insert_with(|| vec![file]);
-    });
-
-    duplicate_mapper
-}
-
-fn print_meta_info(duplicates: &Vec<File>, opts: &Params) {
     println!("Deduplicator v{}", std::env!("CARGO_PKG_VERSION"));
 }
 
@@ -122,20 +111,19 @@ fn process_group_action(duplicates: &Vec<File>, dup_index: usize, dup_size: usiz
 
     match scan_group_confirmation().unwrap() {
         true => {
-            file_manager::delete_files(files_to_delete.collect_vec());
+            file_manager::delete_files(files_to_delete.collect_vec()).ok();
         }
         false => println!("{}", "\nCancelled Delete Operation.".red()),
     }
 }
 
-pub fn interactive(duplicates: Vec<File>, opts: &Params) {
+pub fn interactive(duplicates: DashMap<String, Vec<File>>, opts: &Params) {
-    print_meta_info(&duplicates, opts);
+    print_meta_info();
-    let grouped_duplicates = group_duplicates(duplicates);
+    duplicates
-
+        .clone()
-    grouped_duplicates
+        .into_iter()
-        .iter()
         .enumerate()
-        .for_each(|(gindex, (hash, group))| {
+        .for_each(|(gindex, (_, group))| {
             let mut itable = Table::new();
             itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
             itable.set_titles(row!["index", "filename", "size", "updated_at"]);
@@ -148,18 +136,16 @@ pub fn interactive(duplicates: Vec<File>, opts: &Params) {
                 ]);
             });
 
-            process_group_action(group, gindex, grouped_duplicates.len(), itable);
+            process_group_action(&group, gindex, duplicates.len(), itable);
         });
 }
 
-pub fn print(duplicates: Vec<File>, opts: &Params) {
+pub fn print(duplicates: DashMap<String, Vec<File>>, opts: &Params) {
-    print_meta_info(&duplicates, opts);
+    print_meta_info();
 
     let mut output_table = Table::new();
-    let grouped_duplicates: HashMap<String, Vec<File>> = group_duplicates(duplicates);
-
     output_table.set_titles(row!["hash", "duplicates"]);
-    grouped_duplicates.iter().for_each(|(hash, group)| {
+    duplicates.into_iter().for_each(|(hash, group)| {
         let mut inner_table = Table::new();
         inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
         group.iter().for_each(|file| {

src/scanner.rs

@@ -1,39 +1,57 @@
 use anyhow::Result;
+use dashmap::DashMap;
 use fxhash::hash64 as hasher;
 use glob::glob;
 use indicatif::{ParallelProgressIterator, ProgressStyle};
+use memmap2::Mmap;
 use rayon::prelude::*;
 use std::hash::Hasher;
 use std::{fs, path::PathBuf};
-use memmap2::Mmap;
-use dashmap::DashMap;
 
 use crate::{file_manager::File, params::Params};
 
-pub fn duplicates(app_opts: &Params) -> Result<Vec<File>> {
+#[derive(Clone, Copy)]
-    let scan_results = scan(app_opts)?;
+enum IndexCritera {
-    let index_store = index_files(scan_results)?;
+    Size,
+    Hash,
+}
 
-    let duplicate_files = index_store
+pub fn duplicates(app_opts: &Params) -> Result<DashMap<String, Vec<File>>> {
+    let scan_results = scan(app_opts)?;
+    let size_index_store = index_files(scan_results, IndexCritera::Size)?;
+
+    let sizewize_duplicate_files = size_index_store
         .into_par_iter()
         .filter(|(_, files)| files.len() > 1)
-        .map(|(_, files)| files )
+        .map(|(_, files)| files)
         .flatten()
         .collect::<Vec<File>>();
 
-    Ok(duplicate_files)
+    if sizewize_duplicate_files.len() > 1 {
+        let size_wise_duplicate_paths = sizewize_duplicate_files
+            .into_par_iter()
+            .map(|file| file.path)
+            .collect::<Vec<String>>();
+
+        let hash_index_store = index_files(size_wise_duplicate_paths, IndexCritera::Hash)?;
+        let duplicate_files = hash_index_store
+            .into_par_iter()
+            .filter(|(_, files)| files.len() > 1)
+            .collect();
+
+        Ok(duplicate_files)
+    } else {
+        Ok(DashMap::new())
+    }
 }
 
 fn scan(app_opts: &Params) -> Result<Vec<String>> {
     let glob_patterns: Vec<PathBuf> = app_opts.get_glob_patterns();
     let files: Vec<String> = glob_patterns
         .par_iter()
-        .progress_with_style(
+        .progress_with_style(ProgressStyle::with_template(
-            ProgressStyle::with_template(
+            "{spinner:.green} [scanning files] [{wide_bar:.cyan/blue}] {pos}/{len} files",
-                "{spinner:.green} [scanning files] [{wide_bar:.cyan/blue}] {pos}/{len} files",
+        )?)
-            )
-            .unwrap(),
-        )
         .filter_map(|glob_pattern| glob(glob_pattern.as_os_str().to_str()?).ok())
         .flat_map(|file_vec| {
             file_vec
@@ -50,25 +68,60 @@ fn scan(app_opts: &Params) -> Result<Vec<String>> {
     Ok(files)
 }
 
-fn index_files(files: Vec<String>) -> Result<DashMap<String, Vec<File>>> {
+fn process_file_size_index(fpath: String) -> Result<File> {
+    Ok(File {
+        path: fpath.clone(),
+        size: Some(fs::metadata(fpath)?.len()),
+        hash: None,
+    })
+}
+
+fn process_file_hash_index(fpath: String) -> Result<File> {
+    Ok(File {
+        path: fpath.clone(),
+        size: None,
+        hash: Some(hash_file(&fpath).unwrap_or_default()),
+    })
+}
+
+fn process_file_index(
+    fpath: String,
+    store: &DashMap<String, Vec<File>>,
+    index_criteria: IndexCritera,
+) {
+    match index_criteria {
+        IndexCritera::Size => {
+            let processed_file = process_file_size_index(fpath).unwrap();
+            store
+                .entry(processed_file.size.unwrap_or_default().to_string())
+                .and_modify(|fileset| fileset.push(processed_file.clone()))
+                .or_insert_with(|| vec![processed_file]);
+        }
+        IndexCritera::Hash => {
+            let processed_file = process_file_hash_index(fpath).unwrap();
+            let indexhash = processed_file.clone().hash.unwrap_or_default();
+
+            store
+                .entry(indexhash)
+                .and_modify(|fileset| fileset.push(processed_file.clone()))
+                .or_insert_with(|| vec![processed_file]);
+        }
+    }
+}
+
+fn index_files(
+    files: Vec<String>,
+    index_criteria: IndexCritera,
+) -> Result<DashMap<String, Vec<File>>> {
     let store: DashMap<String, Vec<File>> = DashMap::new();
     files
         .into_par_iter()
-        .progress_with_style(
+        .progress_with_style(ProgressStyle::with_template(
-            ProgressStyle::with_template(
+            "{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files",
-                "{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files",
+        )?)
-            )?,
+        .for_each(|file| process_file_index(file, &store, index_criteria));
-        )
-        .for_each(|file| {
-            let hash = hash_file(&file).unwrap_or_default();
-            let fobj = File { path: file, hash: hash.clone() };
-            store
-                .entry(hash)
-                .and_modify(|fileset| fileset.push(fobj.clone()) )
-                .or_insert_with(|| vec![fobj]);
-        });
 
-        Ok(store)
+    Ok(store)
 }
 
 pub fn incremental_hashing(filepath: &str) -> Result<String> {
@@ -76,9 +129,8 @@ pub fn incremental_hashing(filepath: &str) -> Result<String> {
     let fmap = unsafe { Mmap::map(&file)? };
     let mut inchasher = fxhash::FxHasher::default();
 
-    fmap
+    fmap.chunks(1_000_000)
-        .chunks(1_000)
+        .for_each(|mega| inchasher.write(mega));
-        .for_each(|kilo| { inchasher.write(kilo) });
 
     Ok(format!("{}", inchasher.finish()))
 }
@@ -91,8 +143,9 @@ pub fn standard_hashing(filepath: &str) -> Result<String> {
 pub fn hash_file(filepath: &str) -> Result<String> {
     let filemeta = fs::metadata(filepath)?;
 
-    match filemeta.len() < 1_000_000 {
+    // NOTE: USE INCREMENTAL HASHING ONLY FOR FILES > 100MB
+    match filemeta.len() < 100_000_000 {
         true => standard_hashing(filepath),
-        false => incremental_hashing(filepath)
+        false => incremental_hashing(filepath),
     }
 }