incremental hashing + remove sqlite + mmap file reads

2026-09-01 13:06:43 +00:00 · 2023-01-17 23:40:57 -05:00
parent 175c7579c4
commit 51f5f8e61a
8 changed files with 102 additions and 209 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -70,12 +70,6 @@ version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c"

-[[package]]
-name = "cassowary"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53"
-
 [[package]]
 name = "cc"
 version = "1.0.78"
@@ -223,31 +217,6 @@ dependencies = [
 "cfg-if",
 ]

-[[package]]
-name = "crossterm"
-version = "0.25.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e64e6c0fbe2c17357405f7c758c1ef960fce08bdfb2c03d88d2a18d7e09c4b67"
-dependencies = [
- "bitflags",
- "crossterm_winapi",
- "libc",
- "mio",
- "parking_lot",
- "signal-hook",
- "signal-hook-mio",
- "winapi",
-]
-
-[[package]]
-name = "crossterm_winapi"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ae1b35a484aa10e07fe0638d02301c5ad24de82d310ccbd2f3693da5f09bf1c"
-dependencies = [
- "winapi",
-]
-
 [[package]]
 name = "csv"
 version = "1.1.6"
@@ -314,6 +283,20 @@ dependencies = [
 "syn",
 ]

+[[package]]
+name = "dashmap"
+version = "5.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc"
+dependencies = [
+ "cfg-if",
+ "hashbrown",
+ "lock_api",
+ "once_cell",
+ "parking_lot_core",
+ "rayon",
+]
+
 [[package]]
 name = "deduplicator"
 version = "0.0.9"
@@ -322,18 +305,18 @@ dependencies = [
 "chrono",
 "clap",
 "colored",
- "crossterm",
+ "dashmap",
 "fxhash",
 "glob",
 "humansize",
 "indicatif",
 "itertools",
+ "memmap2",
 "prettytable-rs",
 "rayon",
 "sqlite",
 "thiserror",
 "tokio",
- "tui",
 "unicode-segmentation",
 ]

@@ -423,6 +406,12 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"

+[[package]]
+name = "hashbrown"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+
 [[package]]
 name = "heck"
 version = "0.4.0"
@@ -598,6 +587,15 @@ version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"

+[[package]]
+name = "memmap2"
+version = "0.5.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b182332558b18d807c4ce1ca8ca983b34c3ee32765e47b3f0f69b90355cc1dc"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "memoffset"
 version = "0.7.1"
@@ -855,27 +853,6 @@ version = "1.0.152"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb"

-[[package]]
-name = "signal-hook"
-version = "0.3.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a253b5e89e2698464fc26b545c9edceb338e18a89effeeecfea192c3025be29d"
-dependencies = [
- "libc",
- "signal-hook-registry",
-]
-
-[[package]]
-name = "signal-hook-mio"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af"
-dependencies = [
- "libc",
- "mio",
- "signal-hook",
-]
-
 [[package]]
 name = "signal-hook-registry"
 version = "1.4.0"
@@ -1030,19 +1007,6 @@ dependencies = [
 "syn",
 ]

-[[package]]
-name = "tui"
-version = "0.19.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ccdd26cbd674007e649a272da4475fb666d3aa0ad0531da7136db6fab0e5bad1"
-dependencies = [
- "bitflags",
- "cassowary",
- "crossterm",
- "unicode-segmentation",
- "unicode-width",
-]
-
 [[package]]
 name = "unicode-ident"
 version = "1.0.6"
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -13,16 +13,16 @@ anyhow = "1.0.68"
 chrono = "0.4.23"
 clap = { version = "4.0.32", features = ["derive"] }
 colored = "2.0.0"
-crossterm = "0.25.0"
+dashmap = { version = "5.4.0", features = ["rayon"] }
 fxhash = "0.2.1"
 glob = "0.3.0"
 humansize = "2.1.2"
 indicatif = { version = "0.17.2", features = ["rayon", "tokio"] }
 itertools = "0.10.5"
+memmap2 = "0.5.8"
 prettytable-rs = "0.10.0"
 rayon = "1.6.1"
 sqlite = "0.30.3"
 thiserror = "1.0.38"
 tokio = { version = "1.23.0", features = ["full"] }
-tui = "0.19.0"
 unicode-segmentation = "1.10.0"
--- a/README.md
+++ b/README.md
@@ -16,7 +16,6 @@ Usage: deduplicator [OPTIONS]
 Options:
  -t, --types <TYPES>  Filetypes to deduplicate (default = all)
      --dir <DIR>      Run Deduplicator on dir different from pwd
-  -n, --nocache        Don't use cache for indexing files (default = false)
  -i, --interactive    Delete files interactively
  -h, --help           Print help information
  -V, --version        Print version information
--- a/src/app.rs
+++ b/src/app.rs
@@ -1,4 +1,3 @@
-use crate::database;
 use crate::output;
 use crate::params::Params;
 use crate::scanner;
@@ -8,8 +7,7 @@ pub struct App;

 impl App {
    pub fn init(app_args: &Params) -> Result<()> {
-        let connection = database::get_connection(app_args)?;
-        let duplicates = scanner::duplicates(app_args, &connection)?;
+        let duplicates = scanner::duplicates(app_args)?;
        match app_args.interactive {
            true => output::interactive(duplicates, app_args),
            false => output::print(duplicates, app_args),
--- a/src/database.rs
+++ b/src/database.rs
@@ -1,82 +0,0 @@
-use std::env::temp_dir;
-use anyhow::Result;
-use crate::params::Params;
-use crate::file_manager::File;
-
-fn db_connection_url(args: &Params) -> String {
-    match args.nocache {
-        true => String::from(":memory:"),
-        false => {
-            let temp_dir_path = temp_dir();
-            format!("{}/deduplicator.db", temp_dir_path.display())
-        }
-    }
-}
-
-pub fn get_connection(args: &Params) -> Result<sqlite::Connection, sqlite::Error> {
-    sqlite::open(db_connection_url(args)).and_then(|conn| {
-        setup(&conn).ok();
-        Ok(conn)
-    })
-}
-
-pub fn setup(connection: &sqlite::Connection) -> Result<()> {
-    let query = "CREATE TABLE files (file_identifier STRING, hash STRING)";
-    connection.execute(query).ok();
-    Ok(())
-}
-
-pub fn put(file: &File, connection: &sqlite::Connection) -> Result<()> {
-    let query = format!(
-        "INSERT INTO files (file_identifier, hash) VALUES (\"{}\", \"{}\")",
-        file.path, file.hash
-    );
-    connection.execute(query)?;
-    Ok(())
-}
-
-pub fn indexed_paths(connection: &sqlite::Connection) -> Result<Vec<File>> {
-    let query = "SELECT * FROM files";
-
-    let result: Vec<File> = connection
-        .prepare(query)?
-        .into_iter()
-        .filter_map(|row_result| row_result.ok())
-        .map(|row| {
-            let path = row.read::<&str, _>("file_identifier").to_string();
-            let hash = row.read::<i64, _>("hash").to_string();
-            File { path, hash }
-        })
-        .collect();
-
-    Ok(result)
-}
-
-pub fn duplicate_hashes(connection: &sqlite::Connection, path: &str) -> Result<Vec<File>> {
-    let query = format!(
-        "
-            SELECT a.* FROM files a
-            JOIN (SELECT file_identifier, hash, COUNT(*)
-            FROM files
-            GROUP BY hash
-            HAVING count(*) > 1 ) b
-            ON a.hash = b.hash
-            WHERE a.file_identifier LIKE \"{}%\"
-            ORDER BY a.file_identifier
-        ",
-        path
-    );
-
-    let result: Vec<File> = connection
-        .prepare(query)?
-        .into_iter()
-        .filter_map(|row_result| row_result.ok())
-        .map(|row| {
-            let path = row.read::<&str, _>("file_identifier").to_string();
-            let hash = row.read::<i64, _>("hash").to_string();
-            File { path, hash }
-        })
-        .collect();
-
-    Ok(result)
-}
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,5 +1,4 @@
 mod app;
-mod database;
 mod file_manager;
 mod output;
 mod params;
--- a/src/params.rs
+++ b/src/params.rs
@@ -1,6 +1,6 @@
-use std::{fs, path::PathBuf};
 use anyhow::{anyhow, Result};
 use clap::Parser;
+use std::{fs, path::PathBuf};

 #[derive(Parser, Debug)]
 #[command(author, version, about, long_about = None)]
@@ -11,9 +11,6 @@ pub struct Params {
    /// Run Deduplicator on dir different from pwd
    #[arg(long)]
    pub dir: Option<PathBuf>,
-    /// Don't use cache for indexing files (default = false)
-    #[arg(long, short)]
-    pub nocache: bool,
    /// Delete files interactively
    #[arg(long, short)]
    pub interactive: bool,
@@ -36,4 +33,18 @@ impl Params {

        Ok(dir)
    }
+
+    pub fn get_glob_patterns(&self) -> Vec<PathBuf> {
+        self.types
+            .clone()
+            .unwrap_or_else(|| String::from("*"))
+            .split(',')
+            .map(|filetype| format!("*.{}", filetype))
+            .map(|filetype| {
+                vec![self.get_directory().unwrap(), String::from("**"), filetype]
+                    .iter()
+                    .collect()
+            })
+            .collect()
+    }
 }
--- a/src/scanner.rs
+++ b/src/scanner.rs
@@ -1,46 +1,31 @@
 use anyhow::Result;
-use fxhash::hash32 as hasher;
+use fxhash::hash64 as hasher;
 use glob::glob;
 use indicatif::{ParallelProgressIterator, ProgressStyle};
-use itertools::Itertools;
 use rayon::prelude::*;
+use std::hash::Hasher;
 use std::{fs, path::PathBuf};
+use memmap2::Mmap;
+use dashmap::DashMap;

-use crate::{database, file_manager::File, params::Params};
+use crate::{file_manager::File, params::Params};

-pub fn duplicates(app_opts: &Params, connection: &sqlite::Connection) -> Result<Vec<File>> {
-    let scan_results = scan(app_opts, connection)?;
-    let base_path = app_opts.get_directory()?;
+pub fn duplicates(app_opts: &Params) -> Result<Vec<File>> {
+    let scan_results = scan(app_opts)?;
+    let index_store = index_files(scan_results)?;

-    index_files(scan_results, connection)?;
-    database::duplicate_hashes(connection, &base_path)
+    let duplicate_files = index_store
+        .into_par_iter()
+        .filter(|(_, files)| files.len() > 1)
+        .map(|(_, files)| files )
+        .flatten()
+        .collect::<Vec<File>>();
+
+    Ok(duplicate_files)
 }

-fn get_glob_patterns(opts: &Params, directory: &str) -> Vec<PathBuf> {
-    opts.types
-        .clone()
-        .unwrap_or_else(|| String::from("*"))
-        .split(',')
-        .map(|filetype| format!("*.{}", filetype))
-        .map(|filetype| {
-            vec![directory.to_owned(), String::from("**"), filetype]
-                .iter()
-                .collect()
-        })
-        .collect()
-}
-
-fn is_indexed_file(path: impl Into<String>, indexed: &[File]) -> bool {
-    indexed
-        .iter()
-        .map(|file| file.path.clone())
-        .contains(&path.into())
-}
-
-fn scan(app_opts: &Params, connection: &sqlite::Connection) -> Result<Vec<String>> {
-    let directory = app_opts.get_directory()?;
-    let glob_patterns: Vec<PathBuf> = get_glob_patterns(app_opts, &directory);
-    let indexed_paths = database::indexed_paths(connection)?;
+fn scan(app_opts: &Params) -> Result<Vec<String>> {
+    let glob_patterns: Vec<PathBuf> = app_opts.get_glob_patterns();
    let files: Vec<String> = glob_patterns
        .par_iter()
        .progress_with_style(
@@ -53,7 +38,6 @@ fn scan(app_opts: &Params, connection: &sqlite::Connection) -> Result<Vec<String
        .flat_map(|file_vec| {
            file_vec
                .filter_map(|x| Some(x.ok()?.as_os_str().to_str()?.to_string()))
-                .filter(|fpath| !is_indexed_file(fpath, &indexed_paths))
                .filter(|glob_result| {
                    fs::metadata(glob_result)
                        .map(|f| f.is_file())
@@ -66,29 +50,49 @@ fn scan(app_opts: &Params, connection: &sqlite::Connection) -> Result<Vec<String
    Ok(files)
 }

-fn index_files(files: Vec<String>, connection: &sqlite::Connection) -> Result<()> {
-    let hashed: Vec<File> = files
+fn index_files(files: Vec<String>) -> Result<DashMap<String, Vec<File>>> {
+    let store: DashMap<String, Vec<File>> = DashMap::new();
+    files
        .into_par_iter()
        .progress_with_style(
            ProgressStyle::with_template(
                "{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files",
-            )
-            .unwrap(),
+            )?,
        )
-        .filter_map(|file| {
-            let hash = hash_file(&file).ok()?;
-            Some(File { path: file, hash })
-        })
-        .collect();
+        .for_each(|file| {
+            let hash = hash_file(&file).unwrap_or_default();
+            let fobj = File { path: file, hash: hash.clone() };
+            store
+                .entry(hash)
+                .and_modify(|fileset| fileset.push(fobj.clone()) )
+                .or_insert_with(|| vec![fobj]);
+        });

-    hashed
-        .iter()
-        .try_for_each(|file| database::put(file, connection))
+        Ok(store)
+}
+
+pub fn incremental_hashing(filepath: &str) -> Result<String> {
+    let file = fs::File::open(filepath)?;
+    let fmap = unsafe { Mmap::map(&file)? };
+    let mut inchasher = fxhash::FxHasher::default();
+
+    fmap
+        .chunks(1_000)
+        .for_each(|kilo| { inchasher.write(kilo) });
+
+    Ok(format!("{}", inchasher.finish()))
+}
+
+pub fn standard_hashing(filepath: &str) -> Result<String> {
+    let file = fs::read(filepath)?;
+    Ok(hasher(&*file).to_string())
 }

 pub fn hash_file(filepath: &str) -> Result<String> {
-    let file = fs::read(filepath)?;
-    let hash = hasher(&*file).to_string();
+    let filemeta = fs::metadata(filepath)?;

-    Ok(hash)
+    match filemeta.len() < 1_000_000 {
+        true => standard_hashing(filepath),
+        false => incremental_hashing(filepath)
+    }
 }