diff --git a/Cargo.lock b/Cargo.lock index 7b7a3f2..1240734 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -70,12 +70,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" -[[package]] -name = "cassowary" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" - [[package]] name = "cc" version = "1.0.78" @@ -223,31 +217,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crossterm" -version = "0.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e64e6c0fbe2c17357405f7c758c1ef960fce08bdfb2c03d88d2a18d7e09c4b67" -dependencies = [ - "bitflags", - "crossterm_winapi", - "libc", - "mio", - "parking_lot", - "signal-hook", - "signal-hook-mio", - "winapi", -] - -[[package]] -name = "crossterm_winapi" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ae1b35a484aa10e07fe0638d02301c5ad24de82d310ccbd2f3693da5f09bf1c" -dependencies = [ - "winapi", -] - [[package]] name = "csv" version = "1.1.6" @@ -314,6 +283,20 @@ dependencies = [ "syn", ] +[[package]] +name = "dashmap" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" +dependencies = [ + "cfg-if", + "hashbrown", + "lock_api", + "once_cell", + "parking_lot_core", + "rayon", +] + [[package]] name = "deduplicator" version = "0.0.9" @@ -322,18 +305,18 @@ dependencies = [ "chrono", "clap", "colored", - "crossterm", + "dashmap", "fxhash", "glob", "humansize", "indicatif", "itertools", + "memmap2", "prettytable-rs", "rayon", "sqlite", "thiserror", "tokio", - "tui", "unicode-segmentation", ] @@ -423,6 +406,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "heck" version = "0.4.0" @@ -598,6 +587,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "memmap2" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b182332558b18d807c4ce1ca8ca983b34c3ee32765e47b3f0f69b90355cc1dc" +dependencies = [ + "libc", +] + [[package]] name = "memoffset" version = "0.7.1" @@ -855,27 +853,6 @@ version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" -[[package]] -name = "signal-hook" -version = "0.3.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a253b5e89e2698464fc26b545c9edceb338e18a89effeeecfea192c3025be29d" -dependencies = [ - "libc", - "signal-hook-registry", -] - -[[package]] -name = "signal-hook-mio" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" -dependencies = [ - "libc", - "mio", - "signal-hook", -] - [[package]] name = "signal-hook-registry" version = "1.4.0" @@ -1030,19 +1007,6 @@ dependencies = [ "syn", ] -[[package]] -name = "tui" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccdd26cbd674007e649a272da4475fb666d3aa0ad0531da7136db6fab0e5bad1" -dependencies = [ - "bitflags", - "cassowary", - "crossterm", - "unicode-segmentation", - "unicode-width", -] - [[package]] name = "unicode-ident" version = "1.0.6" diff --git a/Cargo.toml b/Cargo.toml index a38c053..5fd7a11 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,16 +13,16 @@ anyhow = "1.0.68" chrono = "0.4.23" clap = { version = "4.0.32", features = ["derive"] } colored = "2.0.0" -crossterm = "0.25.0" +dashmap = { version = "5.4.0", features = ["rayon"] } fxhash = "0.2.1" glob = "0.3.0" humansize = "2.1.2" indicatif = { version = "0.17.2", features = ["rayon", "tokio"] } itertools = "0.10.5" +memmap2 = "0.5.8" prettytable-rs = "0.10.0" rayon = "1.6.1" sqlite = "0.30.3" thiserror = "1.0.38" tokio = { version = "1.23.0", features = ["full"] } -tui = "0.19.0" unicode-segmentation = "1.10.0" diff --git a/README.md b/README.md index 726c36f..482ac1f 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,6 @@ Usage: deduplicator [OPTIONS] Options: -t, --types Filetypes to deduplicate (default = all) --dir Run Deduplicator on dir different from pwd - -n, --nocache Don't use cache for indexing files (default = false) -i, --interactive Delete files interactively -h, --help Print help information -V, --version Print version information diff --git a/src/app.rs b/src/app.rs index 3356b25..0409738 100644 --- a/src/app.rs +++ b/src/app.rs @@ -1,4 +1,3 @@ -use crate::database; use crate::output; use crate::params::Params; use crate::scanner; @@ -8,8 +7,7 @@ pub struct App; impl App { pub fn init(app_args: &Params) -> Result<()> { - let connection = database::get_connection(app_args)?; - let duplicates = scanner::duplicates(app_args, &connection)?; + let duplicates = scanner::duplicates(app_args)?; match app_args.interactive { true => output::interactive(duplicates, app_args), false => output::print(duplicates, app_args), diff --git a/src/database.rs b/src/database.rs deleted file mode 100644 index 3f8595c..0000000 --- a/src/database.rs +++ /dev/null @@ -1,82 +0,0 @@ -use std::env::temp_dir; -use anyhow::Result; -use crate::params::Params; -use crate::file_manager::File; - -fn db_connection_url(args: &Params) -> String { - match args.nocache { - true => String::from(":memory:"), - false => { - let temp_dir_path = temp_dir(); - format!("{}/deduplicator.db", temp_dir_path.display()) - } - } -} - -pub fn get_connection(args: &Params) -> Result { - sqlite::open(db_connection_url(args)).and_then(|conn| { - setup(&conn).ok(); - Ok(conn) - }) -} - -pub fn setup(connection: &sqlite::Connection) -> Result<()> { - let query = "CREATE TABLE files (file_identifier STRING, hash STRING)"; - connection.execute(query).ok(); - Ok(()) -} - -pub fn put(file: &File, connection: &sqlite::Connection) -> Result<()> { - let query = format!( - "INSERT INTO files (file_identifier, hash) VALUES (\"{}\", \"{}\")", - file.path, file.hash - ); - connection.execute(query)?; - Ok(()) -} - -pub fn indexed_paths(connection: &sqlite::Connection) -> Result> { - let query = "SELECT * FROM files"; - - let result: Vec = connection - .prepare(query)? - .into_iter() - .filter_map(|row_result| row_result.ok()) - .map(|row| { - let path = row.read::<&str, _>("file_identifier").to_string(); - let hash = row.read::("hash").to_string(); - File { path, hash } - }) - .collect(); - - Ok(result) -} - -pub fn duplicate_hashes(connection: &sqlite::Connection, path: &str) -> Result> { - let query = format!( - " - SELECT a.* FROM files a - JOIN (SELECT file_identifier, hash, COUNT(*) - FROM files - GROUP BY hash - HAVING count(*) > 1 ) b - ON a.hash = b.hash - WHERE a.file_identifier LIKE \"{}%\" - ORDER BY a.file_identifier - ", - path - ); - - let result: Vec = connection - .prepare(query)? - .into_iter() - .filter_map(|row_result| row_result.ok()) - .map(|row| { - let path = row.read::<&str, _>("file_identifier").to_string(); - let hash = row.read::("hash").to_string(); - File { path, hash } - }) - .collect(); - - Ok(result) -} diff --git a/src/main.rs b/src/main.rs index 25b703c..4a82824 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,4 @@ mod app; -mod database; mod file_manager; mod output; mod params; diff --git a/src/params.rs b/src/params.rs index 5694eec..0ab59b8 100644 --- a/src/params.rs +++ b/src/params.rs @@ -1,6 +1,6 @@ -use std::{fs, path::PathBuf}; use anyhow::{anyhow, Result}; use clap::Parser; +use std::{fs, path::PathBuf}; #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] @@ -11,9 +11,6 @@ pub struct Params { /// Run Deduplicator on dir different from pwd #[arg(long)] pub dir: Option, - /// Don't use cache for indexing files (default = false) - #[arg(long, short)] - pub nocache: bool, /// Delete files interactively #[arg(long, short)] pub interactive: bool, @@ -36,4 +33,18 @@ impl Params { Ok(dir) } + + pub fn get_glob_patterns(&self) -> Vec { + self.types + .clone() + .unwrap_or_else(|| String::from("*")) + .split(',') + .map(|filetype| format!("*.{}", filetype)) + .map(|filetype| { + vec![self.get_directory().unwrap(), String::from("**"), filetype] + .iter() + .collect() + }) + .collect() + } } diff --git a/src/scanner.rs b/src/scanner.rs index ba0fc6c..e4b8b4d 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -1,46 +1,31 @@ use anyhow::Result; -use fxhash::hash32 as hasher; +use fxhash::hash64 as hasher; use glob::glob; use indicatif::{ParallelProgressIterator, ProgressStyle}; -use itertools::Itertools; use rayon::prelude::*; +use std::hash::Hasher; use std::{fs, path::PathBuf}; +use memmap2::Mmap; +use dashmap::DashMap; -use crate::{database, file_manager::File, params::Params}; +use crate::{file_manager::File, params::Params}; -pub fn duplicates(app_opts: &Params, connection: &sqlite::Connection) -> Result> { - let scan_results = scan(app_opts, connection)?; - let base_path = app_opts.get_directory()?; +pub fn duplicates(app_opts: &Params) -> Result> { + let scan_results = scan(app_opts)?; + let index_store = index_files(scan_results)?; - index_files(scan_results, connection)?; - database::duplicate_hashes(connection, &base_path) + let duplicate_files = index_store + .into_par_iter() + .filter(|(_, files)| files.len() > 1) + .map(|(_, files)| files ) + .flatten() + .collect::>(); + + Ok(duplicate_files) } -fn get_glob_patterns(opts: &Params, directory: &str) -> Vec { - opts.types - .clone() - .unwrap_or_else(|| String::from("*")) - .split(',') - .map(|filetype| format!("*.{}", filetype)) - .map(|filetype| { - vec![directory.to_owned(), String::from("**"), filetype] - .iter() - .collect() - }) - .collect() -} - -fn is_indexed_file(path: impl Into, indexed: &[File]) -> bool { - indexed - .iter() - .map(|file| file.path.clone()) - .contains(&path.into()) -} - -fn scan(app_opts: &Params, connection: &sqlite::Connection) -> Result> { - let directory = app_opts.get_directory()?; - let glob_patterns: Vec = get_glob_patterns(app_opts, &directory); - let indexed_paths = database::indexed_paths(connection)?; +fn scan(app_opts: &Params) -> Result> { + let glob_patterns: Vec = app_opts.get_glob_patterns(); let files: Vec = glob_patterns .par_iter() .progress_with_style( @@ -53,7 +38,6 @@ fn scan(app_opts: &Params, connection: &sqlite::Connection) -> Result Result, connection: &sqlite::Connection) -> Result<()> { - let hashed: Vec = files +fn index_files(files: Vec) -> Result>> { + let store: DashMap> = DashMap::new(); + files .into_par_iter() .progress_with_style( ProgressStyle::with_template( "{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files", - ) - .unwrap(), + )?, ) - .filter_map(|file| { - let hash = hash_file(&file).ok()?; - Some(File { path: file, hash }) - }) - .collect(); + .for_each(|file| { + let hash = hash_file(&file).unwrap_or_default(); + let fobj = File { path: file, hash: hash.clone() }; + store + .entry(hash) + .and_modify(|fileset| fileset.push(fobj.clone()) ) + .or_insert_with(|| vec![fobj]); + }); - hashed - .iter() - .try_for_each(|file| database::put(file, connection)) + Ok(store) +} + +pub fn incremental_hashing(filepath: &str) -> Result { + let file = fs::File::open(filepath)?; + let fmap = unsafe { Mmap::map(&file)? }; + let mut inchasher = fxhash::FxHasher::default(); + + fmap + .chunks(1_000) + .for_each(|kilo| { inchasher.write(kilo) }); + + Ok(format!("{}", inchasher.finish())) +} + +pub fn standard_hashing(filepath: &str) -> Result { + let file = fs::read(filepath)?; + Ok(hasher(&*file).to_string()) } pub fn hash_file(filepath: &str) -> Result { - let file = fs::read(filepath)?; - let hash = hasher(&*file).to_string(); + let filemeta = fs::metadata(filepath)?; - Ok(hash) + match filemeta.len() < 1_000_000 { + true => standard_hashing(filepath), + false => incremental_hashing(filepath) + } }