diff --git a/Cargo.lock b/Cargo.lock index c2ce161..d84c0e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -40,15 +40,6 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" -[[package]] -name = "block-buffer" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" -dependencies = [ - "generic-array", -] - [[package]] name = "bumpalo" version = "3.11.1" @@ -152,15 +143,6 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" -[[package]] -name = "cpufeatures" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" -dependencies = [ - "libc", -] - [[package]] name = "crossbeam-channel" version = "0.5.6" @@ -259,21 +241,11 @@ dependencies = [ "glob", "humansize", "itertools", - "md5", "rayon", - "sha256", "sqlite", "thiserror", "tokio", -] - -[[package]] -name = "digest" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" -dependencies = [ - "generic-array", + "xxhash-rust", ] [[package]] @@ -303,16 +275,6 @@ dependencies = [ "libc", ] -[[package]] -name = "generic-array" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" -dependencies = [ - "typenum", - "version_check", -] - [[package]] name = "glob" version = "0.3.0" @@ -343,12 +305,6 @@ dependencies = [ "libc", ] -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - [[package]] name = "humansize" version = "2.1.2" @@ -474,12 +430,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "md5" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" - [[package]] name = "memchr" version = "2.5.0" @@ -542,12 +492,6 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" -[[package]] -name = "opaque-debug" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" - [[package]] name = "os_str_bytes" version = "6.4.1" @@ -688,29 +632,6 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" -[[package]] -name = "sha2" -version = "0.9.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" -dependencies = [ - "block-buffer", - "cfg-if", - "cpufeatures", - "digest", - "opaque-debug", -] - -[[package]] -name = "sha256" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e334db67871c14c18fc066ad14af13f9fdf5f9a91c61af432d1e3a39c8c6a141" -dependencies = [ - "hex", - "sha2", -] - [[package]] name = "signal-hook-registry" version = "1.4.0" @@ -854,12 +775,6 @@ dependencies = [ "syn", ] -[[package]] -name = "typenum" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" - [[package]] name = "unicode-ident" version = "1.0.6" @@ -1031,3 +946,9 @@ name = "windows_x86_64_msvc" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" + +[[package]] +name = "xxhash-rust" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "735a71d46c4d68d71d4b24d03fdc2b98e38cea81730595801db779c04fe80d70" diff --git a/Cargo.toml b/Cargo.toml index 0bdfc20..d047305 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,9 +13,8 @@ colored = "2.0.0" glob = "0.3.0" humansize = "2.1.2" itertools = "0.10.5" -md5 = "0.7.0" rayon = "1.6.1" -sha256 = "1.1.1" sqlite = "0.30.3" thiserror = "1.0.38" tokio = { version = "1.23.0", features = ["full"] } +xxhash-rust = { version = "0.8.6", features = ["xxh3"] } diff --git a/src/cli.rs b/src/cli.rs index b441a91..ffe1b33 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -12,5 +12,8 @@ pub struct App { pub delete: Option, /// Run Deduplicator on dir different from pwd #[arg(long)] - pub dir: Option + pub dir: Option, + /// Don't use cache for indexing files (default = true) + #[arg(long, short)] + pub nocache: bool } diff --git a/src/database.rs b/src/database.rs index e0e24fd..147d997 100644 --- a/src/database.rs +++ b/src/database.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use crate::cli::App; #[derive(Debug, Clone)] pub struct File { @@ -6,6 +7,18 @@ pub struct File { pub hash: String, } +pub fn get_connection(args: &App) -> Result { + let connection_url = match args.nocache { + false => "/tmp/deduplicator.db", + true => ":memory:" + }; + + sqlite::open(connection_url).and_then(|conn| { + setup(&conn).ok(); + Ok(conn) + }) +} + pub fn setup(connection: &sqlite::Connection) -> Result<()> { let query = "CREATE TABLE files (file_identifier STRING, hash STRING)"; connection.execute(query).ok(); @@ -33,7 +46,7 @@ pub fn indexed_paths(connection: &sqlite::Connection) -> Result> { .map(|row_result| row_result.unwrap()) .map(|row| { let path = row.read::<&str, _>("file_identifier").to_string(); - let hash = row.read::<&str, _>("hash").to_string(); + let hash = row.read::("hash").to_string(); File { path, hash } }) .collect(); @@ -60,7 +73,7 @@ pub fn duplicate_hashes(connection: &sqlite::Connection, path: &String) -> Resul .map(|row_result| row_result.unwrap()) .map(|row| { let path = row.read::<&str, _>("file_identifier").to_string(); - let hash = row.read::<&str, _>("hash").to_string(); + let hash = row.read::("hash").to_string(); File { path, hash } }) .collect(); diff --git a/src/main.rs b/src/main.rs index a0b8c53..8a9a815 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,13 +8,10 @@ use clap::Parser; #[tokio::main] async fn main() -> Result<()> { - let connection = sqlite::open("/tmp/deduplicator.db").and_then(|conn| { - database::setup(&conn).ok(); - Ok(conn) - })?; + let app_args = cli::App::parse(); + let connection = database::get_connection(&app_args)?; + let duplicates = scanner::duplicates(&app_args, &connection)?; - let duplicates = scanner::duplicates(cli::App::parse(), &connection)?; output::print(duplicates); - Ok(()) } diff --git a/src/scanner.rs b/src/scanner.rs index 1fa1c5d..977b302 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -6,10 +6,11 @@ use itertools::Itertools; use rayon::prelude::*; use std::fs; use std::path::PathBuf; +use xxhash_rust::xxh3::xxh3_64 as hasher; -pub fn duplicates(app_opts: App, connection: &sqlite::Connection) -> Result> { - let scan_results = scan(&app_opts, connection)?; - let base_path = get_directory(&app_opts)?; +pub fn duplicates(app_opts: &App, connection: &sqlite::Connection) -> Result> { + let scan_results = scan(app_opts, connection)?; + let base_path = get_directory(app_opts)?; index_files(scan_results, connection); database::duplicate_hashes(connection, &base_path) @@ -92,7 +93,7 @@ fn index_files(files: Vec, connection: &sqlite::Connection) { pub fn hash_file(filepath: &str) -> Result { let file = fs::read(filepath)?; - let hash = sha256::digest(&*file); + let hash = hasher(&*file).to_string(); Ok(hash) }