diff --git a/Cargo.lock b/Cargo.lock index 23e147c..c2ce161 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -161,6 +161,49 @@ dependencies = [ "libc", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" +dependencies = [ + "cfg-if", +] + [[package]] name = "cxx" version = "1.0.85" @@ -217,6 +260,7 @@ dependencies = [ "humansize", "itertools", "md5", + "rayon", "sha256", "sqlite", "thiserror", @@ -442,6 +486,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "memoffset" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +dependencies = [ + "autocfg", +] + [[package]] name = "mio" version = "0.8.5" @@ -578,6 +631,28 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rayon" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + [[package]] name = "redox_syscall" version = "0.2.16" diff --git a/Cargo.toml b/Cargo.toml index 1b52dcc..0bdfc20 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ glob = "0.3.0" humansize = "2.1.2" itertools = "0.10.5" md5 = "0.7.0" +rayon = "1.6.1" sha256 = "1.1.1" sqlite = "0.30.3" thiserror = "1.0.38" diff --git a/src/main.rs b/src/main.rs index f7b003f..7a37c97 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,7 +9,6 @@ use anyhow::Result; #[tokio::main] async fn main() -> Result<()> { let connection = sqlite::open(":memory:")?; - database::setup(&connection)?; let duplicates = scanner::duplicates(cli::App::parse(), &connection)?; diff --git a/src/scanner.rs b/src/scanner.rs index 17cc335..3305f93 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -5,6 +5,7 @@ use crate::database; use anyhow::Result; use glob::glob; use std::fs; +use rayon::prelude::*; pub fn duplicates(app_opts: App, connection: &sqlite::Connection) -> Result> { index_files(scan(app_opts)?, connection); @@ -33,12 +34,13 @@ fn scan(app_opts: App) -> Result> { .collect(); let files: Vec = glob_patterns - .into_iter() + .into_par_iter() .map(|glob_pattern| glob(&glob_pattern.as_os_str().to_str().unwrap())) .map(|glob_result| glob_result.unwrap()) .flat_map(|file_vec| { file_vec .map(|x| x.unwrap().as_os_str().to_str().unwrap().to_string()) + .filter(|glob_result| fs::metadata(glob_result).unwrap().is_file() ) .collect::>() }) .collect();