filetype requirement removed

Cargo.lock

@@ -161,6 +161,49 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "crossbeam-channel"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521"
+dependencies = [
+ "cfg-if",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc"
+dependencies = [
+ "cfg-if",
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a"
+dependencies = [
+ "autocfg",
+ "cfg-if",
+ "crossbeam-utils",
+ "memoffset",
+ "scopeguard",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "cxx"
 version = "1.0.85"
@@ -217,6 +260,7 @@ dependencies = [
  "humansize",
  "itertools",
  "md5",
+ "rayon",
  "sha256",
  "sqlite",
  "thiserror",
@@ -442,6 +486,15 @@ version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
 
+[[package]]
+name = "memoffset"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
+dependencies = [
+ "autocfg",
+]
+
 [[package]]
 name = "mio"
 version = "0.8.5"
@@ -578,6 +631,28 @@ dependencies = [
  "proc-macro2",
 ]
 
+[[package]]
+name = "rayon"
+version = "1.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7"
+dependencies = [
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3"
+dependencies = [
+ "crossbeam-channel",
+ "crossbeam-deque",
+ "crossbeam-utils",
+ "num_cpus",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.2.16"

Cargo.toml

@@ -14,6 +14,7 @@ glob = "0.3.0"
 humansize = "2.1.2"
 itertools = "0.10.5"
 md5 = "0.7.0"
+rayon = "1.6.1"
 sha256 = "1.1.1"
 sqlite = "0.30.3"
 thiserror = "1.0.38"

src/main.rs

@@ -9,7 +9,6 @@ use anyhow::Result;
 #[tokio::main]
 async fn main() -> Result<()> {
     let connection = sqlite::open(":memory:")?;
-
     database::setup(&connection)?;
 
     let duplicates = scanner::duplicates(cli::App::parse(), &connection)?;

src/scanner.rs

@@ -5,6 +5,7 @@ use crate::database;
 use anyhow::Result;
 use glob::glob;
 use std::fs;
+use rayon::prelude::*;
 
 pub fn duplicates(app_opts: App, connection: &sqlite::Connection) -> Result<Vec<File>> {
     index_files(scan(app_opts)?, connection);
@@ -33,12 +34,13 @@ fn scan(app_opts: App) -> Result<Vec<String>> {
         .collect();
 
     let files: Vec<String> = glob_patterns
-        .into_iter()
+        .into_par_iter()
         .map(|glob_pattern| glob(&glob_pattern.as_os_str().to_str().unwrap()))
         .map(|glob_result| glob_result.unwrap())
         .flat_map(|file_vec| {
             file_vec
                 .map(|x| x.unwrap().as_os_str().to_str().unwrap().to_string())
+                .filter(|glob_result| fs::metadata(glob_result).unwrap().is_file() )
                 .collect::<Vec<String>>()
         })
         .collect();