Merge pull request #32 from sreedevk/performance/glob_improvements

Performance/glob improvements

src/output.rs

@@ -40,10 +40,6 @@ fn modified_time(path: &String) -> Result<String> {
     Ok(modified_time.format("%Y-%m-%d %H:%M:%S").to_string())
 }
 
-fn print_meta_info() {
-    println!("Deduplicator v{}", std::env!("CARGO_PKG_VERSION"));
-}
-
 fn scan_group_instruction() -> Result<String> {
     println!("\nEnter the indices of the files you want to delete.");
     println!("You can enter multiple files using commas to seperate file indices.");
@@ -115,8 +111,6 @@ fn process_group_action(duplicates: &Vec<File>, dup_index: usize, dup_size: usiz
 }
 
 pub fn interactive(duplicates: DashMap<String, Vec<File>>, opts: &Params) {
-    print_meta_info();
-
     if duplicates.is_empty() {
         println!(
             "\n{}",
@@ -128,8 +122,8 @@ pub fn interactive(duplicates: DashMap<String, Vec<File>>, opts: &Params) {
     duplicates
         .clone()
         .into_iter()
-        .sorted_unstable_by_key(|f| {
-            -(f.1.first().and_then(|ff| ff.size).unwrap_or_default() as i64)
+        .sorted_unstable_by_key(|(_, f)| {
+            -(f.first().and_then(|ff| ff.size).unwrap_or_default() as i64)
         }) // sort by descending file size in interactive mode
         .enumerate()
         .for_each(|(gindex, (_, group))| {
@@ -140,7 +134,7 @@ pub fn interactive(duplicates: DashMap<String, Vec<File>>, opts: &Params) {
                 itable.add_row(row![
                     index,
                     format_path(&file.path, opts).unwrap_or_default().blue(),
-                    file_size(&file).unwrap_or_default().red(),
+                    file_size(file).unwrap_or_default().red(),
                     modified_time(&file.path).unwrap_or_default().yellow()
                 ]);
             });
@@ -150,8 +144,6 @@ pub fn interactive(duplicates: DashMap<String, Vec<File>>, opts: &Params) {
 }
 
 pub fn print(duplicates: DashMap<String, Vec<File>>, opts: &Params) {
-    print_meta_info();
-
     if duplicates.is_empty() {
         println!(
             "\n{}",
@@ -164,14 +156,14 @@ pub fn print(duplicates: DashMap<String, Vec<File>>, opts: &Params) {
     output_table.set_titles(row!["hash", "duplicates"]);
     duplicates
         .into_iter()
-        .sorted_unstable_by_key(|f| f.1.first().and_then(|ff| ff.size).unwrap_or_default()) // sort by ascending size
+        .sorted_unstable_by_key(|(_, f)| f.first().and_then(|ff| ff.size).unwrap_or_default())
         .for_each(|(hash, group)| {
             let mut inner_table = Table::new();
             inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
             group.iter().for_each(|file| {
                 inner_table.add_row(row![
                     format_path(&file.path, opts).unwrap_or_default().blue(),
-                    file_size(&file).unwrap_or_default().red(),
+                    file_size(file).unwrap_or_default().red(),
                     modified_time(&file.path).unwrap_or_default().yellow()
                 ]);
             });

src/params.rs

@@ -33,8 +33,8 @@ impl Params {
     pub fn get_directory(&self) -> Result<String> {
         let dir_pathbuf: PathBuf = self
             .dir
-            .clone()
-            .unwrap_or(std::env::current_dir()?)
+            .as_ref()
+            .unwrap_or(&std::env::current_dir()?)
             .as_os_str()
             .into();
 
@@ -47,17 +47,18 @@ impl Params {
         Ok(dir)
     }
 
-    pub fn get_glob_patterns(&self) -> Vec<PathBuf> {
-        self.types
-            .clone()
-            .unwrap_or_else(|| String::from("*"))
-            .split(',')
-            .map(|filetype| format!("*.{}", filetype))
-            .map(|filetype| {
-                vec![self.get_directory().unwrap(), String::from("**"), filetype]
-                    .iter()
-                    .collect()
-            })
-            .collect()
+    pub fn get_glob_patterns(&self) -> PathBuf {
+        match self.types.as_ref() {
+            Some(filetypes) => vec![
+                self.get_directory().unwrap(),
+                String::from("**"),
+                format!("{{{}}}", filetypes),
+            ]
+            .iter()
+            .collect::<PathBuf>(),
+            None => vec![self.get_directory().unwrap().as_str(), "**", "*"]
+                .iter()
+                .collect::<PathBuf>(),
+        }
     }
 }

src/scanner.rs

@@ -40,27 +40,22 @@ pub fn duplicates(app_opts: &Params) -> Result<DashMap<String, Vec<File>>> {
 }
 
 fn scan(app_opts: &Params) -> Result<Vec<File>> {
-    let glob_patterns: Vec<PathBuf> = app_opts.get_glob_patterns();
-    let files: Vec<File> = glob_patterns
-        .par_iter()
+    let glob_patterns = app_opts.get_glob_patterns().display().to_string();
+    let glob_iter = glob(&glob_patterns)?;
+    let files = glob_iter
+        .filter(Result::is_ok)
+        .map(|file| file.unwrap())
+        .filter(|fpath| fpath.is_file())
+        .collect::<Vec<PathBuf>>()
+        .into_par_iter()
         .progress_with_style(ProgressStyle::with_template(
-            "{spinner:.green} [scanning files] [{wide_bar:.cyan/blue}] {pos}/{len} files",
+            "{spinner:.green} [processing scan results] [{wide_bar:.cyan/blue}] {pos}/{len} files",
         )?)
-        .filter_map(|glob_pattern| glob(glob_pattern.as_os_str().to_str()?).ok())
-        .flat_map(|file_vec| {
-            file_vec
-                .filter_map(|x| Some(x.ok()?.as_os_str().to_str()?.to_string()))
-                .filter(|glob_result| {
-                    fs::metadata(glob_result)
-                        .map(|f| f.is_file())
-                        .unwrap_or(false)
-                })
-                .collect::<Vec<String>>()
-        })
-        .map(|file_path| File {
-            path: file_path.clone(),
+        .map(|fpath| fpath.display().to_string())
+        .map(|fpath| File {
+            path: fpath.clone(),
             hash: None,
-            size: Some(fs::metadata(file_path).unwrap().len()),
+            size: Some(fs::metadata(fpath).unwrap().len()),
         })
         .filter(|file| filters::is_file_gt_minsize(app_opts, file))
         .collect();
@@ -68,16 +63,8 @@ fn scan(app_opts: &Params) -> Result<Vec<File>> {
     Ok(files)
 }
 
-fn process_file_hash_index(file: &File) -> Result<File> {
-    Ok(File {
-        path: file.path.clone(),
-        size: file.size,
-        hash: Some(hash_file(&file.path).unwrap_or_default()),
-    })
-}
-
 fn process_file_index(
-    file: File,
+    mut file: File,
     store: &DashMap<String, Vec<File>>,
     index_criteria: IndexCritera,
 ) {
@@ -89,13 +76,11 @@ fn process_file_index(
                 .or_insert_with(|| vec![file]);
         }
         IndexCritera::Hash => {
-            let processed_file = process_file_hash_index(&file).unwrap();
-            let indexhash = processed_file.clone().hash.unwrap_or_default();
-
+            file.hash = Some(hash_file(&file.path).unwrap_or_default());
             store
-                .entry(indexhash)
-                .and_modify(|fileset| fileset.push(processed_file.clone()))
-                .or_insert_with(|| vec![processed_file]);
+                .entry(file.clone().hash.unwrap())
+                .and_modify(|fileset| fileset.push(file.clone()))
+                .or_insert_with(|| vec![file]);
         }
     }
 }