v0.2.0

.gitignore

@@ -1,3 +1,2 @@
 /target
-/test_data
+/Cargo.lock
-.envrc

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deduplicator"
-version = "0.1.6"
+version = "0.2.0"
 edition = "2021"
 description = "find,filter,delete Duplicates"
 license = "MIT"
@@ -24,6 +24,7 @@ globwalk = "0.8.1"
 indicatif = { version = "0.17.2", features = ["rayon"] }
 itertools = "0.10.5"
 memmap2 = "0.5.8"
+pathdiff = "0.2.1"
 prettytable-rs = "0.10.0"
 rayon = "1.6.1"
 unicode-segmentation = "1.10.0"
@@ -33,4 +34,3 @@ unicode-segmentation = "1.10.0"
 inherits = "release"
 debug = true
 split-debuginfo = "packed"
-

examples/scanner.rs

@@ -0,0 +1,20 @@
+use std::path::PathBuf;
+
+use anyhow::Result;
+use deduplicator_core::processor::Processor;
+use deduplicator_core::scanner::Scanner;
+
+fn main() -> Result<()> {
+    let scan_results: Vec<_> = Scanner::new()
+        .directory(PathBuf::from("/home/sreedev/Data/books/"))
+        .ignore_links().scan(None)?;
+
+    let processor = Processor::new(scan_results);
+    let final_processor = processor.sizewise()?.hashwise()?;
+    let display_vec = final_processor.files.into_iter().map(|file| file.path ).collect::<Vec<PathBuf>>();
+
+    dbg!(display_vec);
+
+
+    Ok(())
+}

src/app.rs

@@ -1,18 +0,0 @@
-use crate::output;
-use crate::params::Params;
-use crate::scanner;
-use anyhow::Result;
-
-pub struct App;
-
-impl App {
-    pub fn init(app_args: &Params) -> Result<()> {
-        let duplicates = scanner::duplicates(app_args)?;
-        match app_args.interactive {
-            true => output::interactive(duplicates, app_args),
-            false => output::print(duplicates, app_args),
-        }
-
-        Ok(())
-    }
-}

src/file_manager.rs

@@ -1,21 +0,0 @@
-use anyhow::Result;
-use colored::Colorize;
-use std::path::PathBuf;
-
-#[derive(Debug, Clone)]
-pub struct File {
-    pub path: PathBuf,
-    pub size: Option<u64>,
-    pub hash: Option<String>,
-}
-
-pub fn delete_files(files: Vec<File>) -> Result<()> {
-    files.into_iter().for_each(|file| {
-        match std::fs::remove_file(file.path.clone()) {
-            Ok(_) => println!("{}: {}", "DELETED".green(), file.path.display()),
-            Err(_) => println!("{}: {}", "FAILED".red(), file.path.display())
-        }
-    });
-
-    Ok(())
-}

src/fileinfo.rs

@@ -0,0 +1,41 @@
+use anyhow::Result;
+use memmap2::Mmap;
+use std::fs;
+use std::hash::Hasher;
+use std::{fs::Metadata, path::PathBuf};
+
+#[derive(Debug, Clone)]
+pub struct FileInfo {
+    pub path: PathBuf,
+    pub filemeta: Metadata,
+    pub hash: Option<String>,
+    pub size: u64,
+}
+
+impl FileInfo {
+    pub fn hash(&self) -> Result<Self> {
+        let file = fs::File::open(self.path.clone())?;
+        let mapper = unsafe { Mmap::map(&file)? };
+        let mut primhasher = fxhash::FxHasher::default();
+
+        mapper
+            .chunks(1_000_000)
+            .into_iter()
+            .for_each(|chunk| primhasher.write(chunk));
+
+        Ok(Self {
+            hash: Some(primhasher.finish().to_string()),
+            ..self.clone()
+        })
+    }
+
+    pub fn new(path: PathBuf) -> Result<Self> {
+        let filemeta = std::fs::metadata(path.clone())?;
+        Ok(Self {
+            path,
+            filemeta: filemeta.clone(),
+            hash: None,
+            size: filemeta.len(),
+        })
+    }
+}

src/filters.rs

@@ -1,12 +0,0 @@
-use crate::file_manager::File;
-use crate::params::Params;
-
-pub fn is_file_gt_min_size(app_opts: &Params, file: &File) -> bool {
-    match app_opts.get_min_size() {
-        Some(msize) => match file.size {
-            Some(fsize) => fsize >= msize,
-            None => true,
-        },
-        None => true,
-    }
-}

src/formatter.rs

@@ -0,0 +1,111 @@
+pub struct Formatter;
+use crate::fileinfo::FileInfo;
+use crate::params::Params;
+use anyhow::Result;
+use chrono::{DateTime, Utc};
+use colored::Colorize;
+use dashmap::DashMap;
+use indicatif::{ProgressIterator, ParallelProgressIterator, ProgressBar, ProgressFinish, ProgressStyle};
+use pathdiff::diff_paths;
+use prettytable::{format, row, Table};
+use rayon::prelude::*;
+use std::borrow::Cow;
+use std::path::PathBuf;
+use std::time::Duration;
+
+impl Formatter {
+    fn human_path(file: &FileInfo, app_args: &Params, min_path_length: usize) -> Result<String> {
+        let base_directory: PathBuf = app_args.get_directory()?.to_path_buf();
+        let relative_path = diff_paths(file.path.clone(), base_directory).unwrap_or_default();
+
+        let formatted_path = format!(
+            "{:<0width$}",
+            relative_path.to_str().unwrap_or_default().to_string(),
+            width = min_path_length
+        );
+
+        Ok(formatted_path)
+    }
+
+    fn human_filesize(file: &FileInfo) -> Result<String> {
+        Ok(format!("{:>12}", bytesize::ByteSize::b(file.size)))
+    }
+
+    fn human_mtime(file: &FileInfo) -> Result<String> {
+        let modified_time: DateTime<Utc> = file.filemeta.modified()?.into();
+        Ok(modified_time.format("%Y-%m-%d %H:%M:%S").to_string())
+    }
+
+    pub fn generate_table(raw: Vec<FileInfo>, app_args: &Params) -> Result<Table> {
+        let basepath_length = app_args.get_directory()?.to_str().unwrap_or_default().len();
+        let max_filepath_length = raw
+            .iter()
+            .map(|file| file.path.to_str().unwrap_or_default().len())
+            .max()
+            .unwrap_or_default();
+        let min_path_length = max_filepath_length - basepath_length;
+
+        let progress_style = ProgressStyle::with_template(
+            "[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}",
+        )?;
+        let progress_bar = ProgressBar::new(raw.len() as u64);
+        progress_bar.set_style(progress_style);
+        progress_bar.enable_steady_tick(Duration::from_millis(50));
+        progress_bar.set_message("reconciling data");
+
+        let duplicates_table: DashMap<String, Vec<FileInfo>> = DashMap::new();
+        raw.clone()
+            .into_par_iter()
+            .progress_with(progress_bar)
+            .with_finish(ProgressFinish::WithMessage(Cow::from("data reconciled")))
+            .map(|file| file.hash())
+            .filter_map(Result::ok)
+            .for_each(|file| {
+                duplicates_table
+                    .entry(file.hash.clone().unwrap_or_default())
+                    .and_modify(|fileset| fileset.push(file.clone()))
+                    .or_insert_with(|| vec![file]);
+            });
+
+        let mut output_table = Table::new();
+        output_table.set_titles(row!["hash", "duplicates"]);
+
+        let progress_style = ProgressStyle::with_template(
+            "[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}",
+        )?;
+
+        let progress_bar = ProgressBar::new(duplicates_table.len() as u64);
+        progress_bar.set_style(progress_style);
+        progress_bar.enable_steady_tick(Duration::from_millis(50));
+        progress_bar.set_message("generating output");
+
+        duplicates_table
+            .into_iter()
+            .progress_with(progress_bar)
+            .with_finish(ProgressFinish::WithMessage(Cow::from("output generated")))
+            .for_each(|(hash, group)| {
+                let mut inner_table = Table::new();
+                inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
+                group.iter().for_each(|file| {
+                    inner_table.add_row(row![
+                        Self::human_path(&file, &app_args, min_path_length)
+                            .unwrap_or_default()
+                            .blue(),
+                        Self::human_filesize(&file).unwrap_or_default().red(),
+                        Self::human_mtime(&file).unwrap_or_default().yellow()
+                    ]);
+                });
+
+                output_table.add_row(row![hash.green(), inner_table]);
+            });
+
+        Ok(output_table)
+    }
+
+    pub fn print(raw: Vec<FileInfo>, app_args: &Params) -> Result<()> {
+        let output_table = Self::generate_table(raw, app_args)?;
+        output_table.printstd();
+
+        Ok(())
+    }
+}

src/interative.rs

@@ -0,0 +1,85 @@
+use anyhow::Result;
+use colored::Colorize;
+use crate::fileinfo::FileInfo;
+use prettytable::Table;
+use std::io::{self, Write};
+
+pub fn scan_group_confirmation() -> Result<bool> {
+    print!("\nconfirm? [y/N]: ");
+    std::io::stdout().flush()?;
+    let mut user_input = String::new();
+    io::stdin().read_line(&mut user_input)?;
+
+    match user_input.trim() {
+        "Y" | "y" => Ok(true),
+        _ => Ok(false),
+    }
+}
+
+pub fn scan_group_instruction() -> Result<String> {
+    println!("\nEnter the indices of the files you want to delete.");
+    println!("You can enter multiple files using commas to seperate file indices.");
+    println!("example: 1,2");
+    print!("\n> ");
+    std::io::stdout().flush()?;
+    let mut user_input = String::new();
+    io::stdin().read_line(&mut user_input)?;
+
+    Ok(user_input)
+}
+
+pub fn process_group_action(
+    duplicates: &Vec<FileInfo>,
+    dup_index: usize,
+    dup_size: usize,
+    table: Table,
+) {
+    println!("\nDuplicate Set {} of {}\n", dup_index + 1, dup_size);
+    table.printstd();
+    let files_to_delete = scan_group_instruction().unwrap_or_default();
+    let parsed_file_indices = files_to_delete
+        .trim()
+        .split(',')
+        .filter(|element| !element.is_empty())
+        .map(|index| index.parse::<usize>().unwrap_or_default())
+        .collect::<Vec<usize>>();
+
+    if parsed_file_indices
+        .clone()
+        .into_iter()
+        .any(|index| index > (duplicates.len() - 1))
+    {
+        println!("{}", "Err: File Index Out of Bounds!".red());
+        return process_group_action(duplicates, dup_index, dup_size, table);
+    }
+
+    print!("{esc}[2J{esc}[1;1H", esc = 27 as char);
+
+    if parsed_file_indices.is_empty() {
+        return;
+    }
+
+    let files_to_delete = parsed_file_indices
+        .into_iter()
+        .map(|index| duplicates[index].clone());
+
+    println!("\n{}", "The following files will be deleted:".red());
+    files_to_delete
+        .clone()
+        .enumerate()
+        .for_each(|(index, file)| {
+            println!("{}: {}", index.to_string().blue(), file.path.display());
+        });
+
+    match scan_group_confirmation().unwrap() {
+        true => {
+            files_to_delete.into_iter().for_each(|file| {
+                match std::fs::remove_file(file.path.clone()) {
+                    Ok(_) => println!("{}: {}", "DELETED".green(), file.path.display()),
+                    Err(_) => println!("{}: {}", "FAILED".red(), file.path.display()),
+                }
+            });
+        }
+        false => println!("{}", "\nCancelled Delete Operation.".red()),
+    }
+}

src/main.rs

@@ -1,14 +1,23 @@
-mod app;
+mod fileinfo;
-mod file_manager;
+mod processor;
-mod output;
-mod params;
 mod scanner;
-mod filters;
+mod params;
+mod formatter;
+mod interative;
 
 use anyhow::Result;
-use app::App;
+use params::Params;
+use scanner::Scanner;
+use processor::Processor;
 use clap::Parser;
+use formatter::Formatter;
 
 fn main() -> Result<()> {
-    App::init(&params::Params::parse())
+    let app_args = Params::parse();
+    let scan_results = Scanner::build(&app_args)?.scan()?;
+    let processor = Processor::new(scan_results);
+    let results = processor.sizewise()?.hashwise()?;
+
+    Formatter::print(results.files, &app_args)?;
+    Ok(())
 }

src/output.rs

@@ -1,217 +0,0 @@
-use crate::file_manager::{self, File};
-use crate::params::Params;
-use anyhow::Result;
-use chrono::offset::Utc;
-use chrono::DateTime;
-use colored::Colorize;
-use dashmap::DashMap;
-use indicatif::{ProgressBar, ProgressIterator, ProgressStyle};
-use itertools::Itertools;
-use prettytable::{format, row, Table};
-use std::io::Write;
-use std::path::Path;
-use std::time::Duration;
-use std::{fs, io};
-use unicode_segmentation::UnicodeSegmentation;
-
-fn format_path(path: &Path, opts: &Params) -> Result<String> {
-    let display_path = path
-        .to_string_lossy()
-        .replace(opts.get_directory()?.to_string_lossy().as_ref(), "");
-    let display_range = if display_path.chars().count() > 32 {
-        display_path
-            .graphemes(true)
-            .collect::<Vec<&str>>()
-            .into_iter()
-            .rev()
-            .take(32)
-            .rev()
-            .collect()
-    } else {
-        display_path
-    };
-
-    Ok(format!("...{display_range:<32}"))
-}
-
-fn file_size(file: &File) -> Result<String> {
-    Ok(format!("{:>12}", bytesize::ByteSize::b(file.size.unwrap())))
-}
-
-fn modified_time(path: &Path) -> Result<String> {
-    let mdata = fs::metadata(path)?;
-    let modified_time: DateTime<Utc> = mdata.modified()?.into();
-
-    Ok(modified_time.format("%Y-%m-%d %H:%M:%S").to_string())
-}
-
-fn scan_group_instruction() -> Result<String> {
-    println!("\nEnter the indices of the files you want to delete.");
-    println!("You can enter multiple files using commas to seperate file indices.");
-    println!("example: 1,2");
-    print!("\n> ");
-    std::io::stdout().flush()?;
-    let mut user_input = String::new();
-    io::stdin().read_line(&mut user_input)?;
-
-    Ok(user_input)
-}
-
-fn scan_group_confirmation() -> Result<bool> {
-    print!("\nconfirm? [y/N]: ");
-    std::io::stdout().flush()?;
-    let mut user_input = String::new();
-    io::stdin().read_line(&mut user_input)?;
-
-    match user_input.trim() {
-        "Y" | "y" => Ok(true),
-        _ => Ok(false),
-    }
-}
-
-fn process_group_action(duplicates: &Vec<File>, dup_index: usize, dup_size: usize, table: Table) {
-    println!("\nDuplicate Set {} of {}\n", dup_index + 1, dup_size);
-    table.printstd();
-    let files_to_delete = scan_group_instruction().unwrap_or_default();
-    let parsed_file_indices = files_to_delete
-        .trim()
-        .split(',')
-        .filter(|element| !element.is_empty())
-        .map(|index| index.parse::<usize>().unwrap_or_default())
-        .collect::<Vec<usize>>();
-
-    if parsed_file_indices
-        .clone()
-        .into_iter()
-        .any(|index| index > (duplicates.len() - 1))
-    {
-        println!("{}", "Err: File Index Out of Bounds!".red());
-        return process_group_action(duplicates, dup_index, dup_size, table);
-    }
-
-    print!("{esc}[2J{esc}[1;1H", esc = 27 as char);
-
-    if parsed_file_indices.is_empty() {
-        return;
-    }
-
-    let files_to_delete = parsed_file_indices
-        .into_iter()
-        .map(|index| duplicates[index].clone());
-
-    println!("\n{}", "The following files will be deleted:".red());
-    files_to_delete
-        .clone()
-        .enumerate()
-        .for_each(|(index, file)| {
-            println!("{}: {}", index.to_string().blue(), file.path.display());
-        });
-
-    match scan_group_confirmation().unwrap() {
-        true => {
-            file_manager::delete_files(files_to_delete.collect::<Vec<File>>()).ok();
-        }
-        false => println!("{}", "\nCancelled Delete Operation.".red()),
-    }
-}
-
-pub fn interactive(duplicates: DashMap<String, Vec<File>>, opts: &Params) {
-    if duplicates.is_empty() {
-        println!(
-            "\n{}",
-            "No duplicates found matching your search criteria.".green()
-        );
-        return;
-    }
-
-    duplicates
-        .clone()
-        .into_iter()
-        .sorted_unstable_by_key(|(_, f)| {
-            -(f.first().and_then(|ff| ff.size).unwrap_or_default() as i64)
-        }) // sort by descending file size in interactive mode
-        .enumerate()
-        .for_each(|(gindex, (_, group))| {
-            let mut itable = Table::new();
-            itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
-            itable.set_titles(row!["index", "filename", "size", "updated_at"]);
-            group.iter().enumerate().for_each(|(index, file)| {
-                itable.add_row(row![
-                    index,
-                    format_path(&file.path, opts).unwrap_or_default().blue(),
-                    file_size(file).unwrap_or_default().red(),
-                    modified_time(&file.path).unwrap_or_default().yellow()
-                ]);
-            });
-
-            process_group_action(&group, gindex, duplicates.len(), itable);
-        });
-}
-
-pub fn print(duplicates: DashMap<String, Vec<File>>, opts: &Params) {
-    if duplicates.is_empty() {
-        println!(
-            "\n{}",
-            "No duplicates found matching your search criteria.".green()
-        );
-        return;
-    }
-
-    let mut output_table = Table::new();
-    let progress_bar = ProgressBar::new(duplicates.len() as u64);
-    progress_bar.enable_steady_tick(Duration::from_millis(50));
-    let progress_style = ProgressStyle::default_bar()
-        .template("{spinner:.green} [generating output] [{wide_bar:.cyan/blue}] {pos}/{len} files")
-        .unwrap();
-
-    progress_bar.set_style(progress_style);
-    output_table.set_titles(row!["hash", "duplicates"]);
-
-    duplicates
-        .into_iter()
-        .sorted_unstable_by_key(|(_, f)| f.first().and_then(|ff| ff.size).unwrap_or_default())
-        .progress_with(progress_bar)
-        .for_each(|(hash, group)| {
-            let mut inner_table = Table::new();
-            inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
-            group.iter().for_each(|file| {
-                inner_table.add_row(row![
-                    format_path(&file.path, opts).unwrap_or_default().blue(),
-                    file_size(file).unwrap_or_default().red(),
-                    modified_time(&file.path).unwrap_or_default().yellow()
-                ]);
-            });
-            output_table.add_row(row![hash.green(), inner_table]);
-        });
-
-    output_table.printstd();
-}
-
-#[allow(unused)]
-pub fn raw(duplicates: DashMap<String, Vec<File>>, opts: &Params) -> Result<()> {
-    if duplicates.is_empty() {
-        println!(
-            "\n{}",
-            "No duplicates found matching your search criteria.".green()
-        );
-        return Ok(());
-    }
-
-    duplicates
-        .into_iter()
-        .sorted_unstable_by_key(|(_, f)| f.first().and_then(|ff| ff.size).unwrap_or_default())
-        .for_each(|(_hash, group)| {
-            group.iter().for_each(|file| {
-                println!(
-                    "{}\t{}\t{}",
-                    format_path(&file.path, opts).unwrap_or_default().blue(),
-                    file_size(file).unwrap_or_default().red(),
-                    modified_time(&file.path).unwrap_or_default().yellow()
-                )
-            });
-
-            println!("---");
-        });
-
-    Ok(())
-}

src/params.rs

@@ -1,10 +1,9 @@
 use std::{fs, path::PathBuf};
 
-use anyhow::{anyhow, Result};
+use anyhow::Result;
 use clap::{Parser, ValueHint};
-use globwalk::{GlobWalker, GlobWalkerBuilder};
 
-#[derive(Parser, Debug)]
+#[derive(Parser, Debug, Clone)]
 #[command(author, version, about, long_about = None)]
 pub struct Params {
     /// Filetypes to deduplicate [default = all]
@@ -47,42 +46,4 @@ impl Params {
         let dir = fs::canonicalize(dir_path)?;
         Ok(dir)
     }
-
-    fn add_glob_min_depth(&self, builder: GlobWalkerBuilder) -> Result<GlobWalkerBuilder> {
-        match self.min_depth {
-            Some(mindepth) => Ok(builder.min_depth(mindepth)),
-            None => Ok(builder),
-        }
-    }
-
-    fn add_glob_max_depth(&self, builder: GlobWalkerBuilder) -> Result<GlobWalkerBuilder> {
-        match self.max_depth {
-            Some(maxdepth) => Ok(builder.max_depth(maxdepth)),
-            None => Ok(builder),
-        }
-    }
-
-    fn add_glob_follow_links(&self, builder: GlobWalkerBuilder) -> Result<GlobWalkerBuilder> {
-        match self.follow_links {
-            true => Ok(builder.follow_links(true)),
-            false => Ok(builder.follow_links(false)),
-        }
-    }
-
-    pub fn get_glob_walker(&self) -> Result<GlobWalker> {
-        let pattern: String = match self.types.as_ref() {
-            Some(filetypes) => format!("**/*{{{filetypes}}}"),
-            None => "**/*".to_string(),
-        };
-
-        let glob_walker_builder = self
-            .add_glob_min_depth(GlobWalkerBuilder::from_patterns(
-                self.get_directory()?,
-                &[pattern],
-            ))
-            .and_then(|builder| self.add_glob_max_depth(builder))
-            .and_then(|builder| self.add_glob_follow_links(builder))?;
-
-        glob_walker_builder.build().map_err(|e| anyhow!(e))
-    }
 }

src/processor.rs

@@ -0,0 +1,107 @@
+use anyhow::Result;
+use dashmap::DashMap;
+use indicatif::{ParallelProgressIterator, ProgressBar, ProgressStyle, ProgressFinish};
+use rayon::prelude::{IntoParallelIterator, ParallelIterator};
+use std::{time::Duration, borrow::Cow};
+
+use crate::fileinfo::FileInfo;
+
+#[derive(Debug, Clone)]
+pub enum State {
+    Initial,
+    SizeWise,
+    HashWise,
+}
+
+#[derive(Debug, Clone)]
+pub struct Processor {
+    pub files: Vec<FileInfo>,
+    pub state: State,
+}
+
+impl Processor {
+    pub fn new(files: Vec<FileInfo>) -> Self {
+        Self {
+            files,
+            state: State::Initial,
+        }
+    }
+
+    pub fn hashwise(&self) -> Result<Self> {
+        if self.files.is_empty() {
+            return Ok(self.clone());
+        }
+
+        let progress_style = ProgressStyle::with_template("[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}")?;
+        let progress_bar = ProgressBar::new(self.files.len() as u64);
+        progress_bar.set_style(progress_style);
+        progress_bar.enable_steady_tick(Duration::from_millis(50));
+        progress_bar.set_message("indexing file hashes");
+
+        let duplicates_table: DashMap<String, Vec<FileInfo>> = DashMap::new();
+        self.files
+            .clone()
+            .into_par_iter()
+            .progress_with(progress_bar)
+            .with_finish(ProgressFinish::WithMessage(Cow::from("indexed files hashes")))
+            .map(|file| file.hash())
+            .filter_map(Result::ok)
+            .for_each(|file| {
+                duplicates_table
+                    .entry(file.hash.clone().unwrap_or_default())
+                    .and_modify(|fileset| fileset.push(file.clone()))
+                    .or_insert_with(|| vec![file]);
+            });
+
+        let files = duplicates_table
+            .into_read_only()
+            .values()
+            .cloned()
+            .filter(|subfiles| subfiles.len() > 1)
+            .flatten()
+            .collect::<Vec<FileInfo>>();
+
+        Ok(Self {
+            files,
+            state: State::HashWise,
+        })
+    }
+
+    pub fn sizewise(&self) -> Result<Self> {
+        if self.files.is_empty() {
+            return Ok(self.clone());
+        }
+
+        let progress_style = ProgressStyle::with_template("[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}")?;
+        let progress_bar = ProgressBar::new(self.files.len() as u64);
+        progress_bar.set_style(progress_style);
+        progress_bar.enable_steady_tick(Duration::from_millis(50));
+        progress_bar.set_message("indexing file sizes");
+
+        let duplicates_table: DashMap<u64, Vec<FileInfo>> = DashMap::new();
+        self.files
+            .clone()
+            .into_par_iter()
+            .progress_with(progress_bar)
+            .with_finish(ProgressFinish::WithMessage(Cow::from("indexed files sizes")))
+            .for_each(|file| {
+                duplicates_table
+                    .entry(file.size)
+                    .and_modify(|fileset| fileset.push(file.clone()))
+                    .or_insert_with(|| vec![file]);
+            });
+
+        let files = duplicates_table
+            .into_read_only()
+            .values()
+            .cloned()
+            .filter(|subfiles| subfiles.len() > 1)
+            .flatten()
+            .collect::<Vec<FileInfo>>();
+
+        Ok(Self {
+            files,
+            state: State::SizeWise,
+        })
+    }
+}

src/scanner.rs

@@ -1,151 +1,169 @@
-use crate::{file_manager::File, filters, params::Params};
+#![allow(unused)]
+use crate::{fileinfo::FileInfo, params::Params};
 use anyhow::Result;
-use dashmap::DashMap;
+use indicatif::{ProgressBar, ProgressStyle};
-use fxhash::hash64 as hasher;
+use std::{fs, path::PathBuf, time::Duration};
-use indicatif::{ParallelProgressIterator, ProgressBar, ProgressIterator, ProgressStyle};
-use memmap2::Mmap;
-use rayon::prelude::*;
-use std::hash::Hasher;
-use std::time::Duration;
-use std::{
-    fs,
-    path::{Path, PathBuf},
-};
 
-#[derive(Clone, Copy)]
+use globwalk::{GlobWalker, GlobWalkerBuilder};
-enum IndexCritera {
+
-    Size,
+#[derive(Debug, Clone)]
-    Hash,
+pub struct Scanner {
+    pub directory: Option<PathBuf>,
+    pub filetypes: Option<String>,
+    pub min_depth: Option<usize>,
+    pub max_depth: Option<usize>,
+    pub min_size: Option<u64>,
+    pub follow_links: bool,
 }
 
-pub fn duplicates(app_opts: &Params) -> Result<DashMap<String, Vec<File>>> {
+impl Scanner {
-    let scan_results = scan(app_opts)?;
+    pub fn new() -> Self {
-    let size_index_store = index_files(scan_results, IndexCritera::Size)?;
+        Self {
-
+            directory: None,
-    let sizewize_duplicate_files = size_index_store
+            filetypes: None,
-        .into_par_iter()
+            min_depth: None,
-        .filter(|(_, files)| files.len() > 1)
+            max_depth: None,
-        .map(|(_, files)| files)
+            min_size: None,
-        .flatten()
+            follow_links: true,
-        .collect::<Vec<File>>();
+        }
-
-    if sizewize_duplicate_files.len() > 1 {
-        let hash_index_store = index_files(sizewize_duplicate_files, IndexCritera::Hash)?;
-        let duplicate_files = hash_index_store
-            .into_par_iter()
-            .filter(|(_, files)| files.len() > 1)
-            .collect();
-
-        Ok(duplicate_files)
-    } else {
-        Ok(DashMap::new())
     }
-}
 
-fn scan(app_opts: &Params) -> Result<Vec<File>> {
+    pub fn build(app_args: &Params) -> Result<Self> {
-    let walker = app_opts.get_glob_walker()?;
+        let scan_directory = app_args.get_directory()?;
-    let progress = ProgressBar::new_spinner();
+        Ok(Scanner::new())
-    let progress_style =
+            .map(|scanner| scanner.directory(scan_directory))
-        ProgressStyle::with_template("{spinner:.green} [mapping paths] {pos} paths")?;
+            .map(|scanner| match app_args.get_min_size() {
-    progress.set_style(progress_style);
+                Some(min_size) => scanner.min_size(min_size),
-    progress.enable_steady_tick(Duration::from_millis(50));
+                None => scanner,
+            })
+            .map(|scanner| match app_args.min_depth {
+                Some(min_depth) => scanner.min_depth(min_depth),
+                None => scanner,
+            })
+            .map(|scanner| match app_args.max_depth {
+                Some(max_depth) => scanner.max_depth(max_depth),
+                None => scanner,
+            })
+    }
 
-    let files = walker
+    pub fn min_size(&self, min_size: u64) -> Self {
-        .progress_with(progress)
+        Self {
-        .filter_map(Result::ok)
+            min_size: Some(min_size),
-        .map(|file| file.into_path())
+            ..self.clone()
-        .filter(|fpath| fpath.is_file())
+        }
-        .collect::<Vec<PathBuf>>();
+    }
 
-    let scan_progress = ProgressBar::new(files.len() as u64);
+    pub fn min_depth(&self, min_depth: usize) -> Self {
-    let scan_progress_style = ProgressStyle::with_template(
+        Self {
-        "{spinner:.green} [processing mapped paths] [{wide_bar:.cyan/blue}] {pos}/{len} files",
+            min_depth: Some(min_depth),
-    )?;
+            ..self.clone()
-    scan_progress.set_style(scan_progress_style);
+        }
-    scan_progress.enable_steady_tick(Duration::from_millis(50));
+    }
 
-    let scan_results = files
+    pub fn max_depth(&self, max_depth: usize) -> Self {
-        .into_par_iter()
+        Self {
-        .progress_with(scan_progress)
+            max_depth: Some(max_depth),
-        .map(|fpath| File {
+            ..self.clone()
-            path: fpath.clone(),
+        }
-            hash: None,
+    }
-            size: Some(
+
-                fs::metadata(fpath)
+    pub fn directory(&self, dir: PathBuf) -> Self {
-                    .map(|metadata| metadata.len())
+        Self {
-                    .unwrap_or_default(),
+            directory: Some(dir),
-            ),
+            ..self.clone()
+        }
+    }
+
+    pub fn filetypes(&self, patterns: String) -> Self {
+        Self {
+            filetypes: Some(patterns),
+            ..self.clone()
+        }
+    }
+
+    pub fn ignore_links(&self) -> Self {
+        Self {
+            follow_links: false,
+            ..self.clone()
+        }
+    }
+
+    pub fn follow_links(&self) -> Self {
+        Self {
+            follow_links: true,
+            ..self.clone()
+        }
+    }
+
+    fn scan_patterns(&self) -> Result<String> {
+        Ok(match self.filetypes.clone() {
+            Some(ftypes) => format!("**/*{{{ftypes}}}"),
+            None => "**/*".to_string(),
         })
-        .filter(|file| filters::is_file_gt_min_size(app_opts, file))
+    }
-        .collect();
 
-    Ok(scan_results)
+    fn scan_dir(&self) -> Result<PathBuf> {
-}
+        let scan_dir = match self.directory.clone() {
+            Some(path) => path,
+            None => std::env::current_dir()?,
+        };
 
-fn process_file_index(
+        Ok(fs::canonicalize(scan_dir)?)
-    mut file: File,
+    }
-    store: &DashMap<String, Vec<File>>,
+
-    index_criteria: IndexCritera,
+    fn attach_link_opts(&self, walker: GlobWalkerBuilder) -> Result<GlobWalkerBuilder> {
-) {
+        Ok(walker.follow_links(self.follow_links))
-    match index_criteria {
+    }
-        IndexCritera::Size => {
+
-            store
+    fn attach_walker_min_depth(&self, walker: GlobWalkerBuilder) -> Result<GlobWalkerBuilder> {
-                .entry(file.size.unwrap_or_default().to_string())
+        match self.min_depth {
-                .and_modify(|fileset| fileset.push(file.clone()))
+            Some(min_depth) => Ok(walker.min_depth(min_depth)),
-                .or_insert_with(|| vec![file]);
+            None => Ok(walker),
-        }
-        IndexCritera::Hash => {
-            file.hash = Some(hash_file(&file.path).unwrap_or_default());
-            store
-                .entry(file.clone().hash.unwrap_or_default())
-                .and_modify(|fileset| fileset.push(file.clone()))
-                .or_insert_with(|| vec![file]);
         }
     }
-}
 
-fn index_files(
+    fn attach_walker_max_depth(&self, walker: GlobWalkerBuilder) -> Result<GlobWalkerBuilder> {
-    files: Vec<File>,
+        match self.max_depth {
-    index_criteria: IndexCritera,
+            Some(max_depth) => Ok(walker.max_depth(max_depth)),
-) -> Result<DashMap<String, Vec<File>>> {
+            None => Ok(walker),
-    let store: DashMap<String, Vec<File>> = DashMap::new();
+        }
-    let index_progress = ProgressBar::new(files.len() as u64);
+    }
-    let index_progress_style = ProgressStyle::with_template(
-        "{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files",
-    )?;
-    index_progress.set_style(index_progress_style);
-    index_progress.enable_steady_tick(Duration::from_millis(50));
 
-    files
+    fn build_walker(&self) -> Result<GlobWalker> {
-        .into_par_iter()
+        let walker = Ok(GlobWalkerBuilder::from_patterns(
-        .progress_with(index_progress)
+            self.scan_dir()?,
-        .for_each(|file| process_file_index(file, &store, index_criteria));
+            &[self.scan_patterns()?],
+        ))
+        .and_then(|walker| self.attach_walker_min_depth(walker))
+        .and_then(|walker| self.attach_walker_max_depth(walker))
+        .and_then(|walker| self.attach_link_opts(walker))?;
 
-    Ok(store)
+        Ok(walker.build()?)
-}
+    }
 
-fn incremental_hashing(filepath: &Path) -> Result<String> {
+    pub fn scan(&self) -> Result<Vec<FileInfo>> {
-    let file = fs::File::open(filepath)?;
+        let progress_style =
-    let fmap = unsafe { Mmap::map(&file)? };
+            ProgressStyle::with_template("[{elapsed_precise}] {pos:>7} {msg}")?;
-    let mut inchasher = fxhash::FxHasher::default();
+        let progress_bar = ProgressBar::new_spinner();
+        progress_bar.set_style(progress_style);
+        progress_bar.enable_steady_tick(Duration::from_millis(50));
+        progress_bar.set_message("paths mapped");
 
-    fmap.chunks(1_000_000)
+        let results = self
-        .for_each(|mega| inchasher.write(mega));
+            .build_walker()?
+            .filter_map(Result::ok)
+            .map(|entity| entity.into_path())
+            .map(|path| {
+                progress_bar.inc(1);
+                path
+            })
+            .filter(|path| path.is_file())
+            .map(|file| FileInfo::new(file))
+            .filter_map(Result::ok)
+            .collect::<Vec<FileInfo>>();
 
-    Ok(format!("{}", inchasher.finish()))
+        progress_bar.finish_with_message("paths mapped");
-}
 
-fn standard_hashing(filepath: &Path) -> Result<String> {
+        Ok(results)
-    let file = fs::read(filepath)?;
-    Ok(hasher(&*file).to_string())
-}
-
-fn hash_file(filepath: &Path) -> Result<String> {
-    let filemeta = fs::metadata(filepath)?;
-
-    // NOTE: USE INCREMENTAL HASHING ONLY FOR FILES > 100MB
-    match filemeta.len() < 100_000_000 {
-        true => standard_hashing(filepath),
-        false => incremental_hashing(filepath),
     }
 }