Merge pull request #22 from sreedevk/development

Version 0.1.1

.github/workflows/rust.yml

@@ -0,0 +1,20 @@
+name: Rust
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Build
+      run: cargo build --verbose
+    - name: Run tests
+      run: cargo test --verbose

CONTRIBUTING.md

@@ -12,6 +12,7 @@
 * Assign the Issue to yourself (if unassigned) before you start working in order to avoid any conficts. 
 * Open a new GitHub pull request with the patch.
 * Ensure the PR description clearly describes the problem and solution. Include the relevant issue number.
+* Make sure that the PR points to the development branch.
 
 #### **Did you fix whitespace, format code, or make a purely cosmetic patch?**
 

Cargo.lock

@@ -70,12 +70,6 @@ version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c"
 
-[[package]]
-name = "cassowary"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53"
-
 [[package]]
 name = "cc"
 version = "1.0.78"
@@ -223,31 +217,6 @@ dependencies = [
  "cfg-if",
 ]
 
-[[package]]
-name = "crossterm"
-version = "0.25.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e64e6c0fbe2c17357405f7c758c1ef960fce08bdfb2c03d88d2a18d7e09c4b67"
-dependencies = [
- "bitflags",
- "crossterm_winapi",
- "libc",
- "mio",
- "parking_lot",
- "signal-hook",
- "signal-hook-mio",
- "winapi",
-]
-
-[[package]]
-name = "crossterm_winapi"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ae1b35a484aa10e07fe0638d02301c5ad24de82d310ccbd2f3693da5f09bf1c"
-dependencies = [
- "winapi",
-]
-
 [[package]]
 name = "csv"
 version = "1.1.6"
@@ -314,26 +283,40 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "dashmap"
+version = "5.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc"
+dependencies = [
+ "cfg-if",
+ "hashbrown",
+ "lock_api",
+ "once_cell",
+ "parking_lot_core",
+ "rayon",
+]
+
 [[package]]
 name = "deduplicator"
-version = "0.0.8"
+version = "0.1.1"
 dependencies = [
  "anyhow",
  "chrono",
  "clap",
  "colored",
- "crossterm",
+ "dashmap",
  "fxhash",
  "glob",
  "humansize",
  "indicatif",
  "itertools",
+ "memmap2",
  "prettytable-rs",
  "rayon",
- "sqlite",
  "thiserror",
  "tokio",
- "tui",
+ "unicode-segmentation",
 ]
 
 [[package]]
@@ -422,6 +405,12 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
 
+[[package]]
+name = "hashbrown"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+
 [[package]]
 name = "heck"
 version = "0.4.0"
@@ -597,6 +586,15 @@ version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
 
+[[package]]
+name = "memmap2"
+version = "0.5.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b182332558b18d807c4ce1ca8ca983b34c3ee32765e47b3f0f69b90355cc1dc"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "memoffset"
 version = "0.7.1"
@@ -694,12 +692,6 @@ version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
 
-[[package]]
-name = "pkg-config"
-version = "0.3.26"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
-
 [[package]]
 name = "portable-atomic"
 version = "0.3.19"
@@ -854,27 +846,6 @@ version = "1.0.152"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb"
 
-[[package]]
-name = "signal-hook"
-version = "0.3.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a253b5e89e2698464fc26b545c9edceb338e18a89effeeecfea192c3025be29d"
-dependencies = [
- "libc",
- "signal-hook-registry",
-]
-
-[[package]]
-name = "signal-hook-mio"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af"
-dependencies = [
- "libc",
- "mio",
- "signal-hook",
-]
-
 [[package]]
 name = "signal-hook-registry"
 version = "1.4.0"
@@ -900,36 +871,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "sqlite"
-version = "0.30.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12e072cb5fb89b3fe5e9c9584676348feb503f9fb3ae829d9868171bc5372d48"
-dependencies = [
- "libc",
- "sqlite3-sys",
-]
-
-[[package]]
-name = "sqlite3-src"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d1815a7a02c996eb8e5c64f61fcb6fd9b12e593ce265c512c5853b2513635691"
-dependencies = [
- "cc",
- "pkg-config",
-]
-
-[[package]]
-name = "sqlite3-sys"
-version = "0.14.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d47c99824fc55360ba00caf28de0b8a0458369b832e016a64c13af0ad9fbb9ee"
-dependencies = [
- "libc",
- "sqlite3-src",
-]
-
 [[package]]
 name = "strsim"
 version = "0.10.0"
@@ -1029,19 +970,6 @@ dependencies = [
  "syn",
 ]
 
-[[package]]
-name = "tui"
-version = "0.19.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ccdd26cbd674007e649a272da4475fb666d3aa0ad0531da7136db6fab0e5bad1"
-dependencies = [
- "bitflags",
- "cassowary",
- "crossterm",
- "unicode-segmentation",
- "unicode-width",
-]
-
 [[package]]
 name = "unicode-ident"
 version = "1.0.6"

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deduplicator"
-version = "0.0.8"
+version = "0.1.1"
 edition = "2021"
 description = "find,filter,delete Duplicates"
 license = "MIT"
@@ -13,15 +13,15 @@ anyhow = "1.0.68"
 chrono = "0.4.23"
 clap = { version = "4.0.32", features = ["derive"] }
 colored = "2.0.0"
-crossterm = "0.25.0"
+dashmap = { version = "5.4.0", features = ["rayon"] }
 fxhash = "0.2.1"
 glob = "0.3.0"
 humansize = "2.1.2"
 indicatif = { version = "0.17.2", features = ["rayon", "tokio"] }
 itertools = "0.10.5"
+memmap2 = "0.5.8"
 prettytable-rs = "0.10.0"
 rayon = "1.6.1"
-sqlite = "0.30.3"
 thiserror = "1.0.38"
 tokio = { version = "1.23.0", features = ["full"] }
-tui = "0.19.0"
+unicode-segmentation = "1.10.0"

README.md

@@ -16,7 +16,6 @@ Usage: deduplicator [OPTIONS]
 Options:
   -t, --types <TYPES>  Filetypes to deduplicate (default = all)
       --dir <DIR>      Run Deduplicator on dir different from pwd
-  -n, --nocache        Don't use cache for indexing files (default = false)
   -i, --interactive    Delete files interactively
   -h, --help           Print help information
   -V, --version        Print version information
@@ -38,16 +37,13 @@ cargo install deduplicator
 <p align="center">
   Deduplicator uses fxhash (a non-cryptographic hashing algorithm) which is extremely fast. As a result, deduplicator is able to process huge amounts of data in a couple of seconds.</p>
 
-  <p align="center">
+<p align="center">
-    While testing, Deduplicator was able to go through 8.6GB of pdf files and detect duplicates in 2.9 seconds
+  While testing, Deduplicator was able to go through 8.6GB of pdf files and detect duplicates in 2.9 seconds
-  </p>
+</p>
+
 <h2 align="center">Screenshots</h2>
 
-<p align="center>
-<a href="https://asciinema.org/a/oMzGUSlAx32xYXcPBY64uAuLB" target="_blank"><img src="https://asciinema.org/a/oMzGUSlAx32xYXcPBY64uAuLB.svg" /></a>
-</p>
 
-<p align="center">
+<img src="https://user-images.githubusercontent.com/36154121/211948081-63c12b94-6251-487b-a49f-ac5418169d5a.gif" />
-  <img align="center" src="https://user-images.githubusercontent.com/36154121/211458077-90092aa3-496c-492f-a061-618059890d5f.png" width="500" height="400" />
+<img src="https://user-images.githubusercontent.com/36154121/211458077-90092aa3-496c-492f-a061-618059890d5f.png" />
-</p>
 

src/app.rs

@@ -0,0 +1,18 @@
+use crate::output;
+use crate::params::Params;
+use crate::scanner;
+use anyhow::Result;
+
+pub struct App;
+
+impl App {
+    pub fn init(app_args: &Params) -> Result<()> {
+        let duplicates = scanner::duplicates(app_args)?;
+        match app_args.interactive {
+            true => output::interactive(duplicates, app_args),
+            false => output::print(duplicates, app_args),
+        }
+
+        Ok(())
+    }
+}

src/app/event_handler.rs

@@ -1,28 +0,0 @@
-use std::time::Duration;
-
-use anyhow::Result;
-use crossterm::event::{self, KeyCode, KeyEvent};
-
-use super::events;
-
-pub struct EventHandler;
-
-impl EventHandler {
-    pub fn init() -> Result<events::Event> {
-        if crossterm::event::poll(Duration::from_millis(10))? {
-            match event::read()? {
-                event::Event::Key(keycode) => Self::handle_keypress(keycode),
-                _ => Ok(events::Event::Noop),
-            }
-        } else {
-            Ok(events::Event::Noop)
-        }
-    }
-
-    fn handle_keypress(keyevent: KeyEvent) -> Result<events::Event> {
-        match keyevent.code {
-            KeyCode::Char('q') => Ok(events::Event::Exit),
-            _ => Ok(events::Event::Noop),
-        }
-    }
-}

src/app/events.rs

@@ -1,4 +0,0 @@
-pub enum Event {
-    Exit,
-    Noop,
-}

src/app/mod.rs

@@ -1,89 +0,0 @@
-#![allow(unused)]
-
-mod event_handler;
-mod events;
-mod formatter;
-mod ui;
-pub mod file_manager;
-
-use std::{io, thread, time::Duration};
-
-use anyhow::{anyhow, Result};
-use crossterm::{event, execute, terminal};
-use event_handler::EventHandler;
-use tui::{
-    backend::CrosstermBackend,
-    widgets::{Block, Borders, Widget},
-    Terminal,
-};
-use ui::Ui;
-
-use crate::database;
-use crate::output;
-use crate::params::Params;
-use crate::scanner;
-
-pub struct App;
-
-impl App {
-    pub fn init(app_args: &Params) -> Result<()> {
-        // let mut term = Self::init_terminal()?;
-
-        let connection = database::get_connection(app_args)?;
-        let duplicates = scanner::duplicates(app_args, &connection)?;
-
-        // Self::init_render_loop(&mut term)?;
-        // Self::cleanup(&mut term)?;
-
-        match app_args.interactive {
-            true => output::interactive(duplicates, app_args),
-            false => output::print(duplicates, app_args) /* TODO: APP TUI INIT FUNCTION */
-        }
-        
-        Ok(())
-    }
-
-    fn cleanup(term: &mut Terminal<CrosstermBackend<io::Stdout>>) -> Result<()> {
-        terminal::disable_raw_mode()?;
-        execute!(
-            term.backend_mut(),
-            terminal::LeaveAlternateScreen,
-            event::DisableMouseCapture
-        )?;
-
-        term.show_cursor()?;
-        Ok(())
-    }
-
-    fn render_cycle(term: &mut Terminal<CrosstermBackend<io::Stdout>>) -> Result<()> {
-        match EventHandler::init()? {
-            events::Event::Noop => Ui::render_frame(term),
-            events::Event::Exit => Err(anyhow!("Exit")),
-        }
-    }
-
-    fn init_render_loop(term: &mut Terminal<CrosstermBackend<io::Stdout>>) -> Result<()> {
-        // this could be simplified with a `while Self::render_cycle(term).is_ok() {}` in the current state, but maybe
-        // it's good to keep it to handle errors in the future
-        loop {
-            match Self::render_cycle(term) {
-                Ok(_) => continue,
-                Err(_) => break,
-            }
-        }
-
-        Ok(())
-    }
-
-    fn init_terminal() -> Result<Terminal<CrosstermBackend<io::Stdout>>> {
-        terminal::enable_raw_mode()?;
-        let mut stdout = io::stdout();
-        execute!(
-            stdout,
-            terminal::EnterAlternateScreen,
-            event::EnableMouseCapture
-        )?;
-        let backend = CrosstermBackend::new(stdout);
-        Ok(Terminal::new(backend)?)
-    }
-}

src/app/ui.rs

@@ -1,54 +0,0 @@
-use std::io;
-
-use anyhow::Result;
-use tui::{
-    backend::{Backend, CrosstermBackend},
-    layout::{Constraint, Direction, Layout, Rect},
-    style::{Modifier, Style},
-    text::{Span, Spans},
-    widgets::{Block, Borders, List, ListItem, Widget},
-    Frame, Terminal,
-};
-
-pub struct Ui;
-
-impl Ui {
-    fn generate_file_list() -> impl Widget {
-        let tasks: Vec<ListItem> = vec!["Sreedev"; 100]
-            .into_iter()
-            .map(|item| ListItem::new(vec![Spans::from(Span::raw(item))]))
-            .collect();
-
-        List::new(tasks)
-            .block(Block::default().borders(Borders::ALL).title("List"))
-            .highlight_style(Style::default().add_modifier(Modifier::BOLD))
-            .highlight_symbol("> ")
-    }
-
-    fn generate_info_bar() -> impl Widget {
-        Block::default().title("Description").borders(Borders::ALL)
-    }
-
-    fn generate_file_desc() -> impl Widget {
-        Block::default().title("Description").borders(Borders::ALL)
-    }
-
-    pub fn render_frame(term: &mut Terminal<CrosstermBackend<io::Stdout>>) -> Result<()> {
-        term.draw(|f| {
-            let windows = Layout::default()
-                .direction(Direction::Vertical)
-                .constraints([Constraint::Ratio(2, 16), Constraint::Ratio(14, 16)].as_ref())
-                .split(f.size());
-
-            let subwindows = Layout::default()
-                .direction(Direction::Horizontal)
-                .constraints([Constraint::Ratio(1, 4), Constraint::Ratio(3, 4)].as_ref())
-                .split(windows[1]);
-
-            f.render_widget(Self::generate_info_bar(), windows[0]);
-            f.render_widget(Self::generate_file_list(), subwindows[0]);
-            f.render_widget(Self::generate_file_desc(), subwindows[1]);
-        })?;
-        Ok(())
-    }
-}

src/database.rs

@@ -1,89 +0,0 @@
-use std::env::temp_dir;
-
-use anyhow::Result;
-
-use crate::params::Params;
-
-#[derive(Debug, Clone)]
-pub struct File {
-    pub path: String,
-    pub hash: String,
-}
-
-fn db_connection_url(args: &Params) -> String {
-    match args.nocache {
-        true => String::from(":memory:"),
-        false => {
-            let temp_dir_path = temp_dir();
-            format!("{}/deduplicator.db", temp_dir_path.display())
-        }
-    }
-}
-
-pub fn get_connection(args: &Params) -> Result<sqlite::Connection, sqlite::Error> {
-    sqlite::open(db_connection_url(args)).and_then(|conn| {
-        setup(&conn).ok();
-        Ok(conn)
-    })
-}
-
-pub fn setup(connection: &sqlite::Connection) -> Result<()> {
-    let query = "CREATE TABLE files (file_identifier STRING, hash STRING)";
-    connection.execute(query).ok();
-    Ok(())
-}
-
-pub fn put(file: &File, connection: &sqlite::Connection) -> Result<()> {
-    let query = format!(
-        "INSERT INTO files (file_identifier, hash) VALUES (\"{}\", \"{}\")",
-        file.path, file.hash
-    );
-    connection.execute(query)?;
-    Ok(())
-}
-
-pub fn indexed_paths(connection: &sqlite::Connection) -> Result<Vec<File>> {
-    let query = "SELECT * FROM files";
-
-    let result: Vec<File> = connection
-        .prepare(query)?
-        .into_iter()
-        .filter_map(|row_result| row_result.ok())
-        .map(|row| {
-            let path = row.read::<&str, _>("file_identifier").to_string();
-            let hash = row.read::<i64, _>("hash").to_string();
-            File { path, hash }
-        })
-        .collect();
-
-    Ok(result)
-}
-
-pub fn duplicate_hashes(connection: &sqlite::Connection, path: &str) -> Result<Vec<File>> {
-    let query = format!(
-        "
-            SELECT a.* FROM files a
-            JOIN (SELECT file_identifier, hash, COUNT(*)
-            FROM files
-            GROUP BY hash
-            HAVING count(*) > 1 ) b
-            ON a.hash = b.hash
-            WHERE a.file_identifier LIKE \"{}%\"
-            ORDER BY a.file_identifier
-        ",
-        path
-    );
-
-    let result: Vec<File> = connection
-        .prepare(query)?
-        .into_iter()
-        .filter_map(|row_result| row_result.ok())
-        .map(|row| {
-            let path = row.read::<&str, _>("file_identifier").to_string();
-            let hash = row.read::<i64, _>("hash").to_string();
-            File { path, hash }
-        })
-        .collect();
-
-    Ok(result)
-}

src/file_manager.rs

@@ -1,12 +1,18 @@
-use crate::database::File;
 use anyhow::Result;
 use colored::Colorize;
 
+#[derive(Debug, Clone)]
+pub struct File {
+    pub path: String,
+    pub size: Option<u64>,
+    pub hash: Option<String>,
+}
+
 pub fn delete_files(files: Vec<File>) -> Result<()> {
     files.into_iter().for_each(|file| {
         match std::fs::remove_file(file.path.clone()) {
             Ok(_) => println!("{}: {}", "DELETED".green(), file.path),
-            Err(e) => println!("{}: {}", "FAILED".red(), file.path)
+            Err(_) => println!("{}: {}", "FAILED".red(), file.path)
         }
     });
 

src/main.rs

@@ -1,6 +1,5 @@
-#![allow(unused)] // TODO: remove this once TUI is implemented
 mod app;
-mod database;
+mod file_manager;
 mod output;
 mod params;
 mod scanner;

src/output.rs

@@ -1,26 +1,28 @@
-use std::{collections::HashMap, fs, io};
+use crate::file_manager::{self, File};
-use std::io::Write;
+use crate::params::Params;
-
 use anyhow::Result;
 use chrono::offset::Utc;
 use chrono::DateTime;
 use colored::Colorize;
+use dashmap::DashMap;
 use humansize::{format_size, DECIMAL};
 use itertools::Itertools;
-
+use prettytable::{format, row, Table};
-use crate::app::file_manager;
+use std::io::Write;
-use crate::database::File;
+use std::{fs, io};
-use crate::params::Params;
+use unicode_segmentation::UnicodeSegmentation;
-use prettytable::{format, row, Cell, Row, Table};
 
 fn format_path(path: &str, opts: &Params) -> Result<String> {
     let display_path = path.replace(&opts.get_directory()?, "");
-    let text_vec = display_path.chars().collect::<Vec<_>>();
+    let display_range = if display_path.chars().count() > 32 {
-
+        display_path
-    let display_range = if text_vec.len() > 32 {
+            .graphemes(true)
-        text_vec[(display_path.len() - 32)..]
+            .collect::<Vec<&str>>()
-            .iter()
+            .into_iter()
-            .collect::<String>()
+            .rev()
+            .take(32)
+            .rev()
+            .collect()
     } else {
         display_path
     };
@@ -41,19 +43,7 @@ fn modified_time(path: &String) -> Result<String> {
     Ok(modified_time.format("%Y-%m-%d %H:%M:%S").to_string())
 }
 
-fn group_duplicates(duplicates: Vec<File>) -> HashMap<String, Vec<File>> {
+fn print_meta_info() {
-    let mut duplicate_mapper: HashMap<String, Vec<File>> = HashMap::new();
-    duplicates.into_iter().for_each(|file| {
-        duplicate_mapper
-            .entry(file.hash.clone())
-            .and_modify(|value| value.push(file.clone()))
-            .or_insert_with(|| vec![file]);
-    });
-
-    duplicate_mapper
-}
-
-fn print_meta_info(duplicates: &Vec<File>, opts: &Params) {
     println!("Deduplicator v{}", std::env!("CARGO_PKG_VERSION"));
 }
 
@@ -70,14 +60,14 @@ fn scan_group_instruction() -> Result<String> {
 }
 
 fn scan_group_confirmation() -> Result<bool> {
-    print!("\nconfirm? [Y/n]: ");
+    print!("\nconfirm? [y/N]: ");
     std::io::stdout().flush()?;
     let mut user_input = String::new();
     io::stdin().read_line(&mut user_input)?;
 
     match user_input.trim() {
         "Y" | "y" => Ok(true),
-        _ => Ok(false)
+        _ => Ok(false),
     }
 }
 
@@ -103,52 +93,59 @@ fn process_group_action(duplicates: &Vec<File>, dup_index: usize, dup_size: usiz
 
     print!("{esc}[2J{esc}[1;1H", esc = 27 as char);
 
-    if parsed_file_indices.is_empty() { return }
+    if parsed_file_indices.is_empty() {
+        return;
+    }
 
     let files_to_delete = parsed_file_indices
         .into_iter()
         .map(|index| duplicates[index].clone());
 
     println!("\n{}", "The following files will be deleted:".red());
-    files_to_delete.clone().enumerate().for_each(|(index, file)| {
+    files_to_delete
-        println!("{}: {}", index.to_string().blue(), file.path);
+        .clone()
-    });
+        .enumerate()
-    
+        .for_each(|(index, file)| {
+            println!("{}: {}", index.to_string().blue(), file.path);
+        });
+
     match scan_group_confirmation().unwrap() {
-        true => { file_manager::delete_files(files_to_delete.collect_vec()); },
+        true => {
-        false => println!("{}", "\nCancelled Delete Operation.".red())
+            file_manager::delete_files(files_to_delete.collect_vec()).ok();
+        }
+        false => println!("{}", "\nCancelled Delete Operation.".red()),
     }
 }
 
-pub fn interactive(duplicates: Vec<File>, opts: &Params) {
+pub fn interactive(duplicates: DashMap<String, Vec<File>>, opts: &Params) {
-    print_meta_info(&duplicates, opts);
+    print_meta_info();
-    let grouped_duplicates = group_duplicates(duplicates);
+    duplicates
+        .clone()
+        .into_iter()
+        .enumerate()
+        .for_each(|(gindex, (_, group))| {
+            let mut itable = Table::new();
+            itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
+            itable.set_titles(row!["index", "filename", "size", "updated_at"]);
+            group.iter().enumerate().for_each(|(index, file)| {
+                itable.add_row(row![
+                    index,
+                    format_path(&file.path, opts).unwrap_or_default().blue(),
+                    file_size(&file.path).unwrap_or_default().red(),
+                    modified_time(&file.path).unwrap_or_default().yellow()
+                ]);
+            });
 
-    grouped_duplicates.iter().enumerate().for_each(|(gindex, (hash, group))| {
+            process_group_action(&group, gindex, duplicates.len(), itable);
-        let mut itable = Table::new();
-        itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
-        itable.set_titles(row!["index", "filename", "size", "updated_at"]);
-        group.iter().enumerate().for_each(|(index, file)| {
-            itable.add_row(row![
-                index,
-                format_path(&file.path, opts).unwrap_or_default().blue(),
-                file_size(&file.path).unwrap_or_default().red(),
-                modified_time(&file.path).unwrap_or_default().yellow()
-            ]);
         });
-
-        process_group_action(group, gindex, grouped_duplicates.len(), itable);
-    });
 }
 
-pub fn print(duplicates: Vec<File>, opts: &Params) {
+pub fn print(duplicates: DashMap<String, Vec<File>>, opts: &Params) {
-    print_meta_info(&duplicates, opts);
+    print_meta_info();
 
     let mut output_table = Table::new();
-    let grouped_duplicates: HashMap<String, Vec<File>> = group_duplicates(duplicates);
-
     output_table.set_titles(row!["hash", "duplicates"]);
-    grouped_duplicates.iter().for_each(|(hash, group)| {
+    duplicates.into_iter().for_each(|(hash, group)| {
         let mut inner_table = Table::new();
         inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
         group.iter().for_each(|file| {

src/params.rs

@@ -1,7 +1,6 @@
-use std::{fs, path::PathBuf};
-
 use anyhow::{anyhow, Result};
 use clap::Parser;
+use std::{fs, path::PathBuf};
 
 #[derive(Parser, Debug)]
 #[command(author, version, about, long_about = None)]
@@ -12,12 +11,9 @@ pub struct Params {
     /// Run Deduplicator on dir different from pwd
     #[arg(long)]
     pub dir: Option<PathBuf>,
-    /// Don't use cache for indexing files (default = false)
-    #[arg(long, short)]
-    pub nocache: bool,
     /// Delete files interactively
     #[arg(long, short)]
-    pub interactive: bool
+    pub interactive: bool,
 }
 
 impl Params {
@@ -37,4 +33,18 @@ impl Params {
 
         Ok(dir)
     }
+
+    pub fn get_glob_patterns(&self) -> Vec<PathBuf> {
+        self.types
+            .clone()
+            .unwrap_or_else(|| String::from("*"))
+            .split(',')
+            .map(|filetype| format!("*.{}", filetype))
+            .map(|filetype| {
+                vec![self.get_directory().unwrap(), String::from("**"), filetype]
+                    .iter()
+                    .collect()
+            })
+            .collect()
+    }
 }

src/scanner.rs

@@ -1,57 +1,61 @@
-use std::{fs, path::PathBuf};
-use indicatif::{HumanDuration, MultiProgress, ProgressBar, ProgressStyle, ParallelProgressIterator};
 use anyhow::Result;
-use fxhash::hash32 as hasher;
+use dashmap::DashMap;
+use fxhash::hash64 as hasher;
 use glob::glob;
-use itertools::Itertools;
+use indicatif::{ParallelProgressIterator, ProgressStyle};
+use memmap2::Mmap;
 use rayon::prelude::*;
+use std::hash::Hasher;
+use std::{fs, path::PathBuf};
 
-use crate::{
+use crate::{file_manager::File, params::Params};
-    database::{self, File},
-    params::Params,
-};
 
-pub fn duplicates(app_opts: &Params, connection: &sqlite::Connection) -> Result<Vec<File>> {
+#[derive(Clone, Copy)]
-    let scan_results = scan(app_opts, connection)?;
+enum IndexCritera {
-    let base_path = app_opts.get_directory()?;
+    Size,
-
+    Hash,
-    index_files(scan_results, connection)?;
-    database::duplicate_hashes(connection, &base_path)
 }
 
-fn get_glob_patterns(opts: &Params, directory: &str) -> Vec<PathBuf> {
+pub fn duplicates(app_opts: &Params) -> Result<DashMap<String, Vec<File>>> {
-    opts.types
+    let scan_results = scan(app_opts)?;
-        .clone()
+    let size_index_store = index_files(scan_results, IndexCritera::Size)?;
-        .unwrap_or_else(|| String::from("*"))
+
-        .split(',')
+    let sizewize_duplicate_files = size_index_store
-        .map(|filetype| format!("*.{}", filetype))
+        .into_par_iter()
-        .map(|filetype| {
+        .filter(|(_, files)| files.len() > 1)
-            vec![directory.to_owned(), String::from("**"), filetype]
+        .map(|(_, files)| files)
-                .iter()
+        .flatten()
-                .collect()
+        .collect::<Vec<File>>();
-        })
+
-        .collect()
+    if sizewize_duplicate_files.len() > 1 {
+        let size_wise_duplicate_paths = sizewize_duplicate_files
+            .into_par_iter()
+            .map(|file| file.path)
+            .collect::<Vec<String>>();
+
+        let hash_index_store = index_files(size_wise_duplicate_paths, IndexCritera::Hash)?;
+        let duplicate_files = hash_index_store
+            .into_par_iter()
+            .filter(|(_, files)| files.len() > 1)
+            .collect();
+
+        Ok(duplicate_files)
+    } else {
+        Ok(DashMap::new())
+    }
 }
 
-fn is_indexed_file(path: impl Into<String>, indexed: &[File]) -> bool {
+fn scan(app_opts: &Params) -> Result<Vec<String>> {
-    indexed
+    let glob_patterns: Vec<PathBuf> = app_opts.get_glob_patterns();
-        .iter()
-        .map(|file| file.path.clone())
-        .contains(&path.into())
-}
-
-fn scan(app_opts: &Params, connection: &sqlite::Connection) -> Result<Vec<String>> {
-    let directory = app_opts.get_directory()?;
-    let glob_patterns: Vec<PathBuf> = get_glob_patterns(app_opts, &directory);
-    let indexed_paths = database::indexed_paths(connection)?;
     let files: Vec<String> = glob_patterns
         .par_iter()
-        .progress_with_style(ProgressStyle::with_template("{spinner:.green} [scanning files] [{wide_bar:.cyan/blue}] {pos}/{len} files").unwrap())
+        .progress_with_style(ProgressStyle::with_template(
+            "{spinner:.green} [scanning files] [{wide_bar:.cyan/blue}] {pos}/{len} files",
+        )?)
         .filter_map(|glob_pattern| glob(glob_pattern.as_os_str().to_str()?).ok())
         .flat_map(|file_vec| {
             file_vec
                 .filter_map(|x| Some(x.ok()?.as_os_str().to_str()?.to_string()))
-                .filter(|fpath| !is_indexed_file(fpath, &indexed_paths))
                 .filter(|glob_result| {
                     fs::metadata(glob_result)
                         .map(|f| f.is_file())
@@ -64,24 +68,84 @@ fn scan(app_opts: &Params, connection: &sqlite::Connection) -> Result<Vec<String
     Ok(files)
 }
 
-fn index_files(files: Vec<String>, connection: &sqlite::Connection) -> Result<()> {
+fn process_file_size_index(fpath: String) -> Result<File> {
-    let hashed: Vec<File> = files
+    Ok(File {
-        .into_par_iter()
+        path: fpath.clone(),
-        .progress_with_style(ProgressStyle::with_template("{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files").unwrap())
+        size: Some(fs::metadata(fpath)?.len()),
-        .filter_map(|file| {
+        hash: None,
-            let hash = hash_file(&file).ok()?;
+    })
-            Some(database::File { path: file, hash })
+}
-        })
-        .collect();
 
-    hashed
+fn process_file_hash_index(fpath: String) -> Result<File> {
-        .iter()
+    Ok(File {
-        .try_for_each(|file| database::put(file, connection))
+        path: fpath.clone(),
+        size: None,
+        hash: Some(hash_file(&fpath).unwrap_or_default()),
+    })
+}
+
+fn process_file_index(
+    fpath: String,
+    store: &DashMap<String, Vec<File>>,
+    index_criteria: IndexCritera,
+) {
+    match index_criteria {
+        IndexCritera::Size => {
+            let processed_file = process_file_size_index(fpath).unwrap();
+            store
+                .entry(processed_file.size.unwrap_or_default().to_string())
+                .and_modify(|fileset| fileset.push(processed_file.clone()))
+                .or_insert_with(|| vec![processed_file]);
+        }
+        IndexCritera::Hash => {
+            let processed_file = process_file_hash_index(fpath).unwrap();
+            let indexhash = processed_file.clone().hash.unwrap_or_default();
+
+            store
+                .entry(indexhash)
+                .and_modify(|fileset| fileset.push(processed_file.clone()))
+                .or_insert_with(|| vec![processed_file]);
+        }
+    }
+}
+
+fn index_files(
+    files: Vec<String>,
+    index_criteria: IndexCritera,
+) -> Result<DashMap<String, Vec<File>>> {
+    let store: DashMap<String, Vec<File>> = DashMap::new();
+    files
+        .into_par_iter()
+        .progress_with_style(ProgressStyle::with_template(
+            "{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files",
+        )?)
+        .for_each(|file| process_file_index(file, &store, index_criteria));
+
+    Ok(store)
+}
+
+pub fn incremental_hashing(filepath: &str) -> Result<String> {
+    let file = fs::File::open(filepath)?;
+    let fmap = unsafe { Mmap::map(&file)? };
+    let mut inchasher = fxhash::FxHasher::default();
+
+    fmap.chunks(1_000_000)
+        .for_each(|mega| inchasher.write(mega));
+
+    Ok(format!("{}", inchasher.finish()))
+}
+
+pub fn standard_hashing(filepath: &str) -> Result<String> {
+    let file = fs::read(filepath)?;
+    Ok(hasher(&*file).to_string())
 }
 
 pub fn hash_file(filepath: &str) -> Result<String> {
-    let file = fs::read(filepath)?;
+    let filemeta = fs::metadata(filepath)?;
-    let hash = hasher(&*file).to_string();
 
-    Ok(hash)
+    // NOTE: USE INCREMENTAL HASHING ONLY FOR FILES > 100MB
+    match filemeta.len() < 100_000_000 {
+        true => standard_hashing(filepath),
+        false => incremental_hashing(filepath),
+    }
 }