From 0031891b8bc0b5d67e0a9358abc67097fc07a804 Mon Sep 17 00:00:00 2001 From: sreedev Date: Fri, 13 Jan 2023 20:47:51 -0500 Subject: [PATCH 1/6] remove unused code --- src/app.rs | 25 ++++++++++ src/app/event_handler.rs | 28 ----------- src/app/events.rs | 4 -- src/app/formatter.rs | 0 src/app/mod.rs | 89 ----------------------------------- src/app/ui.rs | 54 --------------------- src/{app => }/file_manager.rs | 0 src/main.rs | 2 +- src/output.rs | 7 ++- 9 files changed, 29 insertions(+), 180 deletions(-) create mode 100644 src/app.rs delete mode 100644 src/app/event_handler.rs delete mode 100644 src/app/events.rs delete mode 100644 src/app/formatter.rs delete mode 100644 src/app/mod.rs delete mode 100644 src/app/ui.rs rename src/{app => }/file_manager.rs (100%) diff --git a/src/app.rs b/src/app.rs new file mode 100644 index 0000000..17fc63f --- /dev/null +++ b/src/app.rs @@ -0,0 +1,25 @@ +#![allow(unused)] + +use std::{io, thread, time::Duration}; +use anyhow::{anyhow, Result}; +use crossterm::{event, execute, terminal}; +use crate::database; +use crate::output; +use crate::params::Params; +use crate::scanner; + +pub struct App; + +impl App { + pub fn init(app_args: &Params) -> Result<()> { + let connection = database::get_connection(app_args)?; + let duplicates = scanner::duplicates(app_args, &connection)?; + + match app_args.interactive { + true => output::interactive(duplicates, app_args), + false => output::print(duplicates, app_args) + } + + Ok(()) + } +} diff --git a/src/app/event_handler.rs b/src/app/event_handler.rs deleted file mode 100644 index da7129b..0000000 --- a/src/app/event_handler.rs +++ /dev/null @@ -1,28 +0,0 @@ -use std::time::Duration; - -use anyhow::Result; -use crossterm::event::{self, KeyCode, KeyEvent}; - -use super::events; - -pub struct EventHandler; - -impl EventHandler { - pub fn init() -> Result { - if crossterm::event::poll(Duration::from_millis(10))? { - match event::read()? { - event::Event::Key(keycode) => Self::handle_keypress(keycode), - _ => Ok(events::Event::Noop), - } - } else { - Ok(events::Event::Noop) - } - } - - fn handle_keypress(keyevent: KeyEvent) -> Result { - match keyevent.code { - KeyCode::Char('q') => Ok(events::Event::Exit), - _ => Ok(events::Event::Noop), - } - } -} diff --git a/src/app/events.rs b/src/app/events.rs deleted file mode 100644 index c7167d0..0000000 --- a/src/app/events.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub enum Event { - Exit, - Noop, -} diff --git a/src/app/formatter.rs b/src/app/formatter.rs deleted file mode 100644 index e69de29..0000000 diff --git a/src/app/mod.rs b/src/app/mod.rs deleted file mode 100644 index aa85541..0000000 --- a/src/app/mod.rs +++ /dev/null @@ -1,89 +0,0 @@ -#![allow(unused)] - -mod event_handler; -mod events; -mod formatter; -mod ui; -pub mod file_manager; - -use std::{io, thread, time::Duration}; - -use anyhow::{anyhow, Result}; -use crossterm::{event, execute, terminal}; -use event_handler::EventHandler; -use tui::{ - backend::CrosstermBackend, - widgets::{Block, Borders, Widget}, - Terminal, -}; -use ui::Ui; - -use crate::database; -use crate::output; -use crate::params::Params; -use crate::scanner; - -pub struct App; - -impl App { - pub fn init(app_args: &Params) -> Result<()> { - // let mut term = Self::init_terminal()?; - - let connection = database::get_connection(app_args)?; - let duplicates = scanner::duplicates(app_args, &connection)?; - - // Self::init_render_loop(&mut term)?; - // Self::cleanup(&mut term)?; - - match app_args.interactive { - true => output::interactive(duplicates, app_args), - false => output::print(duplicates, app_args) /* TODO: APP TUI INIT FUNCTION */ - } - - Ok(()) - } - - fn cleanup(term: &mut Terminal>) -> Result<()> { - terminal::disable_raw_mode()?; - execute!( - term.backend_mut(), - terminal::LeaveAlternateScreen, - event::DisableMouseCapture - )?; - - term.show_cursor()?; - Ok(()) - } - - fn render_cycle(term: &mut Terminal>) -> Result<()> { - match EventHandler::init()? { - events::Event::Noop => Ui::render_frame(term), - events::Event::Exit => Err(anyhow!("Exit")), - } - } - - fn init_render_loop(term: &mut Terminal>) -> Result<()> { - // this could be simplified with a `while Self::render_cycle(term).is_ok() {}` in the current state, but maybe - // it's good to keep it to handle errors in the future - loop { - match Self::render_cycle(term) { - Ok(_) => continue, - Err(_) => break, - } - } - - Ok(()) - } - - fn init_terminal() -> Result>> { - terminal::enable_raw_mode()?; - let mut stdout = io::stdout(); - execute!( - stdout, - terminal::EnterAlternateScreen, - event::EnableMouseCapture - )?; - let backend = CrosstermBackend::new(stdout); - Ok(Terminal::new(backend)?) - } -} diff --git a/src/app/ui.rs b/src/app/ui.rs deleted file mode 100644 index 9b8c63b..0000000 --- a/src/app/ui.rs +++ /dev/null @@ -1,54 +0,0 @@ -use std::io; - -use anyhow::Result; -use tui::{ - backend::{Backend, CrosstermBackend}, - layout::{Constraint, Direction, Layout, Rect}, - style::{Modifier, Style}, - text::{Span, Spans}, - widgets::{Block, Borders, List, ListItem, Widget}, - Frame, Terminal, -}; - -pub struct Ui; - -impl Ui { - fn generate_file_list() -> impl Widget { - let tasks: Vec = vec!["Sreedev"; 100] - .into_iter() - .map(|item| ListItem::new(vec![Spans::from(Span::raw(item))])) - .collect(); - - List::new(tasks) - .block(Block::default().borders(Borders::ALL).title("List")) - .highlight_style(Style::default().add_modifier(Modifier::BOLD)) - .highlight_symbol("> ") - } - - fn generate_info_bar() -> impl Widget { - Block::default().title("Description").borders(Borders::ALL) - } - - fn generate_file_desc() -> impl Widget { - Block::default().title("Description").borders(Borders::ALL) - } - - pub fn render_frame(term: &mut Terminal>) -> Result<()> { - term.draw(|f| { - let windows = Layout::default() - .direction(Direction::Vertical) - .constraints([Constraint::Ratio(2, 16), Constraint::Ratio(14, 16)].as_ref()) - .split(f.size()); - - let subwindows = Layout::default() - .direction(Direction::Horizontal) - .constraints([Constraint::Ratio(1, 4), Constraint::Ratio(3, 4)].as_ref()) - .split(windows[1]); - - f.render_widget(Self::generate_info_bar(), windows[0]); - f.render_widget(Self::generate_file_list(), subwindows[0]); - f.render_widget(Self::generate_file_desc(), subwindows[1]); - })?; - Ok(()) - } -} diff --git a/src/app/file_manager.rs b/src/file_manager.rs similarity index 100% rename from src/app/file_manager.rs rename to src/file_manager.rs diff --git a/src/main.rs b/src/main.rs index 21ac960..b07a679 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,9 +1,9 @@ -#![allow(unused)] // TODO: remove this once TUI is implemented mod app; mod database; mod output; mod params; mod scanner; +mod file_manager; use anyhow::Result; use app::App; diff --git a/src/output.rs b/src/output.rs index fcc2518..9987374 100644 --- a/src/output.rs +++ b/src/output.rs @@ -8,15 +8,14 @@ use colored::Colorize; use humansize::{format_size, DECIMAL}; use itertools::Itertools; -use crate::app::file_manager; +use crate::file_manager; use crate::database::File; use crate::params::Params; -use prettytable::{format, row, Cell, Row, Table}; +use prettytable::{format, row, Table}; use unicode_segmentation::UnicodeSegmentation; fn format_path(path: &str, opts: &Params) -> Result { - let display_path = path.replace(&opts.get_directory()?, ""); - + let display_path = path.replace(&opts.get_directory()?, ""); let display_range = if display_path.chars().count() > 32 { display_path .graphemes(true) From 81c96ce3b8d38cb55c3c2776b0643ca883797320 Mon Sep 17 00:00:00 2001 From: sreedev Date: Fri, 13 Jan 2023 20:54:09 -0500 Subject: [PATCH 2/6] code optimizations --- src/app.rs | 9 ++----- src/database.rs | 9 +------ src/file_manager.rs | 7 ++++- src/main.rs | 2 +- src/output.rs | 63 +++++++++++++++++++++++++-------------------- src/params.rs | 3 +-- src/scanner.rs | 25 +++++++++++------- 7 files changed, 62 insertions(+), 56 deletions(-) diff --git a/src/app.rs b/src/app.rs index 17fc63f..3356b25 100644 --- a/src/app.rs +++ b/src/app.rs @@ -1,12 +1,8 @@ -#![allow(unused)] - -use std::{io, thread, time::Duration}; -use anyhow::{anyhow, Result}; -use crossterm::{event, execute, terminal}; use crate::database; use crate::output; use crate::params::Params; use crate::scanner; +use anyhow::Result; pub struct App; @@ -14,10 +10,9 @@ impl App { pub fn init(app_args: &Params) -> Result<()> { let connection = database::get_connection(app_args)?; let duplicates = scanner::duplicates(app_args, &connection)?; - match app_args.interactive { true => output::interactive(duplicates, app_args), - false => output::print(duplicates, app_args) + false => output::print(duplicates, app_args), } Ok(()) diff --git a/src/database.rs b/src/database.rs index 371d41f..3f8595c 100644 --- a/src/database.rs +++ b/src/database.rs @@ -1,14 +1,7 @@ use std::env::temp_dir; - use anyhow::Result; - use crate::params::Params; - -#[derive(Debug, Clone)] -pub struct File { - pub path: String, - pub hash: String, -} +use crate::file_manager::File; fn db_connection_url(args: &Params) -> String { match args.nocache { diff --git a/src/file_manager.rs b/src/file_manager.rs index 4f57d81..9f97318 100644 --- a/src/file_manager.rs +++ b/src/file_manager.rs @@ -1,7 +1,12 @@ -use crate::database::File; use anyhow::Result; use colored::Colorize; +#[derive(Debug, Clone)] +pub struct File { + pub path: String, + pub hash: String, +} + pub fn delete_files(files: Vec) -> Result<()> { files.into_iter().for_each(|file| { match std::fs::remove_file(file.path.clone()) { diff --git a/src/main.rs b/src/main.rs index b07a679..25b703c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,9 +1,9 @@ mod app; mod database; +mod file_manager; mod output; mod params; mod scanner; -mod file_manager; use anyhow::Result; use app::App; diff --git a/src/output.rs b/src/output.rs index 9987374..b52eca5 100644 --- a/src/output.rs +++ b/src/output.rs @@ -1,21 +1,18 @@ -use std::{collections::HashMap, fs, io}; use std::io::Write; - +use std::{collections::HashMap, fs, io}; use anyhow::Result; use chrono::offset::Utc; use chrono::DateTime; use colored::Colorize; use humansize::{format_size, DECIMAL}; use itertools::Itertools; - -use crate::file_manager; -use crate::database::File; +use crate::file_manager::{self, File}; use crate::params::Params; use prettytable::{format, row, Table}; use unicode_segmentation::UnicodeSegmentation; fn format_path(path: &str, opts: &Params) -> Result { - let display_path = path.replace(&opts.get_directory()?, ""); + let display_path = path.replace(&opts.get_directory()?, ""); let display_range = if display_path.chars().count() > 32 { display_path .graphemes(true) @@ -81,7 +78,7 @@ fn scan_group_confirmation() -> Result { match user_input.trim() { "Y" | "y" => Ok(true), - _ => Ok(false) + _ => Ok(false), } } @@ -107,20 +104,27 @@ fn process_group_action(duplicates: &Vec, dup_index: usize, dup_size: usiz print!("{esc}[2J{esc}[1;1H", esc = 27 as char); - if parsed_file_indices.is_empty() { return } + if parsed_file_indices.is_empty() { + return; + } let files_to_delete = parsed_file_indices .into_iter() .map(|index| duplicates[index].clone()); println!("\n{}", "The following files will be deleted:".red()); - files_to_delete.clone().enumerate().for_each(|(index, file)| { - println!("{}: {}", index.to_string().blue(), file.path); - }); - + files_to_delete + .clone() + .enumerate() + .for_each(|(index, file)| { + println!("{}: {}", index.to_string().blue(), file.path); + }); + match scan_group_confirmation().unwrap() { - true => { file_manager::delete_files(files_to_delete.collect_vec()); }, - false => println!("{}", "\nCancelled Delete Operation.".red()) + true => { + file_manager::delete_files(files_to_delete.collect_vec()); + } + false => println!("{}", "\nCancelled Delete Operation.".red()), } } @@ -128,21 +132,24 @@ pub fn interactive(duplicates: Vec, opts: &Params) { print_meta_info(&duplicates, opts); let grouped_duplicates = group_duplicates(duplicates); - grouped_duplicates.iter().enumerate().for_each(|(gindex, (hash, group))| { - let mut itable = Table::new(); - itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR); - itable.set_titles(row!["index", "filename", "size", "updated_at"]); - group.iter().enumerate().for_each(|(index, file)| { - itable.add_row(row![ - index, - format_path(&file.path, opts).unwrap_or_default().blue(), - file_size(&file.path).unwrap_or_default().red(), - modified_time(&file.path).unwrap_or_default().yellow() - ]); - }); + grouped_duplicates + .iter() + .enumerate() + .for_each(|(gindex, (hash, group))| { + let mut itable = Table::new(); + itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR); + itable.set_titles(row!["index", "filename", "size", "updated_at"]); + group.iter().enumerate().for_each(|(index, file)| { + itable.add_row(row![ + index, + format_path(&file.path, opts).unwrap_or_default().blue(), + file_size(&file.path).unwrap_or_default().red(), + modified_time(&file.path).unwrap_or_default().yellow() + ]); + }); - process_group_action(group, gindex, grouped_duplicates.len(), itable); - }); + process_group_action(group, gindex, grouped_duplicates.len(), itable); + }); } pub fn print(duplicates: Vec, opts: &Params) { diff --git a/src/params.rs b/src/params.rs index 11741d9..5694eec 100644 --- a/src/params.rs +++ b/src/params.rs @@ -1,5 +1,4 @@ use std::{fs, path::PathBuf}; - use anyhow::{anyhow, Result}; use clap::Parser; @@ -17,7 +16,7 @@ pub struct Params { pub nocache: bool, /// Delete files interactively #[arg(long, short)] - pub interactive: bool + pub interactive: bool, } impl Params { diff --git a/src/scanner.rs b/src/scanner.rs index 21bb3ab..ba0fc6c 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -1,15 +1,12 @@ -use std::{fs, path::PathBuf}; -use indicatif::{HumanDuration, MultiProgress, ProgressBar, ProgressStyle, ParallelProgressIterator}; use anyhow::Result; use fxhash::hash32 as hasher; use glob::glob; +use indicatif::{ParallelProgressIterator, ProgressStyle}; use itertools::Itertools; use rayon::prelude::*; +use std::{fs, path::PathBuf}; -use crate::{ - database::{self, File}, - params::Params, -}; +use crate::{database, file_manager::File, params::Params}; pub fn duplicates(app_opts: &Params, connection: &sqlite::Connection) -> Result> { let scan_results = scan(app_opts, connection)?; @@ -46,7 +43,12 @@ fn scan(app_opts: &Params, connection: &sqlite::Connection) -> Result = glob_patterns .par_iter() - .progress_with_style(ProgressStyle::with_template("{spinner:.green} [scanning files] [{wide_bar:.cyan/blue}] {pos}/{len} files").unwrap()) + .progress_with_style( + ProgressStyle::with_template( + "{spinner:.green} [scanning files] [{wide_bar:.cyan/blue}] {pos}/{len} files", + ) + .unwrap(), + ) .filter_map(|glob_pattern| glob(glob_pattern.as_os_str().to_str()?).ok()) .flat_map(|file_vec| { file_vec @@ -67,10 +69,15 @@ fn scan(app_opts: &Params, connection: &sqlite::Connection) -> Result, connection: &sqlite::Connection) -> Result<()> { let hashed: Vec = files .into_par_iter() - .progress_with_style(ProgressStyle::with_template("{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files").unwrap()) + .progress_with_style( + ProgressStyle::with_template( + "{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files", + ) + .unwrap(), + ) .filter_map(|file| { let hash = hash_file(&file).ok()?; - Some(database::File { path: file, hash }) + Some(File { path: file, hash }) }) .collect(); From 51f5f8e61af28ac839931f16d8f27dc5576d5f48 Mon Sep 17 00:00:00 2001 From: sreedev Date: Tue, 17 Jan 2023 23:40:57 -0500 Subject: [PATCH 3/6] incremental hashing + remove sqlite + mmap file reads --- Cargo.lock | 98 +++++++++++++++------------------------------- Cargo.toml | 4 +- README.md | 1 - src/app.rs | 4 +- src/database.rs | 82 -------------------------------------- src/main.rs | 1 - src/params.rs | 19 +++++++-- src/scanner.rs | 102 +++++++++++++++++++++++++----------------------- 8 files changed, 102 insertions(+), 209 deletions(-) delete mode 100644 src/database.rs diff --git a/Cargo.lock b/Cargo.lock index 7b7a3f2..1240734 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -70,12 +70,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" -[[package]] -name = "cassowary" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" - [[package]] name = "cc" version = "1.0.78" @@ -223,31 +217,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crossterm" -version = "0.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e64e6c0fbe2c17357405f7c758c1ef960fce08bdfb2c03d88d2a18d7e09c4b67" -dependencies = [ - "bitflags", - "crossterm_winapi", - "libc", - "mio", - "parking_lot", - "signal-hook", - "signal-hook-mio", - "winapi", -] - -[[package]] -name = "crossterm_winapi" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ae1b35a484aa10e07fe0638d02301c5ad24de82d310ccbd2f3693da5f09bf1c" -dependencies = [ - "winapi", -] - [[package]] name = "csv" version = "1.1.6" @@ -314,6 +283,20 @@ dependencies = [ "syn", ] +[[package]] +name = "dashmap" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" +dependencies = [ + "cfg-if", + "hashbrown", + "lock_api", + "once_cell", + "parking_lot_core", + "rayon", +] + [[package]] name = "deduplicator" version = "0.0.9" @@ -322,18 +305,18 @@ dependencies = [ "chrono", "clap", "colored", - "crossterm", + "dashmap", "fxhash", "glob", "humansize", "indicatif", "itertools", + "memmap2", "prettytable-rs", "rayon", "sqlite", "thiserror", "tokio", - "tui", "unicode-segmentation", ] @@ -423,6 +406,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "heck" version = "0.4.0" @@ -598,6 +587,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "memmap2" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b182332558b18d807c4ce1ca8ca983b34c3ee32765e47b3f0f69b90355cc1dc" +dependencies = [ + "libc", +] + [[package]] name = "memoffset" version = "0.7.1" @@ -855,27 +853,6 @@ version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" -[[package]] -name = "signal-hook" -version = "0.3.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a253b5e89e2698464fc26b545c9edceb338e18a89effeeecfea192c3025be29d" -dependencies = [ - "libc", - "signal-hook-registry", -] - -[[package]] -name = "signal-hook-mio" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" -dependencies = [ - "libc", - "mio", - "signal-hook", -] - [[package]] name = "signal-hook-registry" version = "1.4.0" @@ -1030,19 +1007,6 @@ dependencies = [ "syn", ] -[[package]] -name = "tui" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccdd26cbd674007e649a272da4475fb666d3aa0ad0531da7136db6fab0e5bad1" -dependencies = [ - "bitflags", - "cassowary", - "crossterm", - "unicode-segmentation", - "unicode-width", -] - [[package]] name = "unicode-ident" version = "1.0.6" diff --git a/Cargo.toml b/Cargo.toml index a38c053..5fd7a11 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,16 +13,16 @@ anyhow = "1.0.68" chrono = "0.4.23" clap = { version = "4.0.32", features = ["derive"] } colored = "2.0.0" -crossterm = "0.25.0" +dashmap = { version = "5.4.0", features = ["rayon"] } fxhash = "0.2.1" glob = "0.3.0" humansize = "2.1.2" indicatif = { version = "0.17.2", features = ["rayon", "tokio"] } itertools = "0.10.5" +memmap2 = "0.5.8" prettytable-rs = "0.10.0" rayon = "1.6.1" sqlite = "0.30.3" thiserror = "1.0.38" tokio = { version = "1.23.0", features = ["full"] } -tui = "0.19.0" unicode-segmentation = "1.10.0" diff --git a/README.md b/README.md index 726c36f..482ac1f 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,6 @@ Usage: deduplicator [OPTIONS] Options: -t, --types Filetypes to deduplicate (default = all) --dir Run Deduplicator on dir different from pwd - -n, --nocache Don't use cache for indexing files (default = false) -i, --interactive Delete files interactively -h, --help Print help information -V, --version Print version information diff --git a/src/app.rs b/src/app.rs index 3356b25..0409738 100644 --- a/src/app.rs +++ b/src/app.rs @@ -1,4 +1,3 @@ -use crate::database; use crate::output; use crate::params::Params; use crate::scanner; @@ -8,8 +7,7 @@ pub struct App; impl App { pub fn init(app_args: &Params) -> Result<()> { - let connection = database::get_connection(app_args)?; - let duplicates = scanner::duplicates(app_args, &connection)?; + let duplicates = scanner::duplicates(app_args)?; match app_args.interactive { true => output::interactive(duplicates, app_args), false => output::print(duplicates, app_args), diff --git a/src/database.rs b/src/database.rs deleted file mode 100644 index 3f8595c..0000000 --- a/src/database.rs +++ /dev/null @@ -1,82 +0,0 @@ -use std::env::temp_dir; -use anyhow::Result; -use crate::params::Params; -use crate::file_manager::File; - -fn db_connection_url(args: &Params) -> String { - match args.nocache { - true => String::from(":memory:"), - false => { - let temp_dir_path = temp_dir(); - format!("{}/deduplicator.db", temp_dir_path.display()) - } - } -} - -pub fn get_connection(args: &Params) -> Result { - sqlite::open(db_connection_url(args)).and_then(|conn| { - setup(&conn).ok(); - Ok(conn) - }) -} - -pub fn setup(connection: &sqlite::Connection) -> Result<()> { - let query = "CREATE TABLE files (file_identifier STRING, hash STRING)"; - connection.execute(query).ok(); - Ok(()) -} - -pub fn put(file: &File, connection: &sqlite::Connection) -> Result<()> { - let query = format!( - "INSERT INTO files (file_identifier, hash) VALUES (\"{}\", \"{}\")", - file.path, file.hash - ); - connection.execute(query)?; - Ok(()) -} - -pub fn indexed_paths(connection: &sqlite::Connection) -> Result> { - let query = "SELECT * FROM files"; - - let result: Vec = connection - .prepare(query)? - .into_iter() - .filter_map(|row_result| row_result.ok()) - .map(|row| { - let path = row.read::<&str, _>("file_identifier").to_string(); - let hash = row.read::("hash").to_string(); - File { path, hash } - }) - .collect(); - - Ok(result) -} - -pub fn duplicate_hashes(connection: &sqlite::Connection, path: &str) -> Result> { - let query = format!( - " - SELECT a.* FROM files a - JOIN (SELECT file_identifier, hash, COUNT(*) - FROM files - GROUP BY hash - HAVING count(*) > 1 ) b - ON a.hash = b.hash - WHERE a.file_identifier LIKE \"{}%\" - ORDER BY a.file_identifier - ", - path - ); - - let result: Vec = connection - .prepare(query)? - .into_iter() - .filter_map(|row_result| row_result.ok()) - .map(|row| { - let path = row.read::<&str, _>("file_identifier").to_string(); - let hash = row.read::("hash").to_string(); - File { path, hash } - }) - .collect(); - - Ok(result) -} diff --git a/src/main.rs b/src/main.rs index 25b703c..4a82824 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,4 @@ mod app; -mod database; mod file_manager; mod output; mod params; diff --git a/src/params.rs b/src/params.rs index 5694eec..0ab59b8 100644 --- a/src/params.rs +++ b/src/params.rs @@ -1,6 +1,6 @@ -use std::{fs, path::PathBuf}; use anyhow::{anyhow, Result}; use clap::Parser; +use std::{fs, path::PathBuf}; #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] @@ -11,9 +11,6 @@ pub struct Params { /// Run Deduplicator on dir different from pwd #[arg(long)] pub dir: Option, - /// Don't use cache for indexing files (default = false) - #[arg(long, short)] - pub nocache: bool, /// Delete files interactively #[arg(long, short)] pub interactive: bool, @@ -36,4 +33,18 @@ impl Params { Ok(dir) } + + pub fn get_glob_patterns(&self) -> Vec { + self.types + .clone() + .unwrap_or_else(|| String::from("*")) + .split(',') + .map(|filetype| format!("*.{}", filetype)) + .map(|filetype| { + vec![self.get_directory().unwrap(), String::from("**"), filetype] + .iter() + .collect() + }) + .collect() + } } diff --git a/src/scanner.rs b/src/scanner.rs index ba0fc6c..e4b8b4d 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -1,46 +1,31 @@ use anyhow::Result; -use fxhash::hash32 as hasher; +use fxhash::hash64 as hasher; use glob::glob; use indicatif::{ParallelProgressIterator, ProgressStyle}; -use itertools::Itertools; use rayon::prelude::*; +use std::hash::Hasher; use std::{fs, path::PathBuf}; +use memmap2::Mmap; +use dashmap::DashMap; -use crate::{database, file_manager::File, params::Params}; +use crate::{file_manager::File, params::Params}; -pub fn duplicates(app_opts: &Params, connection: &sqlite::Connection) -> Result> { - let scan_results = scan(app_opts, connection)?; - let base_path = app_opts.get_directory()?; +pub fn duplicates(app_opts: &Params) -> Result> { + let scan_results = scan(app_opts)?; + let index_store = index_files(scan_results)?; - index_files(scan_results, connection)?; - database::duplicate_hashes(connection, &base_path) + let duplicate_files = index_store + .into_par_iter() + .filter(|(_, files)| files.len() > 1) + .map(|(_, files)| files ) + .flatten() + .collect::>(); + + Ok(duplicate_files) } -fn get_glob_patterns(opts: &Params, directory: &str) -> Vec { - opts.types - .clone() - .unwrap_or_else(|| String::from("*")) - .split(',') - .map(|filetype| format!("*.{}", filetype)) - .map(|filetype| { - vec![directory.to_owned(), String::from("**"), filetype] - .iter() - .collect() - }) - .collect() -} - -fn is_indexed_file(path: impl Into, indexed: &[File]) -> bool { - indexed - .iter() - .map(|file| file.path.clone()) - .contains(&path.into()) -} - -fn scan(app_opts: &Params, connection: &sqlite::Connection) -> Result> { - let directory = app_opts.get_directory()?; - let glob_patterns: Vec = get_glob_patterns(app_opts, &directory); - let indexed_paths = database::indexed_paths(connection)?; +fn scan(app_opts: &Params) -> Result> { + let glob_patterns: Vec = app_opts.get_glob_patterns(); let files: Vec = glob_patterns .par_iter() .progress_with_style( @@ -53,7 +38,6 @@ fn scan(app_opts: &Params, connection: &sqlite::Connection) -> Result Result, connection: &sqlite::Connection) -> Result<()> { - let hashed: Vec = files +fn index_files(files: Vec) -> Result>> { + let store: DashMap> = DashMap::new(); + files .into_par_iter() .progress_with_style( ProgressStyle::with_template( "{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files", - ) - .unwrap(), + )?, ) - .filter_map(|file| { - let hash = hash_file(&file).ok()?; - Some(File { path: file, hash }) - }) - .collect(); + .for_each(|file| { + let hash = hash_file(&file).unwrap_or_default(); + let fobj = File { path: file, hash: hash.clone() }; + store + .entry(hash) + .and_modify(|fileset| fileset.push(fobj.clone()) ) + .or_insert_with(|| vec![fobj]); + }); - hashed - .iter() - .try_for_each(|file| database::put(file, connection)) + Ok(store) +} + +pub fn incremental_hashing(filepath: &str) -> Result { + let file = fs::File::open(filepath)?; + let fmap = unsafe { Mmap::map(&file)? }; + let mut inchasher = fxhash::FxHasher::default(); + + fmap + .chunks(1_000) + .for_each(|kilo| { inchasher.write(kilo) }); + + Ok(format!("{}", inchasher.finish())) +} + +pub fn standard_hashing(filepath: &str) -> Result { + let file = fs::read(filepath)?; + Ok(hasher(&*file).to_string()) } pub fn hash_file(filepath: &str) -> Result { - let file = fs::read(filepath)?; - let hash = hasher(&*file).to_string(); + let filemeta = fs::metadata(filepath)?; - Ok(hash) + match filemeta.len() < 1_000_000 { + true => standard_hashing(filepath), + false => incremental_hashing(filepath) + } } From ea748c60d8fd74a13b8e65214f8c20ba044dbfbe Mon Sep 17 00:00:00 2001 From: sreedev Date: Wed, 18 Jan 2023 00:24:50 -0500 Subject: [PATCH 4/6] performance tweaks --- Cargo.lock | 37 -------------- Cargo.toml | 1 - src/file_manager.rs | 5 +- src/output.rs | 48 +++++++----------- src/scanner.rs | 121 +++++++++++++++++++++++++++++++------------- 5 files changed, 107 insertions(+), 105 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1240734..9260bf6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -314,7 +314,6 @@ dependencies = [ "memmap2", "prettytable-rs", "rayon", - "sqlite", "thiserror", "tokio", "unicode-segmentation", @@ -693,12 +692,6 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" -[[package]] -name = "pkg-config" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" - [[package]] name = "portable-atomic" version = "0.3.19" @@ -878,36 +871,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "sqlite" -version = "0.30.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12e072cb5fb89b3fe5e9c9584676348feb503f9fb3ae829d9868171bc5372d48" -dependencies = [ - "libc", - "sqlite3-sys", -] - -[[package]] -name = "sqlite3-src" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1815a7a02c996eb8e5c64f61fcb6fd9b12e593ce265c512c5853b2513635691" -dependencies = [ - "cc", - "pkg-config", -] - -[[package]] -name = "sqlite3-sys" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d47c99824fc55360ba00caf28de0b8a0458369b832e016a64c13af0ad9fbb9ee" -dependencies = [ - "libc", - "sqlite3-src", -] - [[package]] name = "strsim" version = "0.10.0" diff --git a/Cargo.toml b/Cargo.toml index 5fd7a11..d883b4e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,6 @@ itertools = "0.10.5" memmap2 = "0.5.8" prettytable-rs = "0.10.0" rayon = "1.6.1" -sqlite = "0.30.3" thiserror = "1.0.38" tokio = { version = "1.23.0", features = ["full"] } unicode-segmentation = "1.10.0" diff --git a/src/file_manager.rs b/src/file_manager.rs index 9f97318..25d5b8d 100644 --- a/src/file_manager.rs +++ b/src/file_manager.rs @@ -4,14 +4,15 @@ use colored::Colorize; #[derive(Debug, Clone)] pub struct File { pub path: String, - pub hash: String, + pub size: Option, + pub hash: Option, } pub fn delete_files(files: Vec) -> Result<()> { files.into_iter().for_each(|file| { match std::fs::remove_file(file.path.clone()) { Ok(_) => println!("{}: {}", "DELETED".green(), file.path), - Err(e) => println!("{}: {}", "FAILED".red(), file.path) + Err(_) => println!("{}: {}", "FAILED".red(), file.path) } }); diff --git a/src/output.rs b/src/output.rs index b52eca5..95c4567 100644 --- a/src/output.rs +++ b/src/output.rs @@ -1,14 +1,15 @@ -use std::io::Write; -use std::{collections::HashMap, fs, io}; +use crate::file_manager::{self, File}; +use crate::params::Params; use anyhow::Result; use chrono::offset::Utc; use chrono::DateTime; use colored::Colorize; +use dashmap::DashMap; use humansize::{format_size, DECIMAL}; use itertools::Itertools; -use crate::file_manager::{self, File}; -use crate::params::Params; use prettytable::{format, row, Table}; +use std::io::Write; +use std::{fs, io}; use unicode_segmentation::UnicodeSegmentation; fn format_path(path: &str, opts: &Params) -> Result { @@ -42,19 +43,7 @@ fn modified_time(path: &String) -> Result { Ok(modified_time.format("%Y-%m-%d %H:%M:%S").to_string()) } -fn group_duplicates(duplicates: Vec) -> HashMap> { - let mut duplicate_mapper: HashMap> = HashMap::new(); - duplicates.into_iter().for_each(|file| { - duplicate_mapper - .entry(file.hash.clone()) - .and_modify(|value| value.push(file.clone())) - .or_insert_with(|| vec![file]); - }); - - duplicate_mapper -} - -fn print_meta_info(duplicates: &Vec, opts: &Params) { +fn print_meta_info() { println!("Deduplicator v{}", std::env!("CARGO_PKG_VERSION")); } @@ -122,20 +111,19 @@ fn process_group_action(duplicates: &Vec, dup_index: usize, dup_size: usiz match scan_group_confirmation().unwrap() { true => { - file_manager::delete_files(files_to_delete.collect_vec()); + file_manager::delete_files(files_to_delete.collect_vec()).ok(); } false => println!("{}", "\nCancelled Delete Operation.".red()), } } -pub fn interactive(duplicates: Vec, opts: &Params) { - print_meta_info(&duplicates, opts); - let grouped_duplicates = group_duplicates(duplicates); - - grouped_duplicates - .iter() +pub fn interactive(duplicates: DashMap>, opts: &Params) { + print_meta_info(); + duplicates + .clone() + .into_iter() .enumerate() - .for_each(|(gindex, (hash, group))| { + .for_each(|(gindex, (_, group))| { let mut itable = Table::new(); itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR); itable.set_titles(row!["index", "filename", "size", "updated_at"]); @@ -148,18 +136,16 @@ pub fn interactive(duplicates: Vec, opts: &Params) { ]); }); - process_group_action(group, gindex, grouped_duplicates.len(), itable); + process_group_action(&group, gindex, duplicates.len(), itable); }); } -pub fn print(duplicates: Vec, opts: &Params) { - print_meta_info(&duplicates, opts); +pub fn print(duplicates: DashMap>, opts: &Params) { + print_meta_info(); let mut output_table = Table::new(); - let grouped_duplicates: HashMap> = group_duplicates(duplicates); - output_table.set_titles(row!["hash", "duplicates"]); - grouped_duplicates.iter().for_each(|(hash, group)| { + duplicates.into_iter().for_each(|(hash, group)| { let mut inner_table = Table::new(); inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR); group.iter().for_each(|file| { diff --git a/src/scanner.rs b/src/scanner.rs index e4b8b4d..c5d1086 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -1,39 +1,57 @@ use anyhow::Result; +use dashmap::DashMap; use fxhash::hash64 as hasher; use glob::glob; use indicatif::{ParallelProgressIterator, ProgressStyle}; +use memmap2::Mmap; use rayon::prelude::*; use std::hash::Hasher; use std::{fs, path::PathBuf}; -use memmap2::Mmap; -use dashmap::DashMap; use crate::{file_manager::File, params::Params}; -pub fn duplicates(app_opts: &Params) -> Result> { - let scan_results = scan(app_opts)?; - let index_store = index_files(scan_results)?; +#[derive(Clone, Copy)] +enum IndexCritera { + Size, + Hash, +} - let duplicate_files = index_store +pub fn duplicates(app_opts: &Params) -> Result>> { + let scan_results = scan(app_opts)?; + let size_index_store = index_files(scan_results, IndexCritera::Size)?; + + let sizewize_duplicate_files = size_index_store .into_par_iter() .filter(|(_, files)| files.len() > 1) - .map(|(_, files)| files ) + .map(|(_, files)| files) .flatten() .collect::>(); - Ok(duplicate_files) + if sizewize_duplicate_files.len() > 1 { + let size_wise_duplicate_paths = sizewize_duplicate_files + .into_par_iter() + .map(|file| file.path) + .collect::>(); + + let hash_index_store = index_files(size_wise_duplicate_paths, IndexCritera::Hash)?; + let duplicate_files = hash_index_store + .into_par_iter() + .filter(|(_, files)| files.len() > 1) + .collect(); + + Ok(duplicate_files) + } else { + Ok(DashMap::new()) + } } fn scan(app_opts: &Params) -> Result> { let glob_patterns: Vec = app_opts.get_glob_patterns(); let files: Vec = glob_patterns .par_iter() - .progress_with_style( - ProgressStyle::with_template( - "{spinner:.green} [scanning files] [{wide_bar:.cyan/blue}] {pos}/{len} files", - ) - .unwrap(), - ) + .progress_with_style(ProgressStyle::with_template( + "{spinner:.green} [scanning files] [{wide_bar:.cyan/blue}] {pos}/{len} files", + )?) .filter_map(|glob_pattern| glob(glob_pattern.as_os_str().to_str()?).ok()) .flat_map(|file_vec| { file_vec @@ -50,25 +68,60 @@ fn scan(app_opts: &Params) -> Result> { Ok(files) } -fn index_files(files: Vec) -> Result>> { +fn process_file_size_index(fpath: String) -> Result { + Ok(File { + path: fpath.clone(), + size: Some(fs::metadata(fpath)?.len()), + hash: None, + }) +} + +fn process_file_hash_index(fpath: String) -> Result { + Ok(File { + path: fpath.clone(), + size: None, + hash: Some(hash_file(&fpath).unwrap_or_default()), + }) +} + +fn process_file_index( + fpath: String, + store: &DashMap>, + index_criteria: IndexCritera, +) { + match index_criteria { + IndexCritera::Size => { + let processed_file = process_file_size_index(fpath).unwrap(); + store + .entry(processed_file.size.unwrap_or_default().to_string()) + .and_modify(|fileset| fileset.push(processed_file.clone())) + .or_insert_with(|| vec![processed_file]); + } + IndexCritera::Hash => { + let processed_file = process_file_hash_index(fpath).unwrap(); + let indexhash = processed_file.clone().hash.unwrap_or_default(); + + store + .entry(indexhash) + .and_modify(|fileset| fileset.push(processed_file.clone())) + .or_insert_with(|| vec![processed_file]); + } + } +} + +fn index_files( + files: Vec, + index_criteria: IndexCritera, +) -> Result>> { let store: DashMap> = DashMap::new(); files .into_par_iter() - .progress_with_style( - ProgressStyle::with_template( - "{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files", - )?, - ) - .for_each(|file| { - let hash = hash_file(&file).unwrap_or_default(); - let fobj = File { path: file, hash: hash.clone() }; - store - .entry(hash) - .and_modify(|fileset| fileset.push(fobj.clone()) ) - .or_insert_with(|| vec![fobj]); - }); + .progress_with_style(ProgressStyle::with_template( + "{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files", + )?) + .for_each(|file| process_file_index(file, &store, index_criteria)); - Ok(store) + Ok(store) } pub fn incremental_hashing(filepath: &str) -> Result { @@ -76,9 +129,8 @@ pub fn incremental_hashing(filepath: &str) -> Result { let fmap = unsafe { Mmap::map(&file)? }; let mut inchasher = fxhash::FxHasher::default(); - fmap - .chunks(1_000) - .for_each(|kilo| { inchasher.write(kilo) }); + fmap.chunks(1_000_000) + .for_each(|mega| inchasher.write(mega)); Ok(format!("{}", inchasher.finish())) } @@ -91,8 +143,9 @@ pub fn standard_hashing(filepath: &str) -> Result { pub fn hash_file(filepath: &str) -> Result { let filemeta = fs::metadata(filepath)?; - match filemeta.len() < 1_000_000 { + // NOTE: USE INCREMENTAL HASHING ONLY FOR FILES > 100MB + match filemeta.len() < 100_000_000 { true => standard_hashing(filepath), - false => incremental_hashing(filepath) + false => incremental_hashing(filepath), } } From 1ea57054741fc51e801034d4a156df5942ea3099 Mon Sep 17 00:00:00 2001 From: sreedev Date: Wed, 18 Jan 2023 00:30:39 -0500 Subject: [PATCH 5/6] fix confusing delete confirmation --- src/output.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/output.rs b/src/output.rs index 95c4567..4767d6f 100644 --- a/src/output.rs +++ b/src/output.rs @@ -60,7 +60,7 @@ fn scan_group_instruction() -> Result { } fn scan_group_confirmation() -> Result { - print!("\nconfirm? [Y/n]: "); + print!("\nconfirm? [y/N]: "); std::io::stdout().flush()?; let mut user_input = String::new(); io::stdin().read_line(&mut user_input)?; From 8ac78fb856b273c7bda51361d0c8c2fe428d1bcf Mon Sep 17 00:00:00 2001 From: sreedev Date: Wed, 18 Jan 2023 00:32:53 -0500 Subject: [PATCH 6/6] version 0.1.1 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9260bf6..249bc8f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -299,7 +299,7 @@ dependencies = [ [[package]] name = "deduplicator" -version = "0.0.9" +version = "0.1.1" dependencies = [ "anyhow", "chrono", diff --git a/Cargo.toml b/Cargo.toml index d883b4e..5abc561 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deduplicator" -version = "0.0.9" +version = "0.1.1" edition = "2021" description = "find,filter,delete Duplicates" license = "MIT"