smolkik_adm/deduplicator
1
0
Fork 0
mirror of https://github.com/sreedevk/deduplicator.git synced 2026-08-26 10:05:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #23 from sreedevk/performance/incremental-hashing

Browse Source 
Performance Improvements
This commit is contained in:
Sreedev Kodichath
2023-01-18 00:31:27 -05:00
committed by GitHub
parent 175c7579c4 1ea5705474
commit 37a4c4d52d
10 changed files with 186 additions and 291 deletions
Download Patch File Download Diff File Expand all files Collapse all files

135
Cargo.lock generated
View File

@@ -70,12 +70,6 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c"
[[package]]
name = "cassowary"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53"
[[package]]
name = "cc"
version = "1.0.78"
@@ -223,31 +217,6 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "crossterm"
version = "0.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e64e6c0fbe2c17357405f7c758c1ef960fce08bdfb2c03d88d2a18d7e09c4b67"
dependencies = [
"bitflags",
"crossterm_winapi",
"libc",
"mio",
"parking_lot",
"signal-hook",
"signal-hook-mio",
"winapi",
]
[[package]]
name = "crossterm_winapi"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ae1b35a484aa10e07fe0638d02301c5ad24de82d310ccbd2f3693da5f09bf1c"
dependencies = [
"winapi",
]
[[package]]
name = "csv"
version = "1.1.6"
@@ -314,6 +283,20 @@ dependencies = [
"syn",
]
[[package]]
name = "dashmap"
version = "5.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc"
dependencies = [
"cfg-if",
"hashbrown",
"lock_api",
"once_cell",
"parking_lot_core",
"rayon",
]
[[package]]
name = "deduplicator"
version = "0.0.9"
@@ -322,18 +305,17 @@ dependencies = [
"chrono",
"clap",
"colored",
"crossterm",
"dashmap",
"fxhash",
"glob",
"humansize",
"indicatif",
"itertools",
"memmap2",
"prettytable-rs",
"rayon",
"sqlite",
"thiserror",
"tokio",
"tui",
"unicode-segmentation",
]
@@ -423,6 +405,12 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
[[package]]
name = "hashbrown"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
[[package]]
name = "heck"
version = "0.4.0"
@@ -598,6 +586,15 @@ version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
[[package]]
name = "memmap2"
version = "0.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b182332558b18d807c4ce1ca8ca983b34c3ee32765e47b3f0f69b90355cc1dc"
dependencies = [
"libc",
]
[[package]]
name = "memoffset"
version = "0.7.1"
@@ -695,12 +692,6 @@ version = "0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
[[package]]
name = "pkg-config"
version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
[[package]]
name = "portable-atomic"
version = "0.3.19"
@@ -855,27 +846,6 @@ version = "1.0.152"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb"
[[package]]
name = "signal-hook"
version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a253b5e89e2698464fc26b545c9edceb338e18a89effeeecfea192c3025be29d"
dependencies = [
"libc",
"signal-hook-registry",
]
[[package]]
name = "signal-hook-mio"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af"
dependencies = [
"libc",
"mio",
"signal-hook",
]
[[package]]
name = "signal-hook-registry"
version = "1.4.0"
@@ -901,36 +871,6 @@ dependencies = [
"winapi",
]
[[package]]
name = "sqlite"
version = "0.30.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12e072cb5fb89b3fe5e9c9584676348feb503f9fb3ae829d9868171bc5372d48"
dependencies = [
"libc",
"sqlite3-sys",
]
[[package]]
name = "sqlite3-src"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d1815a7a02c996eb8e5c64f61fcb6fd9b12e593ce265c512c5853b2513635691"
dependencies = [
"cc",
"pkg-config",
]
[[package]]
name = "sqlite3-sys"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d47c99824fc55360ba00caf28de0b8a0458369b832e016a64c13af0ad9fbb9ee"
dependencies = [
"libc",
"sqlite3-src",
]
[[package]]
name = "strsim"
version = "0.10.0"
@@ -1030,19 +970,6 @@ dependencies = [
"syn",
]
[[package]]
name = "tui"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccdd26cbd674007e649a272da4475fb666d3aa0ad0531da7136db6fab0e5bad1"
dependencies = [
"bitflags",
"cassowary",
"crossterm",
"unicode-segmentation",
"unicode-width",
]
[[package]]
name = "unicode-ident"
version = "1.0.6"

5
Cargo.toml
View File

@@ -13,16 +13,15 @@ anyhow = "1.0.68"
chrono = "0.4.23"
clap = { version = "4.0.32", features = ["derive"] }
colored = "2.0.0"
crossterm = "0.25.0"
dashmap = { version = "5.4.0", features = ["rayon"] }
fxhash = "0.2.1"
glob = "0.3.0"
humansize = "2.1.2"
indicatif = { version = "0.17.2", features = ["rayon", "tokio"] }
itertools = "0.10.5"
memmap2 = "0.5.8"
prettytable-rs = "0.10.0"
rayon = "1.6.1"
sqlite = "0.30.3"
thiserror = "1.0.38"
tokio = { version = "1.23.0", features = ["full"] }
tui = "0.19.0"
unicode-segmentation = "1.10.0"

1
README.md
View File

@@ -16,7 +16,6 @@ Usage: deduplicator [OPTIONS]
Options:
-t, --types <TYPES> Filetypes to deduplicate (default = all)
--dir <DIR> Run Deduplicator on dir different from pwd
-n, --nocache Don't use cache for indexing files (default = false)
-i, --interactive Delete files interactively
-h, --help Print help information
-V, --version Print version information

4
src/app.rs
View File

@@ -1,4 +1,3 @@
use crate::database;
use crate::output;
use crate::params::Params;
use crate::scanner;
@@ -8,8 +7,7 @@ pub struct App;
impl App {
pub fn init(app_args: &Params) -> Result<()> {
let connection = database::get_connection(app_args)?;
let duplicates = scanner::duplicates(app_args, &connection)?;
let duplicates = scanner::duplicates(app_args)?;
match app_args.interactive {
true => output::interactive(duplicates, app_args),
false => output::print(duplicates, app_args),

82
src/database.rs
View File

@@ -1,82 +0,0 @@
use std::env::temp_dir;
use anyhow::Result;
use crate::params::Params;
use crate::file_manager::File;
fn db_connection_url(args: &Params) -> String {
match args.nocache {
true => String::from(":memory:"),
false => {
let temp_dir_path = temp_dir();
format!("{}/deduplicator.db", temp_dir_path.display())
}
}
}
pub fn get_connection(args: &Params) -> Result<sqlite::Connection, sqlite::Error> {
sqlite::open(db_connection_url(args)).and_then(|conn| {
setup(&conn).ok();
Ok(conn)
})
}
pub fn setup(connection: &sqlite::Connection) -> Result<()> {
let query = "CREATE TABLE files (file_identifier STRING, hash STRING)";
connection.execute(query).ok();
Ok(())
}
pub fn put(file: &File, connection: &sqlite::Connection) -> Result<()> {
let query = format!(
"INSERT INTO files (file_identifier, hash) VALUES (\"{}\", \"{}\")",
file.path, file.hash
);
connection.execute(query)?;
Ok(())
}
pub fn indexed_paths(connection: &sqlite::Connection) -> Result<Vec<File>> {
let query = "SELECT * FROM files";
let result: Vec<File> = connection
.prepare(query)?
.into_iter()
.filter_map(|row_result| row_result.ok())
.map(|row| {
let path = row.read::<&str, _>("file_identifier").to_string();
let hash = row.read::<i64, _>("hash").to_string();
File { path, hash }
})
.collect();
Ok(result)
}
pub fn duplicate_hashes(connection: &sqlite::Connection, path: &str) -> Result<Vec<File>> {
let query = format!(
"
SELECT a.* FROM files a
JOIN (SELECT file_identifier, hash, COUNT(*)
FROM files
GROUP BY hash
HAVING count(*) > 1 ) b
ON a.hash = b.hash
WHERE a.file_identifier LIKE \"{}%\"
ORDER BY a.file_identifier
",
path
);
let result: Vec<File> = connection
.prepare(query)?
.into_iter()
.filter_map(|row_result| row_result.ok())
.map(|row| {
let path = row.read::<&str, _>("file_identifier").to_string();
let hash = row.read::<i64, _>("hash").to_string();
File { path, hash }
})
.collect();
Ok(result)
}

5
src/file_manager.rs
View File

@@ -4,14 +4,15 @@ use colored::Colorize;
#[derive(Debug, Clone)]
pub struct File {
pub path: String,
pub hash: String,
pub size: Option<u64>,
pub hash: Option<String>,
}
pub fn delete_files(files: Vec<File>) -> Result<()> {
files.into_iter().for_each(|file| {
match std::fs::remove_file(file.path.clone()) {
Ok(_) => println!("{}: {}", "DELETED".green(), file.path),
Err(e) => println!("{}: {}", "FAILED".red(), file.path)
Err(_) => println!("{}: {}", "FAILED".red(), file.path)
}
});

1
src/main.rs
View File

@@ -1,5 +1,4 @@
mod app;
mod database;
mod file_manager;
mod output;
mod params;

50
src/output.rs
View File

@@ -1,14 +1,15 @@
use std::io::Write;
use std::{collections::HashMap, fs, io};
use crate::file_manager::{self, File};
use crate::params::Params;
use anyhow::Result;
use chrono::offset::Utc;
use chrono::DateTime;
use colored::Colorize;
use dashmap::DashMap;
use humansize::{format_size, DECIMAL};
use itertools::Itertools;
use crate::file_manager::{self, File};
use crate::params::Params;
use prettytable::{format, row, Table};
use std::io::Write;
use std::{fs, io};
use unicode_segmentation::UnicodeSegmentation;
fn format_path(path: &str, opts: &Params) -> Result<String> {
@@ -42,19 +43,7 @@ fn modified_time(path: &String) -> Result<String> {
Ok(modified_time.format("%Y-%m-%d %H:%M:%S").to_string())
}
fn group_duplicates(duplicates: Vec<File>) -> HashMap<String, Vec<File>> {
let mut duplicate_mapper: HashMap<String, Vec<File>> = HashMap::new();
duplicates.into_iter().for_each(|file| {
duplicate_mapper
.entry(file.hash.clone())
.and_modify(|value| value.push(file.clone()))
.or_insert_with(|| vec![file]);
});
duplicate_mapper
}
fn print_meta_info(duplicates: &Vec<File>, opts: &Params) {
fn print_meta_info() {
println!("Deduplicator v{}", std::env!("CARGO_PKG_VERSION"));
}
@@ -71,7 +60,7 @@ fn scan_group_instruction() -> Result<String> {
}
fn scan_group_confirmation() -> Result<bool> {
print!("\nconfirm? [Y/n]: ");
print!("\nconfirm? [y/N]: ");
std::io::stdout().flush()?;
let mut user_input = String::new();
io::stdin().read_line(&mut user_input)?;
@@ -122,20 +111,19 @@ fn process_group_action(duplicates: &Vec<File>, dup_index: usize, dup_size: usiz
match scan_group_confirmation().unwrap() {
true => {
file_manager::delete_files(files_to_delete.collect_vec());
file_manager::delete_files(files_to_delete.collect_vec()).ok();
}
false => println!("{}", "\nCancelled Delete Operation.".red()),
}
}
pub fn interactive(duplicates: Vec<File>, opts: &Params) {
print_meta_info(&duplicates, opts);
let grouped_duplicates = group_duplicates(duplicates);
grouped_duplicates
.iter()
pub fn interactive(duplicates: DashMap<String, Vec<File>>, opts: &Params) {
print_meta_info();
duplicates
.clone()
.into_iter()
.enumerate()
.for_each(|(gindex, (hash, group))| {
.for_each(|(gindex, (_, group))| {
let mut itable = Table::new();
itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
itable.set_titles(row!["index", "filename", "size", "updated_at"]);
@@ -148,18 +136,16 @@ pub fn interactive(duplicates: Vec<File>, opts: &Params) {
]);
});
process_group_action(group, gindex, grouped_duplicates.len(), itable);
process_group_action(&group, gindex, duplicates.len(), itable);
});
}
pub fn print(duplicates: Vec<File>, opts: &Params) {
print_meta_info(&duplicates, opts);
pub fn print(duplicates: DashMap<String, Vec<File>>, opts: &Params) {
print_meta_info();
let mut output_table = Table::new();
let grouped_duplicates: HashMap<String, Vec<File>> = group_duplicates(duplicates);
output_table.set_titles(row!["hash", "duplicates"]);
grouped_duplicates.iter().for_each(|(hash, group)| {
duplicates.into_iter().for_each(|(hash, group)| {
let mut inner_table = Table::new();
inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
group.iter().for_each(|file| {

19
src/params.rs
View File

@@ -1,6 +1,6 @@
use std::{fs, path::PathBuf};
use anyhow::{anyhow, Result};
use clap::Parser;
use std::{fs, path::PathBuf};
#[derive(Parser, Debug)]
#[command(author, version, about, long_about = None)]
@@ -11,9 +11,6 @@ pub struct Params {
/// Run Deduplicator on dir different from pwd
#[arg(long)]
pub dir: Option<PathBuf>,
/// Don't use cache for indexing files (default = false)
#[arg(long, short)]
pub nocache: bool,
/// Delete files interactively
#[arg(long, short)]
pub interactive: bool,
@@ -36,4 +33,18 @@ impl Params {
Ok(dir)
}
pub fn get_glob_patterns(&self) -> Vec<PathBuf> {
self.types
.clone()
.unwrap_or_else(|| String::from("*"))
.split(',')
.map(|filetype| format!("*.{}", filetype))
.map(|filetype| {
vec![self.get_directory().unwrap(), String::from("**"), filetype]
.iter()
.collect()
})
.collect()
}
}

175
src/scanner.rs
View File

@@ -1,59 +1,61 @@
use anyhow::Result;
use fxhash::hash32 as hasher;
use dashmap::DashMap;
use fxhash::hash64 as hasher;
use glob::glob;
use indicatif::{ParallelProgressIterator, ProgressStyle};
use itertools::Itertools;
use memmap2::Mmap;
use rayon::prelude::*;
use std::hash::Hasher;
use std::{fs, path::PathBuf};
use crate::{database, file_manager::File, params::Params};
use crate::{file_manager::File, params::Params};
pub fn duplicates(app_opts: &Params, connection: &sqlite::Connection) -> Result<Vec<File>> {
let scan_results = scan(app_opts, connection)?;
let base_path = app_opts.get_directory()?;
index_files(scan_results, connection)?;
database::duplicate_hashes(connection, &base_path)
#[derive(Clone, Copy)]
enum IndexCritera {
Size,
Hash,
}
fn get_glob_patterns(opts: &Params, directory: &str) -> Vec<PathBuf> {
opts.types
.clone()
.unwrap_or_else(|| String::from("*"))
.split(',')
.map(|filetype| format!("*.{}", filetype))
.map(|filetype| {
vec![directory.to_owned(), String::from("**"), filetype]
.iter()
.collect()
})
.collect()
pub fn duplicates(app_opts: &Params) -> Result<DashMap<String, Vec<File>>> {
let scan_results = scan(app_opts)?;
let size_index_store = index_files(scan_results, IndexCritera::Size)?;
let sizewize_duplicate_files = size_index_store
.into_par_iter()
.filter(|(_, files)| files.len() > 1)
.map(|(_, files)| files)
.flatten()
.collect::<Vec<File>>();
if sizewize_duplicate_files.len() > 1 {
let size_wise_duplicate_paths = sizewize_duplicate_files
.into_par_iter()
.map(|file| file.path)
.collect::<Vec<String>>();
let hash_index_store = index_files(size_wise_duplicate_paths, IndexCritera::Hash)?;
let duplicate_files = hash_index_store
.into_par_iter()
.filter(|(_, files)| files.len() > 1)
.collect();
Ok(duplicate_files)
} else {
Ok(DashMap::new())
}
}
fn is_indexed_file(path: impl Into<String>, indexed: &[File]) -> bool {
indexed
.iter()
.map(|file| file.path.clone())
.contains(&path.into())
}
fn scan(app_opts: &Params, connection: &sqlite::Connection) -> Result<Vec<String>> {
let directory = app_opts.get_directory()?;
let glob_patterns: Vec<PathBuf> = get_glob_patterns(app_opts, &directory);
let indexed_paths = database::indexed_paths(connection)?;
fn scan(app_opts: &Params) -> Result<Vec<String>> {
let glob_patterns: Vec<PathBuf> = app_opts.get_glob_patterns();
let files: Vec<String> = glob_patterns
.par_iter()
.progress_with_style(
ProgressStyle::with_template(
"{spinner:.green} [scanning files] [{wide_bar:.cyan/blue}] {pos}/{len} files",
)
.unwrap(),
)
.progress_with_style(ProgressStyle::with_template(
"{spinner:.green} [scanning files] [{wide_bar:.cyan/blue}] {pos}/{len} files",
)?)
.filter_map(|glob_pattern| glob(glob_pattern.as_os_str().to_str()?).ok())
.flat_map(|file_vec| {
file_vec
.filter_map(|x| Some(x.ok()?.as_os_str().to_str()?.to_string()))
.filter(|fpath| !is_indexed_file(fpath, &indexed_paths))
.filter(|glob_result| {
fs::metadata(glob_result)
.map(|f| f.is_file())
@@ -66,29 +68,84 @@ fn scan(app_opts: &Params, connection: &sqlite::Connection) -> Result<Vec<String
Ok(files)
}
fn index_files(files: Vec<String>, connection: &sqlite::Connection) -> Result<()> {
let hashed: Vec<File> = files
.into_par_iter()
.progress_with_style(
ProgressStyle::with_template(
"{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files",
)
.unwrap(),
)
.filter_map(|file| {
let hash = hash_file(&file).ok()?;
Some(File { path: file, hash })
})
.collect();
fn process_file_size_index(fpath: String) -> Result<File> {
Ok(File {
path: fpath.clone(),
size: Some(fs::metadata(fpath)?.len()),
hash: None,
})
}
hashed
.iter()
.try_for_each(|file| database::put(file, connection))
fn process_file_hash_index(fpath: String) -> Result<File> {
Ok(File {
path: fpath.clone(),
size: None,
hash: Some(hash_file(&fpath).unwrap_or_default()),
})
}
fn process_file_index(
fpath: String,
store: &DashMap<String, Vec<File>>,
index_criteria: IndexCritera,
) {
match index_criteria {
IndexCritera::Size => {
let processed_file = process_file_size_index(fpath).unwrap();
store
.entry(processed_file.size.unwrap_or_default().to_string())
.and_modify(|fileset| fileset.push(processed_file.clone()))
.or_insert_with(|| vec![processed_file]);
}
IndexCritera::Hash => {
let processed_file = process_file_hash_index(fpath).unwrap();
let indexhash = processed_file.clone().hash.unwrap_or_default();
store
.entry(indexhash)
.and_modify(|fileset| fileset.push(processed_file.clone()))
.or_insert_with(|| vec![processed_file]);
}
}
}
fn index_files(
files: Vec<String>,
index_criteria: IndexCritera,
) -> Result<DashMap<String, Vec<File>>> {
let store: DashMap<String, Vec<File>> = DashMap::new();
files
.into_par_iter()
.progress_with_style(ProgressStyle::with_template(
"{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files",
)?)
.for_each(|file| process_file_index(file, &store, index_criteria));
Ok(store)
}
pub fn incremental_hashing(filepath: &str) -> Result<String> {
let file = fs::File::open(filepath)?;
let fmap = unsafe { Mmap::map(&file)? };
let mut inchasher = fxhash::FxHasher::default();
fmap.chunks(1_000_000)
.for_each(|mega| inchasher.write(mega));
Ok(format!("{}", inchasher.finish()))
}
pub fn standard_hashing(filepath: &str) -> Result<String> {
let file = fs::read(filepath)?;
Ok(hasher(&*file).to_string())
}
pub fn hash_file(filepath: &str) -> Result<String> {
let file = fs::read(filepath)?;
let hash = hasher(&*file).to_string();
let filemeta = fs::metadata(filepath)?;
Ok(hash)
// NOTE: USE INCREMENTAL HASHING ONLY FOR FILES > 100MB
match filemeta.len() < 100_000_000 {
true => standard_hashing(filepath),
false => incremental_hashing(filepath),
}
}