From b49940998b8b91ade36a8ae79e48f73fdf871f87 Mon Sep 17 00:00:00 2001 From: sreedev Date: Wed, 25 Jan 2023 21:27:31 -0500 Subject: [PATCH 01/10] version 0.1.6 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 297c101..c83327f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -324,7 +324,7 @@ dependencies = [ [[package]] name = "deduplicator" -version = "0.1.5" +version = "0.1.6" dependencies = [ "anyhow", "bytesize", diff --git a/Cargo.toml b/Cargo.toml index 367fb2b..b7db56e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deduplicator" -version = "0.1.5" +version = "0.1.6" edition = "2021" description = "find,filter,delete Duplicates" license = "MIT" From d06ca7897dc8e72f4e3ccf5116269544b57ad7a1 Mon Sep 17 00:00:00 2001 From: sreedev Date: Wed, 25 Jan 2023 23:12:52 -0500 Subject: [PATCH 02/10] fix critical error --- src/scanner.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/scanner.rs b/src/scanner.rs index 2ac99ae..35a32ca 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -61,7 +61,11 @@ fn scan(app_opts: &Params) -> Result> { .map(|fpath| File { path: fpath.clone(), hash: None, - size: Some(fs::metadata(fpath).unwrap().len()), + size: Some( + fs::metadata(fpath) + .map(|metadata| metadata.len()) + .unwrap_or_default(), + ), }) .filter(|file| filters::is_file_gt_min_size(app_opts, file)) .collect(); @@ -84,7 +88,7 @@ fn process_file_index( IndexCritera::Hash => { file.hash = Some(hash_file(&file.path).unwrap_or_default()); store - .entry(file.clone().hash.unwrap()) + .entry(file.clone().hash.unwrap_or_default()) .and_modify(|fileset| fileset.push(file.clone())) .or_insert_with(|| vec![file]); } From ef1e9a1fce76a69ba0e9ab08daa119ba82b7fb2b Mon Sep 17 00:00:00 2001 From: sreedev Date: Thu, 26 Jan 2023 00:04:17 -0500 Subject: [PATCH 03/10] replace path String in File type with PathBuf --- src/file_manager.rs | 7 ++++--- src/output.rs | 11 +++++++---- src/scanner.rs | 12 +++++++----- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/file_manager.rs b/src/file_manager.rs index 25d5b8d..7a28424 100644 --- a/src/file_manager.rs +++ b/src/file_manager.rs @@ -1,9 +1,10 @@ use anyhow::Result; use colored::Colorize; +use std::path::PathBuf; #[derive(Debug, Clone)] pub struct File { - pub path: String, + pub path: PathBuf, pub size: Option, pub hash: Option, } @@ -11,8 +12,8 @@ pub struct File { pub fn delete_files(files: Vec) -> Result<()> { files.into_iter().for_each(|file| { match std::fs::remove_file(file.path.clone()) { - Ok(_) => println!("{}: {}", "DELETED".green(), file.path), - Err(_) => println!("{}: {}", "FAILED".red(), file.path) + Ok(_) => println!("{}: {}", "DELETED".green(), file.path.display()), + Err(_) => println!("{}: {}", "FAILED".red(), file.path.display()) } }); diff --git a/src/output.rs b/src/output.rs index 8767bfb..0e2201a 100644 --- a/src/output.rs +++ b/src/output.rs @@ -9,11 +9,14 @@ use indicatif::{ProgressBar, ProgressIterator, ProgressStyle}; use itertools::Itertools; use prettytable::{format, row, Table}; use std::io::Write; +use std::path::Path; use std::{fs, io}; use unicode_segmentation::UnicodeSegmentation; -fn format_path(path: &str, opts: &Params) -> Result { - let display_path = path.replace(opts.get_directory()?.to_string_lossy().as_ref(), ""); +fn format_path(path: &Path, opts: &Params) -> Result { + let display_path = path + .to_string_lossy() + .replace(opts.get_directory()?.to_string_lossy().as_ref(), ""); let display_range = if display_path.chars().count() > 32 { display_path .graphemes(true) @@ -34,7 +37,7 @@ fn file_size(file: &File) -> Result { Ok(format!("{:>12}", bytesize::ByteSize::b(file.size.unwrap()))) } -fn modified_time(path: &String) -> Result { +fn modified_time(path: &Path) -> Result { let mdata = fs::metadata(path)?; let modified_time: DateTime = mdata.modified()?.into(); @@ -100,7 +103,7 @@ fn process_group_action(duplicates: &Vec, dup_index: usize, dup_size: usiz .clone() .enumerate() .for_each(|(index, file)| { - println!("{}: {}", index.to_string().blue(), file.path); + println!("{}: {}", index.to_string().blue(), file.path.display()); }); match scan_group_confirmation().unwrap() { diff --git a/src/scanner.rs b/src/scanner.rs index 35a32ca..2138b1c 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -7,7 +7,10 @@ use memmap2::Mmap; use rayon::prelude::*; use std::hash::Hasher; use std::time::Duration; -use std::{fs, path::PathBuf}; +use std::{ + fs, + path::{Path, PathBuf}, +}; #[derive(Clone, Copy)] enum IndexCritera { @@ -57,7 +60,6 @@ fn scan(app_opts: &Params) -> Result> { .progress_with_style(ProgressStyle::with_template( "{spinner:.green} [processing mapped paths] [{wide_bar:.cyan/blue}] {pos}/{len} files", )?) - .map(|fpath| fpath.display().to_string()) .map(|fpath| File { path: fpath.clone(), hash: None, @@ -110,7 +112,7 @@ fn index_files( Ok(store) } -fn incremental_hashing(filepath: &str) -> Result { +fn incremental_hashing(filepath: &Path) -> Result { let file = fs::File::open(filepath)?; let fmap = unsafe { Mmap::map(&file)? }; let mut inchasher = fxhash::FxHasher::default(); @@ -121,12 +123,12 @@ fn incremental_hashing(filepath: &str) -> Result { Ok(format!("{}", inchasher.finish())) } -fn standard_hashing(filepath: &str) -> Result { +fn standard_hashing(filepath: &Path) -> Result { let file = fs::read(filepath)?; Ok(hasher(&*file).to_string()) } -fn hash_file(filepath: &str) -> Result { +fn hash_file(filepath: &Path) -> Result { let filemeta = fs::metadata(filepath)?; // NOTE: USE INCREMENTAL HASHING ONLY FOR FILES > 100MB From f0dbf057059083207af05ce2966416fcceb1b914 Mon Sep 17 00:00:00 2001 From: sreedev Date: Thu, 26 Jan 2023 14:07:50 -0500 Subject: [PATCH 04/10] removed tokio & other unused dependencies --- Cargo.lock | 87 --------------------------------------------------- Cargo.toml | 4 +-- src/main.rs | 3 +- src/output.rs | 6 ++-- 4 files changed, 5 insertions(+), 95 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c83327f..623d40b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -83,12 +83,6 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" -[[package]] -name = "bytes" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" - [[package]] name = "bytesize" version = "1.1.0" @@ -339,8 +333,6 @@ dependencies = [ "memmap2", "prettytable-rs", "rayon", - "thiserror", - "tokio", "unicode-segmentation", ] @@ -535,7 +527,6 @@ dependencies = [ "number_prefix", "portable-atomic", "rayon", - "tokio", "unicode-width", ] @@ -655,18 +646,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "mio" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" -dependencies = [ - "libc", - "log", - "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys", -] - [[package]] name = "num-integer" version = "0.1.45" @@ -714,16 +693,6 @@ version = "6.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" -[[package]] -name = "parking_lot" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" -dependencies = [ - "lock_api", - "parking_lot_core", -] - [[package]] name = "parking_lot_core" version = "0.9.5" @@ -737,12 +706,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "pin-project-lite" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" - [[package]] name = "portable-atomic" version = "0.3.19" @@ -923,31 +886,12 @@ version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" -[[package]] -name = "signal-hook-registry" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" -dependencies = [ - "libc", -] - [[package]] name = "smallvec" version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" -[[package]] -name = "socket2" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "strsim" version = "0.10.0" @@ -1025,37 +969,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "tokio" -version = "1.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38a54aca0c15d014013256222ba0ebed095673f89345dd79119d912eb561b7a8" -dependencies = [ - "autocfg", - "bytes", - "libc", - "memchr", - "mio", - "num_cpus", - "parking_lot", - "pin-project-lite", - "signal-hook-registry", - "socket2", - "tokio-macros", - "windows-sys", -] - -[[package]] -name = "tokio-macros" -version = "1.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "unicode-ident" version = "1.0.6" diff --git a/Cargo.toml b/Cargo.toml index b7db56e..55162a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,11 +21,9 @@ colored = "2.0.0" dashmap = { version = "5.4.0", features = ["rayon"] } fxhash = "0.2.1" globwalk = "0.8.1" -indicatif = { version = "0.17.2", features = ["rayon", "tokio"] } +indicatif = { version = "0.17.2", features = ["rayon"] } itertools = "0.10.5" memmap2 = "0.5.8" prettytable-rs = "0.10.0" rayon = "1.6.1" -thiserror = "1.0.38" -tokio = { version = "1.23.1", features = ["full"] } unicode-segmentation = "1.10.0" diff --git a/src/main.rs b/src/main.rs index 132bae8..a7d1249 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,7 +9,6 @@ use anyhow::Result; use app::App; use clap::Parser; -#[tokio::main] -async fn main() -> Result<()> { +fn main() -> Result<()> { App::init(¶ms::Params::parse()) } diff --git a/src/output.rs b/src/output.rs index 0e2201a..b9716dd 100644 --- a/src/output.rs +++ b/src/output.rs @@ -6,12 +6,12 @@ use chrono::DateTime; use colored::Colorize; use dashmap::DashMap; use indicatif::{ProgressBar, ProgressIterator, ProgressStyle}; -use itertools::Itertools; use prettytable::{format, row, Table}; use std::io::Write; use std::path::Path; use std::{fs, io}; use unicode_segmentation::UnicodeSegmentation; +use itertools::Itertools; fn format_path(path: &Path, opts: &Params) -> Result { let display_path = path @@ -77,7 +77,7 @@ fn process_group_action(duplicates: &Vec, dup_index: usize, dup_size: usiz .split(',') .filter(|element| !element.is_empty()) .map(|index| index.parse::().unwrap_or_default()) - .collect_vec(); + .collect::>(); if parsed_file_indices .clone() @@ -108,7 +108,7 @@ fn process_group_action(duplicates: &Vec, dup_index: usize, dup_size: usiz match scan_group_confirmation().unwrap() { true => { - file_manager::delete_files(files_to_delete.collect_vec()).ok(); + file_manager::delete_files(files_to_delete.collect::>()).ok(); } false => println!("{}", "\nCancelled Delete Operation.".red()), } From 493cac176215e74fc209f3b380ddb68f11586d24 Mon Sep 17 00:00:00 2001 From: sreedev Date: Thu, 26 Jan 2023 14:17:54 -0500 Subject: [PATCH 05/10] upgraded dependencies --- Cargo.lock | 96 +++++++++++++++++++++++++++--------------------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 623d40b..aecc2e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -73,9 +73,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.11.1" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "byteorder" @@ -118,9 +118,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.0.32" +version = "4.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7db700bc935f9e43e88d00b0850dae18a63773cfbec6d8e070fccf7fef89a39" +checksum = "f13b9c79b5d1dd500d20ef541215a6423c75829ef43117e1b4d17fd8af0b5d76" dependencies = [ "bitflags", "clap_derive", @@ -133,9 +133,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.0.21" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0177313f9f02afc995627906bbd8967e2be069f5261954222dac78290c2b9014" +checksum = "684a277d672e91966334af371f1a7b5833f9aa00b07c84e92fbce95e00208ce8" dependencies = [ "heck", "proc-macro-error", @@ -146,9 +146,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d4198f73e42b4936b35b5bb248d81d2b595ecb170da0bac7655c54eedfa8da8" +checksum = "783fe232adfca04f90f56201b26d79682d4cd2625e0bc7290b95123afe558ade" dependencies = [ "os_str_bytes", ] @@ -176,9 +176,9 @@ dependencies = [ [[package]] name = "console" -version = "0.15.4" +version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9b6515d269224923b26b5febea2ed42b2d5f2ce37284a4dd670fedd6cb8347a" +checksum = "c3d79fbe8970a77e3e34151cc13d3b3e248aa0faaecb9f6091fa07ebefe5ad60" dependencies = [ "encode_unicode 0.3.6", "lazy_static", @@ -260,9 +260,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.85" +version = "1.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add3fc1717409d029b20c5b6903fc0c0b02fa6741d820054f4a2efa5e5816fd" +checksum = "b61a7545f753a88bcbe0a70de1fcc0221e10bfc752f576754fa91e663db1622e" dependencies = [ "cc", "cxxbridge-flags", @@ -272,9 +272,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.85" +version = "1.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c87959ba14bc6fbc61df77c3fcfe180fc32b93538c4f1031dd802ccb5f2ff0" +checksum = "f464457d494b5ed6905c63b0c4704842aba319084a0a3561cdc1359536b53200" dependencies = [ "cc", "codespan-reporting", @@ -287,15 +287,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.85" +version = "1.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69a3e162fde4e594ed2b07d0f83c6c67b745e7f28ce58c6df5e6b6bef99dfb59" +checksum = "43c7119ce3a3701ed81aca8410b9acf6fc399d2629d057b87e2efa4e63a3aaea" [[package]] name = "cxxbridge-macro" -version = "1.0.85" +version = "1.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e7e2adeb6a0d4a282e581096b06e1791532b7d576dcde5ccd9382acf55db8e6" +checksum = "65e07508b90551e610910fa648a1878991d367064997a596135b86df30daf07e" dependencies = [ "proc-macro2", "quote", @@ -519,9 +519,9 @@ dependencies = [ [[package]] name = "indicatif" -version = "0.17.2" +version = "0.17.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4295cbb7573c16d310e99e713cf9e75101eb190ab31fccd35f2d2691b4352b19" +checksum = "cef509aa9bc73864d6756f0d34d35504af3cf0844373afe9b8669a5b8005a729" dependencies = [ "console", "number_prefix", @@ -532,9 +532,9 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c" +checksum = "e7d6c6f8c91b4b9ed43484ad1a938e393caf35960fce7f82a040497207bd8e9e" dependencies = [ "libc", "windows-sys", @@ -683,9 +683,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "once_cell" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" +checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" [[package]] name = "os_str_bytes" @@ -695,9 +695,9 @@ checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" [[package]] name = "parking_lot_core" -version = "0.9.5" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ff9f3fef3968a3ec5945535ed654cb38ff72d7495a25619e2247fb15a2ed9ba" +checksum = "ba1ef8814b5c993410bb3adfad7a5ed269563e4a2f90c41f5d85be7fb47133bf" dependencies = [ "cfg-if", "libc", @@ -752,9 +752,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.49" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" +checksum = "6ef7d57beacfaf2d8aee5937dab7b7f28de3cb8b1828479bb5de2a7106f2bae2" dependencies = [ "unicode-ident", ] @@ -780,9 +780,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.10.1" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3" +checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b" dependencies = [ "crossbeam-channel", "crossbeam-deque", @@ -835,9 +835,9 @@ checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" [[package]] name = "rustix" -version = "0.36.5" +version = "0.36.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3807b5d10909833d3e9acd1eb5fb988f79376ff10fce42937de71a449c4c588" +checksum = "d4fdebc4b395b7fbb9ab11e462e20ed9051e7b16e42d24042c776eca0ac81b03" dependencies = [ "bitflags", "errno", @@ -922,9 +922,9 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.1.3" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" dependencies = [ "winapi-util", ] @@ -1118,42 +1118,42 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" +checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" [[package]] name = "windows_aarch64_msvc" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" +checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" [[package]] name = "windows_i686_gnu" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" +checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" [[package]] name = "windows_i686_msvc" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" +checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" [[package]] name = "windows_x86_64_gnu" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" +checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" +checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" [[package]] name = "windows_x86_64_msvc" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" +checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" From 7119f019d331398a32fd6aeec62eea81ff91b766 Mon Sep 17 00:00:00 2001 From: sreedev Date: Fri, 27 Jan 2023 09:52:14 -0500 Subject: [PATCH 06/10] benchmark updates --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index f18ea92..d731bfc 100644 --- a/README.md +++ b/README.md @@ -94,12 +94,12 @@ Deduplicator uses size comparison and fxhash (a non non-cryptographic hashing al ## benchmarks -| Command | Dirsize | Mean [ms] | Min [ms] | Max [ms] | Relative | -|:---|:---|---:|---:|---:|---:| -| `deduplicator --dir ~/Data/tmp` | (~120G) | 27.5 ± 1.0 | 26.0 | 32.1 | 1.70 ± 0.09 | -| `deduplicator --dir ~/Data/books` | (~8.6G) | 21.8 ± 0.7 | 20.5 | 24.4 | 1.35 ± 0.07 | -| `deduplicator --dir ~/Data/books --min-size 10M` | (~8.6G) | 16.1 ± 0.6 | 14.9 | 18.8 | 1.00 | -| `deduplicator --dir ~/Data/ --types pdf,jpg,png,jpeg` | (~290G) | 1857.4 ± 24.5 | 1817.0 | 1895.5 | 115.07 ± 4.64 | +| Command | Dirsize | Filecount | Mean [ms] | Min [ms] | Max [ms] | Relative | +|:---|:---|---:|---:|---:|---:|---:| +| `deduplicator --dir ~/Data/tmp` | (~120G) | 721 files | 82.6 ± 11.0 | 72.8 | 114.3 | 1.00 | +| `deduplicator --dir ~/Data/books` | (~8.6G) | 1419 files | 86.7 ± 17.7 | 71.2 | 144.4 | 1.05 ± 0.26 | +| `deduplicator --dir ~/Data/books --min-size 10M` | (~8.6G) | 1419 files | 97.7 ± 35.0 | 63.1 | 228.8 | 1.18 ± 0.45 | +| `deduplicator --dir ~/Data/ --types pdf,jpg,png,jpeg` | (~290G) | 104222 files | 1253.1 ± 30.6 | 1214.2 | 1301.4 | 15.17 ± 2.05 | * The last entry is lower because of the number of files deduplicator had to go through (~660895 Files). The average size of the files rarely affect the performance of deduplicator. From 5532ded331491d9ef2036ea0fe8b571370cfb6e9 Mon Sep 17 00:00:00 2001 From: sreedev Date: Fri, 27 Jan 2023 09:54:39 -0500 Subject: [PATCH 07/10] updated benchmarks --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d731bfc..138bc13 100644 --- a/README.md +++ b/README.md @@ -96,10 +96,10 @@ Deduplicator uses size comparison and fxhash (a non non-cryptographic hashing al | Command | Dirsize | Filecount | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|:---|---:|---:|---:|---:|---:| -| `deduplicator --dir ~/Data/tmp` | (~120G) | 721 files | 82.6 ± 11.0 | 72.8 | 114.3 | 1.00 | -| `deduplicator --dir ~/Data/books` | (~8.6G) | 1419 files | 86.7 ± 17.7 | 71.2 | 144.4 | 1.05 ± 0.26 | -| `deduplicator --dir ~/Data/books --min-size 10M` | (~8.6G) | 1419 files | 97.7 ± 35.0 | 63.1 | 228.8 | 1.18 ± 0.45 | -| `deduplicator --dir ~/Data/ --types pdf,jpg,png,jpeg` | (~290G) | 104222 files | 1253.1 ± 30.6 | 1214.2 | 1301.4 | 15.17 ± 2.05 | +| `deduplicator --dir ~/Data/tmp` | (~120G) | 721 files | 33.5 ± 28.6 | 25.3 | 151.5 | 1.87 ± 1.60 | +| `deduplicator --dir ~/Data/books` | (~8.6G) | 1419 files | 24.5 ± 1.0 | 22.9 | 28.1 | 1.37 ± 0.08 | +| `deduplicator --dir ~/Data/books --min-size 10M` | (~8.6G) | 1419 files | 17.9 ± 0.7 | 16.8 | 20.0 | 1.00 | +| `deduplicator --dir ~/Data/ --types pdf,jpg,png,jpeg` | (~290G) | 104222 files | 1207.2 ± 37.0 | 1172.2 | 1287.7 | 67.27 ± 3.33 | * The last entry is lower because of the number of files deduplicator had to go through (~660895 Files). The average size of the files rarely affect the performance of deduplicator. From eadffb5feac990cd25c64b3614038066e9c5f556 Mon Sep 17 00:00:00 2001 From: sreedev Date: Fri, 27 Jan 2023 09:56:10 -0500 Subject: [PATCH 08/10] doc fixes --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 138bc13..1b42a66 100644 --- a/README.md +++ b/README.md @@ -96,10 +96,10 @@ Deduplicator uses size comparison and fxhash (a non non-cryptographic hashing al | Command | Dirsize | Filecount | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|:---|---:|---:|---:|---:|---:| -| `deduplicator --dir ~/Data/tmp` | (~120G) | 721 files | 33.5 ± 28.6 | 25.3 | 151.5 | 1.87 ± 1.60 | -| `deduplicator --dir ~/Data/books` | (~8.6G) | 1419 files | 24.5 ± 1.0 | 22.9 | 28.1 | 1.37 ± 0.08 | -| `deduplicator --dir ~/Data/books --min-size 10M` | (~8.6G) | 1419 files | 17.9 ± 0.7 | 16.8 | 20.0 | 1.00 | -| `deduplicator --dir ~/Data/ --types pdf,jpg,png,jpeg` | (~290G) | 104222 files | 1207.2 ± 37.0 | 1172.2 | 1287.7 | 67.27 ± 3.33 | +| `deduplicator ~/Data/tmp` | (~120G) | 721 files | 33.5 ± 28.6 | 25.3 | 151.5 | 1.87 ± 1.60 | +| `deduplicator ~/Data/books` | (~8.6G) | 1419 files | 24.5 ± 1.0 | 22.9 | 28.1 | 1.37 ± 0.08 | +| `deduplicator ~/Data/books --min-size 10M` | (~8.6G) | 1419 files | 17.9 ± 0.7 | 16.8 | 20.0 | 1.00 | +| `deduplicator ~/Data/ --types pdf,jpg,png,jpeg` | (~290G) | 104222 files | 1207.2 ± 37.0 | 1172.2 | 1287.7 | 67.27 ± 3.33 | * The last entry is lower because of the number of files deduplicator had to go through (~660895 Files). The average size of the files rarely affect the performance of deduplicator. From 6e3fe4a37d127b19b02f09e33cc38fc040609c99 Mon Sep 17 00:00:00 2001 From: sreedev Date: Fri, 3 Feb 2023 10:18:26 -0500 Subject: [PATCH 09/10] cargo dist --- .github/workflows/release.yml | 147 +++++++++++++++++++++++++++++----- Cargo.toml | 7 ++ 2 files changed, 133 insertions(+), 21 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cd21307..dade3b4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,31 +1,136 @@ +# CI that: +# +# * checks for a Git Tag that looks like a release ("v1.2.0") +# * creates a Github Release™️ +# * builds binaries/packages with cargo-dist +# * uploads those packages to the Github Release™️ +# +# Note that the Github Release™️ will be created before the packages, +# so there will be a few minutes where the release has no packages +# and then they will slowly trickle in, possibly failing. To make +# this more pleasant we mark the release as a "draft" until all +# artifacts have been successfully uploaded. This allows you to +# choose what to do with partial successes and avoids spamming +# anyone with notifications before the release is actually ready. name: Release -env: - PROJECT_NAME: deduplicator - PROJECT_DESC: "Filter, Sort & Delete Duplicate Files Recursively" - PROJECT_AUTH: "sreedevk" +permissions: + contents: write +# This task will run whenever you push a git tag that looks like +# a version number. We just look for `v` followed by at least one number +# and then whatever. so `v1`, `v1.0.0`, and `v1.0.0-prerelease` all work. +# +# If there's a prerelease-style suffix to the version then the Github Release™️ +# will be marked as a prerelease (handled by taiki-e/create-gh-release-action). +# +# Note that when generating links to uploaded artifacts, cargo-dist will currently +# assume that your git tag is always v{VERSION} where VERSION is the version in +# the published package's Cargo.toml (this is the default behaviour of cargo-release). +# In the future this may be made more robust/configurable. on: - release: - types: - - created + push: + tags: + - v[0-9]+.* + +env: + ALL_CARGO_DIST_TARGET_ARGS: --target=x86_64-unknown-linux-gnu --target=x86_64-apple-darwin --target=x86_64-pc-windows-msvc + ALL_CARGO_DIST_INSTALLER_ARGS: jobs: - upload-assets: - strategy: - matrix: - os: - - ubuntu-latest - - macos-latest - - windows-latest - runs-on: ${{ matrix.os }} + # Create the Github Release™️ so the packages have something to be uploaded to + create-release: + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.create-gh-release.outputs.computed-prefix }}${{ steps.create-gh-release.outputs.version }} steps: - uses: actions/checkout@v3 - - uses: taiki-e/upload-rust-binary-action@v1 + - id: create-gh-release + uses: taiki-e/create-gh-release-action@v1 with: - bin: deduplicator - tar: unix - zip: windows + draft: true + # (required) GitHub token for creating GitHub Releases. token: ${{ secrets.GITHUB_TOKEN }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + + # Build and packages all the things + upload-artifacts: + needs: create-release + strategy: + matrix: + # For these target platforms + include: + - target: x86_64-unknown-linux-gnu + os: ubuntu-20.04 + install-dist: curl --proto '=https' --tlsv1.2 -L -sSf https://github.com/axodotdev/cargo-dist/releases/download/v0.0.2/installer.sh | sh + - target: x86_64-apple-darwin + os: macos-11 + install-dist: curl --proto '=https' --tlsv1.2 -L -sSf https://github.com/axodotdev/cargo-dist/releases/download/v0.0.2/installer.sh | sh + - target: x86_64-pc-windows-msvc + os: windows-2019 + install-dist: irm 'https://github.com/axodotdev/cargo-dist/releases/download/v0.0.2/installer.ps1' | iex + runs-on: ${{ matrix.os }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v3 + - name: Install Rust + run: rustup update stable && rustup default stable + - name: Install cargo-dist + run: ${{ matrix.install-dist }} + - name: Run cargo-dist + # This logic is a bit janky because it's trying to be a polyglot between + # powershell and bash since this will run on windows, macos, and linux! + # The two platforms don't agree on how to talk about env vars but they + # do agree on 'cat' and '$()' so we use that to marshal values between commmands. + run: | + # Actually do builds and make zips and whatnot + cargo dist --target=${{ matrix.target }} --output-format=json > dist-manifest.json + echo "dist ran successfully" + cat dist-manifest.json + # Parse out what we just built and upload it to the Github Release™️ + cat dist-manifest.json | jq --raw-output ".releases[].artifacts[].path" > uploads.txt + echo "uploading..." + cat uploads.txt + gh release upload ${{ needs.create-release.outputs.tag }} $(cat uploads.txt) + echo "uploaded!" + + # Compute and upload the manifest for everything + upload-manifest: + needs: create-release + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v3 + - name: Install Rust + run: rustup update stable && rustup default stable + - name: Install cargo-dist + run: curl --proto '=https' --tlsv1.2 -L -sSf https://github.com/axodotdev/cargo-dist/releases/download/v0.0.2/installer.sh | sh + - name: Run cargo-dist manifest + run: | + # Generate a manifest describing everything + cargo dist manifest --no-local-paths --output-format=json $ALL_CARGO_DIST_TARGET_ARGS $ALL_CARGO_DIST_INSTALLER_ARGS > dist-manifest.json + echo "dist manifest ran successfully" + cat dist-manifest.json + # Upload the manifest to the Github Release™️ + gh release upload ${{ needs.create-release.outputs.tag }} dist-manifest.json + echo "uploaded manifest!" + # Edit the Github Release™️ title/body to match what cargo-dist thinks it should be + CHANGELOG_TITLE=$(cat dist-manifest.json | jq --raw-output ".releases[].changelog_title") + cat dist-manifest.json | jq --raw-output ".releases[].changelog_body" > new_dist_changelog.md + gh release edit ${{ needs.create-release.outputs.tag }} --title="$CHANGELOG_TITLE" --notes-file=new_dist_changelog.md + echo "updated release notes!" + + # Mark the Github Release™️ as a non-draft now that everything has succeeded! + publish-release: + needs: [create-release, upload-artifacts, upload-manifest] + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v3 + - name: mark release as non-draft + run: | + gh release edit ${{ needs.create-release.outputs.tag }} --draft=false + diff --git a/Cargo.toml b/Cargo.toml index 55162a4..95c8b12 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,3 +27,10 @@ memmap2 = "0.5.8" prettytable-rs = "0.10.0" rayon = "1.6.1" unicode-segmentation = "1.10.0" + +# generated by 'cargo dist init' +[profile.dist] +inherits = "release" +debug = true +split-debuginfo = "packed" + From a65e22269cc0baeae697010e1d81e8b4c1ea91cb Mon Sep 17 00:00:00 2001 From: sreedev Date: Mon, 6 Feb 2023 09:35:24 -0500 Subject: [PATCH 10/10] stabilize ui progress bars --- src/output.rs | 33 ++++++++++++++++++++++++++++++++- src/scanner.rs | 30 +++++++++++++++++++++--------- 2 files changed, 53 insertions(+), 10 deletions(-) diff --git a/src/output.rs b/src/output.rs index b9716dd..a8cd7f7 100644 --- a/src/output.rs +++ b/src/output.rs @@ -6,12 +6,13 @@ use chrono::DateTime; use colored::Colorize; use dashmap::DashMap; use indicatif::{ProgressBar, ProgressIterator, ProgressStyle}; +use itertools::Itertools; use prettytable::{format, row, Table}; use std::io::Write; use std::path::Path; +use std::time::Duration; use std::{fs, io}; use unicode_segmentation::UnicodeSegmentation; -use itertools::Itertools; fn format_path(path: &Path, opts: &Params) -> Result { let display_path = path @@ -158,6 +159,7 @@ pub fn print(duplicates: DashMap>, opts: &Params) { let mut output_table = Table::new(); let progress_bar = ProgressBar::new(duplicates.len() as u64); + progress_bar.enable_steady_tick(Duration::from_millis(50)); let progress_style = ProgressStyle::default_bar() .template("{spinner:.green} [generating output] [{wide_bar:.cyan/blue}] {pos}/{len} files") .unwrap(); @@ -184,3 +186,32 @@ pub fn print(duplicates: DashMap>, opts: &Params) { output_table.printstd(); } + +#[allow(unused)] +pub fn raw(duplicates: DashMap>, opts: &Params) -> Result<()> { + if duplicates.is_empty() { + println!( + "\n{}", + "No duplicates found matching your search criteria.".green() + ); + return Ok(()); + } + + duplicates + .into_iter() + .sorted_unstable_by_key(|(_, f)| f.first().and_then(|ff| ff.size).unwrap_or_default()) + .for_each(|(_hash, group)| { + group.iter().for_each(|file| { + println!( + "{}\t{}\t{}", + format_path(&file.path, opts).unwrap_or_default().blue(), + file_size(file).unwrap_or_default().red(), + modified_time(&file.path).unwrap_or_default().yellow() + ) + }); + + println!("---"); + }); + + Ok(()) +} diff --git a/src/scanner.rs b/src/scanner.rs index 2138b1c..4b7ee10 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -48,18 +48,25 @@ fn scan(app_opts: &Params) -> Result> { let progress_style = ProgressStyle::with_template("{spinner:.green} [mapping paths] {pos} paths")?; progress.set_style(progress_style); - progress.enable_steady_tick(Duration::from_millis(100)); + progress.enable_steady_tick(Duration::from_millis(50)); let files = walker .progress_with(progress) .filter_map(Result::ok) .map(|file| file.into_path()) .filter(|fpath| fpath.is_file()) - .collect::>() + .collect::>(); + + let scan_progress = ProgressBar::new(files.len() as u64); + let scan_progress_style = ProgressStyle::with_template( + "{spinner:.green} [processing mapped paths] [{wide_bar:.cyan/blue}] {pos}/{len} files", + )?; + scan_progress.set_style(scan_progress_style); + scan_progress.enable_steady_tick(Duration::from_millis(50)); + + let scan_results = files .into_par_iter() - .progress_with_style(ProgressStyle::with_template( - "{spinner:.green} [processing mapped paths] [{wide_bar:.cyan/blue}] {pos}/{len} files", - )?) + .progress_with(scan_progress) .map(|fpath| File { path: fpath.clone(), hash: None, @@ -72,7 +79,7 @@ fn scan(app_opts: &Params) -> Result> { .filter(|file| filters::is_file_gt_min_size(app_opts, file)) .collect(); - Ok(files) + Ok(scan_results) } fn process_file_index( @@ -102,11 +109,16 @@ fn index_files( index_criteria: IndexCritera, ) -> Result>> { let store: DashMap> = DashMap::new(); + let index_progress = ProgressBar::new(files.len() as u64); + let index_progress_style = ProgressStyle::with_template( + "{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files", + )?; + index_progress.set_style(index_progress_style); + index_progress.enable_steady_tick(Duration::from_millis(50)); + files .into_par_iter() - .progress_with_style(ProgressStyle::with_template( - "{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files", - )?) + .progress_with(index_progress) .for_each(|file| process_file_index(file, &store, index_criteria)); Ok(store)