smolkik_adm/deduplicator
1
0
Fork 0
mirror of https://github.com/sreedevk/deduplicator.git synced 2026-08-26 02:04:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

v0.2.0

Browse Source
This commit is contained in:
sreedev
2023-07-17 13:59:58 -04:00
parent 7bd88f1642
commit e3d48ec505
15 changed files with 871 additions and 771 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@@ -1,3 +1,2 @@
/target
/test_data
.envrc
/Cargo.lock

663
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

4
Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "deduplicator"
version = "0.1.6"
version = "0.2.0"
edition = "2021"
description = "find,filter,delete Duplicates"
license = "MIT"
@@ -24,6 +24,7 @@ globwalk = "0.8.1"
indicatif = { version = "0.17.2", features = ["rayon"] }
itertools = "0.10.5"
memmap2 = "0.5.8"
pathdiff = "0.2.1"
prettytable-rs = "0.10.0"
rayon = "1.6.1"
unicode-segmentation = "1.10.0"
@@ -33,4 +34,3 @@ unicode-segmentation = "1.10.0"
inherits = "release"
debug = true
split-debuginfo = "packed"

20
examples/scanner.rs Normal file
View File

@@ -0,0 +1,20 @@
use std::path::PathBuf;
use anyhow::Result;
use deduplicator_core::processor::Processor;
use deduplicator_core::scanner::Scanner;
fn main() -> Result<()> {
let scan_results: Vec<_> = Scanner::new()
.directory(PathBuf::from("/home/sreedev/Data/books/"))
.ignore_links().scan(None)?;
let processor = Processor::new(scan_results);
let final_processor = processor.sizewise()?.hashwise()?;
let display_vec = final_processor.files.into_iter().map(|file| file.path ).collect::<Vec<PathBuf>>();
dbg!(display_vec);
Ok(())
}

18
src/app.rs
View File

@@ -1,18 +0,0 @@
use crate::output;
use crate::params::Params;
use crate::scanner;
use anyhow::Result;
pub struct App;
impl App {
pub fn init(app_args: &Params) -> Result<()> {
let duplicates = scanner::duplicates(app_args)?;
match app_args.interactive {
true => output::interactive(duplicates, app_args),
false => output::print(duplicates, app_args),
}
Ok(())
}
}

21
src/file_manager.rs
View File

@@ -1,21 +0,0 @@
use anyhow::Result;
use colored::Colorize;
use std::path::PathBuf;
#[derive(Debug, Clone)]
pub struct File {
pub path: PathBuf,
pub size: Option<u64>,
pub hash: Option<String>,
}
pub fn delete_files(files: Vec<File>) -> Result<()> {
files.into_iter().for_each(|file| {
match std::fs::remove_file(file.path.clone()) {
Ok(_) => println!("{}: {}", "DELETED".green(), file.path.display()),
Err(_) => println!("{}: {}", "FAILED".red(), file.path.display())
}
});
Ok(())
}

41
src/fileinfo.rs Normal file
View File

@@ -0,0 +1,41 @@
use anyhow::Result;
use memmap2::Mmap;
use std::fs;
use std::hash::Hasher;
use std::{fs::Metadata, path::PathBuf};
#[derive(Debug, Clone)]
pub struct FileInfo {
pub path: PathBuf,
pub filemeta: Metadata,
pub hash: Option<String>,
pub size: u64,
}
impl FileInfo {
pub fn hash(&self) -> Result<Self> {
let file = fs::File::open(self.path.clone())?;
let mapper = unsafe { Mmap::map(&file)? };
let mut primhasher = fxhash::FxHasher::default();
mapper
.chunks(1_000_000)
.into_iter()
.for_each(|chunk| primhasher.write(chunk));
Ok(Self {
hash: Some(primhasher.finish().to_string()),
..self.clone()
})
}
pub fn new(path: PathBuf) -> Result<Self> {
let filemeta = std::fs::metadata(path.clone())?;
Ok(Self {
path,
filemeta: filemeta.clone(),
hash: None,
size: filemeta.len(),
})
}
}

12
src/filters.rs
View File

@@ -1,12 +0,0 @@
use crate::file_manager::File;
use crate::params::Params;
pub fn is_file_gt_min_size(app_opts: &Params, file: &File) -> bool {
match app_opts.get_min_size() {
Some(msize) => match file.size {
Some(fsize) => fsize >= msize,
None => true,
},
None => true,
}
}

111
src/formatter.rs Normal file
View File

@@ -0,0 +1,111 @@
pub struct Formatter;
use crate::fileinfo::FileInfo;
use crate::params::Params;
use anyhow::Result;
use chrono::{DateTime, Utc};
use colored::Colorize;
use dashmap::DashMap;
use indicatif::{ProgressIterator, ParallelProgressIterator, ProgressBar, ProgressFinish, ProgressStyle};
use pathdiff::diff_paths;
use prettytable::{format, row, Table};
use rayon::prelude::*;
use std::borrow::Cow;
use std::path::PathBuf;
use std::time::Duration;
impl Formatter {
fn human_path(file: &FileInfo, app_args: &Params, min_path_length: usize) -> Result<String> {
let base_directory: PathBuf = app_args.get_directory()?.to_path_buf();
let relative_path = diff_paths(file.path.clone(), base_directory).unwrap_or_default();
let formatted_path = format!(
"{:<0width$}",
relative_path.to_str().unwrap_or_default().to_string(),
width = min_path_length
);
Ok(formatted_path)
}
fn human_filesize(file: &FileInfo) -> Result<String> {
Ok(format!("{:>12}", bytesize::ByteSize::b(file.size)))
}
fn human_mtime(file: &FileInfo) -> Result<String> {
let modified_time: DateTime<Utc> = file.filemeta.modified()?.into();
Ok(modified_time.format("%Y-%m-%d %H:%M:%S").to_string())
}
pub fn generate_table(raw: Vec<FileInfo>, app_args: &Params) -> Result<Table> {
let basepath_length = app_args.get_directory()?.to_str().unwrap_or_default().len();
let max_filepath_length = raw
.iter()
.map(|file| file.path.to_str().unwrap_or_default().len())
.max()
.unwrap_or_default();
let min_path_length = max_filepath_length - basepath_length;
let progress_style = ProgressStyle::with_template(
"[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}",
)?;
let progress_bar = ProgressBar::new(raw.len() as u64);
progress_bar.set_style(progress_style);
progress_bar.enable_steady_tick(Duration::from_millis(50));
progress_bar.set_message("reconciling data");
let duplicates_table: DashMap<String, Vec<FileInfo>> = DashMap::new();
raw.clone()
.into_par_iter()
.progress_with(progress_bar)
.with_finish(ProgressFinish::WithMessage(Cow::from("data reconciled")))
.map(|file| file.hash())
.filter_map(Result::ok)
.for_each(|file| {
duplicates_table
.entry(file.hash.clone().unwrap_or_default())
.and_modify(|fileset| fileset.push(file.clone()))
.or_insert_with(|| vec![file]);
});
let mut output_table = Table::new();
output_table.set_titles(row!["hash", "duplicates"]);
let progress_style = ProgressStyle::with_template(
"[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}",
)?;
let progress_bar = ProgressBar::new(duplicates_table.len() as u64);
progress_bar.set_style(progress_style);
progress_bar.enable_steady_tick(Duration::from_millis(50));
progress_bar.set_message("generating output");
duplicates_table
.into_iter()
.progress_with(progress_bar)
.with_finish(ProgressFinish::WithMessage(Cow::from("output generated")))
.for_each(|(hash, group)| {
let mut inner_table = Table::new();
inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
group.iter().for_each(|file| {
inner_table.add_row(row![
Self::human_path(&file, &app_args, min_path_length)
.unwrap_or_default()
.blue(),
Self::human_filesize(&file).unwrap_or_default().red(),
Self::human_mtime(&file).unwrap_or_default().yellow()
]);
});
output_table.add_row(row![hash.green(), inner_table]);
});
Ok(output_table)
}
pub fn print(raw: Vec<FileInfo>, app_args: &Params) -> Result<()> {
let output_table = Self::generate_table(raw, app_args)?;
output_table.printstd();
Ok(())
}
}

85
src/interative.rs Normal file
View File

@@ -0,0 +1,85 @@
use anyhow::Result;
use colored::Colorize;
use crate::fileinfo::FileInfo;
use prettytable::Table;
use std::io::{self, Write};
pub fn scan_group_confirmation() -> Result<bool> {
print!("\nconfirm? [y/N]: ");
std::io::stdout().flush()?;
let mut user_input = String::new();
io::stdin().read_line(&mut user_input)?;
match user_input.trim() {
"Y" | "y" => Ok(true),
_ => Ok(false),
}
}
pub fn scan_group_instruction() -> Result<String> {
println!("\nEnter the indices of the files you want to delete.");
println!("You can enter multiple files using commas to seperate file indices.");
println!("example: 1,2");
print!("\n> ");
std::io::stdout().flush()?;
let mut user_input = String::new();
io::stdin().read_line(&mut user_input)?;
Ok(user_input)
}
pub fn process_group_action(
duplicates: &Vec<FileInfo>,
dup_index: usize,
dup_size: usize,
table: Table,
) {
println!("\nDuplicate Set {} of {}\n", dup_index + 1, dup_size);
table.printstd();
let files_to_delete = scan_group_instruction().unwrap_or_default();
let parsed_file_indices = files_to_delete
.trim()
.split(',')
.filter(|element| !element.is_empty())
.map(|index| index.parse::<usize>().unwrap_or_default())
.collect::<Vec<usize>>();
if parsed_file_indices
.clone()
.into_iter()
.any(|index| index > (duplicates.len() - 1))
{
println!("{}", "Err: File Index Out of Bounds!".red());
return process_group_action(duplicates, dup_index, dup_size, table);
}
print!("{esc}[2J{esc}[1;1H", esc = 27 as char);
if parsed_file_indices.is_empty() {
return;
}
let files_to_delete = parsed_file_indices
.into_iter()
.map(|index| duplicates[index].clone());
println!("\n{}", "The following files will be deleted:".red());
files_to_delete
.clone()
.enumerate()
.for_each(|(index, file)| {
println!("{}: {}", index.to_string().blue(), file.path.display());
});
match scan_group_confirmation().unwrap() {
true => {
files_to_delete.into_iter().for_each(|file| {
match std::fs::remove_file(file.path.clone()) {
Ok(_) => println!("{}: {}", "DELETED".green(), file.path.display()),
Err(_) => println!("{}: {}", "FAILED".red(), file.path.display()),
}
});
}
false => println!("{}", "\nCancelled Delete Operation.".red()),
}
}

23
src/main.rs
View File

@@ -1,14 +1,23 @@
mod app;
mod file_manager;
mod output;
mod params;
mod fileinfo;
mod processor;
mod scanner;
mod filters;
mod params;
mod formatter;
mod interative;
use anyhow::Result;
use app::App;
use params::Params;
use scanner::Scanner;
use processor::Processor;
use clap::Parser;
use formatter::Formatter;
fn main() -> Result<()> {
App::init(&params::Params::parse())
let app_args = Params::parse();
let scan_results = Scanner::build(&app_args)?.scan()?;
let processor = Processor::new(scan_results);
let results = processor.sizewise()?.hashwise()?;
Formatter::print(results.files, &app_args)?;
Ok(())
}

217
src/output.rs
View File

@@ -1,217 +0,0 @@
use crate::file_manager::{self, File};
use crate::params::Params;
use anyhow::Result;
use chrono::offset::Utc;
use chrono::DateTime;
use colored::Colorize;
use dashmap::DashMap;
use indicatif::{ProgressBar, ProgressIterator, ProgressStyle};
use itertools::Itertools;
use prettytable::{format, row, Table};
use std::io::Write;
use std::path::Path;
use std::time::Duration;
use std::{fs, io};
use unicode_segmentation::UnicodeSegmentation;
fn format_path(path: &Path, opts: &Params) -> Result<String> {
let display_path = path
.to_string_lossy()
.replace(opts.get_directory()?.to_string_lossy().as_ref(), "");
let display_range = if display_path.chars().count() > 32 {
display_path
.graphemes(true)
.collect::<Vec<&str>>()
.into_iter()
.rev()
.take(32)
.rev()
.collect()
} else {
display_path
};
Ok(format!("...{display_range:<32}"))
}
fn file_size(file: &File) -> Result<String> {
Ok(format!("{:>12}", bytesize::ByteSize::b(file.size.unwrap())))
}
fn modified_time(path: &Path) -> Result<String> {
let mdata = fs::metadata(path)?;
let modified_time: DateTime<Utc> = mdata.modified()?.into();
Ok(modified_time.format("%Y-%m-%d %H:%M:%S").to_string())
}
fn scan_group_instruction() -> Result<String> {
println!("\nEnter the indices of the files you want to delete.");
println!("You can enter multiple files using commas to seperate file indices.");
println!("example: 1,2");
print!("\n> ");
std::io::stdout().flush()?;
let mut user_input = String::new();
io::stdin().read_line(&mut user_input)?;
Ok(user_input)
}
fn scan_group_confirmation() -> Result<bool> {
print!("\nconfirm? [y/N]: ");
std::io::stdout().flush()?;
let mut user_input = String::new();
io::stdin().read_line(&mut user_input)?;
match user_input.trim() {
"Y" | "y" => Ok(true),
_ => Ok(false),
}
}
fn process_group_action(duplicates: &Vec<File>, dup_index: usize, dup_size: usize, table: Table) {
println!("\nDuplicate Set {} of {}\n", dup_index + 1, dup_size);
table.printstd();
let files_to_delete = scan_group_instruction().unwrap_or_default();
let parsed_file_indices = files_to_delete
.trim()
.split(',')
.filter(|element| !element.is_empty())
.map(|index| index.parse::<usize>().unwrap_or_default())
.collect::<Vec<usize>>();
if parsed_file_indices
.clone()
.into_iter()
.any(|index| index > (duplicates.len() - 1))
{
println!("{}", "Err: File Index Out of Bounds!".red());
return process_group_action(duplicates, dup_index, dup_size, table);
}
print!("{esc}[2J{esc}[1;1H", esc = 27 as char);
if parsed_file_indices.is_empty() {
return;
}
let files_to_delete = parsed_file_indices
.into_iter()
.map(|index| duplicates[index].clone());
println!("\n{}", "The following files will be deleted:".red());
files_to_delete
.clone()
.enumerate()
.for_each(|(index, file)| {
println!("{}: {}", index.to_string().blue(), file.path.display());
});
match scan_group_confirmation().unwrap() {
true => {
file_manager::delete_files(files_to_delete.collect::<Vec<File>>()).ok();
}
false => println!("{}", "\nCancelled Delete Operation.".red()),
}
}
pub fn interactive(duplicates: DashMap<String, Vec<File>>, opts: &Params) {
if duplicates.is_empty() {
println!(
"\n{}",
"No duplicates found matching your search criteria.".green()
);
return;
}
duplicates
.clone()
.into_iter()
.sorted_unstable_by_key(|(_, f)| {
-(f.first().and_then(|ff| ff.size).unwrap_or_default() as i64)
}) // sort by descending file size in interactive mode
.enumerate()
.for_each(|(gindex, (_, group))| {
let mut itable = Table::new();
itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
itable.set_titles(row!["index", "filename", "size", "updated_at"]);
group.iter().enumerate().for_each(|(index, file)| {
itable.add_row(row![
index,
format_path(&file.path, opts).unwrap_or_default().blue(),
file_size(file).unwrap_or_default().red(),
modified_time(&file.path).unwrap_or_default().yellow()
]);
});
process_group_action(&group, gindex, duplicates.len(), itable);
});
}
pub fn print(duplicates: DashMap<String, Vec<File>>, opts: &Params) {
if duplicates.is_empty() {
println!(
"\n{}",
"No duplicates found matching your search criteria.".green()
);
return;
}
let mut output_table = Table::new();
let progress_bar = ProgressBar::new(duplicates.len() as u64);
progress_bar.enable_steady_tick(Duration::from_millis(50));
let progress_style = ProgressStyle::default_bar()
.template("{spinner:.green} [generating output] [{wide_bar:.cyan/blue}] {pos}/{len} files")
.unwrap();
progress_bar.set_style(progress_style);
output_table.set_titles(row!["hash", "duplicates"]);
duplicates
.into_iter()
.sorted_unstable_by_key(|(_, f)| f.first().and_then(|ff| ff.size).unwrap_or_default())
.progress_with(progress_bar)
.for_each(|(hash, group)| {
let mut inner_table = Table::new();
inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
group.iter().for_each(|file| {
inner_table.add_row(row![
format_path(&file.path, opts).unwrap_or_default().blue(),
file_size(file).unwrap_or_default().red(),
modified_time(&file.path).unwrap_or_default().yellow()
]);
});
output_table.add_row(row![hash.green(), inner_table]);
});
output_table.printstd();
}
#[allow(unused)]
pub fn raw(duplicates: DashMap<String, Vec<File>>, opts: &Params) -> Result<()> {
if duplicates.is_empty() {
println!(
"\n{}",
"No duplicates found matching your search criteria.".green()
);
return Ok(());
}
duplicates
.into_iter()
.sorted_unstable_by_key(|(_, f)| f.first().and_then(|ff| ff.size).unwrap_or_default())
.for_each(|(_hash, group)| {
group.iter().for_each(|file| {
println!(
"{}\t{}\t{}",
format_path(&file.path, opts).unwrap_or_default().blue(),
file_size(file).unwrap_or_default().red(),
modified_time(&file.path).unwrap_or_default().yellow()
)
});
println!("---");
});
Ok(())
}

43
src/params.rs
View File

@@ -1,10 +1,9 @@
use std::{fs, path::PathBuf};
use anyhow::{anyhow, Result};
use anyhow::Result;
use clap::{Parser, ValueHint};
use globwalk::{GlobWalker, GlobWalkerBuilder};
#[derive(Parser, Debug)]
#[derive(Parser, Debug, Clone)]
#[command(author, version, about, long_about = None)]
pub struct Params {
/// Filetypes to deduplicate [default = all]
@@ -47,42 +46,4 @@ impl Params {
let dir = fs::canonicalize(dir_path)?;
Ok(dir)
}
fn add_glob_min_depth(&self, builder: GlobWalkerBuilder) -> Result<GlobWalkerBuilder> {
match self.min_depth {
Some(mindepth) => Ok(builder.min_depth(mindepth)),
None => Ok(builder),
}
}
fn add_glob_max_depth(&self, builder: GlobWalkerBuilder) -> Result<GlobWalkerBuilder> {
match self.max_depth {
Some(maxdepth) => Ok(builder.max_depth(maxdepth)),
None => Ok(builder),
}
}
fn add_glob_follow_links(&self, builder: GlobWalkerBuilder) -> Result<GlobWalkerBuilder> {
match self.follow_links {
true => Ok(builder.follow_links(true)),
false => Ok(builder.follow_links(false)),
}
}
pub fn get_glob_walker(&self) -> Result<GlobWalker> {
let pattern: String = match self.types.as_ref() {
Some(filetypes) => format!("**/*{{{filetypes}}}"),
None => "**/*".to_string(),
};
let glob_walker_builder = self
.add_glob_min_depth(GlobWalkerBuilder::from_patterns(
self.get_directory()?,
&[pattern],
))
.and_then(|builder| self.add_glob_max_depth(builder))
.and_then(|builder| self.add_glob_follow_links(builder))?;
glob_walker_builder.build().map_err(|e| anyhow!(e))
}
}

107
src/processor.rs Normal file
View File

@@ -0,0 +1,107 @@
use anyhow::Result;
use dashmap::DashMap;
use indicatif::{ParallelProgressIterator, ProgressBar, ProgressStyle, ProgressFinish};
use rayon::prelude::{IntoParallelIterator, ParallelIterator};
use std::{time::Duration, borrow::Cow};
use crate::fileinfo::FileInfo;
#[derive(Debug, Clone)]
pub enum State {
Initial,
SizeWise,
HashWise,
}
#[derive(Debug, Clone)]
pub struct Processor {
pub files: Vec<FileInfo>,
pub state: State,
}
impl Processor {
pub fn new(files: Vec<FileInfo>) -> Self {
Self {
files,
state: State::Initial,
}
}
pub fn hashwise(&self) -> Result<Self> {
if self.files.is_empty() {
return Ok(self.clone());
}
let progress_style = ProgressStyle::with_template("[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}")?;
let progress_bar = ProgressBar::new(self.files.len() as u64);
progress_bar.set_style(progress_style);
progress_bar.enable_steady_tick(Duration::from_millis(50));
progress_bar.set_message("indexing file hashes");
let duplicates_table: DashMap<String, Vec<FileInfo>> = DashMap::new();
self.files
.clone()
.into_par_iter()
.progress_with(progress_bar)
.with_finish(ProgressFinish::WithMessage(Cow::from("indexed files hashes")))
.map(|file| file.hash())
.filter_map(Result::ok)
.for_each(|file| {
duplicates_table
.entry(file.hash.clone().unwrap_or_default())
.and_modify(|fileset| fileset.push(file.clone()))
.or_insert_with(|| vec![file]);
});
let files = duplicates_table
.into_read_only()
.values()
.cloned()
.filter(|subfiles| subfiles.len() > 1)
.flatten()
.collect::<Vec<FileInfo>>();
Ok(Self {
files,
state: State::HashWise,
})
}
pub fn sizewise(&self) -> Result<Self> {
if self.files.is_empty() {
return Ok(self.clone());
}
let progress_style = ProgressStyle::with_template("[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}")?;
let progress_bar = ProgressBar::new(self.files.len() as u64);
progress_bar.set_style(progress_style);
progress_bar.enable_steady_tick(Duration::from_millis(50));
progress_bar.set_message("indexing file sizes");
let duplicates_table: DashMap<u64, Vec<FileInfo>> = DashMap::new();
self.files
.clone()
.into_par_iter()
.progress_with(progress_bar)
.with_finish(ProgressFinish::WithMessage(Cow::from("indexed files sizes")))
.for_each(|file| {
duplicates_table
.entry(file.size)
.and_modify(|fileset| fileset.push(file.clone()))
.or_insert_with(|| vec![file]);
});
let files = duplicates_table
.into_read_only()
.values()
.cloned()
.filter(|subfiles| subfiles.len() > 1)
.flatten()
.collect::<Vec<FileInfo>>();
Ok(Self {
files,
state: State::SizeWise,
})
}
}

278
src/scanner.rs
View File

@@ -1,151 +1,169 @@
use crate::{file_manager::File, filters, params::Params};
#![allow(unused)]
use crate::{fileinfo::FileInfo, params::Params};
use anyhow::Result;
use dashmap::DashMap;
use fxhash::hash64 as hasher;
use indicatif::{ParallelProgressIterator, ProgressBar, ProgressIterator, ProgressStyle};
use memmap2::Mmap;
use rayon::prelude::*;
use std::hash::Hasher;
use std::time::Duration;
use std::{
fs,
path::{Path, PathBuf},
};
use indicatif::{ProgressBar, ProgressStyle};
use std::{fs, path::PathBuf, time::Duration};
#[derive(Clone, Copy)]
enum IndexCritera {
Size,
Hash,
use globwalk::{GlobWalker, GlobWalkerBuilder};
#[derive(Debug, Clone)]
pub struct Scanner {
pub directory: Option<PathBuf>,
pub filetypes: Option<String>,
pub min_depth: Option<usize>,
pub max_depth: Option<usize>,
pub min_size: Option<u64>,
pub follow_links: bool,
}
pub fn duplicates(app_opts: &Params) -> Result<DashMap<String, Vec<File>>> {
let scan_results = scan(app_opts)?;
let size_index_store = index_files(scan_results, IndexCritera::Size)?;
let sizewize_duplicate_files = size_index_store
.into_par_iter()
.filter(|(_, files)| files.len() > 1)
.map(|(_, files)| files)
.flatten()
.collect::<Vec<File>>();
if sizewize_duplicate_files.len() > 1 {
let hash_index_store = index_files(sizewize_duplicate_files, IndexCritera::Hash)?;
let duplicate_files = hash_index_store
.into_par_iter()
.filter(|(_, files)| files.len() > 1)
.collect();
Ok(duplicate_files)
} else {
Ok(DashMap::new())
impl Scanner {
pub fn new() -> Self {
Self {
directory: None,
filetypes: None,
min_depth: None,
max_depth: None,
min_size: None,
follow_links: true,
}
}
}
fn scan(app_opts: &Params) -> Result<Vec<File>> {
let walker = app_opts.get_glob_walker()?;
let progress = ProgressBar::new_spinner();
let progress_style =
ProgressStyle::with_template("{spinner:.green} [mapping paths] {pos} paths")?;
progress.set_style(progress_style);
progress.enable_steady_tick(Duration::from_millis(50));
let files = walker
.progress_with(progress)
.filter_map(Result::ok)
.map(|file| file.into_path())
.filter(|fpath| fpath.is_file())
.collect::<Vec<PathBuf>>();
let scan_progress = ProgressBar::new(files.len() as u64);
let scan_progress_style = ProgressStyle::with_template(
"{spinner:.green} [processing mapped paths] [{wide_bar:.cyan/blue}] {pos}/{len} files",
)?;
scan_progress.set_style(scan_progress_style);
scan_progress.enable_steady_tick(Duration::from_millis(50));
let scan_results = files
.into_par_iter()
.progress_with(scan_progress)
.map(|fpath| File {
path: fpath.clone(),
hash: None,
size: Some(
fs::metadata(fpath)
.map(|metadata| metadata.len())
.unwrap_or_default(),
),
pub fn build(app_args: &Params) -> Result<Self> {
let scan_directory = app_args.get_directory()?;
Ok(Scanner::new())
.map(|scanner| scanner.directory(scan_directory))
.map(|scanner| match app_args.get_min_size() {
Some(min_size) => scanner.min_size(min_size),
None => scanner,
})
.map(|scanner| match app_args.min_depth {
Some(min_depth) => scanner.min_depth(min_depth),
None => scanner,
})
.map(|scanner| match app_args.max_depth {
Some(max_depth) => scanner.max_depth(max_depth),
None => scanner,
})
.filter(|file| filters::is_file_gt_min_size(app_opts, file))
.collect();
Ok(scan_results)
}
fn process_file_index(
mut file: File,
store: &DashMap<String, Vec<File>>,
index_criteria: IndexCritera,
) {
match index_criteria {
IndexCritera::Size => {
store
.entry(file.size.unwrap_or_default().to_string())
.and_modify(|fileset| fileset.push(file.clone()))
.or_insert_with(|| vec![file]);
}
IndexCritera::Hash => {
file.hash = Some(hash_file(&file.path).unwrap_or_default());
store
.entry(file.clone().hash.unwrap_or_default())
.and_modify(|fileset| fileset.push(file.clone()))
.or_insert_with(|| vec![file]);
pub fn min_size(&self, min_size: u64) -> Self {
Self {
min_size: Some(min_size),
..self.clone()
}
}
}
fn index_files(
files: Vec<File>,
index_criteria: IndexCritera,
) -> Result<DashMap<String, Vec<File>>> {
let store: DashMap<String, Vec<File>> = DashMap::new();
let index_progress = ProgressBar::new(files.len() as u64);
let index_progress_style = ProgressStyle::with_template(
"{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files",
)?;
index_progress.set_style(index_progress_style);
index_progress.enable_steady_tick(Duration::from_millis(50));
pub fn min_depth(&self, min_depth: usize) -> Self {
Self {
min_depth: Some(min_depth),
..self.clone()
}
}
files
.into_par_iter()
.progress_with(index_progress)
.for_each(|file| process_file_index(file, &store, index_criteria));
pub fn max_depth(&self, max_depth: usize) -> Self {
Self {
max_depth: Some(max_depth),
..self.clone()
}
}
Ok(store)
}
pub fn directory(&self, dir: PathBuf) -> Self {
Self {
directory: Some(dir),
..self.clone()
}
}
fn incremental_hashing(filepath: &Path) -> Result<String> {
let file = fs::File::open(filepath)?;
let fmap = unsafe { Mmap::map(&file)? };
let mut inchasher = fxhash::FxHasher::default();
pub fn filetypes(&self, patterns: String) -> Self {
Self {
filetypes: Some(patterns),
..self.clone()
}
}
fmap.chunks(1_000_000)
.for_each(|mega| inchasher.write(mega));
pub fn ignore_links(&self) -> Self {
Self {
follow_links: false,
..self.clone()
}
}
Ok(format!("{}", inchasher.finish()))
}
pub fn follow_links(&self) -> Self {
Self {
follow_links: true,
..self.clone()
}
}
fn standard_hashing(filepath: &Path) -> Result<String> {
let file = fs::read(filepath)?;
Ok(hasher(&*file).to_string())
}
fn scan_patterns(&self) -> Result<String> {
Ok(match self.filetypes.clone() {
Some(ftypes) => format!("**/*{{{ftypes}}}"),
None => "**/*".to_string(),
})
}
fn hash_file(filepath: &Path) -> Result<String> {
let filemeta = fs::metadata(filepath)?;
fn scan_dir(&self) -> Result<PathBuf> {
let scan_dir = match self.directory.clone() {
Some(path) => path,
None => std::env::current_dir()?,
};
// NOTE: USE INCREMENTAL HASHING ONLY FOR FILES > 100MB
match filemeta.len() < 100_000_000 {
true => standard_hashing(filepath),
false => incremental_hashing(filepath),
Ok(fs::canonicalize(scan_dir)?)
}
fn attach_link_opts(&self, walker: GlobWalkerBuilder) -> Result<GlobWalkerBuilder> {
Ok(walker.follow_links(self.follow_links))
}
fn attach_walker_min_depth(&self, walker: GlobWalkerBuilder) -> Result<GlobWalkerBuilder> {
match self.min_depth {
Some(min_depth) => Ok(walker.min_depth(min_depth)),
None => Ok(walker),
}
}
fn attach_walker_max_depth(&self, walker: GlobWalkerBuilder) -> Result<GlobWalkerBuilder> {
match self.max_depth {
Some(max_depth) => Ok(walker.max_depth(max_depth)),
None => Ok(walker),
}
}
fn build_walker(&self) -> Result<GlobWalker> {
let walker = Ok(GlobWalkerBuilder::from_patterns(
self.scan_dir()?,
&[self.scan_patterns()?],
))
.and_then(|walker| self.attach_walker_min_depth(walker))
.and_then(|walker| self.attach_walker_max_depth(walker))
.and_then(|walker| self.attach_link_opts(walker))?;
Ok(walker.build()?)
}
pub fn scan(&self) -> Result<Vec<FileInfo>> {
let progress_style =
ProgressStyle::with_template("[{elapsed_precise}] {pos:>7} {msg}")?;
let progress_bar = ProgressBar::new_spinner();
progress_bar.set_style(progress_style);
progress_bar.enable_steady_tick(Duration::from_millis(50));
progress_bar.set_message("paths mapped");
let results = self
.build_walker()?
.filter_map(Result::ok)
.map(|entity| entity.into_path())
.map(|path| {
progress_bar.inc(1);
path
})
.filter(|path| path.is_file())
.map(|file| FileInfo::new(file))
.filter_map(Result::ok)
.collect::<Vec<FileInfo>>();
progress_bar.finish_with_message("paths mapped");
Ok(results)
}
}