mirror of
https://github.com/sreedevk/deduplicator.git
synced 2026-08-31 20:45:34 +00:00
parallelization improvements
This commit is contained in:
sreedevk
154
src/processor.rs
154
src/processor.rs
@@ -1,118 +1,72 @@
|
||||
use anyhow::Result;
|
||||
use dashmap::DashMap;
|
||||
use indicatif::{ParallelProgressIterator, ProgressBar, ProgressFinish, ProgressStyle};
|
||||
use rayon::prelude::{IntoParallelIterator, ParallelIterator};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::{borrow::Cow, time::Duration};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex, TryLockError, TryLockResult};
|
||||
|
||||
use crate::fileinfo::FileInfo;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Processor {
|
||||
pub files: Arc<Mutex<Vec<FileInfo>>>,
|
||||
pub hashwise_results: DashMap<String, Vec<FileInfo>>,
|
||||
pub sizewise_results: DashMap<u64, Vec<FileInfo>>,
|
||||
pub max_path_len: usize,
|
||||
}
|
||||
pub struct Processor {}
|
||||
|
||||
impl Processor {
|
||||
pub fn new(files: Arc<Mutex<Vec<FileInfo>>>) -> Self {
|
||||
Self {
|
||||
files,
|
||||
hashwise_results: DashMap::new(),
|
||||
sizewise_results: DashMap::new(),
|
||||
max_path_len: 0,
|
||||
}
|
||||
}
|
||||
pub fn hashwise(
|
||||
sw_store: Arc<DashMap<u64, Vec<FileInfo>>>,
|
||||
hw_store: Arc<DashMap<String, Vec<FileInfo>>>,
|
||||
) -> Result<()> {
|
||||
let keys: Vec<u64> = sw_store.clone().iter().map(|i| *i.key()).collect();
|
||||
|
||||
pub fn hashwise(&mut self) -> Result<()> {
|
||||
let flist_size = {
|
||||
let f = self.files.lock().unwrap();
|
||||
f.len()
|
||||
};
|
||||
if flist_size < 1 {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let progress_style = ProgressStyle::with_template(
|
||||
"[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}",
|
||||
)?;
|
||||
|
||||
let progress_bar = ProgressBar::new(flist_size as u64);
|
||||
progress_bar.set_style(progress_style);
|
||||
progress_bar.enable_steady_tick(Duration::from_millis(50));
|
||||
progress_bar.set_message("indexing file hashes");
|
||||
|
||||
let filelist = self
|
||||
.sizewise_results
|
||||
.clone()
|
||||
.into_read_only()
|
||||
.values()
|
||||
.filter(|&subfiles| subfiles.len() > 1)
|
||||
.flatten()
|
||||
.cloned()
|
||||
.collect::<Vec<FileInfo>>();
|
||||
|
||||
self.max_path_len = filelist
|
||||
.iter()
|
||||
.map(|x| x.path.clone().into_os_string().len())
|
||||
.max()
|
||||
.unwrap_or_default();
|
||||
|
||||
filelist
|
||||
.into_par_iter()
|
||||
.progress_with(progress_bar)
|
||||
.with_finish(ProgressFinish::WithMessage(Cow::from(
|
||||
"indexed files hashes",
|
||||
)))
|
||||
.map(|file| file.hash())
|
||||
.filter_map(Result::ok)
|
||||
.for_each(move |file| {
|
||||
self.hashwise_results
|
||||
.entry(file.hash.clone().unwrap_or_default())
|
||||
.and_modify(|fileset| fileset.push(file.clone()))
|
||||
.or_insert_with(|| vec![file]);
|
||||
});
|
||||
keys.into_iter().for_each(|key| {
|
||||
let group: Vec<FileInfo> = sw_store.get(&key).unwrap().to_vec();
|
||||
if group.len() > 1 {
|
||||
group.into_par_iter().for_each(|file| {
|
||||
hw_store
|
||||
.entry(file.hash.clone().unwrap_or_default())
|
||||
.and_modify(|fileset| fileset.push(file.clone()))
|
||||
.or_insert_with(|| vec![file]);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn sizewise(&mut self) -> Result<()> {
|
||||
let flist_size = {
|
||||
let f = self.files.lock().unwrap();
|
||||
f.len()
|
||||
};
|
||||
|
||||
if flist_size < 1 {
|
||||
return Ok(());
|
||||
pub fn compare_and_update_max_path_len(current: Arc<AtomicU64>, next: u64) -> Result<()> {
|
||||
if current.load(Ordering::Relaxed) < next {
|
||||
current.store(next, Ordering::Release);
|
||||
}
|
||||
|
||||
let progress_style = ProgressStyle::with_template(
|
||||
"[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}",
|
||||
)?;
|
||||
let progress_bar = ProgressBar::new(flist_size as u64);
|
||||
progress_bar.set_style(progress_style);
|
||||
progress_bar.enable_steady_tick(Duration::from_millis(50));
|
||||
progress_bar.set_message("indexing file sizes");
|
||||
|
||||
(0..flist_size)
|
||||
.into_par_iter()
|
||||
.progress_with(progress_bar)
|
||||
.with_finish(ProgressFinish::WithMessage(Cow::from(
|
||||
"indexed files sizes",
|
||||
)))
|
||||
.for_each(|findex| {
|
||||
let file = {
|
||||
let files = self.files.lock().unwrap();
|
||||
files[findex].clone()
|
||||
};
|
||||
|
||||
self.sizewise_results
|
||||
.entry(file.size)
|
||||
.and_modify(|fileset| fileset.push(file.clone()))
|
||||
.or_insert_with(|| vec![file]);
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// TODO: reduce the amount of time files remain locked for
|
||||
pub fn sizewise(
|
||||
scanner_finished: Arc<AtomicBool>,
|
||||
store: Arc<DashMap<u64, Vec<FileInfo>>>,
|
||||
files: Arc<Mutex<Vec<FileInfo>>>,
|
||||
max_file_size: Arc<AtomicU64>,
|
||||
) -> Result<()> {
|
||||
loop {
|
||||
match files.try_lock() {
|
||||
Ok(mut flist) => match flist.pop() {
|
||||
Some(file) => {
|
||||
Self::compare_and_update_max_path_len(
|
||||
max_file_size.clone(),
|
||||
file.path.to_string_lossy().len() as u64,
|
||||
)?;
|
||||
store
|
||||
.entry(file.size)
|
||||
.and_modify(|fileset| fileset.push(file.clone()))
|
||||
.or_insert_with(|| vec![file]);
|
||||
continue;
|
||||
}
|
||||
None => match scanner_finished.load(std::sync::atomic::Ordering::Relaxed) {
|
||||
true => break Ok(()),
|
||||
false => continue,
|
||||
},
|
||||
},
|
||||
TryLockResult::Err(TryLockError::WouldBlock) => continue,
|
||||
_ => break Ok(()),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user