From 4d85f0bbc97683b058bf36c7d89df7aa84d44d10 Mon Sep 17 00:00:00 2001 From: sreedevk Date: Sat, 12 Jul 2025 21:30:05 +0000 Subject: [PATCH] memory usage improvements --- Cargo.lock | 13 ------------- Cargo.toml | 2 -- README.md | 3 +-- src/fileinfo.rs | 10 ++++------ src/formatter.rs | 4 ++-- src/interactive.rs | 2 +- src/params.rs | 3 --- src/scanner.rs | 28 ++++++---------------------- 8 files changed, 14 insertions(+), 51 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 70697e9..df50b21 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -289,8 +289,6 @@ dependencies = [ "pathdiff", "prettytable-rs", "rayon", - "serde", - "serde_json", "threadpool", "unicode-segmentation", ] @@ -750,17 +748,6 @@ dependencies = [ "syn", ] -[[package]] -name = "serde_json" -version = "1.0.120" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" -dependencies = [ - "itoa", - "ryu", - "serde", -] - [[package]] name = "smallvec" version = "1.13.2" diff --git a/Cargo.toml b/Cargo.toml index 6f49017..0e0a606 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,8 +27,6 @@ memmap2 = "0.5.8" pathdiff = "0.2.1" prettytable-rs = "0.10.0" rayon = "1.6.1" -serde = { version = "1.0.192", features = ["derive"] } -serde_json = "1.0.108" threadpool = "1.8.1" unicode-segmentation = "1.10.0" diff --git a/README.md b/README.md index 0b74407..a6ca5cb 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,6 @@ Options: -f, --follow-links Follow links while scanning directories -h, --help Print help information -V, --version Print version information - --json ``` ### Examples @@ -136,6 +135,6 @@ Memory: 31731MiB (~32GiB) ## v0.3 checklist - [x] parallelization of scanning, processing and formatting - [x] reduce cloning values on the heap -- [ ] add a partial hashing mode +- [x] add a partial hashing mode (--strict) - [ ] add an option to use a bloom filter for very large filesystems - [ ] max file path size should use the last set of duplicates diff --git a/src/fileinfo.rs b/src/fileinfo.rs index 409b6e4..05ab80d 100644 --- a/src/fileinfo.rs +++ b/src/fileinfo.rs @@ -1,23 +1,21 @@ use anyhow::Result; use gxhash::GxHasher; use memmap2::Mmap; -use serde::Serialize; use std::fs; use std::hash::Hasher; use std::io::Read; use std::{fs::Metadata, path::PathBuf}; -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone)] pub struct FileInfo { pub path: PathBuf, pub size: u64, - #[serde(skip)] pub filemeta: Metadata, } impl FileInfo { pub fn hash(&self) -> Result { - let file = fs::File::open(self.path.clone())?; + let file = fs::File::open(&self.path)?; let mapper = unsafe { Mmap::map(&file)? }; let mut primhasher = GxHasher::default(); @@ -29,7 +27,7 @@ impl FileInfo { } pub fn initial_page_hash(&self) -> Result { - let file = fs::File::open(self.path.clone())?; + let file = fs::File::open(&self.path)?; let mapper = unsafe { Mmap::map(&file)? }; let mut primhasher = GxHasher::default(); primhasher.write(mapper.take(4096).into_inner()); @@ -38,7 +36,7 @@ impl FileInfo { } pub fn new(path: PathBuf) -> Result { - let filemeta = std::fs::metadata(path.clone())?; + let filemeta = std::fs::metadata(&path)?; Ok(Self { path, filemeta: filemeta.clone(), diff --git a/src/formatter.rs b/src/formatter.rs index 2bdd524..5d4f424 100644 --- a/src/formatter.rs +++ b/src/formatter.rs @@ -5,7 +5,7 @@ use anyhow::Result; use chrono::{DateTime, Utc}; use dashmap::DashMap; use indicatif::{ - ParallelProgressIterator, ProgressBar, ProgressFinish, ProgressIterator, ProgressStyle, + ParallelProgressIterator, ProgressBar, ProgressFinish, ProgressStyle, }; use pathdiff::diff_paths; use prettytable::{format, row, Row, Table}; @@ -22,7 +22,7 @@ impl Formatter { min_path_length: usize, ) -> Result { let base_directory: PathBuf = app_args.get_directory()?; - let relative_path = diff_paths(file.path.clone(), base_directory).unwrap_or_default(); + let relative_path = diff_paths(&file.path, base_directory).unwrap_or_default(); let formatted_path = format!( "{:<0width$}", diff --git a/src/interactive.rs b/src/interactive.rs index f6cebce..c34aea9 100644 --- a/src/interactive.rs +++ b/src/interactive.rs @@ -51,7 +51,7 @@ pub fn init(result: Arc>>, app_args: &Params) -> R ]); }); - process_group_action(&group, gindex, result.len(), itable); + process_group_action(group, gindex, result.len(), itable); }); Ok(()) diff --git a/src/params.rs b/src/params.rs index 5a8ec7c..4026d1e 100644 --- a/src/params.rs +++ b/src/params.rs @@ -27,9 +27,6 @@ pub struct Params { /// Follow links while scanning directories #[arg(long, short)] pub follow_links: bool, - /// print json output - #[arg(long)] - pub json: bool, /// Guarantees that two files are duplicate (performs a full hash) #[arg(long, short = 'f', default_value = "false")] pub strict: bool, diff --git a/src/scanner.rs b/src/scanner.rs index ae0f834..e5cf685 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -7,7 +7,6 @@ use std::{fs, path::PathBuf, time::Duration}; use globwalk::{GlobWalker, GlobWalkerBuilder}; -#[derive(Debug, Clone)] pub struct Scanner { pub directory: Option, pub filetypes: Option, @@ -49,37 +48,22 @@ impl Scanner { scanner.max_depth = app_args.max_depth; } + // TODO: Add option to disable follow links from cli app args + // scanner.follow_links = false; + Ok(scanner) } - pub fn filetypes(&mut self, patterns: String) { - self.filetypes = Some(patterns); - } - - pub fn ignore_links(&self) -> Self { - Self { - follow_links: false, - ..self.clone() - } - } - - pub fn follow_links(&self) -> Self { - Self { - follow_links: true, - ..self.clone() - } - } - fn scan_patterns(&self) -> Result { - Ok(match self.filetypes.clone() { + Ok(match &self.filetypes { Some(ftypes) => format!("**/*{{{ftypes}}}"), None => "**/*".to_string(), }) } fn scan_dir(&self) -> Result { - let scan_dir = match self.directory.clone() { - Some(path) => path, + let scan_dir = match &self.directory { + Some(path) => path.clone(), None => std::env::current_dir()?, };