memory usage improvements

Cargo.lock

@@ -289,8 +289,6 @@ dependencies = [
  "pathdiff",
  "prettytable-rs",
  "rayon",
- "serde",
- "serde_json",
  "threadpool",
  "unicode-segmentation",
 ]
@@ -750,17 +748,6 @@ dependencies = [
  "syn",
 ]
 
-[[package]]
-name = "serde_json"
-version = "1.0.120"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5"
-dependencies = [
- "itoa",
- "ryu",
- "serde",
-]
-
 [[package]]
 name = "smallvec"
 version = "1.13.2"

Cargo.toml

@@ -27,8 +27,6 @@ memmap2 = "0.5.8"
 pathdiff = "0.2.1"
 prettytable-rs = "0.10.0"
 rayon = "1.6.1"
-serde = { version = "1.0.192", features = ["derive"] }
-serde_json = "1.0.108"
 threadpool = "1.8.1"
 unicode-segmentation = "1.10.0"
 

README.md

@@ -21,7 +21,6 @@ Options:
   -f, --follow-links           Follow links while scanning directories
   -h, --help                   Print help information
   -V, --version                Print version information
-      --json                    
 ```
 ### Examples
 
@@ -136,6 +135,6 @@ Memory: 31731MiB (~32GiB)
 ## v0.3 checklist
 - [x] parallelization of scanning, processing and formatting
 - [x] reduce cloning values on the heap
-- [ ] add a partial hashing mode
+- [x] add a partial hashing mode (--strict)
 - [ ] add an option to use a bloom filter for very large filesystems
 - [ ] max file path size should use the last set of duplicates

src/fileinfo.rs

@@ -1,23 +1,21 @@
 use anyhow::Result;
 use gxhash::GxHasher;
 use memmap2::Mmap;
-use serde::Serialize;
 use std::fs;
 use std::hash::Hasher;
 use std::io::Read;
 use std::{fs::Metadata, path::PathBuf};
 
-#[derive(Debug, Clone, Serialize)]
+#[derive(Debug, Clone)]
 pub struct FileInfo {
     pub path: PathBuf,
     pub size: u64,
-    #[serde(skip)]
     pub filemeta: Metadata,
 }
 
 impl FileInfo {
     pub fn hash(&self) -> Result<String> {
-        let file = fs::File::open(self.path.clone())?;
+        let file = fs::File::open(&self.path)?;
         let mapper = unsafe { Mmap::map(&file)? };
         let mut primhasher = GxHasher::default();
 
@@ -29,7 +27,7 @@ impl FileInfo {
     }
 
     pub fn initial_page_hash(&self) -> Result<String> {
-        let file = fs::File::open(self.path.clone())?;
+        let file = fs::File::open(&self.path)?;
         let mapper = unsafe { Mmap::map(&file)? };
         let mut primhasher = GxHasher::default();
         primhasher.write(mapper.take(4096).into_inner());
@@ -38,7 +36,7 @@ impl FileInfo {
     }
 
     pub fn new(path: PathBuf) -> Result<Self> {
-        let filemeta = std::fs::metadata(path.clone())?;
+        let filemeta = std::fs::metadata(&path)?;
         Ok(Self {
             path,
             filemeta: filemeta.clone(),

src/formatter.rs

@@ -5,7 +5,7 @@ use anyhow::Result;
 use chrono::{DateTime, Utc};
 use dashmap::DashMap;
 use indicatif::{
-    ParallelProgressIterator, ProgressBar, ProgressFinish, ProgressIterator, ProgressStyle,
+    ParallelProgressIterator, ProgressBar, ProgressFinish, ProgressStyle,
 };
 use pathdiff::diff_paths;
 use prettytable::{format, row, Row, Table};
@@ -22,7 +22,7 @@ impl Formatter {
         min_path_length: usize,
     ) -> Result<String> {
         let base_directory: PathBuf = app_args.get_directory()?;
-        let relative_path = diff_paths(file.path.clone(), base_directory).unwrap_or_default();
+        let relative_path = diff_paths(&file.path, base_directory).unwrap_or_default();
 
         let formatted_path = format!(
             "{:<0width$}",

src/interactive.rs

@@ -51,7 +51,7 @@ pub fn init(result: Arc<DashMap<String, Vec<FileInfo>>>, app_args: &Params) -> R
             ]);
         });
 
-        process_group_action(&group, gindex, result.len(), itable);
+        process_group_action(group, gindex, result.len(), itable);
     });
 
     Ok(())

src/params.rs

@@ -27,9 +27,6 @@ pub struct Params {
     /// Follow links while scanning directories
     #[arg(long, short)]
     pub follow_links: bool,
-    /// print json output
-    #[arg(long)]
-    pub json: bool,
     /// Guarantees that two files are duplicate (performs a full hash)
     #[arg(long, short = 'f', default_value = "false")]
     pub strict: bool,

src/scanner.rs

@@ -7,7 +7,6 @@ use std::{fs, path::PathBuf, time::Duration};
 
 use globwalk::{GlobWalker, GlobWalkerBuilder};
 
-#[derive(Debug, Clone)]
 pub struct Scanner {
     pub directory: Option<PathBuf>,
     pub filetypes: Option<String>,
@@ -49,37 +48,22 @@ impl Scanner {
             scanner.max_depth = app_args.max_depth;
         }
 
+        // TODO: Add option to disable follow links from cli app args
+        // scanner.follow_links = false;
+
         Ok(scanner)
     }
 
-    pub fn filetypes(&mut self, patterns: String) {
-        self.filetypes = Some(patterns);
-    }
-
-    pub fn ignore_links(&self) -> Self {
-        Self {
-            follow_links: false,
-            ..self.clone()
-        }
-    }
-
-    pub fn follow_links(&self) -> Self {
-        Self {
-            follow_links: true,
-            ..self.clone()
-        }
-    }
-
     fn scan_patterns(&self) -> Result<String> {
-        Ok(match self.filetypes.clone() {
+        Ok(match &self.filetypes {
             Some(ftypes) => format!("**/*{{{ftypes}}}"),
             None => "**/*".to_string(),
         })
     }
 
     fn scan_dir(&self) -> Result<PathBuf> {
-        let scan_dir = match self.directory.clone() {
-            Some(path) => path,
+        let scan_dir = match &self.directory {
+            Some(path) => path.clone(),
             None => std::env::current_dir()?,
         };