From c42538a8c6c0a284b1af33f79f66f707d4d6b3ee Mon Sep 17 00:00:00 2001 From: sreedevk Date: Thu, 10 Jul 2025 01:19:14 +0000 Subject: [PATCH] removed vfs --- Cargo.lock | 60 +++++++++++++++++----------- Cargo.toml | 2 +- src/app.rs | 3 +- src/server/file.rs | 1 + src/server/flags.rs | 7 ---- src/server/mod.rs | 6 +-- src/server/processor.rs | 7 +--- src/server/scanner.rs | 86 +++++++++++++++++++++-------------------- src/server/store.rs | 1 + src/tui/mod.rs | 4 +- 10 files changed, 91 insertions(+), 86 deletions(-) delete mode 100644 src/server/flags.rs diff --git a/Cargo.lock b/Cargo.lock index d1d0408..77c8245 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -292,7 +292,7 @@ dependencies = [ "crossterm_winapi", "mio", "parking_lot", - "rustix", + "rustix 0.38.44", "signal-hook", "signal-hook-mio", "winapi", @@ -398,10 +398,10 @@ dependencies = [ "rayon", "serde", "serde_json", + "tempfile", "threadpool", "unicode-segmentation", "uuid", - "vfs", ] [[package]] @@ -460,16 +460,10 @@ dependencies = [ ] [[package]] -name = "filetime" -version = "0.2.25" +name = "fastrand" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" -dependencies = [ - "cfg-if", - "libc", - "libredox", - "windows-sys 0.59.0", -] +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "fnv" @@ -732,7 +726,6 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ "bitflags 2.6.0", "libc", - "redox_syscall", ] [[package]] @@ -741,6 +734,12 @@ version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" +[[package]] +name = "linux-raw-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" + [[package]] name = "lock_api" version = "0.4.12" @@ -990,7 +989,20 @@ dependencies = [ "bitflags 2.6.0", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" +dependencies = [ + "bitflags 2.6.0", + "errno", + "libc", + "linux-raw-sys 0.9.4", "windows-sys 0.59.0", ] @@ -1133,6 +1145,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tempfile" +version = "3.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" +dependencies = [ + "fastrand", + "getrandom 0.3.3", + "once_cell", + "rustix 1.0.7", + "windows-sys 0.59.0", +] + [[package]] name = "term" version = "0.7.0" @@ -1225,15 +1250,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "vfs" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ec343ec20aa715908fd028a4b8e7c99a349d13143224222e4d61c316d1e7f0a" -dependencies = [ - "filetime", -] - [[package]] name = "walkdir" version = "2.5.0" diff --git a/Cargo.toml b/Cargo.toml index 37bd261..5a47efb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,10 +31,10 @@ ratatui = "0.29.0" rayon = "1.6.1" serde = { version = "1.0.192", features = ["derive"] } serde_json = "1.0.108" +tempfile = "3.20.0" threadpool = "1.8.1" unicode-segmentation = "1.10.0" uuid = { version = "1.17.0", features = ["v4"] } -vfs = "0.12.1" [profile.release] strip = true diff --git a/src/app.rs b/src/app.rs index 714f432..f12df2e 100644 --- a/src/app.rs +++ b/src/app.rs @@ -1,5 +1,4 @@ -use std::collections::VecDeque; -use std::sync::mpsc::{channel, Receiver, Sender}; +use std::sync::mpsc::channel; use std::sync::Arc; use anyhow::{anyhow, Result}; diff --git a/src/server/file.rs b/src/server/file.rs index 98a1bc4..e5ef448 100644 --- a/src/server/file.rs +++ b/src/server/file.rs @@ -7,6 +7,7 @@ use uuid::Uuid; const PARTIAL_SIZE: u64 = 4096; +#[allow(unused)] #[derive(Clone, Debug)] pub struct FileMeta { pub id: Uuid, diff --git a/src/server/flags.rs b/src/server/flags.rs deleted file mode 100644 index def31e1..0000000 --- a/src/server/flags.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub struct ScannerFlags { - -} - -pub struct Flags { - scanner_flags: ScannerFlags -} diff --git a/src/server/mod.rs b/src/server/mod.rs index b028d44..a7c31bb 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -1,5 +1,4 @@ pub mod file; -mod flags; mod processor; mod scanner; mod store; @@ -10,7 +9,6 @@ use std::sync::mpsc::channel; use std::sync::Arc; use std::sync::Mutex; use threadpool::ThreadPool; -use vfs::PhysicalFS; use self::processor::Processor; use self::scanner::Scanner; @@ -43,7 +41,6 @@ impl Server { let processor_fq = self.fq.clone(); let processor_store = self.dupstore.clone(); let (processor_tx, processor_rx) = channel::(); - let (server_tx, server_rx) = channel::(); self.tpool.execute(move || { Processor::new(processor_fq, processor_store, processor_rx) @@ -53,9 +50,8 @@ impl Server { let scanner_fq = self.fq.clone(); let (scanner_tx, scanner_rx) = channel::(); - let filesystem = Arc::new(PhysicalFS::new("/")); self.tpool.execute(move || { - Scanner::new(scanner_fq, scanner_rx, filesystem) + Scanner::new(scanner_fq, scanner_rx) .index() .expect("scanner indexing interrupted."); }); diff --git a/src/server/processor.rs b/src/server/processor.rs index 5ce4dc4..4881abe 100644 --- a/src/server/processor.rs +++ b/src/server/processor.rs @@ -1,14 +1,11 @@ use anyhow::Result; -use std::collections::HashMap; -use std::sync::mpsc::{Receiver, Sender}; -use std::sync::{Arc, Mutex}; +use std::sync::mpsc::Receiver; +use std::sync::Arc; use super::file::FileMeta; use super::store::{Index, Store}; use super::{FileQueue, Message}; -const BATCH_SIZE: usize = 10; - pub struct Processor { files: FileQueue, duplicates: Arc, diff --git a/src/server/scanner.rs b/src/server/scanner.rs index 411f24f..b298cc4 100644 --- a/src/server/scanner.rs +++ b/src/server/scanner.rs @@ -4,22 +4,19 @@ use std::fs; use std::sync::mpsc::{self, Receiver}; use std::sync::Arc; use std::sync::Mutex; -use vfs::FileSystem; -pub struct Scanner { +pub struct Scanner { files: FileQueue, proc_queue: FileQueue, msg_rx: Receiver, - root: Arc, } -impl Scanner { - pub fn new(fq: FileQueue, rx: Receiver, fsys: Arc) -> Self { +impl Scanner { + pub fn new(fq: FileQueue, rx: Receiver) -> Self { Self { files: fq, proc_queue: Arc::new(Mutex::new(vec![])), msg_rx: rx, - root: fsys, } } @@ -44,22 +41,34 @@ impl Scanner { match npath { None => continue, Some(path) => { - self.root.read_dir(path.as_ref())?.for_each(|entry| { - let mdata = fs::metadata(&entry).expect("unable to read file metadata."); - let mpath = entry.into_boxed_str(); - match mdata.is_dir() { - true => { - let mut pq = - self.proc_queue.lock().expect("proc queue lock acq failed."); - pq.push(mpath); + std::fs::read_dir(path.as_ref())? + .filter_map(Result::ok) + .for_each(|entry: fs::DirEntry| { + let mdata = + fs::metadata(entry.path()).expect("unable to read file metadata."); + + let mpath = entry + .path() + .into_os_string() + .into_string() + .expect("invalid path conversion failed.") + .into_boxed_str(); + + match mdata.is_dir() { + true => { + let mut pq = self + .proc_queue + .lock() + .expect("proc queue lock acq failed."); + pq.push(mpath); + } + false => { + let mut fq = + self.files.lock().expect("file queue lock acq failed."); + fq.push(mpath) + } } - false => { - let mut fq = - self.files.lock().expect("file queue lock acq failed."); - fq.push(mpath) - } - } - }); + }); } } } @@ -72,44 +81,39 @@ impl Scanner { mod tests { use super::*; use anyhow::Result; + use std::io::Write; use std::sync::mpsc::channel; - use std::thread; - use vfs::MemoryFS; + use tempfile::{tempdir, tempfile_in}; #[test] - fn scanner_scans_files_on_vfs() -> Result<()> { - let files = [ - ("hello.txt", "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum"), - + fn scanner_scans_files() -> Result<()> { + let files = + [("hello.txt", "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum"), ("hello_dup.txt", "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum")]; let file_queue: FileQueue = Arc::new(Mutex::new(vec![])); - let filesystem: Arc = Arc::new(MemoryFS::new()); - let fs_root = String::from("/root"); let (tx, rx) = channel::(); + let root = tempdir()?; - filesystem.create_dir(&fs_root)?; - - let fs_root_box = fs_root.into_boxed_str(); - for (filename, content) in files.into_iter() { - filesystem - .create_file(&format!("{}/{}", fs_root_box.as_ref(), filename))? - .write_all(content.as_bytes())?; + for (_filename, content) in files.into_iter() { + let mut tf = tempfile_in(root.path())?; + tf.write_all(content.as_bytes())?; } - let scanner = Scanner::new(file_queue.clone(), rx, filesystem); - tx.send(Message::AddScanDirectory(fs_root_box))?; + let scanner = Scanner::new(file_queue.clone(), rx); + let rpath = root.path().to_str().unwrap().to_string().into_boxed_str(); + + tx.send(Message::AddScanDirectory(rpath))?; + tx.send(Message::Exit)?; scanner.index()?; - tx.send(Message::Exit)?; - let fq_len = { let v = file_queue.lock().unwrap(); v.len() }; - // assert_eq!(files.len(), fq_len); + assert_eq!(files.len(), fq_len); Ok(()) } diff --git a/src/server/store.rs b/src/server/store.rs index 2b25fe8..54940b0 100644 --- a/src/server/store.rs +++ b/src/server/store.rs @@ -2,6 +2,7 @@ use super::file::FileMeta; use dashmap::DashMap; use std::sync::Arc; +#[allow(unused)] #[derive(Debug, Hash, PartialEq, Eq)] pub enum Index { Size(u64), diff --git a/src/tui/mod.rs b/src/tui/mod.rs index be482a2..639e854 100644 --- a/src/tui/mod.rs +++ b/src/tui/mod.rs @@ -1,11 +1,9 @@ -use std::collections::VecDeque; use std::sync::mpsc::Sender; -use std::sync::Mutex; use std::time::Duration; use anyhow::{Context, Result}; use ratatui::crossterm::event::{self, Event, KeyCode}; -use ratatui::widgets::{Block, Borders, List, ListItem, ListState, Paragraph}; +use ratatui::widgets::{Block, Borders, List, ListItem}; use ratatui::{DefaultTerminal, Frame}; use crate::server::{Message, Server};