smolkik_adm/deduplicator
1
0
Fork 0
mirror of https://github.com/sreedevk/deduplicator.git synced 2026-09-05 23:10:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

removed vfs

Browse Source
This commit is contained in:
sreedevk
2025-07-10 01:19:14 +00:00
parent 05437e0256
commit c42538a8c6
10 changed files with 91 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
Cargo.lock generated
View File

@@ -292,7 +292,7 @@ dependencies = [
"crossterm_winapi",
"mio",
"parking_lot",
"rustix",
"rustix 0.38.44",
"signal-hook",
"signal-hook-mio",
"winapi",
@@ -398,10 +398,10 @@ dependencies = [
"rayon",
"serde",
"serde_json",
"tempfile",
"threadpool",
"unicode-segmentation",
"uuid",
"vfs",
]
[[package]]
@@ -460,16 +460,10 @@ dependencies = [
]
[[package]]
name = "filetime"
version = "0.2.25"
name = "fastrand"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586"
dependencies = [
"cfg-if",
"libc",
"libredox",
"windows-sys 0.59.0",
]
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "fnv"
@@ -732,7 +726,6 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d"
dependencies = [
"bitflags 2.6.0",
"libc",
"redox_syscall",
]
[[package]]
@@ -741,6 +734,12 @@ version = "0.4.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
[[package]]
name = "linux-raw-sys"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12"
[[package]]
name = "lock_api"
version = "0.4.12"
@@ -990,7 +989,20 @@ dependencies = [
"bitflags 2.6.0",
"errno",
"libc",
"linux-raw-sys",
"linux-raw-sys 0.4.15",
"windows-sys 0.59.0",
]
[[package]]
name = "rustix"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266"
dependencies = [
"bitflags 2.6.0",
"errno",
"libc",
"linux-raw-sys 0.9.4",
"windows-sys 0.59.0",
]
@@ -1133,6 +1145,19 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "tempfile"
version = "3.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1"
dependencies = [
"fastrand",
"getrandom 0.3.3",
"once_cell",
"rustix 1.0.7",
"windows-sys 0.59.0",
]
[[package]]
name = "term"
version = "0.7.0"
@@ -1225,15 +1250,6 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "vfs"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ec343ec20aa715908fd028a4b8e7c99a349d13143224222e4d61c316d1e7f0a"
dependencies = [
"filetime",
]
[[package]]
name = "walkdir"
version = "2.5.0"

2
Cargo.toml
View File

@@ -31,10 +31,10 @@ ratatui = "0.29.0"
rayon = "1.6.1"
serde = { version = "1.0.192", features = ["derive"] }
serde_json = "1.0.108"
tempfile = "3.20.0"
threadpool = "1.8.1"
unicode-segmentation = "1.10.0"
uuid = { version = "1.17.0", features = ["v4"] }
vfs = "0.12.1"
[profile.release]
strip = true

3
src/app.rs
View File

@@ -1,5 +1,4 @@
use std::collections::VecDeque;
use std::sync::mpsc::{channel, Receiver, Sender};
use std::sync::mpsc::channel;
use std::sync::Arc;
use anyhow::{anyhow, Result};

1
src/server/file.rs
View File

@@ -7,6 +7,7 @@ use uuid::Uuid;
const PARTIAL_SIZE: u64 = 4096;
#[allow(unused)]
#[derive(Clone, Debug)]
pub struct FileMeta {
pub id: Uuid,

7
src/server/flags.rs
View File

@@ -1,7 +0,0 @@
pub struct ScannerFlags {
}
pub struct Flags {
scanner_flags: ScannerFlags
}

6
src/server/mod.rs
View File

@@ -1,5 +1,4 @@
pub mod file;
mod flags;
mod processor;
mod scanner;
mod store;
@@ -10,7 +9,6 @@ use std::sync::mpsc::channel;
use std::sync::Arc;
use std::sync::Mutex;
use threadpool::ThreadPool;
use vfs::PhysicalFS;
use self::processor::Processor;
use self::scanner::Scanner;
@@ -43,7 +41,6 @@ impl Server {
let processor_fq = self.fq.clone();
let processor_store = self.dupstore.clone();
let (processor_tx, processor_rx) = channel::<Message>();
let (server_tx, server_rx) = channel::<Message>();
self.tpool.execute(move || {
Processor::new(processor_fq, processor_store, processor_rx)
@@ -53,9 +50,8 @@ impl Server {
let scanner_fq = self.fq.clone();
let (scanner_tx, scanner_rx) = channel::<Message>();
let filesystem = Arc::new(PhysicalFS::new("/"));
self.tpool.execute(move || {
Scanner::new(scanner_fq, scanner_rx, filesystem)
Scanner::new(scanner_fq, scanner_rx)
.index()
.expect("scanner indexing interrupted.");
});

7
src/server/processor.rs
View File

@@ -1,14 +1,11 @@
use anyhow::Result;
use std::collections::HashMap;
use std::sync::mpsc::{Receiver, Sender};
use std::sync::{Arc, Mutex};
use std::sync::mpsc::Receiver;
use std::sync::Arc;
use super::file::FileMeta;
use super::store::{Index, Store};
use super::{FileQueue, Message};
const BATCH_SIZE: usize = 10;
pub struct Processor {
files: FileQueue,
duplicates: Arc<Store>,

86
src/server/scanner.rs
View File

@@ -4,22 +4,19 @@ use std::fs;
use std::sync::mpsc::{self, Receiver};
use std::sync::Arc;
use std::sync::Mutex;
use vfs::FileSystem;
pub struct Scanner<T: FileSystem> {
pub struct Scanner {
files: FileQueue,
proc_queue: FileQueue,
msg_rx: Receiver<Message>,
root: Arc<T>,
}
impl<T: FileSystem> Scanner<T> {
pub fn new(fq: FileQueue, rx: Receiver<Message>, fsys: Arc<T>) -> Self {
impl Scanner {
pub fn new(fq: FileQueue, rx: Receiver<Message>) -> Self {
Self {
files: fq,
proc_queue: Arc::new(Mutex::new(vec![])),
msg_rx: rx,
root: fsys,
}
}
@@ -44,22 +41,34 @@ impl<T: FileSystem> Scanner<T> {
match npath {
None => continue,
Some(path) => {
self.root.read_dir(path.as_ref())?.for_each(|entry| {
let mdata = fs::metadata(&entry).expect("unable to read file metadata.");
let mpath = entry.into_boxed_str();
match mdata.is_dir() {
true => {
let mut pq =
self.proc_queue.lock().expect("proc queue lock acq failed.");
pq.push(mpath);
std::fs::read_dir(path.as_ref())?
.filter_map(Result::ok)
.for_each(|entry: fs::DirEntry| {
let mdata =
fs::metadata(entry.path()).expect("unable to read file metadata.");
let mpath = entry
.path()
.into_os_string()
.into_string()
.expect("invalid path conversion failed.")
.into_boxed_str();
match mdata.is_dir() {
true => {
let mut pq = self
.proc_queue
.lock()
.expect("proc queue lock acq failed.");
pq.push(mpath);
}
false => {
let mut fq =
self.files.lock().expect("file queue lock acq failed.");
fq.push(mpath)
}
}
false => {
let mut fq =
self.files.lock().expect("file queue lock acq failed.");
fq.push(mpath)
}
}
});
});
}
}
}
@@ -72,44 +81,39 @@ impl<T: FileSystem> Scanner<T> {
mod tests {
use super::*;
use anyhow::Result;
use std::io::Write;
use std::sync::mpsc::channel;
use std::thread;
use vfs::MemoryFS;
use tempfile::{tempdir, tempfile_in};
#[test]
fn scanner_scans_files_on_vfs() -> Result<()> {
let files = [
("hello.txt", "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum"),
fn scanner_scans_files() -> Result<()> {
let files =
[("hello.txt", "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum"),
("hello_dup.txt", "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum")];
let file_queue: FileQueue = Arc::new(Mutex::new(vec![]));
let filesystem: Arc<MemoryFS> = Arc::new(MemoryFS::new());
let fs_root = String::from("/root");
let (tx, rx) = channel::<Message>();
let root = tempdir()?;
filesystem.create_dir(&fs_root)?;
let fs_root_box = fs_root.into_boxed_str();
for (filename, content) in files.into_iter() {
filesystem
.create_file(&format!("{}/{}", fs_root_box.as_ref(), filename))?
.write_all(content.as_bytes())?;
for (_filename, content) in files.into_iter() {
let mut tf = tempfile_in(root.path())?;
tf.write_all(content.as_bytes())?;
}
let scanner = Scanner::new(file_queue.clone(), rx, filesystem);
tx.send(Message::AddScanDirectory(fs_root_box))?;
let scanner = Scanner::new(file_queue.clone(), rx);
let rpath = root.path().to_str().unwrap().to_string().into_boxed_str();
tx.send(Message::AddScanDirectory(rpath))?;
tx.send(Message::Exit)?;
scanner.index()?;
tx.send(Message::Exit)?;
let fq_len = {
let v = file_queue.lock().unwrap();
v.len()
};
// assert_eq!(files.len(), fq_len);
assert_eq!(files.len(), fq_len);
Ok(())
}

1
src/server/store.rs
View File

@@ -2,6 +2,7 @@ use super::file::FileMeta;
use dashmap::DashMap;
use std::sync::Arc;
#[allow(unused)]
#[derive(Debug, Hash, PartialEq, Eq)]
pub enum Index {
Size(u64),

4
src/tui/mod.rs
View File

@@ -1,11 +1,9 @@
use std::collections::VecDeque;
use std::sync::mpsc::Sender;
use std::sync::Mutex;
use std::time::Duration;
use anyhow::{Context, Result};
use ratatui::crossterm::event::{self, Event, KeyCode};
use ratatui::widgets::{Block, Borders, List, ListItem, ListState, Paragraph};
use ratatui::widgets::{Block, Borders, List, ListItem};
use ratatui::{DefaultTerminal, Frame};
use crate::server::{Message, Server};