smolkik_adm/deduplicator
1
0
Fork 0
mirror of https://github.com/sreedevk/deduplicator.git synced 2026-08-26 18:15:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

added application module

Browse Source
This commit is contained in:
sreedevk
2025-07-07 01:49:36 +00:00
parent 5ce0a2774a
commit 8a5b540e40
10 changed files with 430 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

354
Cargo.lock generated
View File

@@ -11,6 +11,12 @@ dependencies = [
"memchr",
]
[[package]]
name = "allocator-api2"
version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
[[package]]
name = "android-tzdata"
version = "0.1.1"
@@ -121,6 +127,21 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3e368af43e418a04d52505cf3dbc23dda4e3407ae2fa99fd0e4f308ce546acc"
[[package]]
name = "cassowary"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53"
[[package]]
name = "castaway"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0abae9be0aaf9ea96a3b1b8b1b55c602ca751eba1b1500220cea4ecbafe7c0d5"
dependencies = [
"rustversion",
]
[[package]]
name = "cc"
version = "1.0.104"
@@ -203,6 +224,20 @@ dependencies = [
"windows-sys 0.48.0",
]
[[package]]
name = "compact_str"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b79c4069c6cad78e2e0cdfcbd26275770669fb39fd308a752dc110e83b9af32"
dependencies = [
"castaway",
"cfg-if",
"itoa",
"rustversion",
"ryu",
"static_assertions",
]
[[package]]
name = "console"
version = "0.15.8"
@@ -212,7 +247,7 @@ dependencies = [
"encode_unicode 0.3.6",
"lazy_static",
"libc",
"unicode-width",
"unicode-width 0.1.13",
"windows-sys 0.52.0",
]
@@ -247,6 +282,31 @@ version = "0.8.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
[[package]]
name = "crossterm"
version = "0.28.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6"
dependencies = [
"bitflags 2.6.0",
"crossterm_winapi",
"mio",
"parking_lot",
"rustix",
"signal-hook",
"signal-hook-mio",
"winapi",
]
[[package]]
name = "crossterm_winapi"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b"
dependencies = [
"winapi",
]
[[package]]
name = "csv"
version = "1.3.0"
@@ -268,6 +328,41 @@ dependencies = [
"memchr",
]
[[package]]
name = "darling"
version = "0.20.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee"
dependencies = [
"darling_core",
"darling_macro",
]
[[package]]
name = "darling_core"
version = "0.20.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e"
dependencies = [
"fnv",
"ident_case",
"proc-macro2",
"quote",
"strsim",
"syn",
]
[[package]]
name = "darling_macro"
version = "0.20.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead"
dependencies = [
"darling_core",
"quote",
"syn",
]
[[package]]
name = "dashmap"
version = "5.5.3"
@@ -275,7 +370,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856"
dependencies = [
"cfg-if",
"hashbrown",
"hashbrown 0.14.5",
"lock_api",
"once_cell",
"parking_lot_core",
@@ -295,10 +390,11 @@ dependencies = [
"globwalk",
"gxhash",
"indicatif",
"itertools",
"itertools 0.10.5",
"memmap2",
"pathdiff",
"prettytable-rs",
"ratatui",
"rayon",
"serde",
"serde_json",
@@ -346,6 +442,34 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
[[package]]
name = "equivalent"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
[[package]]
name = "errno"
version = "0.3.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad"
dependencies = [
"libc",
"windows-sys 0.52.0",
]
[[package]]
name = "fnv"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "foldhash"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
[[package]]
name = "getrandom"
version = "0.2.15"
@@ -408,6 +532,17 @@ version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
[[package]]
name = "hashbrown"
version = "0.15.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5"
dependencies = [
"allocator-api2",
"equivalent",
"foldhash",
]
[[package]]
name = "heck"
version = "0.5.0"
@@ -449,6 +584,12 @@ dependencies = [
"cc",
]
[[package]]
name = "ident_case"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
[[package]]
name = "ignore"
version = "0.4.22"
@@ -476,7 +617,26 @@ dependencies = [
"number_prefix",
"portable-atomic",
"rayon",
"unicode-width",
"unicode-width 0.1.13",
]
[[package]]
name = "indoc"
version = "2.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd"
[[package]]
name = "instability"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bf9fed6d91cfb734e7476a06bde8300a1b94e217e1b523b6f0cd1a01998c71d"
dependencies = [
"darling",
"indoc",
"proc-macro2",
"quote",
"syn",
]
[[package]]
@@ -514,6 +674,15 @@ dependencies = [
"either",
]
[[package]]
name = "itertools"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
dependencies = [
"either",
]
[[package]]
name = "itoa"
version = "1.0.11"
@@ -552,6 +721,12 @@ dependencies = [
"libc",
]
[[package]]
name = "linux-raw-sys"
version = "0.4.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
[[package]]
name = "lock_api"
version = "0.4.12"
@@ -568,6 +743,15 @@ version = "0.4.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
[[package]]
name = "lru"
version = "0.12.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38"
dependencies = [
"hashbrown 0.15.4",
]
[[package]]
name = "memchr"
version = "2.7.4"
@@ -583,6 +767,18 @@ dependencies = [
"libc",
]
[[package]]
name = "mio"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c"
dependencies = [
"libc",
"log",
"wasi 0.11.0+wasi-snapshot-preview1",
"windows-sys 0.59.0",
]
[[package]]
name = "num-traits"
version = "0.2.19"
@@ -614,6 +810,16 @@ version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
[[package]]
name = "parking_lot"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"
dependencies = [
"lock_api",
"parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.9.10"
@@ -627,6 +833,12 @@ dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "paste"
version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
[[package]]
name = "pathdiff"
version = "0.2.1"
@@ -650,23 +862,23 @@ dependencies = [
"is-terminal",
"lazy_static",
"term",
"unicode-width",
"unicode-width 0.1.13",
]
[[package]]
name = "proc-macro2"
version = "1.0.86"
version = "1.0.95"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77"
checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.36"
version = "1.0.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d"
dependencies = [
"proc-macro2",
]
@@ -677,6 +889,27 @@ version = "5.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
[[package]]
name = "ratatui"
version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b"
dependencies = [
"bitflags 2.6.0",
"cassowary",
"compact_str",
"crossterm",
"indoc",
"instability",
"itertools 0.13.0",
"lru",
"paste",
"strum",
"unicode-segmentation",
"unicode-truncate",
"unicode-width 0.2.0",
]
[[package]]
name = "rayon"
version = "1.10.0"
@@ -734,6 +967,19 @@ version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
[[package]]
name = "rustix"
version = "0.38.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
dependencies = [
"bitflags 2.6.0",
"errno",
"libc",
"linux-raw-sys",
"windows-sys 0.52.0",
]
[[package]]
name = "rustversion"
version = "1.0.17"
@@ -792,12 +1038,48 @@ dependencies = [
"serde",
]
[[package]]
name = "signal-hook"
version = "0.3.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2"
dependencies = [
"libc",
"signal-hook-registry",
]
[[package]]
name = "signal-hook-mio"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34db1a06d485c9142248b7a054f034b349b212551f3dfd19c94d45a754a217cd"
dependencies = [
"libc",
"mio",
"signal-hook",
]
[[package]]
name = "signal-hook-registry"
version = "1.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410"
dependencies = [
"libc",
]
[[package]]
name = "smallvec"
version = "1.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
[[package]]
name = "static_assertions"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
[[package]]
name = "strsim"
version = "0.11.1"
@@ -805,10 +1087,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
[[package]]
name = "syn"
version = "2.0.68"
name = "strum"
version = "0.26.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9"
checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06"
dependencies = [
"strum_macros",
]
[[package]]
name = "strum_macros"
version = "0.26.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be"
dependencies = [
"heck",
"proc-macro2",
"quote",
"rustversion",
"syn",
]
[[package]]
name = "syn"
version = "2.0.104"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40"
dependencies = [
"proc-macro2",
"quote",
@@ -867,12 +1171,29 @@ version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202"
[[package]]
name = "unicode-truncate"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3644627a5af5fa321c95b9b235a72fd24cd29c648c2c379431e6628655627bf"
dependencies = [
"itertools 0.13.0",
"unicode-segmentation",
"unicode-width 0.1.13",
]
[[package]]
name = "unicode-width"
version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d"
[[package]]
name = "unicode-width"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd"
[[package]]
name = "utf8parse"
version = "0.2.2"
@@ -1031,6 +1352,15 @@ dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "windows-sys"
version = "0.59.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "windows-targets"
version = "0.48.5"

1
Cargo.toml
View File

@@ -27,6 +27,7 @@ itertools = "0.10.5"
memmap2 = "0.5.8"
pathdiff = "0.2.1"
prettytable-rs = "0.10.0"
ratatui = "0.29.0"
rayon = "1.6.1"
serde = { version = "1.0.192", features = ["derive"] }
serde_json = "1.0.108"

25
src/app.rs Normal file
View File

@@ -0,0 +1,25 @@
use anyhow::Result;
use threadpool::ThreadPool;
use crate::pipeline::Server;
use crate::tui::Tui;
pub struct App {
tpool: ThreadPool,
server: Server,
ui: Tui,
}
impl App {
pub fn new() -> Self {
Self {
tpool: ThreadPool::new(8),
server: Server::new().expect("server init failed"),
ui: Tui {},
}
}
pub fn start() -> Result<()> {
Ok(())
}
}

37
src/main.rs
View File

@@ -4,33 +4,32 @@ mod interactive;
mod params;
mod processor;
mod scanner;
mod app;
/* version 2.0 modules*/
mod pipeline;
mod cli;
mod pipeline;
mod tui;
use anyhow::Result;
use clap::Parser;
use formatter::Formatter;
use params::Params;
use processor::Processor;
use scanner::Scanner;
// use clap::Parser;
// use formatter::Formatter;
// use params::Params;
// use processor::Processor;
// use scanner::Scanner;
use pipeline::Server;
fn main() -> Result<()> {
let app_args = Params::parse();
let scan_results = Scanner::build(&app_args)?.scan()?;
let processor = Processor::new(scan_results);
let results = processor.sizewise()?.hashwise()?;
let server = Server::new();
// let app_args = Params::parse();
// let scan_results = Scanner::build(&app_args)?.scan()?;
// let processor = Processor::new(scan_results);
// let results = processor.sizewise()?.hashwise()?;
match app_args.interactive {
false => {
Formatter::print(results.files, &app_args)?;
}
true => {
interactive::init(results.files, &app_args)?;
}
}
// match app_args.interactive {
// false => Formatter::print(results.files, &app_args)?,
// true => interactive::init(results.files, &app_args)?,
// }
Ok(())
}

13
src/pipeline/file.rs
View File

@@ -7,13 +7,14 @@ use uuid::Uuid;
const PARTIAL_SIZE: u64 = 4096;
#[derive(Clone, Debug)]
pub struct FileMeta {
id: Uuid,
path: Box<str>,
size: u64,
modtime: i64,
partial: Arc<[u8]>,
full_hash: Arc<[u8]>,
pub id: Uuid,
pub path: Box<str>,
pub size: u64,
pub modtime: i64,
pub partial: Arc<[u8]>,
pub full_hash: Arc<[u8]>,
}
impl FileMeta {

7
src/pipeline/flags.rs Normal file
View File

@@ -0,0 +1,7 @@
pub struct ScannerFlags {
}
pub struct Flags {
scanner_flags: ScannerFlags
}

1
src/pipeline/mod.rs
View File

@@ -2,6 +2,7 @@ mod file;
mod processor;
mod scanner;
mod store;
mod flags;
use anyhow::{anyhow, Result};
use std::collections::HashMap;

20
src/pipeline/processor.rs
View File

@@ -4,7 +4,7 @@ use std::sync::mpsc::Receiver;
use std::sync::{Arc, Mutex};
use super::file::FileMeta;
use super::Store;
use super::store::{Index, Store};
use super::{FileQueue, Message};
const BATCH_SIZE: usize = 10;
@@ -30,6 +30,24 @@ impl Processor {
Ok(Message::Exit) | Err(_) => break,
_ => {}
}
let next_file = {
let mut cfiles = self.files.lock().unwrap();
cfiles.pop()
};
match next_file {
None => continue,
Some(file_res) => match FileMeta::new(file_res) {
Err(_) => continue,
Ok(fm) => {
// self.duplicates.add(Index::Partial(fm_arc.partial), fm_arc.clone());
let fm_arc = Arc::new(fm);
self.duplicates
.add(Index::Size(fm_arc.size), fm_arc.clone());
}
},
}
}
Ok(())

8
src/pipeline/store.rs
View File

@@ -5,8 +5,8 @@ use std::sync::{Arc, Mutex};
#[derive(Debug, Hash, PartialEq, Eq)]
pub enum Index {
Size(Box<str>),
Partial(Box<str>),
Size(u64),
Partial(Arc<[u8]>),
Full(Box<str>),
}
@@ -21,12 +21,10 @@ impl Store {
}
}
pub fn add(&self, index: Index, file: Arc<FileMeta>) -> Result<()> {
pub fn add(&self, index: Index, file: Arc<FileMeta>) {
let mut imut = self.internal.lock().unwrap();
imut.entry(index)
.and_modify(|fg| fg.push(file.clone()))
.or_insert(vec![file]);
Ok(())
}
}

7
src/tui/mod.rs
View File

@@ -0,0 +1,7 @@
pub struct Tui {
};
impl Tui {
pub fn init(&self) { }
}