mirror of
https://github.com/sreedevk/deduplicator.git
synced 2026-10-20 01:55:13 +00:00
Sreedev Kodichath
3ca931fa8e
The deduplication core was a three-stage streaming producer/consumer (scan -> group-by-size -> group-by-hash) orchestrated by a Server struct that ran the stages concurrently on a threadpool. Coordination relied on AtomicBool flags polled in busy-wait loops, an Arc<Mutex<Vec>> hand-off queue, DashMap stores, and a per-file Arc<Mutex<FileState>>. Every stage had to receive its own hand-cloned Arcs with ad-hoc names, and the busy-wait branches burned a core spinning while waiting for the producer. Replace it with a staged batch pipeline over owned collections: pipeline::run(&Params) drives scan -> group_by_size -> group_by_hash in sequence, with rayon supplying parallelism. With no shared mutable state between stages, all the Arc plumbing, the AtomicBool coordination, the Mutex queue, the per-file lock, server.rs, and the threadpool/dashmap dependencies are gone. FileInfo becomes plain data and each stage is a pure, independently testable function. Behavior is preserved except for three authorized deviations: progress spinners render sequentially rather than concurrently under -p; the interactive empty-result message prints once instead of twice; and the interactive "Duplicate Set X of Y" total now counts the confirmed duplicate groups shown instead of the internal candidate-hash store size. This lands as the foundation for the cache, CLI, and TUI work that follows.
58 lines
1.3 KiB
TOML
58 lines
1.3 KiB
TOML
[package]
|
|
name = "deduplicator"
|
|
version = "0.3.2"
|
|
edition = "2021"
|
|
description = "find,filter and delete duplicate files"
|
|
repository = "https://github.com/sreedevk/deduplicator"
|
|
license = "MIT"
|
|
authors = [
|
|
"Sreedev Kodichath <sreedevpadmakumar@gmail.com>",
|
|
"Valentin Bersier <vbersier@gmail.com>",
|
|
"Dhruva Sagar <dhruva.sagar@gmail.com>",
|
|
]
|
|
|
|
[[bin]]
|
|
name = "deduplicator"
|
|
path = "src/main.rs"
|
|
|
|
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
|
[dependencies]
|
|
anyhow = "1.0.68"
|
|
bytesize = "2.0.1"
|
|
chrono = "0.4.23"
|
|
clap = { version = "4.0.32", features = ["derive"] }
|
|
globwalk = "0.9.1"
|
|
gxhash = { version = "3.4.1", default-features = false }
|
|
indicatif = { version = "0.18.0", features = ["rayon"] }
|
|
memmap2 = "0.9.7"
|
|
pathdiff = "0.2.1"
|
|
prettytable-rs = "0.10.0"
|
|
rand = "0.9.1"
|
|
rayon = "1.6.1"
|
|
unicode-segmentation = "1.12.0"
|
|
|
|
[profile.release]
|
|
strip = true
|
|
opt-level = 3
|
|
lto = "thin"
|
|
debug = false
|
|
codegen-units = 1
|
|
|
|
# generated by 'cargo dist init'
|
|
[profile.dist]
|
|
inherits = "release"
|
|
|
|
[workspace.metadata.dist]
|
|
rust-toolchain-version = "1.87.0"
|
|
ci = ["github"]
|
|
targets = [
|
|
"x86_64-unknown-linux-gnu",
|
|
"x86_64-apple-darwin",
|
|
"x86_64-pc-windows-msvc",
|
|
"aarch64-apple-darwin",
|
|
]
|
|
cargo-dist-version = "0.0.7"
|
|
|
|
[dev-dependencies]
|
|
tempfile = "3.20.0"
|