mirror of
https://github.com/sreedevk/deduplicator.git
synced 2026-08-26 18:15:33 +00:00
Compare commits
15 Commits
v0.3.0
...
experiment
| Author | SHA1 | Date | |
|---|---|---|---|
|
9effdc9681 | ||
|
|
936c631623 | ||
|
|
0d62361132 | ||
|
|
b103d7d634 | ||
|
|
c42538a8c6 | ||
|
|
05437e0256 | ||
|
|
5942cf0b6c | ||
|
|
79c0c2be3d | ||
|
|
49b46f37f3 | ||
|
|
a37d59308f | ||
|
|
8a5b540e40 | ||
|
|
5ce0a2774a | ||
|
|
8d32ac5151 | ||
|
|
0c70e27835 | ||
|
|
53282c50df |
2
.gitignore
vendored
2
.gitignore
vendored
@@ -1,4 +1,4 @@
|
||||
/target
|
||||
/Cargo.lock
|
||||
/result-bin
|
||||
/.bacon-locations
|
||||
/result-bin
|
||||
|
||||
585
Cargo.lock
generated
585
Cargo.lock
generated
@@ -11,6 +11,12 @@ dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "allocator-api2"
|
||||
version = "0.2.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
|
||||
|
||||
[[package]]
|
||||
name = "android-tzdata"
|
||||
version = "0.1.1"
|
||||
@@ -62,7 +68,7 @@ version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ad186efb764318d35165f1758e7dcef3b10628e26d41a44bc5550652e6804391"
|
||||
dependencies = [
|
||||
"windows-sys",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -72,7 +78,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"windows-sys",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -121,6 +127,21 @@ version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a3e368af43e418a04d52505cf3dbc23dda4e3407ae2fa99fd0e4f308ce546acc"
|
||||
|
||||
[[package]]
|
||||
name = "cassowary"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53"
|
||||
|
||||
[[package]]
|
||||
name = "castaway"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0abae9be0aaf9ea96a3b1b8b1b55c602ca751eba1b1500220cea4ecbafe7c0d5"
|
||||
dependencies = [
|
||||
"rustversion",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.0.104"
|
||||
@@ -144,7 +165,7 @@ dependencies = [
|
||||
"js-sys",
|
||||
"num-traits",
|
||||
"wasm-bindgen",
|
||||
"windows-targets",
|
||||
"windows-targets 0.52.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -193,6 +214,30 @@ version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422"
|
||||
|
||||
[[package]]
|
||||
name = "colored"
|
||||
version = "2.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "compact_str"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3b79c4069c6cad78e2e0cdfcbd26275770669fb39fd308a752dc110e83b9af32"
|
||||
dependencies = [
|
||||
"castaway",
|
||||
"cfg-if",
|
||||
"itoa",
|
||||
"rustversion",
|
||||
"ryu",
|
||||
"static_assertions",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "console"
|
||||
version = "0.15.8"
|
||||
@@ -202,8 +247,8 @@ dependencies = [
|
||||
"encode_unicode 0.3.6",
|
||||
"lazy_static",
|
||||
"libc",
|
||||
"unicode-width",
|
||||
"windows-sys",
|
||||
"unicode-width 0.1.13",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -237,6 +282,31 @@ version = "0.8.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
|
||||
|
||||
[[package]]
|
||||
name = "crossterm"
|
||||
version = "0.28.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6"
|
||||
dependencies = [
|
||||
"bitflags 2.6.0",
|
||||
"crossterm_winapi",
|
||||
"mio",
|
||||
"parking_lot",
|
||||
"rustix 0.38.44",
|
||||
"signal-hook",
|
||||
"signal-hook-mio",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossterm_winapi"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b"
|
||||
dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "csv"
|
||||
version = "1.3.0"
|
||||
@@ -258,6 +328,41 @@ dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling"
|
||||
version = "0.20.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee"
|
||||
dependencies = [
|
||||
"darling_core",
|
||||
"darling_macro",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling_core"
|
||||
version = "0.20.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e"
|
||||
dependencies = [
|
||||
"fnv",
|
||||
"ident_case",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"strsim",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling_macro"
|
||||
version = "0.20.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead"
|
||||
dependencies = [
|
||||
"darling_core",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dashmap"
|
||||
version = "5.5.3"
|
||||
@@ -265,7 +370,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"hashbrown",
|
||||
"hashbrown 0.14.5",
|
||||
"lock_api",
|
||||
"once_cell",
|
||||
"parking_lot_core",
|
||||
@@ -274,23 +379,29 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "deduplicator"
|
||||
version = "0.3.0"
|
||||
version = "0.2.2"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bytesize",
|
||||
"chrono",
|
||||
"clap",
|
||||
"colored",
|
||||
"dashmap",
|
||||
"globwalk",
|
||||
"gxhash",
|
||||
"indicatif",
|
||||
"itertools 0.10.5",
|
||||
"memmap2",
|
||||
"pathdiff",
|
||||
"prettytable-rs",
|
||||
"rand",
|
||||
"ratatui",
|
||||
"rayon",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
"threadpool",
|
||||
"unicode-segmentation",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -332,6 +443,12 @@ version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
|
||||
|
||||
[[package]]
|
||||
name = "equivalent"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
|
||||
|
||||
[[package]]
|
||||
name = "errno"
|
||||
version = "0.3.13"
|
||||
@@ -339,7 +456,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -348,6 +465,18 @@ version = "2.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
|
||||
|
||||
[[package]]
|
||||
name = "fnv"
|
||||
version = "1.0.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
|
||||
|
||||
[[package]]
|
||||
name = "foldhash"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.15"
|
||||
@@ -410,6 +539,17 @@ version = "0.14.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.15.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5"
|
||||
dependencies = [
|
||||
"allocator-api2",
|
||||
"equivalent",
|
||||
"foldhash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.5.0"
|
||||
@@ -451,6 +591,12 @@ dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ident_case"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
|
||||
|
||||
[[package]]
|
||||
name = "ignore"
|
||||
version = "0.4.22"
|
||||
@@ -478,7 +624,26 @@ dependencies = [
|
||||
"number_prefix",
|
||||
"portable-atomic",
|
||||
"rayon",
|
||||
"unicode-width",
|
||||
"unicode-width 0.1.13",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "indoc"
|
||||
version = "2.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd"
|
||||
|
||||
[[package]]
|
||||
name = "instability"
|
||||
version = "0.3.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0bf9fed6d91cfb734e7476a06bde8300a1b94e217e1b523b6f0cd1a01998c71d"
|
||||
dependencies = [
|
||||
"darling",
|
||||
"indoc",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -498,7 +663,7 @@ checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b"
|
||||
dependencies = [
|
||||
"hermit-abi 0.3.9",
|
||||
"libc",
|
||||
"windows-sys",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -507,6 +672,24 @@ version = "1.70.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800"
|
||||
|
||||
[[package]]
|
||||
name = "itertools"
|
||||
version = "0.10.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
|
||||
dependencies = [
|
||||
"either",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itertools"
|
||||
version = "0.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
|
||||
dependencies = [
|
||||
"either",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.11"
|
||||
@@ -515,10 +698,11 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
|
||||
|
||||
[[package]]
|
||||
name = "js-sys"
|
||||
version = "0.3.69"
|
||||
version = "0.3.77"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d"
|
||||
checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
@@ -544,6 +728,12 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
version = "0.4.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
|
||||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
version = "0.9.4"
|
||||
@@ -566,6 +756,15 @@ version = "0.4.22"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
|
||||
|
||||
[[package]]
|
||||
name = "lru"
|
||||
version = "0.12.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38"
|
||||
dependencies = [
|
||||
"hashbrown 0.15.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.7.4"
|
||||
@@ -581,6 +780,18 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mio"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"log",
|
||||
"wasi 0.11.0+wasi-snapshot-preview1",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.2.19"
|
||||
@@ -612,6 +823,16 @@ version = "1.19.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.12.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"
|
||||
dependencies = [
|
||||
"lock_api",
|
||||
"parking_lot_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot_core"
|
||||
version = "0.9.10"
|
||||
@@ -622,9 +843,15 @@ dependencies = [
|
||||
"libc",
|
||||
"redox_syscall",
|
||||
"smallvec",
|
||||
"windows-targets",
|
||||
"windows-targets 0.52.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "paste"
|
||||
version = "1.0.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
|
||||
|
||||
[[package]]
|
||||
name = "pathdiff"
|
||||
version = "0.2.1"
|
||||
@@ -637,15 +864,6 @@ version = "1.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0"
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
|
||||
dependencies = [
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prettytable-rs"
|
||||
version = "0.10.0"
|
||||
@@ -657,23 +875,23 @@ dependencies = [
|
||||
"is-terminal",
|
||||
"lazy_static",
|
||||
"term",
|
||||
"unicode-width",
|
||||
"unicode-width 0.1.13",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.86"
|
||||
version = "1.0.95"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77"
|
||||
checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.36"
|
||||
version = "1.0.40"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
|
||||
checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
@@ -685,32 +903,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.9.1"
|
||||
name = "ratatui"
|
||||
version = "0.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97"
|
||||
checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b"
|
||||
dependencies = [
|
||||
"rand_chacha",
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_chacha"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
|
||||
dependencies = [
|
||||
"ppv-lite86",
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.9.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"
|
||||
dependencies = [
|
||||
"getrandom 0.3.3",
|
||||
"bitflags 2.6.0",
|
||||
"cassowary",
|
||||
"compact_str",
|
||||
"crossterm",
|
||||
"indoc",
|
||||
"instability",
|
||||
"itertools 0.13.0",
|
||||
"lru",
|
||||
"paste",
|
||||
"strum",
|
||||
"unicode-segmentation",
|
||||
"unicode-truncate",
|
||||
"unicode-width 0.2.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -770,6 +980,19 @@ version = "0.8.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "0.38.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
|
||||
dependencies = [
|
||||
"bitflags 2.6.0",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys 0.4.15",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "1.0.7"
|
||||
@@ -779,8 +1002,8 @@ dependencies = [
|
||||
"bitflags 2.6.0",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys",
|
||||
"linux-raw-sys 0.9.4",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -830,12 +1053,59 @@ dependencies = [
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.120"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"ryu",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "signal-hook"
|
||||
version = "0.3.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"signal-hook-registry",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "signal-hook-mio"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "34db1a06d485c9142248b7a054f034b349b212551f3dfd19c94d45a754a217cd"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"mio",
|
||||
"signal-hook",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "signal-hook-registry"
|
||||
version = "1.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.13.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
|
||||
|
||||
[[package]]
|
||||
name = "static_assertions"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.11.1"
|
||||
@@ -843,10 +1113,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.68"
|
||||
name = "strum"
|
||||
version = "0.26.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9"
|
||||
checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06"
|
||||
dependencies = [
|
||||
"strum_macros",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "strum_macros"
|
||||
version = "0.26.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"rustversion",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.104"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -862,8 +1154,8 @@ dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.3.3",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys",
|
||||
"rustix 1.0.7",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -912,18 +1204,52 @@ version = "1.0.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-segmentation"
|
||||
version = "1.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-truncate"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b3644627a5af5fa321c95b9b235a72fd24cd29c648c2c379431e6628655627bf"
|
||||
dependencies = [
|
||||
"itertools 0.13.0",
|
||||
"unicode-segmentation",
|
||||
"unicode-width 0.1.13",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-width"
|
||||
version = "0.1.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-width"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd"
|
||||
|
||||
[[package]]
|
||||
name = "utf8parse"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.17.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d"
|
||||
dependencies = [
|
||||
"getrandom 0.3.3",
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "walkdir"
|
||||
version = "2.5.0"
|
||||
@@ -951,23 +1277,24 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen"
|
||||
version = "0.2.92"
|
||||
version = "0.2.100"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8"
|
||||
checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
"rustversion",
|
||||
"wasm-bindgen-macro",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-backend"
|
||||
version = "0.2.92"
|
||||
version = "0.2.100"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da"
|
||||
checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"log",
|
||||
"once_cell",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
@@ -976,9 +1303,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro"
|
||||
version = "0.2.92"
|
||||
version = "0.2.100"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726"
|
||||
checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"wasm-bindgen-macro-support",
|
||||
@@ -986,9 +1313,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro-support"
|
||||
version = "0.2.92"
|
||||
version = "0.2.100"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"
|
||||
checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -999,9 +1326,12 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-shared"
|
||||
version = "0.2.92"
|
||||
version = "0.2.100"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96"
|
||||
checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
@@ -1025,7 +1355,7 @@ version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b"
|
||||
dependencies = [
|
||||
"windows-sys",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1040,7 +1370,16 @@ version = "0.52.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
|
||||
dependencies = [
|
||||
"windows-targets",
|
||||
"windows-targets 0.52.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.48.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
|
||||
dependencies = [
|
||||
"windows-targets 0.48.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1049,7 +1388,31 @@ version = "0.52.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
|
||||
dependencies = [
|
||||
"windows-targets",
|
||||
"windows-targets 0.52.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.59.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
|
||||
dependencies = [
|
||||
"windows-targets 0.52.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-targets"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
|
||||
dependencies = [
|
||||
"windows_aarch64_gnullvm 0.48.5",
|
||||
"windows_aarch64_msvc 0.48.5",
|
||||
"windows_i686_gnu 0.48.5",
|
||||
"windows_i686_msvc 0.48.5",
|
||||
"windows_x86_64_gnu 0.48.5",
|
||||
"windows_x86_64_gnullvm 0.48.5",
|
||||
"windows_x86_64_msvc 0.48.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1058,28 +1421,46 @@ version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
|
||||
dependencies = [
|
||||
"windows_aarch64_gnullvm",
|
||||
"windows_aarch64_msvc",
|
||||
"windows_i686_gnu",
|
||||
"windows_aarch64_gnullvm 0.52.6",
|
||||
"windows_aarch64_msvc 0.52.6",
|
||||
"windows_i686_gnu 0.52.6",
|
||||
"windows_i686_gnullvm",
|
||||
"windows_i686_msvc",
|
||||
"windows_x86_64_gnu",
|
||||
"windows_x86_64_gnullvm",
|
||||
"windows_x86_64_msvc",
|
||||
"windows_i686_msvc 0.52.6",
|
||||
"windows_x86_64_gnu 0.52.6",
|
||||
"windows_x86_64_gnullvm 0.52.6",
|
||||
"windows_x86_64_msvc 0.52.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_gnullvm"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_gnullvm"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_msvc"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_msvc"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
version = "0.52.6"
|
||||
@@ -1092,24 +1473,48 @@ version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_msvc"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_msvc"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnu"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnu"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnullvm"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnullvm"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
version = "0.52.6"
|
||||
@@ -1124,23 +1529,3 @@ checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
|
||||
dependencies = [
|
||||
"bitflags 2.6.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.8.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f"
|
||||
dependencies = [
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.8.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
43
Cargo.toml
43
Cargo.toml
@@ -1,58 +1,51 @@
|
||||
[package]
|
||||
name = "deduplicator"
|
||||
version = "0.3.0"
|
||||
version = "0.2.2"
|
||||
edition = "2021"
|
||||
description = "find,filter and delete duplicate files"
|
||||
description = "find,filter,delete Duplicates"
|
||||
repository = "https://github.com/sreedevk/deduplicator"
|
||||
license = "MIT"
|
||||
authors = [
|
||||
"Sreedev Kodichath <sreedevpadmakumar@gmail.com>",
|
||||
"Valentin Bersier <vbersier@gmail.com>",
|
||||
"Dhruva Sagar <dhruva.sagar@gmail.com>",
|
||||
"Sreedev Kodichath <sreedevpadmakumar@gmail.com>",
|
||||
"Valentin Bersier <vbersier@gmail.com>",
|
||||
"Dhruva Sagar <dhruva.sagar@gmail.com>",
|
||||
]
|
||||
|
||||
[[bin]]
|
||||
name = "deduplicator"
|
||||
path = "src/main.rs"
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
anyhow = "1.0.68"
|
||||
bytesize = "1.1.0"
|
||||
chrono = "0.4.23"
|
||||
clap = { version = "4.0.32", features = ["derive"] }
|
||||
colored = "2.0.0"
|
||||
dashmap = { version = "5.4.0", features = ["rayon"] }
|
||||
globwalk = "0.8.1"
|
||||
gxhash = { version = "3.4.1", default-features = false }
|
||||
gxhash = "3.4.1"
|
||||
indicatif = { version = "0.17.2", features = ["rayon"] }
|
||||
itertools = "0.10.5"
|
||||
memmap2 = "0.5.8"
|
||||
pathdiff = "0.2.1"
|
||||
prettytable-rs = "0.10.0"
|
||||
rand = "0.9.1"
|
||||
ratatui = "0.29.0"
|
||||
rayon = "1.6.1"
|
||||
serde = { version = "1.0.192", features = ["derive"] }
|
||||
serde_json = "1.0.108"
|
||||
tempfile = "3.20.0"
|
||||
threadpool = "1.8.1"
|
||||
unicode-segmentation = "1.10.0"
|
||||
uuid = { version = "1.17.0", features = ["v4"] }
|
||||
|
||||
[profile.release]
|
||||
strip = true
|
||||
opt-level = 3
|
||||
lto = "thin"
|
||||
debug = false
|
||||
codegen-units = 1
|
||||
|
||||
# generated by 'cargo dist init'
|
||||
[profile.dist]
|
||||
inherits = "release"
|
||||
lto = "thin"
|
||||
|
||||
[workspace.metadata.dist]
|
||||
rust-toolchain-version = "1.87.0"
|
||||
rust-toolchain-version = "1.78.0"
|
||||
ci = ["github"]
|
||||
targets = [
|
||||
"x86_64-unknown-linux-gnu",
|
||||
"x86_64-apple-darwin",
|
||||
"x86_64-pc-windows-msvc",
|
||||
"aarch64-apple-darwin",
|
||||
]
|
||||
targets = ["x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc", "aarch64-apple-darwin"]
|
||||
cargo-dist-version = "0.0.7"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3.20.0"
|
||||
|
||||
185
README.md
185
README.md
@@ -7,8 +7,6 @@
|
||||
## Usage
|
||||
|
||||
```bash
|
||||
find,filter and delete duplicate files
|
||||
|
||||
Usage: deduplicator [OPTIONS] [scan_dir_path]
|
||||
|
||||
Arguments:
|
||||
@@ -17,14 +15,13 @@ Arguments:
|
||||
Options:
|
||||
-t, --types <TYPES> Filetypes to deduplicate [default = all]
|
||||
-i, --interactive Delete files interactively
|
||||
-m, --min-size <MIN_SIZE> Minimum filesize of duplicates to scan (e.g., 100B/1K/2M/3G/4T) [default: 1b]
|
||||
-D, --max-depth <MAX_DEPTH> Max Depth to scan while looking for duplicates
|
||||
-d, --min-depth <MIN_DEPTH> Min Depth to scan while looking for duplicates
|
||||
-s, --min-size <MIN_SIZE> Minimum filesize of duplicates to scan (e.g., 100B/1K/2M/3G/4T) [default: 1b]
|
||||
-d, --max-depth <MAX_DEPTH> Max Depth to scan while looking for duplicates
|
||||
--min-depth <MIN_DEPTH> Min Depth to scan while looking for duplicates
|
||||
-f, --follow-links Follow links while scanning directories
|
||||
-s, --strict Guarantees that two files are duplicate (performs a full hash)
|
||||
-p, --progress Show Progress spinners & metrics
|
||||
-h, --help Print help
|
||||
-V, --version Print version
|
||||
-h, --help Print help information
|
||||
-V, --version Print version information
|
||||
--json
|
||||
```
|
||||
### Examples
|
||||
|
||||
@@ -45,118 +42,94 @@ deduplicator ~/.config --follow-links
|
||||
deduplicator ~/Media --min-size 100mb
|
||||
```
|
||||
|
||||
## Demo
|
||||
|
||||
|
||||
|
||||
|
||||
## Installation
|
||||
Currently, you can only install deduplicator using cargo package manager.
|
||||
|
||||
### Cargo
|
||||
> GxHash relies on aes hardware acceleration, so please set `RUSTFLAGS` to `"-C target-feature=+aes"` or `"-C target-cpu=native"` before
|
||||
> installing.
|
||||
### Cargo Install
|
||||
|
||||
#### Stable
|
||||
|
||||
> [!WARNING] Note from GxHash: GxHash relies on aes hardware acceleration, you must make sure the aes feature is enabled when building (otherwise it won't build). This can be done by setting the RUSTFLAGS environment variable to -C target-feature=+aes or -C target-cpu=native (the latter should work if your CPU is properly recognized by rustc, which is the case most of the time).
|
||||
> please install version `0.2.1` if you are unable to install `0.2.2`
|
||||
|
||||
#### install from crates.io
|
||||
```bash
|
||||
$ RUSTFLAGS="-C target-cpu=native" cargo install deduplicator
|
||||
```
|
||||
|
||||
#### install from git
|
||||
> [!]
|
||||
|
||||
#### Nightly
|
||||
|
||||
if you'd like to install with nightly features, you can use
|
||||
|
||||
```bash
|
||||
$ RUSTFLAGS="-C target-cpu=native" cargo install deduplicator --git https://github.com/sreedevk/deduplicator
|
||||
$ cargo install --git https://github.com/sreedevk/deduplicator
|
||||
```
|
||||
Please note that if you use a version manager to install rust (like asdf), you need to reshim (`asdf reshim rust`).
|
||||
|
||||
### Linux (Pre-built Binary)
|
||||
|
||||
you can download the pre-built binary from the [Releases](https://github.com/sreedevk/deduplicator/releases) page.
|
||||
download the `deduplicator-x86_64-unknown-linux-gnu.tar.gz` for linux. Once you have the tarball file with the executable,
|
||||
you can follow these steps to install:
|
||||
|
||||
```bash
|
||||
$ tar -zxvf deduplicator-x86_64-unknown-linux-gnu.tar.gz
|
||||
$ sudo mv deduplicator /usr/bin/
|
||||
```
|
||||
|
||||
### Mac OS (Pre-built Binary)
|
||||
|
||||
you can download the pre-build binary from the [Releases](https://github.com/sreedevk/deduplicator/releases) page.
|
||||
download the `deduplicator-x86_64-apple-darwin.tar.gz` tarball for mac os. Once you have the tarball file with the executable, you can follow these steps to install:
|
||||
|
||||
```bash
|
||||
$ tar -zxvf deduplicator-x86_64-unknown-linux-gnu.tar.gz
|
||||
$ sudo mv deduplicator /usr/bin/
|
||||
```
|
||||
|
||||
### Windows (Pre-built Binary)
|
||||
|
||||
you can download the pre-build binary from the [Releases](https://github.com/sreedevk/deduplicator/releases) page.
|
||||
download the `deduplicator-x86_64-pc-windows-msvc.zip` zip file for windows. unzip the `zip` file & move the `deduplicator.exe` to a location in the PATH system environment variable.
|
||||
|
||||
Note: If you Run into an msvc error, please install MSCV from [here](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist?view=msvc-170)
|
||||
|
||||
## Performance
|
||||
Deduplicator uses size comparison and [GxHash](https://docs.rs/gxhash/latest/gxhash/) to quickly check a large number of files to find duplicates. its also heavily parallelized. The default behavior of deduplicator is to only hash the first page (4K) of the file. This is to ensure that performance is the default priority. You can modify this behavior by using the `--strict` flag which will hash the whole file and ensure that 2 files are indeed duplicates. I'll add benchmarks in future versions.
|
||||
|
||||
### Benchmarks
|
||||
I've used hyperfine to run deduplicator on files generated by the rake file at `rakelib/benchmark.rake`. The Benchmarking accuracy can further be improved by isolating runs inside restricted docker containers. I'll include that in the future. For now, here's the hyperfine output on my i7-12800H laptop with 32G of RAM.
|
||||
Deduplicator uses size comparison and fxhash (a non non-cryptographic hashing algo) to quickly scan through large number of files to find duplicates. its also highly parallel (uses rayon and dashmap). I was able to scan through 120GB of files (Videos, PDFs, Images) in ~300ms. checkout the benchmarks
|
||||
|
||||
## benchmarks
|
||||
|
||||
| Command | Dirsize | Filecount | Mean [ms] | Min [ms] | Max [ms] | Relative |
|
||||
|:---|:---|---:|---:|---:|---:|---:|
|
||||
| `deduplicator ~/Data/tmp` | (~120G) | 721 files | 33.5 ± 28.6 | 25.3 | 151.5 | 1.87 ± 1.60 |
|
||||
| `deduplicator ~/Data/books` | (~8.6G) | 1419 files | 24.5 ± 1.0 | 22.9 | 28.1 | 1.37 ± 0.08 |
|
||||
| `deduplicator ~/Data/books --min-size 10M` | (~8.6G) | 1419 files | 17.9 ± 0.7 | 16.8 | 20.0 | 1.00 |
|
||||
| `deduplicator ~/Data/ --types pdf,jpg,png,jpeg` | (~290G) | 104222 files | 1207.2 ± 37.0 | 1172.2 | 1287.7 | 67.27 ± 3.33 |
|
||||
|
||||
* The last entry is lower because of the number of files deduplicator had to go through (~660895 Files). The average size of the files rarely affect the performance of deduplicator.
|
||||
|
||||
These benchmarks were run using [hyperfine](https://github.com/sharkdp/hyperfine). Here are the specs of the machine used to benchmark deduplicator:
|
||||
|
||||
#### Fewer Large Files
|
||||
```
|
||||
# hyperfine -N --warmup 80 './target/release/deduplicator bench_artifacts'
|
||||
Benchmark 1: ./target/release/deduplicator bench_artifacts
|
||||
Time (mean ± σ): 2.5 ms ± 0.4 ms [User: 2.2 ms, System: 4.8 ms]
|
||||
Range (min … max): 1.9 ms … 6.8 ms 1322 runs
|
||||
|
||||
# dust 'bench_artifacts'
|
||||
37M ┌── file_1_fwds.bin │██ │ 1%
|
||||
201M ├── file_0_fwds.bin │██████████ │ 8%
|
||||
390M ├── file_0_fwdcbss.bin│████████████████████ │ 15%
|
||||
390M ├── file_0_fwscas.bin │████████████████████ │ 15%
|
||||
390M ├── file_0_fwss.bin │████████████████████ │ 15%
|
||||
390M ├── file_1_fwdcbss.bin│████████████████████ │ 15%
|
||||
390M ├── file_1_fwscas.bin │████████████████████ │ 15%
|
||||
390M ├── file_1_fwss.bin │████████████████████ │ 15%
|
||||
2.5G ┌─┴ bench_artifacts │███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ │ 100%
|
||||
OS: Arch Linux x86_64
|
||||
Host: Precision 5540
|
||||
Kernel: 5.15.89-1-lts
|
||||
Uptime: 4 hours, 44 mins
|
||||
Shell: zsh 5.9
|
||||
Terminal: kitty
|
||||
CPU: Intel i9-9880H (16) @ 4.800GHz
|
||||
GPU: NVIDIA Quadro T2000 Mobile / Max-Q
|
||||
GPU: Intel CoffeeLake-H GT2 [UHD Graphics 630]
|
||||
Memory: 31731MiB (~32GiB)
|
||||
```
|
||||
|
||||
#### Many Small Files
|
||||
```
|
||||
# hyperfine --warmup 20 './target/release/deduplicator bench_artifacts'
|
||||
Benchmark 1: ./target/release/deduplicator bench_artifacts
|
||||
Time (mean ± σ): 22.3 ms ± 1.7 ms [User: 35.8 ms, System: 46.3 ms]
|
||||
Range (min … max): 18.9 ms … 27.2 ms 112 runs
|
||||
## Screenshots
|
||||
|
||||
# dust 'bench_artifacts'
|
||||
3.9M ┌── file_992_fwss.bin │█ │ 0%
|
||||
3.9M ├── file_993_fwdcbss.bin│█ │ 0%
|
||||
3.9M ├── file_993_fwscas.bin │█ │ 0%
|
||||
3.9M ├── file_993_fwss.bin │█ │ 0%
|
||||
3.9M ├── file_994_fwdcbss.bin│█ │ 0%
|
||||
3.9M ├── file_994_fwscas.bin │█ │ 0%
|
||||
3.9M ├── file_994_fwss.bin │█ │ 0%
|
||||
3.9M ├── file_995_fwdcbss.bin│█ │ 0%
|
||||
3.9M ├── file_995_fwscas.bin │█ │ 0%
|
||||
3.9M ├── file_995_fwss.bin │█ │ 0%
|
||||
3.9M ├── file_996_fwdcbss.bin│█ │ 0%
|
||||
3.9M ├── file_996_fwscas.bin │█ │ 0%
|
||||
3.9M ├── file_996_fwss.bin │█ │ 0%
|
||||
3.9M ├── file_997_fwdcbss.bin│█ │ 0%
|
||||
3.9M ├── file_997_fwscas.bin │█ │ 0%
|
||||
3.9M ├── file_997_fwss.bin │█ │ 0%
|
||||
3.9M ├── file_998_fwdcbss.bin│█ │ 0%
|
||||
3.9M ├── file_998_fwscas.bin │█ │ 0%
|
||||
3.9M ├── file_998_fwss.bin │█ │ 0%
|
||||
3.9M ├── file_999_fwdcbss.bin│█ │ 0%
|
||||
3.9M ├── file_999_fwscas.bin │█ │ 0%
|
||||
3.9M ├── file_999_fwss.bin │█ │ 0%
|
||||
3.9M ├── file_99_fwdcbss.bin │█ │ 0%
|
||||
3.9M ├── file_99_fwscas.bin │█ │ 0%
|
||||
3.9M ├── file_99_fwss.bin │█ │ 0%
|
||||
3.9M ├── file_9_fwdcbss.bin │█ │ 0%
|
||||
3.9M ├── file_9_fwscas.bin │█ │ 0%
|
||||
3.9M ├── file_9_fwss.bin │█ │ 0%
|
||||
11G ┌─┴ bench_artifacts │█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ │ 100%
|
||||
```
|
||||
|
||||
|
||||
## proposed
|
||||
- [ ] parallelization
|
||||
- [ ] (scanning + processing sw + processing hw) & formatting & printing
|
||||
- [ ] scanning + processing sw + processing hw + formatting + printing
|
||||
- [ ] max file path size should use the last set of duplicates
|
||||
- [ ] add more unit tests
|
||||
- [ ] test against different filesystems
|
||||
- [ ] test against different file name encodings
|
||||
- [ ] restore json output (was removed in 0.3)
|
||||
- [ ] fix memory leak on very large filesystems
|
||||
- [ ] maybe use a bloom filter
|
||||
- [ ] reduce FileInfo size
|
||||
- [ ] output in a tree format
|
||||
- [ ] tui
|
||||
- [ ] change the default hashing method to include the first & last page of a file (8K)
|
||||
|
||||
## v0.3
|
||||
- [x] parallelization
|
||||
- [x] (scanning) + (processing sw & processing hw & formatting & printing)
|
||||
- [x] reduce cloning values on the heap
|
||||
- [x] add a partial hashing mode (--strict)
|
||||
- [x] add unit tests
|
||||
- [x] add silent mode
|
||||
- [x] update documentation
|
||||
- [x] remove color output
|
||||
- [x] progress bar improvements
|
||||
- [x] use progress bar groups
|
||||
- [x] remove broken json rendering
|
||||
- [x] add benchmarks
|
||||
## Roadmap
|
||||
- Tree format output for duplicate file listing
|
||||
- GUI
|
||||
- Packages for different operating system repositories (currently only installable via cargo)
|
||||
- TUI: Improve Key Handling
|
||||
|
||||
@@ -1,93 +0,0 @@
|
||||
require 'tempfile'
|
||||
require 'fileutils'
|
||||
require 'securerandom'
|
||||
|
||||
namespace :benchmark do
|
||||
file 'target/release/deduplicator' do
|
||||
sh "cargo build --release"
|
||||
end
|
||||
|
||||
task :few_large_files => 'target/release/deduplicator' do
|
||||
# Benchmark 1: ./target/release/deduplicator bench_artifacts
|
||||
# Time (mean ± σ): 2.5 ms ± 0.6 ms [User: 2.4 ms, System: 4.8 ms]
|
||||
# Range (min … max): 1.8 ms … 9.7 ms 1474 runs
|
||||
|
||||
root = "bench_artifacts"
|
||||
Dir.mkdir(root)
|
||||
|
||||
# files with same size
|
||||
2.times.map do |i|
|
||||
File.open(File.join(root, "file_#{i}_fwss.bin"), 'wb') do |f|
|
||||
f.write(SecureRandom.bytes(4096 * 100_000))
|
||||
end
|
||||
end
|
||||
|
||||
# files with different sizes
|
||||
2.times.map do |i|
|
||||
File.open(File.join(root, "file_#{i}_fwds.bin"), 'wb') do |f|
|
||||
f.write(SecureRandom.bytes(4096 * (rand * 100_000).ceil))
|
||||
end
|
||||
end
|
||||
|
||||
# files with same content & size
|
||||
2.times.each do |i|
|
||||
File.open(File.join(root, "file_#{i}_fwscas.bin"), 'wb') do |f|
|
||||
f.write("\0" * (4096 * 100_000))
|
||||
end
|
||||
end
|
||||
|
||||
# files with different content but same size
|
||||
2.times.each do |i|
|
||||
File.open(File.join(root, "file_#{i}_fwdcbss.bin"), 'wb') do |f|
|
||||
f.write(SecureRandom.bytes(4096 * 100_000))
|
||||
end
|
||||
end
|
||||
|
||||
sh("hyperfine -N --warmup 80 './target/release/deduplicator #{root}'")
|
||||
sh("dust '#{root}'")
|
||||
|
||||
FileUtils.rm_rf(root)
|
||||
end
|
||||
|
||||
task :many_small_files => 'target/release/deduplicator' do
|
||||
# Benchmark 1: ./target/release/deduplicator bench_artifacts
|
||||
# Time (mean ± σ): 10.6 ms ± 1.0 ms [User: 20.0 ms, System: 22.5 ms]
|
||||
# Range (min … max): 8.4 ms … 14.2 ms 235 runs
|
||||
|
||||
root = "bench_artifacts"
|
||||
Dir.mkdir(root)
|
||||
|
||||
# files with same size
|
||||
1000.times.each do |i|
|
||||
File.open(File.join(root, "file_#{i}_fwss.bin"), 'wb') do |f|
|
||||
f.write(SecureRandom.bytes(4096 * 1000))
|
||||
end
|
||||
end
|
||||
|
||||
# files with different sizes
|
||||
1000.times.each do |i|
|
||||
File.open(File.join(root, "file_#{i}_fwds.bin"), 'wb') do |f|
|
||||
f.write(SecureRandom.bytes(4096 * (rand * 100).ceil))
|
||||
end
|
||||
end
|
||||
|
||||
# files with same content & size
|
||||
1000.times.each do |i|
|
||||
File.open(File.join(root, "file_#{i}_fwscas.bin"), 'wb') do |f|
|
||||
f.write("\0" * (4096 * 1000))
|
||||
end
|
||||
end
|
||||
|
||||
# files with different content but same size
|
||||
1000.times.each do |i|
|
||||
File.open(File.join(root, "file_#{i}_fwdcbss.bin"), 'wb') do |f|
|
||||
f.write(SecureRandom.bytes(4096 * 1000))
|
||||
end
|
||||
end
|
||||
|
||||
sh("hyperfine --warmup 20 './target/release/deduplicator #{root}'")
|
||||
sh("dust '#{root}'")
|
||||
|
||||
FileUtils.rm_rf(root)
|
||||
end
|
||||
end
|
||||
72
src/app.rs
Normal file
72
src/app.rs
Normal file
@@ -0,0 +1,72 @@
|
||||
use std::sync::mpsc::channel;
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::{anyhow, Result};
|
||||
use threadpool::ThreadPool;
|
||||
|
||||
use crate::params::Params;
|
||||
use crate::server::{Message, Server};
|
||||
use crate::tui::Tui;
|
||||
|
||||
pub struct App {
|
||||
tpool: ThreadPool,
|
||||
server: Arc<Server>,
|
||||
app_opts: Arc<Params>,
|
||||
}
|
||||
|
||||
impl App {
|
||||
pub fn new(app_opts: Arc<Params>) -> Self {
|
||||
Self {
|
||||
tpool: ThreadPool::new(8),
|
||||
server: Arc::new(Server::new().expect("server init failed")),
|
||||
app_opts,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn start(&self) -> Result<()> {
|
||||
let (server_tx, server_rx) = channel::<Message>();
|
||||
let (app_tx, app_rx) = channel::<Message>();
|
||||
|
||||
let server_ptr = self.server.clone();
|
||||
let tui_server_ptr = self.server.clone();
|
||||
let mut ui = Tui::new(app_tx.clone(), tui_server_ptr);
|
||||
let root_dir = self
|
||||
.app_opts
|
||||
.get_directory()?
|
||||
.into_os_string()
|
||||
.into_string()
|
||||
.map_err(|_| anyhow!("path to str conv failed"))?
|
||||
.into_boxed_str();
|
||||
|
||||
self.tpool.execute(move || {
|
||||
server_ptr.start(server_rx).expect("server init failed");
|
||||
});
|
||||
|
||||
self.tpool.execute(move || {
|
||||
ui.start().expect("ui init failed");
|
||||
});
|
||||
|
||||
self.tpool.execute(move || loop {
|
||||
match app_rx.try_recv() {
|
||||
Ok(Message::Exit) => {
|
||||
server_tx
|
||||
.send(Message::Exit)
|
||||
.expect("message passing to app from ui failed.");
|
||||
break;
|
||||
}
|
||||
Ok(Message::AddScanDirectory(dir)) => {
|
||||
server_tx
|
||||
.send(Message::AddScanDirectory(dir))
|
||||
.expect("message passing to server failed.");
|
||||
}
|
||||
_ => continue,
|
||||
}
|
||||
});
|
||||
|
||||
app_tx.send(Message::AddScanDirectory(root_dir))?;
|
||||
|
||||
self.tpool.join();
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
0
src/cli/mod.rs
Normal file
0
src/cli/mod.rs
Normal file
@@ -1,45 +1,43 @@
|
||||
use anyhow::Result;
|
||||
use gxhash::gxhash128;
|
||||
use gxhash::GxHasher;
|
||||
use memmap2::Mmap;
|
||||
use std::{
|
||||
fs,
|
||||
io::Read,
|
||||
path::{Path, PathBuf},
|
||||
time::SystemTime,
|
||||
};
|
||||
use serde::Serialize;
|
||||
use std::fs;
|
||||
use std::hash::Hasher;
|
||||
use std::{fs::Metadata, path::PathBuf};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct FileInfo {
|
||||
pub path: Box<Path>,
|
||||
pub path: PathBuf,
|
||||
pub hash: Option<String>,
|
||||
pub size: u64,
|
||||
pub modified: SystemTime,
|
||||
#[serde(skip)]
|
||||
pub filemeta: Metadata,
|
||||
}
|
||||
|
||||
impl FileInfo {
|
||||
pub fn hash(&self, seed: i64) -> Result<u128> {
|
||||
let file = fs::File::open(&self.path)?;
|
||||
pub fn hash(&self) -> Result<Self> {
|
||||
let file = fs::File::open(self.path.clone())?;
|
||||
let mapper = unsafe { Mmap::map(&file)? };
|
||||
let final_hash = mapper.chunks(4096).fold(0u128, |acc, chunk: &[u8]| {
|
||||
acc.wrapping_add(gxhash128(chunk, seed))
|
||||
});
|
||||
let mut primhasher = GxHasher::default();
|
||||
|
||||
Ok(final_hash)
|
||||
}
|
||||
mapper
|
||||
.chunks(1_000_000)
|
||||
.for_each(|chunk| primhasher.write(chunk));
|
||||
|
||||
pub fn initial_page_hash(&self, seed: i64) -> Result<u128> {
|
||||
let mut file = fs::File::open(&self.path)?;
|
||||
let mut buffer = [0; 4096];
|
||||
let bytes_read = file.read(&mut buffer)?;
|
||||
|
||||
Ok(gxhash128(&buffer[..bytes_read], seed))
|
||||
Ok(Self {
|
||||
hash: Some(primhasher.finish().to_string()),
|
||||
..self.clone()
|
||||
})
|
||||
}
|
||||
|
||||
pub fn new(path: PathBuf) -> Result<Self> {
|
||||
let filemeta = std::fs::metadata(&path)?;
|
||||
let filemeta = std::fs::metadata(path.clone())?;
|
||||
Ok(Self {
|
||||
path: path.into_boxed_path(),
|
||||
path,
|
||||
filemeta: filemeta.clone(),
|
||||
hash: None,
|
||||
size: filemeta.len(),
|
||||
modified: filemeta.modified()?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
131
src/formatter.rs
131
src/formatter.rs
@@ -1,14 +1,20 @@
|
||||
use crate::{fileinfo::FileInfo, params::Params};
|
||||
pub struct Formatter;
|
||||
use crate::fileinfo::FileInfo;
|
||||
use crate::params::Params;
|
||||
use anyhow::Result;
|
||||
use chrono::{DateTime, Utc};
|
||||
use colored::Colorize;
|
||||
use dashmap::DashMap;
|
||||
use indicatif::{ParallelProgressIterator, ProgressBar, ProgressFinish, ProgressStyle};
|
||||
use indicatif::{
|
||||
ParallelProgressIterator, ProgressBar, ProgressFinish, ProgressIterator, ProgressStyle,
|
||||
};
|
||||
use pathdiff::diff_paths;
|
||||
use prettytable::{format, row, Row, Table};
|
||||
use prettytable::{format, row, Table};
|
||||
use rayon::prelude::*;
|
||||
use std::{borrow::Cow, path::PathBuf, sync::Arc, time::Duration};
|
||||
use std::borrow::Cow;
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
|
||||
pub struct Formatter;
|
||||
impl Formatter {
|
||||
pub fn human_path(
|
||||
file: &FileInfo,
|
||||
@@ -16,7 +22,7 @@ impl Formatter {
|
||||
min_path_length: usize,
|
||||
) -> Result<String> {
|
||||
let base_directory: PathBuf = app_args.get_directory()?;
|
||||
let relative_path = diff_paths(&file.path, base_directory).unwrap_or_default();
|
||||
let relative_path = diff_paths(file.path.clone(), base_directory).unwrap_or_default();
|
||||
|
||||
let formatted_path = format!(
|
||||
"{:<0width$}",
|
||||
@@ -32,69 +38,96 @@ impl Formatter {
|
||||
}
|
||||
|
||||
pub fn human_mtime(file: &FileInfo) -> Result<String> {
|
||||
let modified_time: DateTime<Utc> = file.modified.into();
|
||||
let modified_time: DateTime<Utc> = file.filemeta.modified()?.into();
|
||||
Ok(modified_time.format("%Y-%m-%d %H:%M:%S").to_string())
|
||||
}
|
||||
|
||||
pub fn gen_sub_tbl(items: Vec<FileInfo>, app_args: &Params, max_path_len: u64) -> Table {
|
||||
let mut inner_table = Table::new();
|
||||
inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
|
||||
items.iter().for_each(|file| {
|
||||
inner_table.add_row(row![
|
||||
Self::human_path(file, app_args, max_path_len as usize).unwrap_or_default(),
|
||||
Self::human_filesize(file).unwrap_or_default(),
|
||||
Self::human_mtime(file).unwrap_or_default()
|
||||
]);
|
||||
});
|
||||
inner_table
|
||||
}
|
||||
pub fn generate_table(raw: Vec<FileInfo>, app_args: &Params) -> Result<Table> {
|
||||
let basepath_length = app_args.get_directory()?.to_str().unwrap_or_default().len();
|
||||
let max_filepath_length = raw
|
||||
.iter()
|
||||
.map(|file| file.path.to_str().unwrap_or_default().len())
|
||||
.max()
|
||||
.unwrap_or_default();
|
||||
|
||||
pub fn generate_table(
|
||||
raw: Arc<DashMap<u128, Vec<FileInfo>>>,
|
||||
mpath_len: u64,
|
||||
args: &Params,
|
||||
) -> Result<Table> {
|
||||
let progress_bar = match args.progress {
|
||||
true => ProgressBar::new_spinner(),
|
||||
false => ProgressBar::hidden(),
|
||||
let min_path_length = if max_filepath_length > basepath_length {
|
||||
max_filepath_length - basepath_length
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
let progress_style = ProgressStyle::with_template("[{elapsed_precise}] {pos:>7} {msg}")?;
|
||||
let progress_style = ProgressStyle::with_template(
|
||||
"[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}",
|
||||
)?;
|
||||
let progress_bar = ProgressBar::new(raw.len() as u64);
|
||||
progress_bar.set_style(progress_style);
|
||||
progress_bar.enable_steady_tick(Duration::from_millis(50));
|
||||
progress_bar.set_message("reconciling data");
|
||||
|
||||
let duplicates_table: DashMap<String, Vec<FileInfo>> = DashMap::new();
|
||||
raw.into_par_iter()
|
||||
.progress_with(progress_bar)
|
||||
.with_finish(ProgressFinish::WithMessage(Cow::from("data reconciled")))
|
||||
.map(|file| file.hash())
|
||||
.filter_map(Result::ok)
|
||||
.for_each(|file| {
|
||||
duplicates_table
|
||||
.entry(file.hash.clone().unwrap_or_default())
|
||||
.and_modify(|fileset| fileset.push(file.clone()))
|
||||
.or_insert_with(|| vec![file]);
|
||||
});
|
||||
|
||||
let mut output_table = Table::new();
|
||||
output_table.set_titles(row!["hash", "duplicates"]);
|
||||
|
||||
let progress_style = ProgressStyle::with_template(
|
||||
"[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}",
|
||||
)?;
|
||||
|
||||
let progress_bar = ProgressBar::new(duplicates_table.len() as u64);
|
||||
progress_bar.set_style(progress_style);
|
||||
progress_bar.enable_steady_tick(Duration::from_millis(50));
|
||||
progress_bar.set_message("generating output");
|
||||
|
||||
let rows = raw
|
||||
.par_iter_mut()
|
||||
duplicates_table
|
||||
.into_iter()
|
||||
.progress_with(progress_bar)
|
||||
.with_finish(ProgressFinish::WithMessage(Cow::from("output generated")))
|
||||
.filter(|i| i.value().len() > 1)
|
||||
.map(|i| {
|
||||
row![
|
||||
i.key(),
|
||||
Self::gen_sub_tbl(i.value().to_vec(), args, mpath_len)
|
||||
]
|
||||
})
|
||||
.collect::<Vec<Row>>();
|
||||
.for_each(|(hash, group)| {
|
||||
let mut inner_table = Table::new();
|
||||
inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
|
||||
group.iter().for_each(|file| {
|
||||
inner_table.add_row(row![
|
||||
Self::human_path(file, app_args, min_path_length)
|
||||
.unwrap_or_default()
|
||||
.blue(),
|
||||
Self::human_filesize(file).unwrap_or_default().red(),
|
||||
Self::human_mtime(file).unwrap_or_default().yellow()
|
||||
]);
|
||||
});
|
||||
|
||||
output_table.add_row(row![hash.green(), inner_table]);
|
||||
});
|
||||
|
||||
let mut output_table = Table::new();
|
||||
output_table.set_titles(row!["hash", "duplicates"]);
|
||||
output_table.extend(rows);
|
||||
Ok(output_table)
|
||||
}
|
||||
|
||||
pub fn print(
|
||||
raw: Arc<DashMap<u128, Vec<FileInfo>>>,
|
||||
max_path_len: u64,
|
||||
app_args: &Params,
|
||||
) -> Result<()> {
|
||||
pub fn print(raw: Vec<FileInfo>, app_args: &Params) -> Result<()> {
|
||||
if raw.is_empty() {
|
||||
println!("\n\nNo duplicates found matching your search criteria.\n");
|
||||
println!(
|
||||
"\n\n{}\n",
|
||||
"No duplicates found matching your search criteria.".green()
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let output_table = Self::generate_table(raw, max_path_len, app_args)?;
|
||||
output_table.printstd();
|
||||
if app_args.json {
|
||||
let output_json = serde_json::to_string_pretty(&raw)?;
|
||||
println!("{}", output_json);
|
||||
} else {
|
||||
let output_table = Self::generate_table(raw, app_args)?;
|
||||
output_table.printstd();
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1,125 +1,154 @@
|
||||
use crate::{fileinfo::FileInfo, formatter::Formatter, params::Params};
|
||||
use crate::formatter::Formatter;
|
||||
use crate::{fileinfo::FileInfo, params::Params};
|
||||
use anyhow::Result;
|
||||
use colored::Colorize;
|
||||
use dashmap::DashMap;
|
||||
use indicatif::{ParallelProgressIterator, ProgressBar, ProgressFinish, ProgressStyle};
|
||||
use prettytable::{format, row, Table};
|
||||
use rayon::prelude::*;
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
io::{self, Write},
|
||||
sync::Arc,
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
pub struct Interactive;
|
||||
pub fn scan_group_confirmation() -> Result<bool> {
|
||||
print!("\nconfirm? [y/N]: ");
|
||||
std::io::stdout().flush()?;
|
||||
let mut user_input = String::new();
|
||||
io::stdin().read_line(&mut user_input)?;
|
||||
|
||||
impl Interactive {
|
||||
pub fn init(result: Arc<DashMap<u128, Vec<FileInfo>>>, app_args: &Params) -> Result<()> {
|
||||
result
|
||||
.clone()
|
||||
.iter()
|
||||
.filter(|i| i.value().len() > 1)
|
||||
.enumerate()
|
||||
.for_each(|(gindex, i)| {
|
||||
let group = i.value();
|
||||
let mut itable = Table::new();
|
||||
itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
|
||||
itable.set_titles(row!["index", "filename", "size", "updated_at"]);
|
||||
|
||||
let max_path_size = group
|
||||
.iter()
|
||||
.map(|f| f.path.iter().count())
|
||||
.max()
|
||||
.unwrap_or_default();
|
||||
|
||||
group.iter().enumerate().for_each(|(index, file)| {
|
||||
itable.add_row(row![
|
||||
index,
|
||||
Formatter::human_path(file, app_args, max_path_size).unwrap_or_default(),
|
||||
Formatter::human_filesize(file).unwrap_or_default(),
|
||||
Formatter::human_mtime(file).unwrap_or_default()
|
||||
]);
|
||||
});
|
||||
|
||||
Self::process_group_action(group, gindex, result.len(), itable);
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn scan_group_confirmation() -> Result<bool> {
|
||||
print!("\nconfirm? [y/N]: ");
|
||||
std::io::stdout().flush()?;
|
||||
let mut user_input = String::new();
|
||||
io::stdin().read_line(&mut user_input)?;
|
||||
|
||||
match user_input.trim() {
|
||||
"Y" | "y" => Ok(true),
|
||||
_ => Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn scan_group_instruction() -> Result<String> {
|
||||
println!("\nEnter the indices of the files you want to delete.");
|
||||
println!("You can enter multiple files using commas to seperate file indices.");
|
||||
println!("example: 1,2");
|
||||
print!("\n> ");
|
||||
std::io::stdout().flush()?;
|
||||
let mut user_input = String::new();
|
||||
io::stdin().read_line(&mut user_input)?;
|
||||
|
||||
Ok(user_input)
|
||||
}
|
||||
|
||||
pub fn process_group_action(
|
||||
duplicates: &Vec<FileInfo>,
|
||||
dup_index: usize,
|
||||
dup_size: usize,
|
||||
table: Table,
|
||||
) {
|
||||
println!("\nDuplicate Set {} of {}\n", dup_index + 1, dup_size);
|
||||
table.printstd();
|
||||
let files_to_delete = Self::scan_group_instruction().unwrap_or_default();
|
||||
let parsed_file_indices = files_to_delete
|
||||
.trim()
|
||||
.split(',')
|
||||
.filter(|element| !element.is_empty())
|
||||
.map(|index| index.parse::<usize>().unwrap_or_default())
|
||||
.collect::<Vec<usize>>();
|
||||
|
||||
if parsed_file_indices
|
||||
.clone()
|
||||
.into_iter()
|
||||
.any(|index| index > (duplicates.len() - 1))
|
||||
{
|
||||
println!("Err: File Index Out of Bounds!");
|
||||
return Self::process_group_action(duplicates, dup_index, dup_size, table);
|
||||
}
|
||||
|
||||
print!("{esc}[2J{esc}[1;1H", esc = 27 as char);
|
||||
|
||||
if parsed_file_indices.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let files_to_delete = parsed_file_indices
|
||||
.into_iter()
|
||||
.map(|index| duplicates[index].clone());
|
||||
|
||||
println!("\nThe following files will be deleted:");
|
||||
files_to_delete
|
||||
.clone()
|
||||
.enumerate()
|
||||
.for_each(|(index, file)| {
|
||||
println!("{}: {}", index, file.path.display());
|
||||
});
|
||||
|
||||
match Self::scan_group_confirmation().unwrap() {
|
||||
true => {
|
||||
files_to_delete.into_iter().for_each(|file| {
|
||||
match std::fs::remove_file(file.path.clone()) {
|
||||
Ok(_) => println!("DELETED: {}", file.path.display()),
|
||||
Err(_) => println!("FAILED: {}", file.path.display()),
|
||||
}
|
||||
});
|
||||
}
|
||||
false => println!("\nCancelled Delete Operation."),
|
||||
}
|
||||
match user_input.trim() {
|
||||
"Y" | "y" => Ok(true),
|
||||
_ => Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn scan_group_instruction() -> Result<String> {
|
||||
println!("\nEnter the indices of the files you want to delete.");
|
||||
println!("You can enter multiple files using commas to seperate file indices.");
|
||||
println!("example: 1,2");
|
||||
print!("\n> ");
|
||||
std::io::stdout().flush()?;
|
||||
let mut user_input = String::new();
|
||||
io::stdin().read_line(&mut user_input)?;
|
||||
|
||||
Ok(user_input)
|
||||
}
|
||||
|
||||
pub fn init(result: Vec<FileInfo>, app_args: &Params) -> Result<()> {
|
||||
let basepath_length = app_args.get_directory()?.to_str().unwrap_or_default().len();
|
||||
let max_filepath_length = result
|
||||
.iter()
|
||||
.map(|file| file.path.to_str().unwrap_or_default().len())
|
||||
.max()
|
||||
.unwrap_or_default();
|
||||
|
||||
let min_path_length = if max_filepath_length > basepath_length {
|
||||
max_filepath_length - basepath_length
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
let progress_style = ProgressStyle::with_template(
|
||||
"[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}",
|
||||
)?;
|
||||
let progress_bar = ProgressBar::new(result.len() as u64);
|
||||
progress_bar.set_style(progress_style);
|
||||
progress_bar.enable_steady_tick(Duration::from_millis(50));
|
||||
progress_bar.set_message("reconciling data");
|
||||
|
||||
let duplicates: DashMap<String, Vec<FileInfo>> = DashMap::new();
|
||||
result
|
||||
.into_par_iter()
|
||||
.progress_with(progress_bar)
|
||||
.with_finish(ProgressFinish::WithMessage(Cow::from("data reconciled")))
|
||||
.map(|file| file.hash())
|
||||
.filter_map(Result::ok)
|
||||
.for_each(|file| {
|
||||
duplicates
|
||||
.entry(file.hash.clone().unwrap_or_default())
|
||||
.and_modify(|fileset| fileset.push(file.clone()))
|
||||
.or_insert_with(|| vec![file]);
|
||||
});
|
||||
|
||||
duplicates
|
||||
.clone()
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.for_each(|(gindex, (_, group))| {
|
||||
let mut itable = Table::new();
|
||||
itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
|
||||
itable.set_titles(row!["index", "filename", "size", "updated_at"]);
|
||||
group.iter().enumerate().for_each(|(index, file)| {
|
||||
itable.add_row(row![
|
||||
index,
|
||||
Formatter::human_path(file, app_args, min_path_length)
|
||||
.unwrap_or_default()
|
||||
.blue(),
|
||||
Formatter::human_filesize(file).unwrap_or_default().red(),
|
||||
Formatter::human_mtime(file).unwrap_or_default().yellow()
|
||||
]);
|
||||
});
|
||||
|
||||
process_group_action(&group, gindex, duplicates.len(), itable);
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn process_group_action(
|
||||
duplicates: &Vec<FileInfo>,
|
||||
dup_index: usize,
|
||||
dup_size: usize,
|
||||
table: Table,
|
||||
) {
|
||||
println!("\nDuplicate Set {} of {}\n", dup_index + 1, dup_size);
|
||||
table.printstd();
|
||||
let files_to_delete = scan_group_instruction().unwrap_or_default();
|
||||
let parsed_file_indices = files_to_delete
|
||||
.trim()
|
||||
.split(',')
|
||||
.filter(|element| !element.is_empty())
|
||||
.map(|index| index.parse::<usize>().unwrap_or_default())
|
||||
.collect::<Vec<usize>>();
|
||||
|
||||
if parsed_file_indices
|
||||
.clone()
|
||||
.into_iter()
|
||||
.any(|index| index > (duplicates.len() - 1))
|
||||
{
|
||||
println!("{}", "Err: File Index Out of Bounds!".red());
|
||||
return process_group_action(duplicates, dup_index, dup_size, table);
|
||||
}
|
||||
|
||||
print!("{esc}[2J{esc}[1;1H", esc = 27 as char);
|
||||
|
||||
if parsed_file_indices.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let files_to_delete = parsed_file_indices
|
||||
.into_iter()
|
||||
.map(|index| duplicates[index].clone());
|
||||
|
||||
println!("\n{}", "The following files will be deleted:".red());
|
||||
files_to_delete
|
||||
.clone()
|
||||
.enumerate()
|
||||
.for_each(|(index, file)| {
|
||||
println!("{}: {}", index.to_string().blue(), file.path.display());
|
||||
});
|
||||
|
||||
match scan_group_confirmation().unwrap() {
|
||||
true => {
|
||||
files_to_delete.into_iter().for_each(|file| {
|
||||
match std::fs::remove_file(file.path.clone()) {
|
||||
Ok(_) => println!("{}: {}", "DELETED".green(), file.path.display()),
|
||||
Err(_) => println!("{}: {}", "FAILED".red(), file.path.display()),
|
||||
}
|
||||
});
|
||||
}
|
||||
false => println!("{}", "\nCancelled Delete Operation.".red()),
|
||||
}
|
||||
}
|
||||
|
||||
39
src/main.rs
39
src/main.rs
@@ -1,35 +1,40 @@
|
||||
mod app;
|
||||
mod fileinfo;
|
||||
mod formatter;
|
||||
mod interactive;
|
||||
mod params;
|
||||
mod processor;
|
||||
mod scanner;
|
||||
mod server;
|
||||
|
||||
use self::{formatter::Formatter, interactive::Interactive, server::Server};
|
||||
/* version 2.0 modules*/
|
||||
mod cli;
|
||||
mod server;
|
||||
mod tui;
|
||||
|
||||
use anyhow::Result;
|
||||
|
||||
use self::app::App;
|
||||
use clap::Parser;
|
||||
use params::Params;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::sync::Arc;
|
||||
// use formatter::Formatter;
|
||||
// use processor::Processor;
|
||||
// use scanner::Scanner;
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let app_args = Params::parse();
|
||||
let server = Server::new(app_args.clone());
|
||||
// let scan_results = Scanner::build(&app_args)?.scan()?;
|
||||
// let processor = Processor::new(scan_results);
|
||||
// let results = processor.sizewise()?.hashwise()?;
|
||||
|
||||
server.start()?;
|
||||
// match app_args.interactive {
|
||||
// false => Formatter::print(results.files, &app_args)?,
|
||||
// true => interactive::init(results.files, &app_args)?,
|
||||
// }
|
||||
|
||||
match app_args.interactive {
|
||||
false => {
|
||||
Formatter::print(
|
||||
server.hw_duplicate_set,
|
||||
server.max_file_path_len.load(Ordering::Acquire),
|
||||
&app_args,
|
||||
)?;
|
||||
}
|
||||
true => {
|
||||
Interactive::init(server.hw_duplicate_set, &app_args)?;
|
||||
}
|
||||
};
|
||||
App::new(Arc::new(app_args))
|
||||
.start()
|
||||
.expect("app init failed.");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ use std::{fs, path::PathBuf};
|
||||
use anyhow::Result;
|
||||
use clap::{Parser, ValueHint};
|
||||
|
||||
#[derive(Parser, Debug, Default, Clone)]
|
||||
#[derive(Parser, Debug, Clone)]
|
||||
#[command(author, version, about, long_about = None)]
|
||||
pub struct Params {
|
||||
/// Filetypes to deduplicate [default = all]
|
||||
@@ -16,23 +16,20 @@ pub struct Params {
|
||||
#[arg(long, short)]
|
||||
pub interactive: bool,
|
||||
/// Minimum filesize of duplicates to scan (e.g., 100B/1K/2M/3G/4T).
|
||||
#[arg(long, short = 'm', default_value = "1b")]
|
||||
#[arg(long, short = 's', default_value = "1b")]
|
||||
pub min_size: Option<String>,
|
||||
/// Max Depth to scan while looking for duplicates
|
||||
#[arg(long, short = 'D')]
|
||||
#[arg(long, short = 'd')]
|
||||
pub max_depth: Option<usize>,
|
||||
/// Min Depth to scan while looking for duplicates
|
||||
#[arg(long, short = 'd')]
|
||||
#[arg(long)]
|
||||
pub min_depth: Option<usize>,
|
||||
/// Follow links while scanning directories
|
||||
#[arg(long, short)]
|
||||
pub follow_links: bool,
|
||||
/// Guarantees that two files are duplicate (performs a full hash)
|
||||
#[arg(long, short = 's', default_value = "false")]
|
||||
pub strict: bool,
|
||||
/// Show Progress spinners & metrics
|
||||
#[arg(long, short = 'p', default_value = "false")]
|
||||
pub progress: bool,
|
||||
/// print json output
|
||||
#[arg(long)]
|
||||
pub json: bool,
|
||||
}
|
||||
|
||||
impl Params {
|
||||
|
||||
424
src/processor.rs
424
src/processor.rs
@@ -1,369 +1,107 @@
|
||||
use anyhow::Result;
|
||||
use dashmap::DashMap;
|
||||
use indicatif::{
|
||||
MultiProgress, ParallelProgressIterator, ProgressBar, ProgressFinish, ProgressStyle,
|
||||
};
|
||||
use indicatif::{ParallelProgressIterator, ProgressBar, ProgressStyle, ProgressFinish};
|
||||
use rayon::prelude::{IntoParallelIterator, ParallelIterator};
|
||||
use std::borrow::Cow;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex, TryLockError, TryLockResult};
|
||||
use std::time::Duration;
|
||||
use std::{time::Duration, borrow::Cow};
|
||||
|
||||
use crate::fileinfo::FileInfo;
|
||||
use crate::params::Params;
|
||||
|
||||
pub struct Processor {}
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum State {
|
||||
Initial,
|
||||
SizeWise,
|
||||
HashWise,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Processor {
|
||||
pub files: Vec<FileInfo>,
|
||||
pub state: State,
|
||||
}
|
||||
|
||||
impl Processor {
|
||||
pub fn hashwise(
|
||||
app_args: Arc<Params>,
|
||||
sw_store: Arc<DashMap<u64, Vec<FileInfo>>>,
|
||||
hw_store: Arc<DashMap<u128, Vec<FileInfo>>>,
|
||||
progress_bar_box: Arc<MultiProgress>,
|
||||
max_file_size: Arc<AtomicU64>,
|
||||
seed: i64,
|
||||
) -> Result<()> {
|
||||
let progress_bar = match app_args.progress {
|
||||
true => progress_bar_box.add(ProgressBar::new_spinner()),
|
||||
false => ProgressBar::hidden(),
|
||||
};
|
||||
pub fn new(files: Vec<FileInfo>) -> Self {
|
||||
Self {
|
||||
files,
|
||||
state: State::Initial,
|
||||
}
|
||||
}
|
||||
|
||||
let keys: Vec<u64> = sw_store.clone().iter().map(|i| *i.key()).collect();
|
||||
pub fn hashwise(&self) -> Result<Self> {
|
||||
if self.files.is_empty() {
|
||||
return Ok(self.clone());
|
||||
}
|
||||
|
||||
let progress_style = ProgressStyle::with_template("[{elapsed_precise}] {pos:>7} {msg}")?;
|
||||
let progress_style = ProgressStyle::with_template("[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}")?;
|
||||
let progress_bar = ProgressBar::new(self.files.len() as u64);
|
||||
progress_bar.set_style(progress_style);
|
||||
progress_bar.enable_steady_tick(Duration::from_millis(50));
|
||||
progress_bar.set_message("files grouped by hash.");
|
||||
progress_bar.set_message("indexing file hashes");
|
||||
|
||||
keys.into_par_iter()
|
||||
let duplicates_table: DashMap<String, Vec<FileInfo>> = DashMap::new();
|
||||
self.files
|
||||
.clone()
|
||||
.into_par_iter()
|
||||
.progress_with(progress_bar)
|
||||
.with_finish(ProgressFinish::WithMessage(Cow::from(
|
||||
"files grouped by hash.",
|
||||
)))
|
||||
.for_each(|key| {
|
||||
let group: Vec<FileInfo> = sw_store.get(&key).unwrap().to_vec();
|
||||
if group.len() > 1 {
|
||||
group.into_par_iter().for_each(|file| {
|
||||
let fhash = if app_args.strict {
|
||||
file.hash(seed).expect("hashing file failed.")
|
||||
} else {
|
||||
file.initial_page_hash(seed).expect("hashing file failed.")
|
||||
};
|
||||
|
||||
Self::compare_and_update_max_path_len(
|
||||
max_file_size.clone(),
|
||||
file.path.to_string_lossy().len() as u64,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
hw_store
|
||||
.entry(fhash)
|
||||
.and_modify(|fileset| fileset.push(file.clone()))
|
||||
.or_insert_with(|| vec![file]);
|
||||
});
|
||||
}
|
||||
.with_finish(ProgressFinish::WithMessage(Cow::from("indexed files hashes")))
|
||||
.map(|file| file.hash())
|
||||
.filter_map(Result::ok)
|
||||
.for_each(|file| {
|
||||
duplicates_table
|
||||
.entry(file.hash.clone().unwrap_or_default())
|
||||
.and_modify(|fileset| fileset.push(file.clone()))
|
||||
.or_insert_with(|| vec![file]);
|
||||
});
|
||||
|
||||
Ok(())
|
||||
let files = duplicates_table
|
||||
.into_read_only()
|
||||
.values()
|
||||
.cloned()
|
||||
.filter(|subfiles| subfiles.len() > 1)
|
||||
.flatten()
|
||||
.collect::<Vec<FileInfo>>();
|
||||
|
||||
Ok(Self {
|
||||
files,
|
||||
state: State::HashWise,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn compare_and_update_max_path_len(current: Arc<AtomicU64>, next: u64) -> Result<()> {
|
||||
if current.load(Ordering::Relaxed) < next {
|
||||
current.store(next, Ordering::Release);
|
||||
pub fn sizewise(&self) -> Result<Self> {
|
||||
if self.files.is_empty() {
|
||||
return Ok(self.clone());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn sizewise(
|
||||
app_args: Arc<Params>,
|
||||
scanner_finished: Arc<AtomicBool>,
|
||||
store: Arc<DashMap<u64, Vec<FileInfo>>>,
|
||||
files: Arc<Mutex<Vec<FileInfo>>>,
|
||||
progress_bar_box: Arc<MultiProgress>,
|
||||
) -> Result<()> {
|
||||
let progress_bar = match app_args.progress {
|
||||
true => progress_bar_box.add(ProgressBar::new_spinner()),
|
||||
false => ProgressBar::hidden(),
|
||||
};
|
||||
|
||||
let progress_style = ProgressStyle::with_template("[{elapsed_precise}] {pos:>7} {msg}")?;
|
||||
let progress_style = ProgressStyle::with_template("[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}")?;
|
||||
let progress_bar = ProgressBar::new(self.files.len() as u64);
|
||||
progress_bar.set_style(progress_style);
|
||||
progress_bar.enable_steady_tick(Duration::from_millis(50));
|
||||
progress_bar.set_message("files grouped by size");
|
||||
progress_bar.set_message("indexing file sizes");
|
||||
|
||||
loop {
|
||||
let fileopt: Option<FileInfo> = {
|
||||
match files.try_lock() {
|
||||
Ok(mut flist) => flist.pop(),
|
||||
TryLockResult::Err(TryLockError::WouldBlock) => None,
|
||||
_ => None,
|
||||
}
|
||||
};
|
||||
let duplicates_table: DashMap<u64, Vec<FileInfo>> = DashMap::new();
|
||||
self.files
|
||||
.clone()
|
||||
.into_par_iter()
|
||||
.progress_with(progress_bar)
|
||||
.with_finish(ProgressFinish::WithMessage(Cow::from("indexed files sizes")))
|
||||
.for_each(|file| {
|
||||
duplicates_table
|
||||
.entry(file.size)
|
||||
.and_modify(|fileset| fileset.push(file.clone()))
|
||||
.or_insert_with(|| vec![file]);
|
||||
});
|
||||
|
||||
match fileopt {
|
||||
Some(file) => {
|
||||
progress_bar.inc(1);
|
||||
store
|
||||
.entry(file.size)
|
||||
.and_modify(|fileset| fileset.push(file.clone()))
|
||||
.or_insert_with(|| vec![file]);
|
||||
continue;
|
||||
}
|
||||
None => match scanner_finished.load(std::sync::atomic::Ordering::Relaxed) {
|
||||
true => {
|
||||
progress_bar.finish_with_message("files grouped by size");
|
||||
break Ok(());
|
||||
}
|
||||
false => continue,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use anyhow::Result;
|
||||
use dashmap::DashMap;
|
||||
use indicatif::MultiProgress;
|
||||
use rand::Rng;
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tempfile::TempDir;
|
||||
|
||||
use crate::{fileinfo::FileInfo, params::Params};
|
||||
|
||||
use super::Processor;
|
||||
|
||||
fn generate_bytes(size: usize) -> Vec<u8> {
|
||||
let mut rng = rand::rng();
|
||||
(0..size).map(|_| rng.random::<u8>()).collect::<Vec<u8>>()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hashwise_sorting_two_files_with_identical_init_page_only_strict_mode() -> Result<()> {
|
||||
let root = TempDir::new()?;
|
||||
let content = generate_bytes(4096);
|
||||
|
||||
let mut content_x = content.clone();
|
||||
let mut content_y = content.clone();
|
||||
|
||||
content_x.extend(generate_bytes(1720320));
|
||||
content_y.extend(generate_bytes(1720320));
|
||||
|
||||
let files = [
|
||||
(root.path().join("fileone.bin"), content_x),
|
||||
(root.path().join("filetwo.bin"), content_y),
|
||||
];
|
||||
|
||||
for (fpath, content) in files.iter() {
|
||||
let mut f = File::create_new(fpath)?;
|
||||
f.write_all(content)?;
|
||||
}
|
||||
|
||||
let dupstore = Arc::new(DashMap::new());
|
||||
let file_queue = Arc::new(Mutex::new(
|
||||
files
|
||||
.iter()
|
||||
.map(|f| FileInfo::new(f.0.clone()).unwrap())
|
||||
.collect::<Vec<FileInfo>>(),
|
||||
));
|
||||
|
||||
let hw_dupstore = Arc::new(DashMap::new());
|
||||
Processor::sizewise(
|
||||
Arc::new(Params::default()),
|
||||
Arc::new(AtomicBool::new(true)),
|
||||
dupstore.clone(),
|
||||
file_queue,
|
||||
Arc::new(MultiProgress::new()),
|
||||
)?;
|
||||
|
||||
let args = Params {
|
||||
strict: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
Processor::hashwise(
|
||||
Arc::new(args),
|
||||
dupstore.clone(),
|
||||
hw_dupstore.clone(),
|
||||
Arc::new(MultiProgress::new()),
|
||||
Arc::new(AtomicU64::new(32)),
|
||||
300,
|
||||
)?;
|
||||
|
||||
assert_eq!(hw_dupstore.len(), 2);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hashwise_sorting_two_files_with_identical_init_page_only_fast_mode() -> Result<()> {
|
||||
let root = TempDir::new()?;
|
||||
let content = generate_bytes(4096);
|
||||
|
||||
let mut content_x = content.clone();
|
||||
let mut content_y = content.clone();
|
||||
|
||||
content_x.extend(generate_bytes(1720320));
|
||||
content_y.extend(generate_bytes(1720320));
|
||||
|
||||
let files = [
|
||||
(root.path().join("fileone.bin"), content_x),
|
||||
(root.path().join("filetwo.bin"), content_y),
|
||||
];
|
||||
|
||||
for (fpath, content) in files.iter() {
|
||||
let mut f = File::create_new(fpath)?;
|
||||
f.write_all(content)?;
|
||||
}
|
||||
|
||||
let dupstore = Arc::new(DashMap::new());
|
||||
let file_queue = Arc::new(Mutex::new(
|
||||
files
|
||||
.iter()
|
||||
.map(|f| FileInfo::new(f.0.clone()).unwrap())
|
||||
.collect::<Vec<FileInfo>>(),
|
||||
));
|
||||
|
||||
let hw_dupstore = Arc::new(DashMap::new());
|
||||
Processor::sizewise(
|
||||
Arc::new(Params::default()),
|
||||
Arc::new(AtomicBool::new(true)),
|
||||
dupstore.clone(),
|
||||
file_queue,
|
||||
Arc::new(MultiProgress::new()),
|
||||
)?;
|
||||
|
||||
Processor::hashwise(
|
||||
Arc::new(Params::default()),
|
||||
dupstore.clone(),
|
||||
hw_dupstore.clone(),
|
||||
Arc::new(MultiProgress::new()),
|
||||
Arc::new(AtomicU64::new(32)),
|
||||
300,
|
||||
)?;
|
||||
|
||||
assert_eq!(hw_dupstore.len(), 1);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hashwise_sorting_two_files_with_identical_data() -> Result<()> {
|
||||
let root = TempDir::new()?;
|
||||
let content = generate_bytes(282624);
|
||||
let files = [
|
||||
(root.path().join("fileone.bin"), content.clone()),
|
||||
(root.path().join("filetwo.bin"), content.clone()),
|
||||
];
|
||||
|
||||
for (fpath, content) in files.iter() {
|
||||
let mut f = File::create_new(fpath)?;
|
||||
f.write_all(content)?;
|
||||
}
|
||||
|
||||
let dupstore = Arc::new(DashMap::new());
|
||||
let file_queue = Arc::new(Mutex::new(
|
||||
files
|
||||
.iter()
|
||||
.map(|f| FileInfo::new(f.0.clone()).unwrap())
|
||||
.collect::<Vec<FileInfo>>(),
|
||||
));
|
||||
|
||||
let hw_dupstore = Arc::new(DashMap::new());
|
||||
Processor::sizewise(
|
||||
Arc::new(Params::default()),
|
||||
Arc::new(AtomicBool::new(true)),
|
||||
dupstore.clone(),
|
||||
file_queue,
|
||||
Arc::new(MultiProgress::new()),
|
||||
)?;
|
||||
|
||||
Processor::hashwise(
|
||||
Arc::new(Params::default()),
|
||||
dupstore.clone(),
|
||||
hw_dupstore.clone(),
|
||||
Arc::new(MultiProgress::new()),
|
||||
Arc::new(AtomicU64::new(32)),
|
||||
300,
|
||||
)?;
|
||||
|
||||
assert_eq!(hw_dupstore.len(), 1);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sizewise_sorting_two_files_of_different_sizes() -> Result<()> {
|
||||
let root = TempDir::new()?;
|
||||
let files = [
|
||||
(root.path().join("fileone.bin"), generate_bytes(282624)),
|
||||
(root.path().join("filetwo.bin"), generate_bytes(1720320)),
|
||||
];
|
||||
|
||||
for (fpath, content) in files.iter() {
|
||||
let mut f = File::create_new(fpath)?;
|
||||
f.write_all(content)?;
|
||||
}
|
||||
|
||||
let file_queue = Arc::new(Mutex::new(
|
||||
files
|
||||
.iter()
|
||||
.map(|f| FileInfo::new(f.0.clone()).unwrap())
|
||||
.collect::<Vec<FileInfo>>(),
|
||||
));
|
||||
|
||||
let dupstore = Arc::new(DashMap::new());
|
||||
|
||||
Processor::sizewise(
|
||||
Arc::new(Params::default()),
|
||||
Arc::new(AtomicBool::new(true)),
|
||||
dupstore.clone(),
|
||||
file_queue,
|
||||
Arc::new(MultiProgress::new()),
|
||||
)?;
|
||||
|
||||
assert_eq!(dupstore.len(), 2);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sizewise_sorting_two_files_of_same_size() -> Result<()> {
|
||||
let root = TempDir::new()?;
|
||||
let files = [
|
||||
(root.path().join("fileone.bin"), generate_bytes(282624)),
|
||||
(root.path().join("filetwo.bin"), generate_bytes(282624)),
|
||||
];
|
||||
|
||||
for (fpath, content) in files.iter() {
|
||||
let mut f = File::create_new(fpath)?;
|
||||
f.write_all(content)?;
|
||||
}
|
||||
|
||||
let file_queue = Arc::new(Mutex::new(
|
||||
files
|
||||
.iter()
|
||||
.map(|f| FileInfo::new(f.0.clone()).unwrap())
|
||||
.collect::<Vec<FileInfo>>(),
|
||||
));
|
||||
|
||||
let dupstore = Arc::new(DashMap::new());
|
||||
|
||||
Processor::sizewise(
|
||||
Arc::new(Params::default()),
|
||||
Arc::new(AtomicBool::new(true)),
|
||||
dupstore.clone(),
|
||||
file_queue,
|
||||
Arc::new(MultiProgress::new()),
|
||||
)?;
|
||||
|
||||
assert_eq!(dupstore.len(), 1);
|
||||
|
||||
Ok(())
|
||||
let files = duplicates_table
|
||||
.into_read_only()
|
||||
.values()
|
||||
.cloned()
|
||||
.filter(|subfiles| subfiles.len() > 1)
|
||||
.flatten()
|
||||
.collect::<Vec<FileInfo>>();
|
||||
|
||||
Ok(Self {
|
||||
files,
|
||||
state: State::SizeWise,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
141
src/scanner.rs
141
src/scanner.rs
@@ -1,41 +1,120 @@
|
||||
#![allow(unused)]
|
||||
use crate::{fileinfo::FileInfo, params::Params};
|
||||
use anyhow::Result;
|
||||
use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::{path::Path, time::Duration};
|
||||
use indicatif::{ProgressBar, ProgressStyle};
|
||||
use std::{fs, path::PathBuf, time::Duration};
|
||||
|
||||
use globwalk::{GlobWalker, GlobWalkerBuilder};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Scanner {
|
||||
pub directory: Box<Path>,
|
||||
pub directory: Option<PathBuf>,
|
||||
pub filetypes: Option<String>,
|
||||
pub min_depth: Option<usize>,
|
||||
pub max_depth: Option<usize>,
|
||||
pub min_size: Option<u64>,
|
||||
pub follow_links: bool,
|
||||
pub progress: bool,
|
||||
}
|
||||
|
||||
impl Scanner {
|
||||
pub fn new(app_args: Arc<Params>) -> Result<Self> {
|
||||
Ok(Self {
|
||||
directory: app_args.get_directory()?.into_boxed_path(),
|
||||
filetypes: app_args.get_types(),
|
||||
min_depth: app_args.min_depth,
|
||||
max_depth: app_args.max_depth,
|
||||
min_size: app_args.get_min_size(),
|
||||
follow_links: app_args.follow_links,
|
||||
progress: app_args.progress,
|
||||
})
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
directory: None,
|
||||
filetypes: None,
|
||||
min_depth: None,
|
||||
max_depth: None,
|
||||
min_size: None,
|
||||
follow_links: true,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn build(app_args: &Params) -> Result<Self> {
|
||||
let scan_directory = app_args.get_directory()?;
|
||||
Ok(Scanner::new())
|
||||
.map(|scanner| scanner.directory(scan_directory))
|
||||
.map(|scanner| match app_args.get_min_size() {
|
||||
Some(min_size) => scanner.min_size(min_size),
|
||||
None => scanner,
|
||||
})
|
||||
.map(|scanner| match app_args.get_types() {
|
||||
Some(ftypes) => scanner.filetypes(ftypes),
|
||||
None => scanner,
|
||||
})
|
||||
.map(|scanner| match app_args.min_depth {
|
||||
Some(min_depth) => scanner.min_depth(min_depth),
|
||||
None => scanner,
|
||||
})
|
||||
.map(|scanner| match app_args.max_depth {
|
||||
Some(max_depth) => scanner.max_depth(max_depth),
|
||||
None => scanner,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn min_size(&self, min_size: u64) -> Self {
|
||||
Self {
|
||||
min_size: Some(min_size),
|
||||
..self.clone()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn min_depth(&self, min_depth: usize) -> Self {
|
||||
Self {
|
||||
min_depth: Some(min_depth),
|
||||
..self.clone()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn max_depth(&self, max_depth: usize) -> Self {
|
||||
Self {
|
||||
max_depth: Some(max_depth),
|
||||
..self.clone()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn directory(&self, dir: PathBuf) -> Self {
|
||||
Self {
|
||||
directory: Some(dir),
|
||||
..self.clone()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn filetypes(&self, patterns: String) -> Self {
|
||||
Self {
|
||||
filetypes: Some(patterns),
|
||||
..self.clone()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn ignore_links(&self) -> Self {
|
||||
Self {
|
||||
follow_links: false,
|
||||
..self.clone()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn follow_links(&self) -> Self {
|
||||
Self {
|
||||
follow_links: true,
|
||||
..self.clone()
|
||||
}
|
||||
}
|
||||
|
||||
fn scan_patterns(&self) -> Result<String> {
|
||||
Ok(match &self.filetypes {
|
||||
Ok(match self.filetypes.clone() {
|
||||
Some(ftypes) => format!("**/*{{{ftypes}}}"),
|
||||
None => "**/*".to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
fn scan_dir(&self) -> Result<PathBuf> {
|
||||
let scan_dir = match self.directory.clone() {
|
||||
Some(path) => path,
|
||||
None => std::env::current_dir()?,
|
||||
};
|
||||
|
||||
Ok(fs::canonicalize(scan_dir)?)
|
||||
}
|
||||
|
||||
fn attach_link_opts(&self, walker: GlobWalkerBuilder) -> Result<GlobWalkerBuilder> {
|
||||
Ok(walker.follow_links(self.follow_links))
|
||||
}
|
||||
@@ -55,7 +134,7 @@ impl Scanner {
|
||||
}
|
||||
fn build_walker(&self) -> Result<GlobWalker> {
|
||||
let walker = Ok(GlobWalkerBuilder::from_patterns(
|
||||
self.directory.clone(),
|
||||
self.scan_dir()?,
|
||||
&[self.scan_patterns()?],
|
||||
))
|
||||
.and_then(|walker| self.attach_walker_min_depth(walker))
|
||||
@@ -65,36 +144,30 @@ impl Scanner {
|
||||
Ok(walker.build()?)
|
||||
}
|
||||
|
||||
pub fn scan(
|
||||
&self,
|
||||
files: Arc<Mutex<Vec<FileInfo>>>,
|
||||
progress_bar_box: Arc<MultiProgress>,
|
||||
) -> Result<()> {
|
||||
let progress_bar = match self.progress {
|
||||
true => progress_bar_box.add(ProgressBar::new_spinner()),
|
||||
false => ProgressBar::hidden(),
|
||||
};
|
||||
|
||||
pub fn scan(&self) -> Result<Vec<FileInfo>> {
|
||||
let progress_style = ProgressStyle::with_template("[{elapsed_precise}] {pos:>7} {msg}")?;
|
||||
let progress_bar = ProgressBar::new_spinner();
|
||||
progress_bar.set_style(progress_style);
|
||||
progress_bar.enable_steady_tick(Duration::from_millis(50));
|
||||
progress_bar.set_message("paths mapped");
|
||||
let min_size = self.min_size.unwrap_or_default();
|
||||
|
||||
self.build_walker()?
|
||||
let results = self
|
||||
.build_walker()?
|
||||
.filter_map(Result::ok)
|
||||
.map(|entity| entity.into_path())
|
||||
.inspect(|_path| progress_bar.inc(1))
|
||||
.map(|path| {
|
||||
progress_bar.inc(1);
|
||||
path
|
||||
})
|
||||
.filter(|path| path.is_file())
|
||||
.map(FileInfo::new)
|
||||
.filter_map(Result::ok)
|
||||
.filter(|file| file.size > min_size)
|
||||
.for_each(|file| {
|
||||
let mut flock = files.lock().unwrap();
|
||||
flock.push(file);
|
||||
});
|
||||
.collect::<Vec<FileInfo>>();
|
||||
|
||||
progress_bar.finish_with_message("paths mapped");
|
||||
Ok(())
|
||||
|
||||
Ok(results)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,99 +0,0 @@
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use crate::processor::Processor;
|
||||
use crate::scanner::Scanner;
|
||||
use anyhow::Result;
|
||||
use dashmap::DashMap;
|
||||
use indicatif::{MultiProgress, ProgressDrawTarget};
|
||||
use rand::Rng;
|
||||
use threadpool::ThreadPool;
|
||||
|
||||
use crate::fileinfo::FileInfo;
|
||||
use crate::params::Params;
|
||||
|
||||
pub struct Server {
|
||||
filequeue: Arc<Mutex<Vec<FileInfo>>>,
|
||||
sw_duplicate_set: Arc<DashMap<u64, Vec<FileInfo>>>,
|
||||
pub hw_duplicate_set: Arc<DashMap<u128, Vec<FileInfo>>>,
|
||||
threadpool: ThreadPool,
|
||||
app_args: Arc<Params>,
|
||||
pub max_file_path_len: Arc<AtomicU64>,
|
||||
}
|
||||
|
||||
impl Server {
|
||||
pub fn new(opts: Params) -> Self {
|
||||
Self {
|
||||
filequeue: Arc::new(Mutex::new(Vec::new())),
|
||||
sw_duplicate_set: Arc::new(DashMap::new()),
|
||||
hw_duplicate_set: Arc::new(DashMap::new()),
|
||||
threadpool: ThreadPool::new(4),
|
||||
app_args: Arc::new(opts),
|
||||
max_file_path_len: Arc::new(AtomicU64::new(0)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn start(&self) -> Result<()> {
|
||||
let progbarbox = Arc::new(MultiProgress::new());
|
||||
let mut rng = rand::rng();
|
||||
let seed: i64 = rng.random();
|
||||
|
||||
if !self.app_args.progress {
|
||||
progbarbox.set_draw_target(ProgressDrawTarget::hidden());
|
||||
}
|
||||
|
||||
let app_args_clone_for_sc = self.app_args.clone();
|
||||
let app_args_clone_for_pr = self.app_args.clone();
|
||||
let file_queue_clone_sc = self.filequeue.clone();
|
||||
let file_queue_clone_pr = self.filequeue.clone();
|
||||
let scanner_finished = Arc::new(AtomicBool::new(false));
|
||||
|
||||
let sfin_sc_tr_cl = scanner_finished.clone();
|
||||
let sfin_pr_tr_cl = scanner_finished.clone();
|
||||
|
||||
let store_dupl_sw_for_sw = self.sw_duplicate_set.clone();
|
||||
let store_dupl_sw_for_hw = self.sw_duplicate_set.clone();
|
||||
let store_dupl_hw = self.hw_duplicate_set.clone();
|
||||
let max_file_path_len_clone = self.max_file_path_len.clone();
|
||||
|
||||
let progbarbox_sc_clone = progbarbox.clone();
|
||||
|
||||
self.threadpool.execute(move || {
|
||||
Scanner::new(app_args_clone_for_sc)
|
||||
.unwrap()
|
||||
.scan(file_queue_clone_sc, progbarbox_sc_clone)
|
||||
.unwrap();
|
||||
|
||||
sfin_sc_tr_cl.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||
});
|
||||
|
||||
let progbarbox_pr_clone = progbarbox.clone();
|
||||
|
||||
self.threadpool.execute(move || {
|
||||
Processor::sizewise(
|
||||
app_args_clone_for_pr.clone(),
|
||||
sfin_pr_tr_cl,
|
||||
store_dupl_sw_for_sw,
|
||||
file_queue_clone_pr,
|
||||
progbarbox_pr_clone.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
Processor::hashwise(
|
||||
app_args_clone_for_pr,
|
||||
store_dupl_sw_for_hw,
|
||||
store_dupl_hw,
|
||||
progbarbox_pr_clone,
|
||||
max_file_path_len_clone,
|
||||
seed,
|
||||
)
|
||||
.unwrap();
|
||||
});
|
||||
|
||||
progbarbox.clear()?;
|
||||
|
||||
self.threadpool.join();
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
44
src/server/file.rs
Normal file
44
src/server/file.rs
Normal file
@@ -0,0 +1,44 @@
|
||||
use anyhow::Result;
|
||||
use std::fs::File;
|
||||
use std::io::Read;
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
use std::sync::Arc;
|
||||
use uuid::Uuid;
|
||||
|
||||
const PARTIAL_SIZE: u64 = 4096;
|
||||
|
||||
#[allow(unused)]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct FileMeta {
|
||||
pub id: Uuid,
|
||||
pub path: Box<str>,
|
||||
pub size: u64,
|
||||
pub modtime: i64,
|
||||
pub partial: Arc<[u8]>,
|
||||
pub full_hash: Arc<[u8]>,
|
||||
}
|
||||
|
||||
impl FileMeta {
|
||||
fn create_partial(path: &str) -> Result<Arc<[u8]>> {
|
||||
let mut partial_take = File::open(path)?.take(PARTIAL_SIZE);
|
||||
let mut partial_buffer = Vec::with_capacity(PARTIAL_SIZE as usize);
|
||||
partial_take.read_to_end(&mut partial_buffer)?;
|
||||
|
||||
Ok(Arc::from(partial_buffer))
|
||||
}
|
||||
|
||||
pub fn new(path: Box<str>) -> Result<Self> {
|
||||
let pstr = path.to_string();
|
||||
let filemeta = std::fs::metadata(&pstr)?;
|
||||
let partial = Self::create_partial(&pstr)?;
|
||||
|
||||
Ok(Self {
|
||||
id: Uuid::new_v4(),
|
||||
size: filemeta.size(),
|
||||
modtime: filemeta.mtime(),
|
||||
full_hash: Arc::new([]),
|
||||
path,
|
||||
partial,
|
||||
})
|
||||
}
|
||||
}
|
||||
79
src/server/mod.rs
Normal file
79
src/server/mod.rs
Normal file
@@ -0,0 +1,79 @@
|
||||
pub mod file;
|
||||
mod processor;
|
||||
mod scanner;
|
||||
mod store;
|
||||
|
||||
use anyhow::Result;
|
||||
use std::sync::mpsc;
|
||||
use std::sync::mpsc::channel;
|
||||
use std::sync::Arc;
|
||||
use std::sync::Mutex;
|
||||
use threadpool::ThreadPool;
|
||||
|
||||
use self::processor::Processor;
|
||||
use self::scanner::Scanner;
|
||||
use self::store::Store;
|
||||
|
||||
pub type FileQueue = Arc<Mutex<Vec<Box<str>>>>;
|
||||
|
||||
pub enum Message {
|
||||
AddScanDirectory(Box<str>),
|
||||
Exit,
|
||||
None,
|
||||
}
|
||||
|
||||
pub struct Server {
|
||||
pub fq: FileQueue,
|
||||
pub dupstore: Arc<Store>,
|
||||
pub tpool: ThreadPool,
|
||||
}
|
||||
|
||||
impl Server {
|
||||
pub fn new() -> Result<Self> {
|
||||
Ok(Self {
|
||||
fq: Arc::new(Mutex::new(vec![])),
|
||||
dupstore: Arc::new(Store::new()),
|
||||
tpool: ThreadPool::new(4),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn start(&self, rx: mpsc::Receiver<Message>) -> Result<()> {
|
||||
let processor_fq = self.fq.clone();
|
||||
let processor_store = self.dupstore.clone();
|
||||
let (processor_tx, processor_rx) = channel::<Message>();
|
||||
|
||||
self.tpool.execute(move || {
|
||||
Processor::new(processor_fq, processor_store, processor_rx)
|
||||
.process()
|
||||
.expect("processer execution interrupted.");
|
||||
});
|
||||
|
||||
let scanner_fq = self.fq.clone();
|
||||
let (scanner_tx, scanner_rx) = channel::<Message>();
|
||||
self.tpool.execute(move || {
|
||||
Scanner::new(scanner_fq, scanner_rx)
|
||||
.index()
|
||||
.expect("scanner indexing interrupted.");
|
||||
});
|
||||
|
||||
self.tpool.execute(move || loop {
|
||||
match rx.recv() {
|
||||
Ok(Message::AddScanDirectory(path)) => {
|
||||
scanner_tx
|
||||
.send(Message::AddScanDirectory(path))
|
||||
.expect("scanner tx message passing failed.");
|
||||
}
|
||||
Ok(Message::None) => {}
|
||||
Ok(Message::Exit) | Err(_) => {
|
||||
scanner_tx.send(Message::Exit).unwrap_or_default();
|
||||
processor_tx.send(Message::Exit).unwrap_or_default();
|
||||
break;
|
||||
}
|
||||
};
|
||||
});
|
||||
|
||||
self.tpool.join();
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
146
src/server/processor.rs
Normal file
146
src/server/processor.rs
Normal file
@@ -0,0 +1,146 @@
|
||||
use anyhow::Result;
|
||||
use std::sync::mpsc::{Receiver, TryRecvError};
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::file::FileMeta;
|
||||
use super::store::{Index, Store};
|
||||
use super::{FileQueue, Message};
|
||||
|
||||
pub struct Processor {
|
||||
files: FileQueue,
|
||||
duplicates: Arc<Store>,
|
||||
msg_rx: Receiver<Message>,
|
||||
}
|
||||
|
||||
impl Processor {
|
||||
pub fn new(files: FileQueue, duplicates: Arc<Store>, msg_rx: Receiver<Message>) -> Self {
|
||||
Self {
|
||||
files,
|
||||
duplicates,
|
||||
msg_rx,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn process(&self) -> Result<()> {
|
||||
loop {
|
||||
match self.msg_rx.try_recv() {
|
||||
Ok(Message::Exit) => break,
|
||||
Err(TryRecvError::Empty) => {},
|
||||
_ => {}
|
||||
}
|
||||
|
||||
let next_file = {
|
||||
let mut cfiles = self.files.lock().unwrap();
|
||||
cfiles.pop()
|
||||
};
|
||||
|
||||
match next_file {
|
||||
None => continue,
|
||||
Some(file_res) => match FileMeta::new(file_res) {
|
||||
Err(_) => continue,
|
||||
Ok(fm) => {
|
||||
let fm_arc = Arc::new(fm);
|
||||
self.duplicates
|
||||
.add(Index::Size(fm_arc.size), fm_arc.clone());
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use anyhow::Result;
|
||||
use std::sync::mpsc;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::thread;
|
||||
use tempfile::TempDir;
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
|
||||
#[test]
|
||||
fn processor_differentiates_files_with_different_sizes() -> Result<()> {
|
||||
let file_queue = Arc::new(Mutex::new(vec![]));
|
||||
let (tx, rx) = mpsc::channel::<Message>();
|
||||
let root = TempDir::new()?;
|
||||
let store = Arc::new(Store::new());
|
||||
|
||||
let fq_c = file_queue.clone();
|
||||
let store_c = store.clone();
|
||||
let proc_thread = thread::spawn(move || {
|
||||
let processor = Processor::new(fq_c, store_c, rx);
|
||||
processor.process().expect("processor failed");
|
||||
});
|
||||
|
||||
let files =
|
||||
[("hello.txt", "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat"),
|
||||
("hello_dup.txt", "Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum")];
|
||||
|
||||
for (filename, content) in files.into_iter() {
|
||||
let fpath = root.path().join(filename);
|
||||
let mut tf = File::create(fpath.clone())?;
|
||||
tf.write_all(content.as_bytes())?;
|
||||
|
||||
let mut mfq = file_queue.lock().unwrap();
|
||||
mfq.push(fpath.into_os_string().into_string().unwrap().into_boxed_str());
|
||||
}
|
||||
|
||||
for _ in 0..10 {
|
||||
if store.entries().len() < 2 {
|
||||
thread::sleep(std::time::Duration::from_millis(100));
|
||||
}
|
||||
}
|
||||
|
||||
tx.send(Message::Exit).expect("unable to send msg to processor");
|
||||
proc_thread.join().expect("failed to join on thread!");
|
||||
|
||||
assert!(store.entries().len() == 2);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn processor_groups_files_with_same_size() -> Result<()> {
|
||||
let file_queue = Arc::new(Mutex::new(vec![]));
|
||||
let (tx, rx) = mpsc::channel::<Message>();
|
||||
let root = TempDir::new()?;
|
||||
let store = Arc::new(Store::new());
|
||||
|
||||
let fq_c = file_queue.clone();
|
||||
let store_c = store.clone();
|
||||
let proc_thread = thread::spawn(move || {
|
||||
let processor = Processor::new(fq_c, store_c, rx);
|
||||
processor.process().expect("processor failed");
|
||||
});
|
||||
|
||||
let files =
|
||||
[("hello.txt", "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum"),
|
||||
("hello_dup.txt", "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum")];
|
||||
|
||||
for (filename, content) in files.into_iter() {
|
||||
let fpath = root.path().join(filename);
|
||||
let mut tf = File::create(fpath.clone())?;
|
||||
tf.write_all(content.as_bytes())?;
|
||||
|
||||
let mut mfq = file_queue.lock().unwrap();
|
||||
mfq.push(fpath.into_os_string().into_string().unwrap().into_boxed_str());
|
||||
}
|
||||
|
||||
for _ in 0..10 {
|
||||
if store.entries().is_empty() {
|
||||
thread::sleep(std::time::Duration::from_millis(100));
|
||||
}
|
||||
}
|
||||
|
||||
tx.send(Message::Exit).expect("unable to send msg to processor");
|
||||
proc_thread.join().expect("failed to join on thread!");
|
||||
|
||||
assert!(store.entries().len() == 1);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
128
src/server/scanner.rs
Normal file
128
src/server/scanner.rs
Normal file
@@ -0,0 +1,128 @@
|
||||
use super::{FileQueue, Message};
|
||||
use anyhow::Result;
|
||||
use std::fs;
|
||||
use std::sync::mpsc::{self, Receiver};
|
||||
use std::sync::Arc;
|
||||
use std::sync::Mutex;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Scanner {
|
||||
files: FileQueue,
|
||||
proc_queue: FileQueue,
|
||||
msg_rx: Receiver<Message>,
|
||||
}
|
||||
|
||||
impl Scanner {
|
||||
pub fn new(fq: FileQueue, rx: Receiver<Message>) -> Self {
|
||||
Self {
|
||||
files: fq,
|
||||
proc_queue: Arc::new(Mutex::new(vec![])),
|
||||
msg_rx: rx,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn index(&self) -> Result<()> {
|
||||
loop {
|
||||
match self.msg_rx.try_recv() {
|
||||
Ok(Message::AddScanDirectory(path)) => {
|
||||
let mut mfq = self.proc_queue.lock().unwrap();
|
||||
mfq.push(path);
|
||||
}
|
||||
Err(mpsc::TryRecvError::Empty) | Ok(Message::None) => {}
|
||||
Ok(Message::Exit) | Err(_) => break,
|
||||
}
|
||||
|
||||
let npath = {
|
||||
match self.proc_queue.try_lock() {
|
||||
Ok(mut q) => q.pop(),
|
||||
Err(_) => None,
|
||||
}
|
||||
};
|
||||
|
||||
match npath {
|
||||
None => continue,
|
||||
Some(path) => {
|
||||
std::fs::read_dir(path.as_ref())?
|
||||
.filter_map(Result::ok)
|
||||
.for_each(|entry: fs::DirEntry| {
|
||||
let mdata =
|
||||
fs::metadata(entry.path()).expect("unable to read file metadata.");
|
||||
|
||||
let mpath = entry
|
||||
.path()
|
||||
.into_os_string()
|
||||
.into_string()
|
||||
.expect("invalid path conversion failed.")
|
||||
.into_boxed_str();
|
||||
|
||||
match mdata.is_dir() {
|
||||
true => {
|
||||
let mut pq = self
|
||||
.proc_queue
|
||||
.lock()
|
||||
.expect("proc queue lock acq failed.");
|
||||
pq.push(mpath);
|
||||
}
|
||||
false => {
|
||||
let mut fq = self
|
||||
.files
|
||||
.lock()
|
||||
.expect("file queue lock acq failed.");
|
||||
fq.push(mpath)
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use anyhow::Result;
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use std::sync::mpsc::channel;
|
||||
use tempfile::TempDir;
|
||||
use std::thread;
|
||||
|
||||
#[test]
|
||||
fn scanner_scans_files() -> Result<()> {
|
||||
let files =
|
||||
[("hello.txt", "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum"),
|
||||
("hello_dup.txt", "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum")];
|
||||
|
||||
let file_queue: FileQueue = Arc::new(Mutex::new(vec![]));
|
||||
let (tx, rx) = channel::<Message>();
|
||||
let root = TempDir::new()?;
|
||||
|
||||
for (filename, content) in files.into_iter() {
|
||||
let mut tf = File::create(root.path().join(filename))?;
|
||||
tf.write_all(content.as_bytes())?;
|
||||
}
|
||||
|
||||
let rpath = root.path().to_str().unwrap().to_string().into_boxed_str();
|
||||
let fq_clone = file_queue.clone();
|
||||
let scanner_thread = thread::spawn(move || {
|
||||
let scanner = Scanner::new(fq_clone, rx);
|
||||
scanner.index().unwrap();
|
||||
});
|
||||
|
||||
tx.send(Message::AddScanDirectory(rpath))?;
|
||||
tx.send(Message::Exit)?;
|
||||
scanner_thread.join().unwrap();
|
||||
|
||||
let fq_len = {
|
||||
let v = file_queue.lock().unwrap();
|
||||
v.len()
|
||||
};
|
||||
|
||||
assert_eq!(files.len(), fq_len);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
35
src/server/store.rs
Normal file
35
src/server/store.rs
Normal file
@@ -0,0 +1,35 @@
|
||||
use super::file::FileMeta;
|
||||
use dashmap::DashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[allow(unused)]
|
||||
#[derive(Debug, Hash, PartialEq, Eq)]
|
||||
pub enum Index {
|
||||
Size(u64),
|
||||
Partial(Arc<[u8]>),
|
||||
Full(Box<str>),
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Store {
|
||||
internal: Arc<DashMap<Index, Vec<Arc<FileMeta>>>>,
|
||||
}
|
||||
|
||||
impl Store {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
internal: Arc::new(DashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn entries(&self) -> Arc<DashMap<Index, Vec<Arc<FileMeta>>>> {
|
||||
self.internal.clone()
|
||||
}
|
||||
|
||||
pub fn add(&self, index: Index, file: Arc<FileMeta>) {
|
||||
self.internal
|
||||
.entry(index)
|
||||
.and_modify(|fg| fg.push(file.clone()))
|
||||
.or_insert(vec![file]);
|
||||
}
|
||||
}
|
||||
99
src/tui/mod.rs
Normal file
99
src/tui/mod.rs
Normal file
@@ -0,0 +1,99 @@
|
||||
use std::sync::mpsc::Sender;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use ratatui::crossterm::event::{self, Event, KeyCode};
|
||||
use ratatui::layout::{Constraint, Direction, Layout};
|
||||
use ratatui::widgets::{Block, Borders, List, ListItem, Paragraph};
|
||||
use ratatui::{DefaultTerminal, Frame};
|
||||
|
||||
use crate::server::{Message, Server};
|
||||
use std::sync::Arc;
|
||||
|
||||
pub struct Tui {
|
||||
app_tx: Sender<Message>,
|
||||
server: Arc<Server>,
|
||||
}
|
||||
|
||||
impl Tui {
|
||||
pub fn new(app_tx: Sender<Message>, server: Arc<Server>) -> Self {
|
||||
Self { app_tx, server }
|
||||
}
|
||||
|
||||
pub fn start(&mut self) -> Result<()> {
|
||||
let terminal = ratatui::init();
|
||||
self.run(terminal).expect("ui loop failed.");
|
||||
ratatui::restore();
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn poll_events() -> Result<Message> {
|
||||
match event::poll(Duration::from_millis(100)).context("event polling failed.")? {
|
||||
true => match event::read().context("event read failed.")? {
|
||||
Event::Key(key) => match key.code {
|
||||
KeyCode::Char('q') => Ok(Message::Exit),
|
||||
_ => Ok(Message::None),
|
||||
},
|
||||
_ => Ok(Message::None),
|
||||
},
|
||||
false => Ok(Message::None),
|
||||
}
|
||||
}
|
||||
|
||||
fn handle_events(&self) -> Result<Message> {
|
||||
match Self::poll_events() {
|
||||
Ok(Message::Exit) => {
|
||||
self.app_tx
|
||||
.send(Message::Exit)
|
||||
.expect("app event send failed.");
|
||||
Ok(Message::Exit)
|
||||
}
|
||||
_ => Ok(Message::None),
|
||||
}
|
||||
}
|
||||
|
||||
fn draw(&mut self, frame: &mut Frame) {
|
||||
let listitems = self
|
||||
.server
|
||||
.dupstore
|
||||
.entries()
|
||||
.iter()
|
||||
.flat_map(|val| {
|
||||
let mut group = vec![ListItem::new(format!("{:?}", val.key()))];
|
||||
val.value().iter().for_each(|f| {
|
||||
group.push(ListItem::new(format!("|-{}", f.path)));
|
||||
});
|
||||
group
|
||||
})
|
||||
.collect::<Vec<ListItem>>();
|
||||
|
||||
let list = List::new(listitems.clone());
|
||||
let layout_chunks = Layout::default()
|
||||
.direction(Direction::Horizontal)
|
||||
.constraints([Constraint::Percentage(50), Constraint::Percentage(50)])
|
||||
.split(frame.area());
|
||||
|
||||
frame.render_widget(
|
||||
list.block(Block::new().borders(Borders::ALL)),
|
||||
layout_chunks[0],
|
||||
);
|
||||
|
||||
frame.render_widget(
|
||||
Paragraph::new(String::new()).block(Block::new().borders(Borders::ALL)),
|
||||
layout_chunks[1],
|
||||
);
|
||||
}
|
||||
|
||||
fn run(&mut self, mut terminal: DefaultTerminal) -> Result<()> {
|
||||
loop {
|
||||
terminal.draw(|f| self.draw(f))?;
|
||||
match self.handle_events() {
|
||||
Ok(Message::Exit) => break,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user