diff --git a/.gitignore b/.gitignore index 345ad75..4fffb2f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ /target -/test_data -.envrc +/Cargo.lock diff --git a/Cargo.lock b/Cargo.lock index aecc2e2..5fb9f8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,6 +11,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "aho-corasick" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" +dependencies = [ + "memchr", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -21,22 +36,60 @@ dependencies = [ ] [[package]] -name = "anyhow" -version = "1.0.68" +name = "anstream" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61" +checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is-terminal", + "utf8parse", +] [[package]] -name = "atty" -version = "0.2.14" +name = "anstyle" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" + +[[package]] +name = "anstyle-parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", + "utf8parse", ] +[[package]] +name = "anstyle-query" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +dependencies = [ + "windows-sys 0.48.0", +] + +[[package]] +name = "anstyle-wincon" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" +dependencies = [ + "anstyle", + "windows-sys 0.48.0", +] + +[[package]] +name = "anyhow" +version = "1.0.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" + [[package]] name = "autocfg" version = "1.1.0" @@ -50,22 +103,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] -name = "bstr" -version = "0.2.17" +name = "bitflags" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" -dependencies = [ - "lazy_static", - "memchr", - "regex-automata", - "serde", -] +checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" [[package]] name = "bstr" -version = "1.1.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45ea9b00a7b3f2988e9a65ad3917e62123c38dba709b666506207be96d1790b" +checksum = "c3d4260bcc2e8fc9df1eac4919a720effeb63a3f0952f5bf4944adfa18897f09" dependencies = [ "memchr", "serde", @@ -73,9 +120,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.12.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" [[package]] name = "byteorder" @@ -85,15 +132,15 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytesize" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c58ec36aac5066d5ca17df51b3e70279f5670a72102f5752cb7e7c856adfc70" +checksum = "38fcc2979eff34a4b84e1cf9a1e3da42a7d44b3b690a40cdcb23e3d556cfb2e5" [[package]] name = "cc" -version = "1.0.78" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" [[package]] name = "cfg-if" @@ -103,13 +150,13 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.23" +version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" +checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5" dependencies = [ + "android-tzdata", "iana-time-zone", "js-sys", - "num-integer", "num-traits", "time", "wasm-bindgen", @@ -118,27 +165,34 @@ dependencies = [ [[package]] name = "clap" -version = "4.1.4" +version = "4.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f13b9c79b5d1dd500d20ef541215a6423c75829ef43117e1b4d17fd8af0b5d76" +checksum = "3eab9e8ceb9afdade1ab3f0fd8dbce5b1b2f468ad653baf10e771781b2b67b73" dependencies = [ - "bitflags", + "clap_builder", "clap_derive", - "clap_lex", - "is-terminal", "once_cell", +] + +[[package]] +name = "clap_builder" +version = "4.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f2763db829349bf00cfc06251268865ed4363b93a943174f638daf3ecdba2cd" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", "strsim", - "termcolor", ] [[package]] name = "clap_derive" -version = "4.1.0" +version = "4.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "684a277d672e91966334af371f1a7b5833f9aa00b07c84e92fbce95e00208ce8" +checksum = "54a9bb5758fc5dfe728d1019941681eccaf0cf8a4189b692a0ee2f2ecf90a050" dependencies = [ "heck", - "proc-macro-error", "proc-macro2", "quote", "syn", @@ -146,58 +200,51 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.3.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "783fe232adfca04f90f56201b26d79682d4cd2625e0bc7290b95123afe558ade" -dependencies = [ - "os_str_bytes", -] +checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" [[package]] -name = "codespan-reporting" -version = "0.11.1" +name = "colorchoice" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" -dependencies = [ - "termcolor", - "unicode-width", -] +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] name = "colored" -version = "2.0.0" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3616f750b84d8f0de8a58bda93e08e2a81ad3f523089b05f1dffecab48c6cbd" +checksum = "2674ec482fbc38012cf31e6c42ba0177b431a0cb6f15fe40efa5aab1bda516f6" dependencies = [ - "atty", + "is-terminal", "lazy_static", - "winapi", + "windows-sys 0.48.0", ] [[package]] name = "console" -version = "0.15.5" +version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d79fbe8970a77e3e34151cc13d3b3e248aa0faaecb9f6091fa07ebefe5ad60" +checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" dependencies = [ "encode_unicode 0.3.6", "lazy_static", "libc", "unicode-width", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] name = "core-foundation-sys" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "crossbeam-channel" -version = "0.5.6" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" dependencies = [ "cfg-if", "crossbeam-utils", @@ -205,9 +252,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" dependencies = [ "cfg-if", "crossbeam-epoch", @@ -216,9 +263,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.13" +version = "0.9.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" +checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" dependencies = [ "autocfg", "cfg-if", @@ -229,20 +276,19 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.14" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" +checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" dependencies = [ "cfg-if", ] [[package]] name = "csv" -version = "1.1.6" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" +checksum = "626ae34994d3d8d668f4269922248239db4ae42d538b14c398b74a52208e8086" dependencies = [ - "bstr 0.2.17", "csv-core", "itoa", "ryu", @@ -258,50 +304,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "cxx" -version = "1.0.87" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b61a7545f753a88bcbe0a70de1fcc0221e10bfc752f576754fa91e663db1622e" -dependencies = [ - "cc", - "cxxbridge-flags", - "cxxbridge-macro", - "link-cplusplus", -] - -[[package]] -name = "cxx-build" -version = "1.0.87" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f464457d494b5ed6905c63b0c4704842aba319084a0a3561cdc1359536b53200" -dependencies = [ - "cc", - "codespan-reporting", - "once_cell", - "proc-macro2", - "quote", - "scratch", - "syn", -] - -[[package]] -name = "cxxbridge-flags" -version = "1.0.87" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43c7119ce3a3701ed81aca8410b9acf6fc399d2629d057b87e2efa4e63a3aaea" - -[[package]] -name = "cxxbridge-macro" -version = "1.0.87" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65e07508b90551e610910fa648a1878991d367064997a596135b86df30daf07e" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "dashmap" version = "5.4.0" @@ -318,7 +320,7 @@ dependencies = [ [[package]] name = "deduplicator" -version = "0.1.6" +version = "0.2.0" dependencies = [ "anyhow", "bytesize", @@ -331,6 +333,7 @@ dependencies = [ "indicatif", "itertools", "memmap2", + "pathdiff", "prettytable-rs", "rayon", "unicode-segmentation", @@ -359,9 +362,9 @@ dependencies = [ [[package]] name = "either" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" [[package]] name = "encode_unicode" @@ -377,13 +380,13 @@ checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" [[package]] name = "errno" -version = "0.2.8" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" dependencies = [ "errno-dragonfly", "libc", - "winapi", + "windows-sys 0.48.0", ] [[package]] @@ -413,9 +416,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.8" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ "cfg-if", "libc", @@ -428,8 +431,8 @@ version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "029d74589adefde59de1a0c4f4732695c32805624aec7b68d91503d4dba79afc" dependencies = [ - "aho-corasick", - "bstr 1.1.0", + "aho-corasick 0.7.20", + "bstr", "fnv", "log", "regex", @@ -441,7 +444,7 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93e3af942408868f6934a7b85134a3230832b9977cf66125df2f9edcfce4ddcc" dependencies = [ - "bitflags", + "bitflags 1.3.2", "ignore", "walkdir", ] @@ -454,18 +457,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "heck" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" - -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" @@ -477,27 +471,32 @@ dependencies = [ ] [[package]] -name = "iana-time-zone" -version = "0.1.53" +name = "hermit-abi" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" + +[[package]] +name = "iana-time-zone" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "winapi", + "windows", ] [[package]] name = "iana-time-zone-haiku" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" dependencies = [ - "cxx", - "cxx-build", + "cc", ] [[package]] @@ -519,11 +518,12 @@ dependencies = [ [[package]] name = "indicatif" -version = "0.17.3" +version = "0.17.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cef509aa9bc73864d6756f0d34d35504af3cf0844373afe9b8669a5b8005a729" +checksum = "8ff8cc23a7393a397ed1d7f56e6365cba772aba9f9912ab968b03043c395d057" dependencies = [ "console", + "instant", "number_prefix", "portable-atomic", "rayon", @@ -531,25 +531,23 @@ dependencies = [ ] [[package]] -name = "io-lifetimes" -version = "1.0.4" +name = "instant" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7d6c6f8c91b4b9ed43484ad1a938e393caf35960fce7f82a040497207bd8e9e" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ - "libc", - "windows-sys", + "cfg-if", ] [[package]] name = "is-terminal" -version = "0.4.2" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28dfb6c8100ccc63462345b67d1bbc3679177c75ee4bf59bf29c8b1d110b8189" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ - "hermit-abi 0.2.6", - "io-lifetimes", + "hermit-abi 0.3.2", "rustix", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -563,15 +561,15 @@ dependencies = [ [[package]] name = "itoa" -version = "0.4.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "js-sys" -version = "0.3.60" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" dependencies = [ "wasm-bindgen", ] @@ -584,24 +582,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.139" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" - -[[package]] -name = "link-cplusplus" -version = "1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" -dependencies = [ - "cc", -] +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "linux-raw-sys" -version = "0.1.4" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" +checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0" [[package]] name = "lock_api" @@ -630,32 +619,22 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memmap2" -version = "0.5.8" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b182332558b18d807c4ce1ca8ca983b34c3ee32765e47b3f0f69b90355cc1dc" +checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" dependencies = [ "libc", ] [[package]] name = "memoffset" -version = "0.7.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" dependencies = [ "autocfg", ] -[[package]] -name = "num-integer" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" -dependencies = [ - "autocfg", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.15" @@ -683,34 +662,34 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "once_cell" -version = "1.17.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" - -[[package]] -name = "os_str_bytes" -version = "6.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "parking_lot_core" -version = "0.9.6" +version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba1ef8814b5c993410bb3adfad7a5ed269563e4a2f90c41f5d85be7fb47133bf" +checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] -name = "portable-atomic" -version = "0.3.19" +name = "pathdiff" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26f6a7b87c2e435a3241addceeeff740ff8b7e76b74c13bf9acb17fa454ea00b" +checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" + +[[package]] +name = "portable-atomic" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edc55135a600d700580e406b4de0d59cb9ad25e344a3a091a97ded2622ec4ec6" [[package]] name = "prettytable-rs" @@ -726,53 +705,29 @@ dependencies = [ "unicode-width", ] -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2", - "quote", - "version_check", -] - [[package]] name = "proc-macro2" -version = "1.0.50" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ef7d57beacfaf2d8aee5937dab7b7f28de3cb8b1828479bb5de2a7106f2bae2" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.23" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +checksum = "5fe8a65d69dd0808184ebb5f836ab526bb259db23c657efa38711b1072ee47f0" dependencies = [ "proc-macro2", ] [[package]] name = "rayon" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" +checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" dependencies = [ "either", "rayon-core", @@ -780,9 +735,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.10.2" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b" +checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" dependencies = [ "crossbeam-channel", "crossbeam-deque", @@ -796,7 +751,7 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -812,52 +767,45 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.1" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" +checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370" dependencies = [ - "aho-corasick", + "aho-corasick 1.0.1", "memchr", "regex-syntax", ] -[[package]] -name = "regex-automata" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" - [[package]] name = "regex-syntax" -version = "0.6.28" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" +checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" [[package]] name = "rustix" -version = "0.36.7" +version = "0.38.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4fdebc4b395b7fbb9ab11e462e20ed9051e7b16e42d24042c776eca0ac81b03" +checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5" dependencies = [ - "bitflags", + "bitflags 2.3.3", "errno", - "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] name = "rustversion" -version = "1.0.11" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "ryu" -version = "1.0.12" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "same-file" @@ -874,17 +822,11 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" -[[package]] -name = "scratch" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" - [[package]] name = "serde" -version = "1.0.152" +version = "1.0.160" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" [[package]] name = "smallvec" @@ -900,9 +842,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.107" +version = "2.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +checksum = "45c3457aacde3c65315de5031ec191ce46604304d2446e803d71ade03308d970" dependencies = [ "proc-macro2", "quote", @@ -920,29 +862,20 @@ dependencies = [ "winapi", ] -[[package]] -name = "termcolor" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" -dependencies = [ - "winapi-util", -] - [[package]] name = "thiserror" -version = "1.0.38" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" +checksum = "a35fc5b8971143ca348fa6df4f024d4d55264f3468c71ad1c2f365b0a4d58c42" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.38" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" +checksum = "463fe12d7993d3b327787537ce8dd4dfa058de32fc2b195ef3cde03dc4771e8f" dependencies = [ "proc-macro2", "quote", @@ -951,10 +884,11 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.1.4" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" dependencies = [ + "cfg-if", "once_cell", ] @@ -971,15 +905,15 @@ dependencies = [ [[package]] name = "unicode-ident" -version = "1.0.6" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" [[package]] name = "unicode-segmentation" -version = "1.10.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" [[package]] name = "unicode-width" @@ -988,19 +922,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" [[package]] -name = "version_check" -version = "0.9.4" +name = "utf8parse" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "walkdir" -version = "2.3.2" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" dependencies = [ "same-file", - "winapi", "winapi-util", ] @@ -1018,9 +951,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.83" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -1028,9 +961,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.83" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" dependencies = [ "bumpalo", "log", @@ -1043,9 +976,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.83" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1053,9 +986,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.83" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", @@ -1066,9 +999,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.83" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" [[package]] name = "winapi" @@ -1102,58 +1035,142 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows-sys" -version = "0.42.0" +name = "windows" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows-targets 0.48.1", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.1", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.48.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" +dependencies = [ + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", ] [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" [[package]] name = "windows_aarch64_msvc" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" [[package]] name = "windows_i686_gnu" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" [[package]] name = "windows_i686_msvc" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" [[package]] name = "windows_x86_64_gnu" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" [[package]] name = "windows_x86_64_msvc" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" diff --git a/Cargo.toml b/Cargo.toml index 95c8b12..1e537db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deduplicator" -version = "0.1.6" +version = "0.2.0" edition = "2021" description = "find,filter,delete Duplicates" license = "MIT" @@ -24,6 +24,7 @@ globwalk = "0.8.1" indicatif = { version = "0.17.2", features = ["rayon"] } itertools = "0.10.5" memmap2 = "0.5.8" +pathdiff = "0.2.1" prettytable-rs = "0.10.0" rayon = "1.6.1" unicode-segmentation = "1.10.0" @@ -33,4 +34,3 @@ unicode-segmentation = "1.10.0" inherits = "release" debug = true split-debuginfo = "packed" - diff --git a/src/app.rs b/src/app.rs deleted file mode 100644 index 0409738..0000000 --- a/src/app.rs +++ /dev/null @@ -1,18 +0,0 @@ -use crate::output; -use crate::params::Params; -use crate::scanner; -use anyhow::Result; - -pub struct App; - -impl App { - pub fn init(app_args: &Params) -> Result<()> { - let duplicates = scanner::duplicates(app_args)?; - match app_args.interactive { - true => output::interactive(duplicates, app_args), - false => output::print(duplicates, app_args), - } - - Ok(()) - } -} diff --git a/src/file_manager.rs b/src/file_manager.rs deleted file mode 100644 index 7a28424..0000000 --- a/src/file_manager.rs +++ /dev/null @@ -1,21 +0,0 @@ -use anyhow::Result; -use colored::Colorize; -use std::path::PathBuf; - -#[derive(Debug, Clone)] -pub struct File { - pub path: PathBuf, - pub size: Option, - pub hash: Option, -} - -pub fn delete_files(files: Vec) -> Result<()> { - files.into_iter().for_each(|file| { - match std::fs::remove_file(file.path.clone()) { - Ok(_) => println!("{}: {}", "DELETED".green(), file.path.display()), - Err(_) => println!("{}: {}", "FAILED".red(), file.path.display()) - } - }); - - Ok(()) -} diff --git a/src/fileinfo.rs b/src/fileinfo.rs new file mode 100644 index 0000000..140fb52 --- /dev/null +++ b/src/fileinfo.rs @@ -0,0 +1,40 @@ +use anyhow::Result; +use memmap2::Mmap; +use std::fs; +use std::hash::Hasher; +use std::{fs::Metadata, path::PathBuf}; + +#[derive(Debug, Clone)] +pub struct FileInfo { + pub path: PathBuf, + pub filemeta: Metadata, + pub hash: Option, + pub size: u64, +} + +impl FileInfo { + pub fn hash(&self) -> Result { + let file = fs::File::open(self.path.clone())?; + let mapper = unsafe { Mmap::map(&file)? }; + let mut primhasher = fxhash::FxHasher::default(); + + mapper + .chunks(1_000_000) + .for_each(|chunk| primhasher.write(chunk)); + + Ok(Self { + hash: Some(primhasher.finish().to_string()), + ..self.clone() + }) + } + + pub fn new(path: PathBuf) -> Result { + let filemeta = std::fs::metadata(path.clone())?; + Ok(Self { + path, + filemeta: filemeta.clone(), + hash: None, + size: filemeta.len(), + }) + } +} diff --git a/src/filters.rs b/src/filters.rs deleted file mode 100644 index 71e67e4..0000000 --- a/src/filters.rs +++ /dev/null @@ -1,12 +0,0 @@ -use crate::file_manager::File; -use crate::params::Params; - -pub fn is_file_gt_min_size(app_opts: &Params, file: &File) -> bool { - match app_opts.get_min_size() { - Some(msize) => match file.size { - Some(fsize) => fsize >= msize, - None => true, - }, - None => true, - } -} diff --git a/src/formatter.rs b/src/formatter.rs new file mode 100644 index 0000000..fe6bcdb --- /dev/null +++ b/src/formatter.rs @@ -0,0 +1,129 @@ +pub struct Formatter; +use crate::fileinfo::FileInfo; +use crate::params::Params; +use anyhow::Result; +use chrono::{DateTime, Utc}; +use colored::Colorize; +use dashmap::DashMap; +use indicatif::{ + ParallelProgressIterator, ProgressBar, ProgressFinish, ProgressIterator, ProgressStyle, +}; +use pathdiff::diff_paths; +use prettytable::{format, row, Table}; +use rayon::prelude::*; +use std::borrow::Cow; +use std::path::PathBuf; +use std::time::Duration; + +impl Formatter { + pub fn human_path( + file: &FileInfo, + app_args: &Params, + min_path_length: usize, + ) -> Result { + let base_directory: PathBuf = app_args.get_directory()?; + let relative_path = diff_paths(file.path.clone(), base_directory).unwrap_or_default(); + + let formatted_path = format!( + "{:<0width$}", + relative_path.to_str().unwrap_or_default().to_string(), + width = min_path_length + ); + + Ok(formatted_path) + } + + pub fn human_filesize(file: &FileInfo) -> Result { + Ok(format!("{:>12}", bytesize::ByteSize::b(file.size))) + } + + pub fn human_mtime(file: &FileInfo) -> Result { + let modified_time: DateTime = file.filemeta.modified()?.into(); + Ok(modified_time.format("%Y-%m-%d %H:%M:%S").to_string()) + } + + pub fn generate_table(raw: Vec, app_args: &Params) -> Result { + let basepath_length = app_args.get_directory()?.to_str().unwrap_or_default().len(); + let max_filepath_length = raw + .iter() + .map(|file| file.path.to_str().unwrap_or_default().len()) + .max() + .unwrap_or_default(); + + let min_path_length = if max_filepath_length > basepath_length { + max_filepath_length - basepath_length + } else { + 0 + }; + + let progress_style = ProgressStyle::with_template( + "[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}", + )?; + let progress_bar = ProgressBar::new(raw.len() as u64); + progress_bar.set_style(progress_style); + progress_bar.enable_steady_tick(Duration::from_millis(50)); + progress_bar.set_message("reconciling data"); + + let duplicates_table: DashMap> = DashMap::new(); + raw.into_par_iter() + .progress_with(progress_bar) + .with_finish(ProgressFinish::WithMessage(Cow::from("data reconciled"))) + .map(|file| file.hash()) + .filter_map(Result::ok) + .for_each(|file| { + duplicates_table + .entry(file.hash.clone().unwrap_or_default()) + .and_modify(|fileset| fileset.push(file.clone())) + .or_insert_with(|| vec![file]); + }); + + let mut output_table = Table::new(); + output_table.set_titles(row!["hash", "duplicates"]); + + let progress_style = ProgressStyle::with_template( + "[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}", + )?; + + let progress_bar = ProgressBar::new(duplicates_table.len() as u64); + progress_bar.set_style(progress_style); + progress_bar.enable_steady_tick(Duration::from_millis(50)); + progress_bar.set_message("generating output"); + + duplicates_table + .into_iter() + .progress_with(progress_bar) + .with_finish(ProgressFinish::WithMessage(Cow::from("output generated"))) + .for_each(|(hash, group)| { + let mut inner_table = Table::new(); + inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR); + group.iter().for_each(|file| { + inner_table.add_row(row![ + Self::human_path(file, app_args, min_path_length) + .unwrap_or_default() + .blue(), + Self::human_filesize(file).unwrap_or_default().red(), + Self::human_mtime(file).unwrap_or_default().yellow() + ]); + }); + + output_table.add_row(row![hash.green(), inner_table]); + }); + + Ok(output_table) + } + + pub fn print(raw: Vec, app_args: &Params) -> Result<()> { + if raw.is_empty() { + println!( + "\n\n{}\n", + "No duplicates found matching your search criteria.".green() + ); + return Ok(()); + } + + let output_table = Self::generate_table(raw, app_args)?; + output_table.printstd(); + + Ok(()) + } +} diff --git a/src/interactive.rs b/src/interactive.rs new file mode 100644 index 0000000..9e6c2b0 --- /dev/null +++ b/src/interactive.rs @@ -0,0 +1,154 @@ +use crate::formatter::Formatter; +use crate::{fileinfo::FileInfo, params::Params}; +use anyhow::Result; +use colored::Colorize; +use dashmap::DashMap; +use indicatif::{ParallelProgressIterator, ProgressBar, ProgressFinish, ProgressStyle}; +use prettytable::{format, row, Table}; +use rayon::prelude::*; +use std::{ + borrow::Cow, + io::{self, Write}, + time::Duration, +}; + +pub fn scan_group_confirmation() -> Result { + print!("\nconfirm? [y/N]: "); + std::io::stdout().flush()?; + let mut user_input = String::new(); + io::stdin().read_line(&mut user_input)?; + + match user_input.trim() { + "Y" | "y" => Ok(true), + _ => Ok(false), + } +} + +pub fn scan_group_instruction() -> Result { + println!("\nEnter the indices of the files you want to delete."); + println!("You can enter multiple files using commas to seperate file indices."); + println!("example: 1,2"); + print!("\n> "); + std::io::stdout().flush()?; + let mut user_input = String::new(); + io::stdin().read_line(&mut user_input)?; + + Ok(user_input) +} + +pub fn init(result: Vec, app_args: &Params) -> Result<()> { + let basepath_length = app_args.get_directory()?.to_str().unwrap_or_default().len(); + let max_filepath_length = result + .iter() + .map(|file| file.path.to_str().unwrap_or_default().len()) + .max() + .unwrap_or_default(); + + let min_path_length = if max_filepath_length > basepath_length { + max_filepath_length - basepath_length + } else { + 0 + }; + + let progress_style = ProgressStyle::with_template( + "[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}", + )?; + let progress_bar = ProgressBar::new(result.len() as u64); + progress_bar.set_style(progress_style); + progress_bar.enable_steady_tick(Duration::from_millis(50)); + progress_bar.set_message("reconciling data"); + + let duplicates: DashMap> = DashMap::new(); + result + .into_par_iter() + .progress_with(progress_bar) + .with_finish(ProgressFinish::WithMessage(Cow::from("data reconciled"))) + .map(|file| file.hash()) + .filter_map(Result::ok) + .for_each(|file| { + duplicates + .entry(file.hash.clone().unwrap_or_default()) + .and_modify(|fileset| fileset.push(file.clone())) + .or_insert_with(|| vec![file]); + }); + + duplicates + .clone() + .into_iter() + .enumerate() + .for_each(|(gindex, (_, group))| { + let mut itable = Table::new(); + itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR); + itable.set_titles(row!["index", "filename", "size", "updated_at"]); + group.iter().enumerate().for_each(|(index, file)| { + itable.add_row(row![ + index, + Formatter::human_path(file, app_args, min_path_length) + .unwrap_or_default() + .blue(), + Formatter::human_filesize(file).unwrap_or_default().red(), + Formatter::human_mtime(file).unwrap_or_default().yellow() + ]); + }); + + process_group_action(&group, gindex, duplicates.len(), itable); + }); + + Ok(()) +} + +pub fn process_group_action( + duplicates: &Vec, + dup_index: usize, + dup_size: usize, + table: Table, +) { + println!("\nDuplicate Set {} of {}\n", dup_index + 1, dup_size); + table.printstd(); + let files_to_delete = scan_group_instruction().unwrap_or_default(); + let parsed_file_indices = files_to_delete + .trim() + .split(',') + .filter(|element| !element.is_empty()) + .map(|index| index.parse::().unwrap_or_default()) + .collect::>(); + + if parsed_file_indices + .clone() + .into_iter() + .any(|index| index > (duplicates.len() - 1)) + { + println!("{}", "Err: File Index Out of Bounds!".red()); + return process_group_action(duplicates, dup_index, dup_size, table); + } + + print!("{esc}[2J{esc}[1;1H", esc = 27 as char); + + if parsed_file_indices.is_empty() { + return; + } + + let files_to_delete = parsed_file_indices + .into_iter() + .map(|index| duplicates[index].clone()); + + println!("\n{}", "The following files will be deleted:".red()); + files_to_delete + .clone() + .enumerate() + .for_each(|(index, file)| { + println!("{}: {}", index.to_string().blue(), file.path.display()); + }); + + match scan_group_confirmation().unwrap() { + true => { + files_to_delete.into_iter().for_each(|file| { + match std::fs::remove_file(file.path.clone()) { + Ok(_) => println!("{}: {}", "DELETED".green(), file.path.display()), + Err(_) => println!("{}: {}", "FAILED".red(), file.path.display()), + } + }); + } + false => println!("{}", "\nCancelled Delete Operation.".red()), + } +} diff --git a/src/main.rs b/src/main.rs index a7d1249..f462459 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,14 +1,27 @@ -mod app; -mod file_manager; -mod output; +mod fileinfo; +mod formatter; +mod interactive; mod params; +mod processor; mod scanner; -mod filters; use anyhow::Result; -use app::App; use clap::Parser; +use formatter::Formatter; +use params::Params; +use processor::Processor; +use scanner::Scanner; fn main() -> Result<()> { - App::init(¶ms::Params::parse()) + let app_args = Params::parse(); + let scan_results = Scanner::build(&app_args)?.scan()?; + let processor = Processor::new(scan_results); + let results = processor.sizewise()?.hashwise()?; + + match app_args.interactive { + false => { Formatter::print(results.files, &app_args)?; } + true => { interactive::init(results.files, &app_args)?; } + } + + Ok(()) } diff --git a/src/output.rs b/src/output.rs deleted file mode 100644 index a8cd7f7..0000000 --- a/src/output.rs +++ /dev/null @@ -1,217 +0,0 @@ -use crate::file_manager::{self, File}; -use crate::params::Params; -use anyhow::Result; -use chrono::offset::Utc; -use chrono::DateTime; -use colored::Colorize; -use dashmap::DashMap; -use indicatif::{ProgressBar, ProgressIterator, ProgressStyle}; -use itertools::Itertools; -use prettytable::{format, row, Table}; -use std::io::Write; -use std::path::Path; -use std::time::Duration; -use std::{fs, io}; -use unicode_segmentation::UnicodeSegmentation; - -fn format_path(path: &Path, opts: &Params) -> Result { - let display_path = path - .to_string_lossy() - .replace(opts.get_directory()?.to_string_lossy().as_ref(), ""); - let display_range = if display_path.chars().count() > 32 { - display_path - .graphemes(true) - .collect::>() - .into_iter() - .rev() - .take(32) - .rev() - .collect() - } else { - display_path - }; - - Ok(format!("...{display_range:<32}")) -} - -fn file_size(file: &File) -> Result { - Ok(format!("{:>12}", bytesize::ByteSize::b(file.size.unwrap()))) -} - -fn modified_time(path: &Path) -> Result { - let mdata = fs::metadata(path)?; - let modified_time: DateTime = mdata.modified()?.into(); - - Ok(modified_time.format("%Y-%m-%d %H:%M:%S").to_string()) -} - -fn scan_group_instruction() -> Result { - println!("\nEnter the indices of the files you want to delete."); - println!("You can enter multiple files using commas to seperate file indices."); - println!("example: 1,2"); - print!("\n> "); - std::io::stdout().flush()?; - let mut user_input = String::new(); - io::stdin().read_line(&mut user_input)?; - - Ok(user_input) -} - -fn scan_group_confirmation() -> Result { - print!("\nconfirm? [y/N]: "); - std::io::stdout().flush()?; - let mut user_input = String::new(); - io::stdin().read_line(&mut user_input)?; - - match user_input.trim() { - "Y" | "y" => Ok(true), - _ => Ok(false), - } -} - -fn process_group_action(duplicates: &Vec, dup_index: usize, dup_size: usize, table: Table) { - println!("\nDuplicate Set {} of {}\n", dup_index + 1, dup_size); - table.printstd(); - let files_to_delete = scan_group_instruction().unwrap_or_default(); - let parsed_file_indices = files_to_delete - .trim() - .split(',') - .filter(|element| !element.is_empty()) - .map(|index| index.parse::().unwrap_or_default()) - .collect::>(); - - if parsed_file_indices - .clone() - .into_iter() - .any(|index| index > (duplicates.len() - 1)) - { - println!("{}", "Err: File Index Out of Bounds!".red()); - return process_group_action(duplicates, dup_index, dup_size, table); - } - - print!("{esc}[2J{esc}[1;1H", esc = 27 as char); - - if parsed_file_indices.is_empty() { - return; - } - - let files_to_delete = parsed_file_indices - .into_iter() - .map(|index| duplicates[index].clone()); - - println!("\n{}", "The following files will be deleted:".red()); - files_to_delete - .clone() - .enumerate() - .for_each(|(index, file)| { - println!("{}: {}", index.to_string().blue(), file.path.display()); - }); - - match scan_group_confirmation().unwrap() { - true => { - file_manager::delete_files(files_to_delete.collect::>()).ok(); - } - false => println!("{}", "\nCancelled Delete Operation.".red()), - } -} - -pub fn interactive(duplicates: DashMap>, opts: &Params) { - if duplicates.is_empty() { - println!( - "\n{}", - "No duplicates found matching your search criteria.".green() - ); - return; - } - - duplicates - .clone() - .into_iter() - .sorted_unstable_by_key(|(_, f)| { - -(f.first().and_then(|ff| ff.size).unwrap_or_default() as i64) - }) // sort by descending file size in interactive mode - .enumerate() - .for_each(|(gindex, (_, group))| { - let mut itable = Table::new(); - itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR); - itable.set_titles(row!["index", "filename", "size", "updated_at"]); - group.iter().enumerate().for_each(|(index, file)| { - itable.add_row(row![ - index, - format_path(&file.path, opts).unwrap_or_default().blue(), - file_size(file).unwrap_or_default().red(), - modified_time(&file.path).unwrap_or_default().yellow() - ]); - }); - - process_group_action(&group, gindex, duplicates.len(), itable); - }); -} - -pub fn print(duplicates: DashMap>, opts: &Params) { - if duplicates.is_empty() { - println!( - "\n{}", - "No duplicates found matching your search criteria.".green() - ); - return; - } - - let mut output_table = Table::new(); - let progress_bar = ProgressBar::new(duplicates.len() as u64); - progress_bar.enable_steady_tick(Duration::from_millis(50)); - let progress_style = ProgressStyle::default_bar() - .template("{spinner:.green} [generating output] [{wide_bar:.cyan/blue}] {pos}/{len} files") - .unwrap(); - - progress_bar.set_style(progress_style); - output_table.set_titles(row!["hash", "duplicates"]); - - duplicates - .into_iter() - .sorted_unstable_by_key(|(_, f)| f.first().and_then(|ff| ff.size).unwrap_or_default()) - .progress_with(progress_bar) - .for_each(|(hash, group)| { - let mut inner_table = Table::new(); - inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR); - group.iter().for_each(|file| { - inner_table.add_row(row![ - format_path(&file.path, opts).unwrap_or_default().blue(), - file_size(file).unwrap_or_default().red(), - modified_time(&file.path).unwrap_or_default().yellow() - ]); - }); - output_table.add_row(row![hash.green(), inner_table]); - }); - - output_table.printstd(); -} - -#[allow(unused)] -pub fn raw(duplicates: DashMap>, opts: &Params) -> Result<()> { - if duplicates.is_empty() { - println!( - "\n{}", - "No duplicates found matching your search criteria.".green() - ); - return Ok(()); - } - - duplicates - .into_iter() - .sorted_unstable_by_key(|(_, f)| f.first().and_then(|ff| ff.size).unwrap_or_default()) - .for_each(|(_hash, group)| { - group.iter().for_each(|file| { - println!( - "{}\t{}\t{}", - format_path(&file.path, opts).unwrap_or_default().blue(), - file_size(file).unwrap_or_default().red(), - modified_time(&file.path).unwrap_or_default().yellow() - ) - }); - - println!("---"); - }); - - Ok(()) -} diff --git a/src/params.rs b/src/params.rs index b467695..e37dd1d 100644 --- a/src/params.rs +++ b/src/params.rs @@ -1,10 +1,9 @@ use std::{fs, path::PathBuf}; -use anyhow::{anyhow, Result}; +use anyhow::Result; use clap::{Parser, ValueHint}; -use globwalk::{GlobWalker, GlobWalkerBuilder}; -#[derive(Parser, Debug)] +#[derive(Parser, Debug, Clone)] #[command(author, version, about, long_about = None)] pub struct Params { /// Filetypes to deduplicate [default = all] @@ -48,41 +47,7 @@ impl Params { Ok(dir) } - fn add_glob_min_depth(&self, builder: GlobWalkerBuilder) -> Result { - match self.min_depth { - Some(mindepth) => Ok(builder.min_depth(mindepth)), - None => Ok(builder), - } - } - - fn add_glob_max_depth(&self, builder: GlobWalkerBuilder) -> Result { - match self.max_depth { - Some(maxdepth) => Ok(builder.max_depth(maxdepth)), - None => Ok(builder), - } - } - - fn add_glob_follow_links(&self, builder: GlobWalkerBuilder) -> Result { - match self.follow_links { - true => Ok(builder.follow_links(true)), - false => Ok(builder.follow_links(false)), - } - } - - pub fn get_glob_walker(&self) -> Result { - let pattern: String = match self.types.as_ref() { - Some(filetypes) => format!("**/*{{{filetypes}}}"), - None => "**/*".to_string(), - }; - - let glob_walker_builder = self - .add_glob_min_depth(GlobWalkerBuilder::from_patterns( - self.get_directory()?, - &[pattern], - )) - .and_then(|builder| self.add_glob_max_depth(builder)) - .and_then(|builder| self.add_glob_follow_links(builder))?; - - glob_walker_builder.build().map_err(|e| anyhow!(e)) + pub fn get_types(&self) -> Option { + self.types.clone() } } diff --git a/src/processor.rs b/src/processor.rs new file mode 100644 index 0000000..6ff1120 --- /dev/null +++ b/src/processor.rs @@ -0,0 +1,107 @@ +use anyhow::Result; +use dashmap::DashMap; +use indicatif::{ParallelProgressIterator, ProgressBar, ProgressStyle, ProgressFinish}; +use rayon::prelude::{IntoParallelIterator, ParallelIterator}; +use std::{time::Duration, borrow::Cow}; + +use crate::fileinfo::FileInfo; + +#[derive(Debug, Clone)] +pub enum State { + Initial, + SizeWise, + HashWise, +} + +#[derive(Debug, Clone)] +pub struct Processor { + pub files: Vec, + pub state: State, +} + +impl Processor { + pub fn new(files: Vec) -> Self { + Self { + files, + state: State::Initial, + } + } + + pub fn hashwise(&self) -> Result { + if self.files.is_empty() { + return Ok(self.clone()); + } + + let progress_style = ProgressStyle::with_template("[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}")?; + let progress_bar = ProgressBar::new(self.files.len() as u64); + progress_bar.set_style(progress_style); + progress_bar.enable_steady_tick(Duration::from_millis(50)); + progress_bar.set_message("indexing file hashes"); + + let duplicates_table: DashMap> = DashMap::new(); + self.files + .clone() + .into_par_iter() + .progress_with(progress_bar) + .with_finish(ProgressFinish::WithMessage(Cow::from("indexed files hashes"))) + .map(|file| file.hash()) + .filter_map(Result::ok) + .for_each(|file| { + duplicates_table + .entry(file.hash.clone().unwrap_or_default()) + .and_modify(|fileset| fileset.push(file.clone())) + .or_insert_with(|| vec![file]); + }); + + let files = duplicates_table + .into_read_only() + .values() + .cloned() + .filter(|subfiles| subfiles.len() > 1) + .flatten() + .collect::>(); + + Ok(Self { + files, + state: State::HashWise, + }) + } + + pub fn sizewise(&self) -> Result { + if self.files.is_empty() { + return Ok(self.clone()); + } + + let progress_style = ProgressStyle::with_template("[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}")?; + let progress_bar = ProgressBar::new(self.files.len() as u64); + progress_bar.set_style(progress_style); + progress_bar.enable_steady_tick(Duration::from_millis(50)); + progress_bar.set_message("indexing file sizes"); + + let duplicates_table: DashMap> = DashMap::new(); + self.files + .clone() + .into_par_iter() + .progress_with(progress_bar) + .with_finish(ProgressFinish::WithMessage(Cow::from("indexed files sizes"))) + .for_each(|file| { + duplicates_table + .entry(file.size) + .and_modify(|fileset| fileset.push(file.clone())) + .or_insert_with(|| vec![file]); + }); + + let files = duplicates_table + .into_read_only() + .values() + .cloned() + .filter(|subfiles| subfiles.len() > 1) + .flatten() + .collect::>(); + + Ok(Self { + files, + state: State::SizeWise, + }) + } +} diff --git a/src/scanner.rs b/src/scanner.rs index 4b7ee10..25f49d9 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -1,151 +1,173 @@ -use crate::{file_manager::File, filters, params::Params}; +#![allow(unused)] +use crate::{fileinfo::FileInfo, params::Params}; use anyhow::Result; -use dashmap::DashMap; -use fxhash::hash64 as hasher; -use indicatif::{ParallelProgressIterator, ProgressBar, ProgressIterator, ProgressStyle}; -use memmap2::Mmap; -use rayon::prelude::*; -use std::hash::Hasher; -use std::time::Duration; -use std::{ - fs, - path::{Path, PathBuf}, -}; +use indicatif::{ProgressBar, ProgressStyle}; +use std::{fs, path::PathBuf, time::Duration}; -#[derive(Clone, Copy)] -enum IndexCritera { - Size, - Hash, +use globwalk::{GlobWalker, GlobWalkerBuilder}; + +#[derive(Debug, Clone)] +pub struct Scanner { + pub directory: Option, + pub filetypes: Option, + pub min_depth: Option, + pub max_depth: Option, + pub min_size: Option, + pub follow_links: bool, } -pub fn duplicates(app_opts: &Params) -> Result>> { - let scan_results = scan(app_opts)?; - let size_index_store = index_files(scan_results, IndexCritera::Size)?; - - let sizewize_duplicate_files = size_index_store - .into_par_iter() - .filter(|(_, files)| files.len() > 1) - .map(|(_, files)| files) - .flatten() - .collect::>(); - - if sizewize_duplicate_files.len() > 1 { - let hash_index_store = index_files(sizewize_duplicate_files, IndexCritera::Hash)?; - let duplicate_files = hash_index_store - .into_par_iter() - .filter(|(_, files)| files.len() > 1) - .collect(); - - Ok(duplicate_files) - } else { - Ok(DashMap::new()) +impl Scanner { + pub fn new() -> Self { + Self { + directory: None, + filetypes: None, + min_depth: None, + max_depth: None, + min_size: None, + follow_links: true, + } } -} -fn scan(app_opts: &Params) -> Result> { - let walker = app_opts.get_glob_walker()?; - let progress = ProgressBar::new_spinner(); - let progress_style = - ProgressStyle::with_template("{spinner:.green} [mapping paths] {pos} paths")?; - progress.set_style(progress_style); - progress.enable_steady_tick(Duration::from_millis(50)); + pub fn build(app_args: &Params) -> Result { + let scan_directory = app_args.get_directory()?; + Ok(Scanner::new()) + .map(|scanner| scanner.directory(scan_directory)) + .map(|scanner| match app_args.get_min_size() { + Some(min_size) => scanner.min_size(min_size), + None => scanner, + }) + .map(|scanner| match app_args.get_types() { + Some(ftypes) => scanner.filetypes(ftypes), + None => scanner, + }) + .map(|scanner| match app_args.min_depth { + Some(min_depth) => scanner.min_depth(min_depth), + None => scanner, + }) + .map(|scanner| match app_args.max_depth { + Some(max_depth) => scanner.max_depth(max_depth), + None => scanner, + }) + } - let files = walker - .progress_with(progress) - .filter_map(Result::ok) - .map(|file| file.into_path()) - .filter(|fpath| fpath.is_file()) - .collect::>(); + pub fn min_size(&self, min_size: u64) -> Self { + Self { + min_size: Some(min_size), + ..self.clone() + } + } - let scan_progress = ProgressBar::new(files.len() as u64); - let scan_progress_style = ProgressStyle::with_template( - "{spinner:.green} [processing mapped paths] [{wide_bar:.cyan/blue}] {pos}/{len} files", - )?; - scan_progress.set_style(scan_progress_style); - scan_progress.enable_steady_tick(Duration::from_millis(50)); + pub fn min_depth(&self, min_depth: usize) -> Self { + Self { + min_depth: Some(min_depth), + ..self.clone() + } + } - let scan_results = files - .into_par_iter() - .progress_with(scan_progress) - .map(|fpath| File { - path: fpath.clone(), - hash: None, - size: Some( - fs::metadata(fpath) - .map(|metadata| metadata.len()) - .unwrap_or_default(), - ), + pub fn max_depth(&self, max_depth: usize) -> Self { + Self { + max_depth: Some(max_depth), + ..self.clone() + } + } + + pub fn directory(&self, dir: PathBuf) -> Self { + Self { + directory: Some(dir), + ..self.clone() + } + } + + pub fn filetypes(&self, patterns: String) -> Self { + Self { + filetypes: Some(patterns), + ..self.clone() + } + } + + pub fn ignore_links(&self) -> Self { + Self { + follow_links: false, + ..self.clone() + } + } + + pub fn follow_links(&self) -> Self { + Self { + follow_links: true, + ..self.clone() + } + } + + fn scan_patterns(&self) -> Result { + Ok(match self.filetypes.clone() { + Some(ftypes) => format!("**/*{{{ftypes}}}"), + None => "**/*".to_string(), }) - .filter(|file| filters::is_file_gt_min_size(app_opts, file)) - .collect(); + } - Ok(scan_results) -} + fn scan_dir(&self) -> Result { + let scan_dir = match self.directory.clone() { + Some(path) => path, + None => std::env::current_dir()?, + }; -fn process_file_index( - mut file: File, - store: &DashMap>, - index_criteria: IndexCritera, -) { - match index_criteria { - IndexCritera::Size => { - store - .entry(file.size.unwrap_or_default().to_string()) - .and_modify(|fileset| fileset.push(file.clone())) - .or_insert_with(|| vec![file]); - } - IndexCritera::Hash => { - file.hash = Some(hash_file(&file.path).unwrap_or_default()); - store - .entry(file.clone().hash.unwrap_or_default()) - .and_modify(|fileset| fileset.push(file.clone())) - .or_insert_with(|| vec![file]); + Ok(fs::canonicalize(scan_dir)?) + } + + fn attach_link_opts(&self, walker: GlobWalkerBuilder) -> Result { + Ok(walker.follow_links(self.follow_links)) + } + + fn attach_walker_min_depth(&self, walker: GlobWalkerBuilder) -> Result { + match self.min_depth { + Some(min_depth) => Ok(walker.min_depth(min_depth)), + None => Ok(walker), } } -} -fn index_files( - files: Vec, - index_criteria: IndexCritera, -) -> Result>> { - let store: DashMap> = DashMap::new(); - let index_progress = ProgressBar::new(files.len() as u64); - let index_progress_style = ProgressStyle::with_template( - "{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files", - )?; - index_progress.set_style(index_progress_style); - index_progress.enable_steady_tick(Duration::from_millis(50)); + fn attach_walker_max_depth(&self, walker: GlobWalkerBuilder) -> Result { + match self.max_depth { + Some(max_depth) => Ok(walker.max_depth(max_depth)), + None => Ok(walker), + } + } + fn build_walker(&self) -> Result { + let walker = Ok(GlobWalkerBuilder::from_patterns( + self.scan_dir()?, + &[self.scan_patterns()?], + )) + .and_then(|walker| self.attach_walker_min_depth(walker)) + .and_then(|walker| self.attach_walker_max_depth(walker)) + .and_then(|walker| self.attach_link_opts(walker))?; - files - .into_par_iter() - .progress_with(index_progress) - .for_each(|file| process_file_index(file, &store, index_criteria)); + Ok(walker.build()?) + } - Ok(store) -} + pub fn scan(&self) -> Result> { + let progress_style = ProgressStyle::with_template("[{elapsed_precise}] {pos:>7} {msg}")?; + let progress_bar = ProgressBar::new_spinner(); + progress_bar.set_style(progress_style); + progress_bar.enable_steady_tick(Duration::from_millis(50)); + progress_bar.set_message("paths mapped"); + let min_size = self.min_size.unwrap_or_default(); -fn incremental_hashing(filepath: &Path) -> Result { - let file = fs::File::open(filepath)?; - let fmap = unsafe { Mmap::map(&file)? }; - let mut inchasher = fxhash::FxHasher::default(); + let results = self + .build_walker()? + .filter_map(Result::ok) + .map(|entity| entity.into_path()) + .map(|path| { + progress_bar.inc(1); + path + }) + .filter(|path| path.is_file()) + .map(FileInfo::new) + .filter_map(Result::ok) + .filter(|file| file.size > min_size) + .collect::>(); - fmap.chunks(1_000_000) - .for_each(|mega| inchasher.write(mega)); + progress_bar.finish_with_message("paths mapped"); - Ok(format!("{}", inchasher.finish())) -} - -fn standard_hashing(filepath: &Path) -> Result { - let file = fs::read(filepath)?; - Ok(hasher(&*file).to_string()) -} - -fn hash_file(filepath: &Path) -> Result { - let filemeta = fs::metadata(filepath)?; - - // NOTE: USE INCREMENTAL HASHING ONLY FOR FILES > 100MB - match filemeta.len() < 100_000_000 { - true => standard_hashing(filepath), - false => incremental_hashing(filepath), + Ok(results) } }