smolkik_adm/deduplicator
1
0
Fork 0
mirror of https://github.com/sreedevk/deduplicator.git synced 2026-08-26 02:04:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: switch to globwalk

Browse Source
This commit is contained in:
beeb
2023-01-23 21:53:44 +01:00
parent 876fdc69ac
commit 6f8b1d55df
5 changed files with 122 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

116
Cargo.lock generated
View File

@@ -2,6 +2,15 @@
# It is not intended for manual editing.
version = 3
[[package]]
name = "aho-corasick"
version = "0.7.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac"
dependencies = [
"memchr",
]
[[package]]
name = "android_system_properties"
version = "0.1.5"
@@ -52,6 +61,16 @@ dependencies = [
"serde",
]
[[package]]
name = "bstr"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b45ea9b00a7b3f2988e9a65ad3917e62123c38dba709b666506207be96d1790b"
dependencies = [
"memchr",
"serde",
]
[[package]]
name = "bumpalo"
version = "3.11.1"
@@ -229,7 +248,7 @@ version = "1.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1"
dependencies = [
"bstr",
"bstr 0.2.17",
"csv-core",
"itoa",
"ryu",
@@ -314,7 +333,7 @@ dependencies = [
"colored",
"dashmap",
"fxhash",
"glob",
"globwalk",
"indicatif",
"itertools",
"memmap2",
@@ -385,6 +404,12 @@ dependencies = [
"libc",
]
[[package]]
name = "fnv"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "fxhash"
version = "0.2.1"
@@ -406,10 +431,28 @@ dependencies = [
]
[[package]]
name = "glob"
version = "0.3.0"
name = "globset"
version = "0.4.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
checksum = "029d74589adefde59de1a0c4f4732695c32805624aec7b68d91503d4dba79afc"
dependencies = [
"aho-corasick",
"bstr 1.1.0",
"fnv",
"log",
"regex",
]
[[package]]
name = "globwalk"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93e3af942408868f6934a7b85134a3230832b9977cf66125df2f9edcfce4ddcc"
dependencies = [
"bitflags",
"ignore",
"walkdir",
]
[[package]]
name = "hashbrown"
@@ -465,6 +508,23 @@ dependencies = [
"cxx-build",
]
[[package]]
name = "ignore"
version = "0.4.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbe7873dab538a9a44ad79ede1faf5f30d49f9a5c883ddbab48bce81b64b7492"
dependencies = [
"globset",
"lazy_static",
"log",
"memchr",
"regex",
"same-file",
"thread_local",
"walkdir",
"winapi-util",
]
[[package]]
name = "indicatif"
version = "0.17.2"
@@ -787,12 +847,29 @@ dependencies = [
"thiserror",
]
[[package]]
name = "regex"
version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
]
[[package]]
name = "regex-automata"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
[[package]]
name = "regex-syntax"
version = "0.6.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"
[[package]]
name = "rustix"
version = "0.36.5"
@@ -819,6 +896,15 @@ version = "1.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde"
[[package]]
name = "same-file"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
dependencies = [
"winapi-util",
]
[[package]]
name = "scopeguard"
version = "1.1.0"
@@ -919,6 +1005,15 @@ dependencies = [
"syn",
]
[[package]]
name = "thread_local"
version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180"
dependencies = [
"once_cell",
]
[[package]]
name = "time"
version = "0.1.45"
@@ -985,6 +1080,17 @@ version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
[[package]]
name = "walkdir"
version = "2.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56"
dependencies = [
"same-file",
"winapi",
"winapi-util",
]
[[package]]
name = "wasi"
version = "0.10.0+wasi-snapshot-preview1"

8
Cargo.toml
View File

@@ -4,7 +4,11 @@ version = "0.1.3"
edition = "2021"
description = "find,filter,delete Duplicates"
license = "MIT"
authors = ["Sreedev Kodichath <sreedevpadmakumar@gmail.com>", "Valentin Bersier <vbersier@gmail.com>", "Dhruva Sagar <dhruva.sagar@gmail.com>"]
authors = [
"Sreedev Kodichath <sreedevpadmakumar@gmail.com>",
"Valentin Bersier <vbersier@gmail.com>",
"Dhruva Sagar <dhruva.sagar@gmail.com>",
]
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -16,7 +20,7 @@ clap = { version = "4.0.32", features = ["derive"] }
colored = "2.0.0"
dashmap = { version = "5.4.0", features = ["rayon"] }
fxhash = "0.2.1"
glob = "0.3.0"
globwalk = "0.8.1"
indicatif = { version = "0.17.2", features = ["rayon", "tokio"] }
itertools = "0.10.5"
memmap2 = "0.5.8"

2
src/output.rs
View File

@@ -26,7 +26,7 @@ fn format_path(path: &str, opts: &Params) -> Result<String> {
display_path
};
Ok(format!("...{:<32}", display_range))
Ok(format!("...{display_range:<32}"))
}
fn file_size(file: &File) -> Result<String> {

2
src/params.rs
View File

@@ -52,7 +52,7 @@ impl Params {
Some(filetypes) => vec![
self.get_directory().unwrap(),
String::from("**"),
format!("{{{}}}", filetypes),
format!("*.{{{filetypes}}}"),
]
.iter()
.collect::<PathBuf>(),

7
src/scanner.rs
View File

@@ -2,7 +2,6 @@ use crate::{file_manager::File, filters, params::Params};
use anyhow::Result;
use dashmap::DashMap;
use fxhash::hash64 as hasher;
use glob::glob;
use indicatif::{ParallelProgressIterator, ProgressStyle};
use memmap2::Mmap;
use rayon::prelude::*;
@@ -41,10 +40,10 @@ pub fn duplicates(app_opts: &Params) -> Result<DashMap<String, Vec<File>>> {
fn scan(app_opts: &Params) -> Result<Vec<File>> {
let glob_patterns = app_opts.get_glob_patterns().display().to_string();
let glob_iter = glob(&glob_patterns)?;
let glob_iter = globwalk::glob(glob_patterns)?;
let files = glob_iter
.filter(Result::is_ok)
.map(|file| file.unwrap())
.filter_map(Result::ok)
.map(|file| file.into_path())
.filter(|fpath| fpath.is_file())
.collect::<Vec<PathBuf>>()
.into_par_iter()