From 30398c3ca927057bf1b7e81c4ef0c17fd993f973 Mon Sep 17 00:00:00 2001 From: sreedev Date: Thu, 19 Jan 2023 23:25:16 -0500 Subject: [PATCH 1/6] min opts --- src/output.rs | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/src/output.rs b/src/output.rs index 188a51e..44def2d 100644 --- a/src/output.rs +++ b/src/output.rs @@ -40,10 +40,6 @@ fn modified_time(path: &String) -> Result { Ok(modified_time.format("%Y-%m-%d %H:%M:%S").to_string()) } -fn print_meta_info() { - println!("Deduplicator v{}", std::env!("CARGO_PKG_VERSION")); -} - fn scan_group_instruction() -> Result { println!("\nEnter the indices of the files you want to delete."); println!("You can enter multiple files using commas to seperate file indices."); @@ -115,8 +111,6 @@ fn process_group_action(duplicates: &Vec, dup_index: usize, dup_size: usiz } pub fn interactive(duplicates: DashMap>, opts: &Params) { - print_meta_info(); - if duplicates.is_empty() { println!( "\n{}", @@ -128,8 +122,8 @@ pub fn interactive(duplicates: DashMap>, opts: &Params) { duplicates .clone() .into_iter() - .sorted_unstable_by_key(|f| { - -(f.1.first().and_then(|ff| ff.size).unwrap_or_default() as i64) + .sorted_unstable_by_key(|(_, f)| { + -(f.first().and_then(|ff| ff.size).unwrap_or_default() as i64) }) // sort by descending file size in interactive mode .enumerate() .for_each(|(gindex, (_, group))| { @@ -140,7 +134,7 @@ pub fn interactive(duplicates: DashMap>, opts: &Params) { itable.add_row(row![ index, format_path(&file.path, opts).unwrap_or_default().blue(), - file_size(&file).unwrap_or_default().red(), + file_size(file).unwrap_or_default().red(), modified_time(&file.path).unwrap_or_default().yellow() ]); }); @@ -150,8 +144,6 @@ pub fn interactive(duplicates: DashMap>, opts: &Params) { } pub fn print(duplicates: DashMap>, opts: &Params) { - print_meta_info(); - if duplicates.is_empty() { println!( "\n{}", @@ -164,14 +156,14 @@ pub fn print(duplicates: DashMap>, opts: &Params) { output_table.set_titles(row!["hash", "duplicates"]); duplicates .into_iter() - .sorted_unstable_by_key(|f| f.1.first().and_then(|ff| ff.size).unwrap_or_default()) // sort by ascending size + .sorted_unstable_by_key(|(_, f)| f.first().and_then(|ff| ff.size).unwrap_or_default()) .for_each(|(hash, group)| { let mut inner_table = Table::new(); inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR); group.iter().for_each(|file| { inner_table.add_row(row![ format_path(&file.path, opts).unwrap_or_default().blue(), - file_size(&file).unwrap_or_default().red(), + file_size(file).unwrap_or_default().red(), modified_time(&file.path).unwrap_or_default().yellow() ]); }); From 7b275d40d0df519337dce8b786a9b1b5bd25e0a4 Mon Sep 17 00:00:00 2001 From: sreedev Date: Thu, 19 Jan 2023 23:27:40 -0500 Subject: [PATCH 2/6] min opts 2 --- src/params.rs | 29 ++++++++++++++-------------- src/scanner.rs | 51 ++++++++++++++++++-------------------------------- 2 files changed, 33 insertions(+), 47 deletions(-) diff --git a/src/params.rs b/src/params.rs index dd21f05..adca974 100644 --- a/src/params.rs +++ b/src/params.rs @@ -33,8 +33,8 @@ impl Params { pub fn get_directory(&self) -> Result { let dir_pathbuf: PathBuf = self .dir - .clone() - .unwrap_or(std::env::current_dir()?) + .as_ref() + .unwrap_or(&std::env::current_dir()?) .as_os_str() .into(); @@ -47,17 +47,18 @@ impl Params { Ok(dir) } - pub fn get_glob_patterns(&self) -> Vec { - self.types - .clone() - .unwrap_or_else(|| String::from("*")) - .split(',') - .map(|filetype| format!("*.{}", filetype)) - .map(|filetype| { - vec![self.get_directory().unwrap(), String::from("**"), filetype] - .iter() - .collect() - }) - .collect() + pub fn get_glob_patterns(&self) -> PathBuf { + match self.types.as_ref() { + Some(filetypes) => vec![ + self.get_directory().unwrap(), + String::from("**"), + format!("{{{}}}", filetypes), + ] + .iter() + .collect::(), + None => vec![self.get_directory().unwrap().as_str(), "**", "*"] + .iter() + .collect::(), + } } } diff --git a/src/scanner.rs b/src/scanner.rs index 5383267..b69534e 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -40,27 +40,22 @@ pub fn duplicates(app_opts: &Params) -> Result>> { } fn scan(app_opts: &Params) -> Result> { - let glob_patterns: Vec = app_opts.get_glob_patterns(); - let files: Vec = glob_patterns - .par_iter() + let glob_patterns = app_opts.get_glob_patterns().display().to_string(); + let glob_iter = glob(&glob_patterns)?; + let files = glob_iter + .filter(Result::is_ok) + .map(|file| file.unwrap()) + .filter(|fpath| fpath.is_file()) + .collect::>() + .into_par_iter() .progress_with_style(ProgressStyle::with_template( - "{spinner:.green} [scanning files] [{wide_bar:.cyan/blue}] {pos}/{len} files", + "{spinner:.green} [processing scan results] [{wide_bar:.cyan/blue}] {pos}/{len} files", )?) - .filter_map(|glob_pattern| glob(glob_pattern.as_os_str().to_str()?).ok()) - .flat_map(|file_vec| { - file_vec - .filter_map(|x| Some(x.ok()?.as_os_str().to_str()?.to_string())) - .filter(|glob_result| { - fs::metadata(glob_result) - .map(|f| f.is_file()) - .unwrap_or(false) - }) - .collect::>() - }) - .map(|file_path| File { - path: file_path.clone(), + .map(|fpath| fpath.display().to_string()) + .map(|fpath| File { + path: fpath.clone(), hash: None, - size: Some(fs::metadata(file_path).unwrap().len()), + size: Some(fs::metadata(fpath).unwrap().len()), }) .filter(|file| filters::is_file_gt_minsize(app_opts, file)) .collect(); @@ -68,16 +63,8 @@ fn scan(app_opts: &Params) -> Result> { Ok(files) } -fn process_file_hash_index(file: &File) -> Result { - Ok(File { - path: file.path.clone(), - size: file.size, - hash: Some(hash_file(&file.path).unwrap_or_default()), - }) -} - fn process_file_index( - file: File, + mut file: File, store: &DashMap>, index_criteria: IndexCritera, ) { @@ -89,13 +76,11 @@ fn process_file_index( .or_insert_with(|| vec![file]); } IndexCritera::Hash => { - let processed_file = process_file_hash_index(&file).unwrap(); - let indexhash = processed_file.clone().hash.unwrap_or_default(); - + file.hash = Some(hash_file(&file.path).unwrap_or_default()); store - .entry(indexhash) - .and_modify(|fileset| fileset.push(processed_file.clone())) - .or_insert_with(|| vec![processed_file]); + .entry(file.clone().hash.unwrap()) + .and_modify(|fileset| fileset.push(file.clone())) + .or_insert_with(|| vec![file]); } } } From ccff95dfc5c636c7075c0955f81658eac18d413f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 21 Jan 2023 01:57:43 +0000 Subject: [PATCH 3/6] build(deps): bump tokio from 1.23.0 to 1.23.1 Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.23.0 to 1.23.1. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.23.0...tokio-1.23.1) --- updated-dependencies: - dependency-name: tokio dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 051de52..6880478 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -932,9 +932,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.23.0" +version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eab6d665857cc6ca78d6e80303a02cea7a7851e85dfbd77cbdc09bd129f1ef46" +checksum = "38a54aca0c15d014013256222ba0ebed095673f89345dd79119d912eb561b7a8" dependencies = [ "autocfg", "bytes", diff --git a/Cargo.toml b/Cargo.toml index 673cb14..8c6edc1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,5 +23,5 @@ memmap2 = "0.5.8" prettytable-rs = "0.10.0" rayon = "1.6.1" thiserror = "1.0.38" -tokio = { version = "1.23.0", features = ["full"] } +tokio = { version = "1.23.1", features = ["full"] } unicode-segmentation = "1.10.0" From 0674ab55bfad4a2e3a00bfc37166418b9b3d0829 Mon Sep 17 00:00:00 2001 From: Sreedev Kodichath Date: Thu, 19 Jan 2023 23:37:13 -0500 Subject: [PATCH 4/6] Update README.md --- README.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/README.md b/README.md index 908b807..e5502b9 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,4 @@ cargo install deduplicator

Screenshots

- - - - + From 8acc3f8c8f90ef49b838249f3a5d6e9013066c85 Mon Sep 17 00:00:00 2001 From: sreedev Date: Mon, 23 Jan 2023 02:18:23 -0500 Subject: [PATCH 5/6] added github workflow for building binaries with releases --- .github/workflows/build_and_release.yml | 44 +++++++++++++++++++++++++ .github/workflows/rust.yml | 20 ----------- 2 files changed, 44 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/build_and_release.yml delete mode 100644 .github/workflows/rust.yml diff --git a/.github/workflows/build_and_release.yml b/.github/workflows/build_and_release.yml new file mode 100644 index 0000000..4a2b97e --- /dev/null +++ b/.github/workflows/build_and_release.yml @@ -0,0 +1,44 @@ +name: Build and release + +on: + push: + branches: + - main + release: + types: [created] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Setup Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + profile: minimal + override: true + + - name: Build for Windows + run: | + cargo build --release --target x86_64-pc-windows-gnu + + - name: Build for Linux + run: | + cargo build --release --target x86_64-unknown-linux-gnu + + - name: Build for MacOS + run: | + cargo build --release --target x86_64-apple-darwin + + - name: Create release + uses: actions/create-release@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref }} + release_name: Release ${{ github.ref }} + draft: false + prerelease: false diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml deleted file mode 100644 index 656f3d7..0000000 --- a/.github/workflows/rust.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Rust - -on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - -env: - CARGO_TERM_COLOR: always - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Build - run: cargo build --verbose - - name: Run tests - run: cargo test --verbose From 876fdc69ac60b642e8e62b02e9f38303109419a0 Mon Sep 17 00:00:00 2001 From: sreedev Date: Mon, 23 Jan 2023 02:20:42 -0500 Subject: [PATCH 6/6] added version changes --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6880478..3c59fd7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -305,7 +305,7 @@ dependencies = [ [[package]] name = "deduplicator" -version = "0.1.2" +version = "0.1.3" dependencies = [ "anyhow", "bytesize", diff --git a/Cargo.toml b/Cargo.toml index 8c6edc1..da1018a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deduplicator" -version = "0.1.2" +version = "0.1.3" edition = "2021" description = "find,filter,delete Duplicates" license = "MIT"