typo fix

reduce memuse: fileinfo does not store full metadata

.cargo/config.toml

@@ -0,0 +1,2 @@
+[build]
+rustflags = ["-C", "target-feature=+aes,+sse2"]

.github/workflows/release.yml

@@ -1,121 +1,137 @@
-name: Deduplicator Release Build CI Pipeline
+# CI that:
+#
+# * checks for a Git Tag that looks like a release
+# * creates a Github Release™ and fills in its text
+# * builds artifacts with cargo-dist (executable-zips, installers)
+# * uploads those artifacts to the Github Release™
+#
+# Note that the Github Release™ will be created before the artifacts,
+# so there will be a few minutes where the release has no artifacts
+# and then they will slowly trickle in, possibly failing. To make
+# this more pleasant we mark the release as a "draft" until all
+# artifacts have been successfully uploaded. This allows you to
+# choose what to do with partial successes and avoids spamming
+# anyone with notifications before the release is actually ready.
+name: Release
 
+permissions:
+  contents: write
+
+# This task will run whenever you push a git tag that looks like a version
+# like "v1", "v1.2.0", "v0.1.0-prerelease01", "my-app-v1.0.0", etc.
+# The version will be roughly parsed as ({PACKAGE_NAME}-)?v{VERSION}, where
+# PACKAGE_NAME must be the name of a Cargo package in your workspace, and VERSION
+# must be a Cargo-style SemVer Version.
+#
+# If PACKAGE_NAME is specified, then we will create a Github Release™ for that
+# package (erroring out if it doesn't have the given version or isn't cargo-dist-able).
+#
+# If PACKAGE_NAME isn't specified, then we will create a Github Release™ for all
+# (cargo-dist-able) packages in the workspace with that version (this is mode is
+# intended for workspaces with only one dist-able package, or with all dist-able
+# packages versioned/released in lockstep).
+#
+# If you push multiple tags at once, separate instances of this workflow will
+# spin up, creating an independent Github Release™ for each one.
+#
+# If there's a prerelease-style suffix to the version then the Github Release™
+# will be marked as a prerelease.
 on:
   push:
     tags:
-      - 'v*.*.*'
+      - '*-?v[0-9]+*'
 
 jobs:
-  lint_test:
+  # Create the Github Release™ so the packages have something to be uploaded to
+  create-release:
     runs-on: ubuntu-latest
+    outputs:
+      has-releases: ${{ steps.create-release.outputs.has-releases }}
     env:
-      RUSTFLAGS: "-C target-feature=+aes,+sse2"
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
+      - uses: actions/checkout@v3
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@stable
-        with:
-          components: clippy
+        run: rustup update 1.71.0 --no-self-update && rustup default 1.71.0
+      - name: Install cargo-dist
+        run: curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.0.7/cargo-dist-installer.sh | sh
+      - id: create-release
+        run: |
+          cargo dist plan --tag=${{ github.ref_name }} --output-format=json > dist-manifest.json
+          echo "dist plan ran successfully"
+          cat dist-manifest.json
 
-      - name: Run cargo check
-        run: cargo check
+          # Create the Github Release™ based on what cargo-dist thinks it should be
+          ANNOUNCEMENT_TITLE=$(jq --raw-output ".announcement_title" dist-manifest.json)
+          IS_PRERELEASE=$(jq --raw-output ".announcement_is_prerelease" dist-manifest.json)
+          jq --raw-output ".announcement_github_body" dist-manifest.json > new_dist_announcement.md
+          gh release create ${{ github.ref_name }} --draft --prerelease="$IS_PRERELEASE" --title="$ANNOUNCEMENT_TITLE" --notes-file=new_dist_announcement.md
+          echo "created announcement!"
 
-      - name: Run cargo clippy
-        run: cargo clippy -- -D warnings
+          # Upload the manifest to the Github Release™
+          gh release upload ${{ github.ref_name }} dist-manifest.json
+          echo "uploaded manifest!"
 
-      - name: Run tests
-        run: cargo test -- --test-threads=1
+          # Disable all the upload-artifacts tasks if we have no actual releases
+          HAS_RELEASES=$(jq --raw-output ".releases != null" dist-manifest.json)
+          echo "has-releases=$HAS_RELEASES" >> "$GITHUB_OUTPUT"
 
-  build:
-    needs: lint_test
-    runs-on: ${{ matrix.os }}
+  # Build and packages all the things
+  upload-artifacts:
+    # Let the initial task tell us to not run (currently very blunt)
+    needs: create-release
+    if: ${{ needs.create-release.outputs.has-releases == 'true' }}
     strategy:
       matrix:
+        # For these target platforms
         include:
-          - target: aarch64-unknown-linux-gnu
-            os: ubuntu-latest
-            artifact_name: linux-aarch64
-            rustflags: "-C target-feature=+aes,+neon"
-          - target: x86_64-unknown-linux-gnu
-            os: ubuntu-latest
-            artifact_name: linux-amd64
-            rustflags: "-C target-feature=+aes,+sse2"
-          - target: x86_64-apple-darwin
-            os: macos-14
-            artifact_name: macos-amd64
-            rustflags: "-C target-feature=+aes,+sse2"
-          - target: aarch64-apple-darwin
-            os: macos-14
-            artifact_name: macos-arm64
-            rustflags: "-C target-feature=+aes,+neon"
+        - os: macos-11
+          dist-args: --artifacts=local --target=aarch64-apple-darwin --target=x86_64-apple-darwin
+          install-dist: curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.0.7/cargo-dist-installer.sh | sh
+        - os: ubuntu-20.04
+          dist-args: --artifacts=local --target=x86_64-unknown-linux-gnu
+          install-dist: curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.0.7/cargo-dist-installer.sh | sh
+        - os: windows-2019
+          dist-args: --artifacts=local --target=x86_64-pc-windows-msvc
+          install-dist: irm  https://github.com/axodotdev/cargo-dist/releases/download/v0.0.7/cargo-dist-installer.ps1 | iex
 
+    runs-on: ${{ matrix.os }}
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
+      - uses: actions/checkout@v3
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@stable
-
-      - name: Add target
-        run: rustup target add ${{ matrix.target }}
-
-      - name: Install cross-compilation toolchain (Linux ARM only)
-        if: matrix.target == 'aarch64-unknown-linux-gnu'
+        run: rustup update 1.71.0 --no-self-update && rustup default 1.71.0
+      - name: Install cargo-dist
+        run: ${{ matrix.install-dist }}
+      - name: Run cargo-dist
+        # This logic is a bit janky because it's trying to be a polyglot between
+        # powershell and bash since this will run on windows, macos, and linux!
+        # The two platforms don't agree on how to talk about env vars but they
+        # do agree on 'cat' and '$()' so we use that to marshal values between commands.
         run: |
-          sudo apt-get update
-          sudo apt-get install -y gcc-aarch64-linux-gnu
+          # Actually do builds and make zips and whatnot
+          cargo dist build --tag=${{ github.ref_name }} --output-format=json ${{ matrix.dist-args }} > dist-manifest.json
+          echo "dist ran successfully"
+          cat dist-manifest.json
 
-      - name: Build release binary
-        env:
-          RUSTFLAGS: ${{ matrix.rustflags }}
-          CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: ${{ matrix.target == 'aarch64-unknown-linux-gnu' && 'aarch64-linux-gnu-gcc' || '' }}
-        run: |
-          if [ "${{ matrix.target }}" = "aarch64-unknown-linux-gnu" ]; then
-            export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc
-          fi
-          cargo build --release --target ${{ matrix.target }}
+          # Parse out what we just built and upload it to the Github Release™
+          jq --raw-output ".artifacts[]?.path | select( . != null )" dist-manifest.json > uploads.txt
+          echo "uploading..."
+          cat uploads.txt
+          gh release upload ${{ github.ref_name }} $(cat uploads.txt)
+          echo "uploaded!"
 
-      - name: Package binary
-        run: |
-          if [[ "${{ matrix.os }}" == macos* ]]; then
-            BINARY_NAME=$(find target/${{ matrix.target }}/release -maxdepth 1 -type f -perm +111 -print0 | xargs -0 basename | head -1)
-          else
-            BINARY_NAME=$(find target/${{ matrix.target }}/release -maxdepth 1 -type f -executable -print0 | xargs -0 basename | head -1)
-          fi
-          
-          echo "Packaging binary: $BINARY_NAME"
-          mkdir -p release
-          cp target/${{ matrix.target }}/release/$BINARY_NAME release/$BINARY_NAME
-          tar -C release -czf ${{ matrix.artifact_name }}.tar.gz $BINARY_NAME
-
-      - name: Upload artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{ matrix.artifact_name }}-binary
-          path: ${{ matrix.artifact_name }}.tar.gz
-
-  create_release:
-    needs: build
+  # Mark the Github Release™ as a non-draft now that everything has succeeded!
+  publish-release:
+    # Only run after all the other tasks, but it's ok if upload-artifacts was skipped
+    needs: [create-release, upload-artifacts]
+    if: ${{ always() && needs.create-release.result == 'success' && (needs.upload-artifacts.result == 'skipped' || needs.upload-artifacts.result == 'success') }}
     runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Download all artifacts
-        uses: actions/download-artifact@v4
-        with:
-          path: artifacts
-
-      - name: Create GitHub Release
-        id: create_release
-        uses: softprops/action-gh-release@v1
-        with:
-          tag_name: ${{ github.ref_name }}
-          name: Release ${{ github.ref_name }}
-          body: "Automated release for version ${{ github.ref_name }}"
-          files: |
-            artifacts/*/*.tar.gz
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/checkout@v3
+      - name: mark release as non-draft
+        run: |
+          gh release edit ${{ github.ref_name }} --draft=false

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deduplicator"
-version = "0.3.1"
+version = "0.3.0"
 edition = "2021"
 description = "find,filter and delete duplicate files"
 repository = "https://github.com/sreedevk/deduplicator"
@@ -18,14 +18,14 @@ path = "src/main.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 [dependencies]
 anyhow = "1.0.68"
-bytesize = "2.0.1"
+bytesize = "1.1.0"
 chrono = "0.4.23"
 clap = { version = "4.0.32", features = ["derive"] }
-dashmap = { version = "6.1.0", features = ["rayon"] }
-globwalk = "0.9.1"
+dashmap = { version = "5.4.0", features = ["rayon"] }
+globwalk = "0.8.1"
 gxhash = { version = "3.4.1", default-features = false }
-indicatif = { version = "0.18.0", features = ["rayon"] }
-memmap2 = "0.9.7"
+indicatif = { version = "0.17.2", features = ["rayon"] }
+memmap2 = "0.5.8"
 pathdiff = "0.2.1"
 prettytable-rs = "0.10.0"
 rand = "0.9.1"

README.md

@@ -15,17 +15,16 @@ Arguments:
   [scan_dir_path]  Run Deduplicator on dir different from pwd (e.g., ~/Pictures )
 
 Options:
-  -T, --exclude-types <EXCLUDE_TYPES>  Exclude Filetypes [default = none]
-  -t, --types <TYPES>                  Filetypes to deduplicate [default = all]
-  -i, --interactive                    Delete files interactively
-  -m, --min-size <MIN_SIZE>            Minimum filesize of duplicates to scan (e.g., 100B/1K/2M/3G/4T) [default: 1b]
-  -D, --max-depth <MAX_DEPTH>          Max Depth to scan while looking for duplicates
-  -d, --min-depth <MIN_DEPTH>          Min Depth to scan while looking for duplicates
-  -f, --follow-links                   Follow links while scanning directories
-  -s, --strict                         Guarantees that two files are duplicate (performs a full hash)
-  -p, --progress                       Show Progress spinners & metrics
-  -h, --help                           Print help
-  -V, --version                        Print version
+  -t, --types <TYPES>          Filetypes to deduplicate [default = all]
+  -i, --interactive            Delete files interactively
+  -m, --min-size <MIN_SIZE>    Minimum filesize of duplicates to scan (e.g., 100B/1K/2M/3G/4T) [default: 1b]
+  -D, --max-depth <MAX_DEPTH>  Max Depth to scan while looking for duplicates
+  -d, --min-depth <MIN_DEPTH>  Min Depth to scan while looking for duplicates
+  -f, --follow-links           Follow links while scanning directories
+  -s, --strict                 Guarantees that two files are duplicate (performs a full hash)
+  -p, --progress               Show Progress spinners & metrics
+  -h, --help                   Print help
+  -V, --version                Print version
 ```
 ### Examples
 
@@ -33,9 +32,6 @@ Options:
 # Scan for duplicates recursively from the current dir, only look for png, jpg & pdf file types & interactively delete files
 deduplicator -t pdf,jpg,png -i
 
-# Scan for duplicates recursively from current dir, exclude png and jpg file types
-deduplicator -T jpg,png
-
 # Scan for duplicates recursively from the ~/Pictures dir, only look for png, jpeg, jpg & pdf file types & interactively delete files
 deduplicator ~/Pictures/ -t png,jpeg,jpg,pdf -i
 
@@ -61,23 +57,14 @@ Currently, you can only install deduplicator using cargo package manager.
 > GxHash relies on aes hardware acceleration, so please set `RUSTFLAGS` to `"-C target-feature=+aes"` or `"-C target-cpu=native"` before
 > installing.
 
-#### install from crates.io (stable)
-
+#### install from crates.io
 ```bash
 $ RUSTFLAGS="-C target-cpu=native" cargo install deduplicator
-
-# or
-
-$ RUSTFLAGS="-C target-feature=+aes,+sse2" cargo install deduplicator
 ```
 
-#### install from git (nightly)
+#### install from git
 ```bash
 $ RUSTFLAGS="-C target-cpu=native" cargo install deduplicator --git https://github.com/sreedevk/deduplicator
-
-# or
-
-$ RUSTFLAGS="-C target-feature=+aes,+sse2" cargo install --git https://github.com/sreedevk/deduplicator
 ```
 
 ## Performance
@@ -90,98 +77,77 @@ I've used hyperfine to run deduplicator on files generated by the rake file at `
 ```
 # hyperfine -N --warmup 80 './target/release/deduplicator bench_artifacts'
 Benchmark 1: ./target/release/deduplicator bench_artifacts
-  Time (mean ± σ):       2.2 ms ±   0.4 ms    [User: 2.2 ms, System: 4.4 ms]
-  Range (min … max):     1.3 ms …   7.1 ms    1522 runs
+  Time (mean ± σ):       2.5 ms ±   0.4 ms    [User: 2.2 ms, System: 4.8 ms]
+  Range (min … max):     1.9 ms …   6.8 ms    1322 runs
 
-dust 'bench_artifacts'
-
- 54M   ┌── file_0_fwds.bin   │████                                                               │   2%
-122M   ├── file_1_fwds.bin   │████████                                                           │   5%
-390M   ├── file_0_fwdcbss.bin│██████████████████████████                                         │  15%
-390M   ├── file_0_fwscas.bin │██████████████████████████                                         │  15%
-390M   ├── file_0_fwss.bin   │██████████████████████████                                         │  15%
-390M   ├── file_1_fwdcbss.bin│██████████████████████████                                         │  15%
-390M   ├── file_1_fwscas.bin │██████████████████████████                                         │  15%
-390M   ├── file_1_fwss.bin   │██████████████████████████                                         │  15%
-2.5G ┌─┴ bench_artifacts     │██████████████████████████████████████████████████████████████████ │ 100%
+# dust 'bench_artifacts'
+ 37M   ┌── file_1_fwds.bin   │██                                                                                                                              │   1%
+201M   ├── file_0_fwds.bin   │██████████                                                                                                                      │   8%
+390M   ├── file_0_fwdcbss.bin│████████████████████                                                                                                            │  15%
+390M   ├── file_0_fwscas.bin │████████████████████                                                                                                            │  15%
+390M   ├── file_0_fwss.bin   │████████████████████                                                                                                            │  15%
+390M   ├── file_1_fwdcbss.bin│████████████████████                                                                                                            │  15%
+390M   ├── file_1_fwscas.bin │████████████████████                                                                                                            │  15%
+390M   ├── file_1_fwss.bin   │████████████████████                                                                                                            │  15%
+2.5G ┌─┴ bench_artifacts     │███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ │ 100%
 ```
 
 #### Many Small Files
 ```
 # hyperfine --warmup 20 './target/release/deduplicator bench_artifacts'
 Benchmark 1: ./target/release/deduplicator bench_artifacts
-  Time (mean ± σ):      40.1 ms ±   2.3 ms    [User: 251.0 ms, System: 277.3 ms]
-  Range (min … max):    35.0 ms …  45.9 ms    72 runs
+  Time (mean ± σ):      22.3 ms ±   1.7 ms    [User: 35.8 ms, System: 46.3 ms]
+  Range (min … max):    18.9 ms …  27.2 ms    112 runs
 
-dust 'bench_artifacts'
-3.9M   ┌── file_992_fwscas.bin │█                                                                │   0%
-3.9M   ├── file_992_fwss.bin   │█                                                                │   0%
-3.9M   ├── file_993_fwdcbss.bin│█                                                                │   0%
-3.9M   ├── file_993_fwscas.bin │█                                                                │   0%
-3.9M   ├── file_993_fwss.bin   │█                                                                │   0%
-3.9M   ├── file_994_fwdcbss.bin│█                                                                │   0%
-3.9M   ├── file_994_fwscas.bin │█                                                                │   0%
-3.9M   ├── file_994_fwss.bin   │█                                                                │   0%
-3.9M   ├── file_995_fwdcbss.bin│█                                                                │   0%
-3.9M   ├── file_995_fwscas.bin │█                                                                │   0%
-3.9M   ├── file_995_fwss.bin   │█                                                                │   0%
-3.9M   ├── file_996_fwdcbss.bin│█                                                                │   0%
-3.9M   ├── file_996_fwscas.bin │█                                                                │   0%
-3.9M   ├── file_996_fwss.bin   │█                                                                │   0%
-3.9M   ├── file_997_fwdcbss.bin│█                                                                │   0%
-3.9M   ├── file_997_fwscas.bin │█                                                                │   0%
-3.9M   ├── file_997_fwss.bin   │█                                                                │   0%
-3.9M   ├── file_998_fwdcbss.bin│█                                                                │   0%
-3.9M   ├── file_998_fwscas.bin │█                                                                │   0%
-3.9M   ├── file_998_fwss.bin   │█                                                                │   0%
-3.9M   ├── file_999_fwdcbss.bin│█                                                                │   0%
-3.9M   ├── file_999_fwscas.bin │█                                                                │   0%
-3.9M   ├── file_999_fwss.bin   │█                                                                │   0%
-3.9M   ├── file_99_fwdcbss.bin │█                                                                │   0%
-3.9M   ├── file_99_fwscas.bin  │█                                                                │   0%
-3.9M   ├── file_99_fwss.bin    │█                                                                │   0%
-3.9M   ├── file_9_fwdcbss.bin  │█                                                                │   0%
-3.9M   ├── file_9_fwscas.bin   │█                                                                │   0%
-3.9M   ├── file_9_fwss.bin     │█                                                                │   0%
- 11G ┌─┴ bench_artifacts       │████████████████████████████████████████████████████████████████ │ 100%
+# dust 'bench_artifacts'
+3.9M   ┌── file_992_fwss.bin   │█                                                                                                                                             │   0%
+3.9M   ├── file_993_fwdcbss.bin│█                                                                                                                                             │   0%
+3.9M   ├── file_993_fwscas.bin │█                                                                                                                                             │   0%
+3.9M   ├── file_993_fwss.bin   │█                                                                                                                                             │   0%
+3.9M   ├── file_994_fwdcbss.bin│█                                                                                                                                             │   0%
+3.9M   ├── file_994_fwscas.bin │█                                                                                                                                             │   0%
+3.9M   ├── file_994_fwss.bin   │█                                                                                                                                             │   0%
+3.9M   ├── file_995_fwdcbss.bin│█                                                                                                                                             │   0%
+3.9M   ├── file_995_fwscas.bin │█                                                                                                                                             │   0%
+3.9M   ├── file_995_fwss.bin   │█                                                                                                                                             │   0%
+3.9M   ├── file_996_fwdcbss.bin│█                                                                                                                                             │   0%
+3.9M   ├── file_996_fwscas.bin │█                                                                                                                                             │   0%
+3.9M   ├── file_996_fwss.bin   │█                                                                                                                                             │   0%
+3.9M   ├── file_997_fwdcbss.bin│█                                                                                                                                             │   0%
+3.9M   ├── file_997_fwscas.bin │█                                                                                                                                             │   0%
+3.9M   ├── file_997_fwss.bin   │█                                                                                                                                             │   0%
+3.9M   ├── file_998_fwdcbss.bin│█                                                                                                                                             │   0%
+3.9M   ├── file_998_fwscas.bin │█                                                                                                                                             │   0%
+3.9M   ├── file_998_fwss.bin   │█                                                                                                                                             │   0%
+3.9M   ├── file_999_fwdcbss.bin│█                                                                                                                                             │   0%
+3.9M   ├── file_999_fwscas.bin │█                                                                                                                                             │   0%
+3.9M   ├── file_999_fwss.bin   │█                                                                                                                                             │   0%
+3.9M   ├── file_99_fwdcbss.bin │█                                                                                                                                             │   0%
+3.9M   ├── file_99_fwscas.bin  │█                                                                                                                                             │   0%
+3.9M   ├── file_99_fwss.bin    │█                                                                                                                                             │   0%
+3.9M   ├── file_9_fwdcbss.bin  │█                                                                                                                                             │   0%
+3.9M   ├── file_9_fwscas.bin   │█                                                                                                                                             │   0%
+3.9M   ├── file_9_fwss.bin     │█                                                                                                                                             │   0%
+ 11G ┌─┴ bench_artifacts       │█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ │ 100%
 ```
 
 ## proposed
 - [ ] parallelization
+    - [ ] (scanning + processing sw + processing hw) & formatting & printing
     - [ ] scanning + processing sw + processing hw + formatting + printing
 - [ ] max file path size should use the last set of duplicates
 - [ ] add more unit tests
     - [ ] test against different filesystems
     - [ ] test against different file name encodings
-- [ ] restore json output (was removed in 0.3 due to quality issues)
+- [ ] restore json output (was removed in 0.3)
 - [ ] fix memory leak on very large filesystems
     - [ ] maybe use a bloom filter
     - [ ] reduce FileInfo size
+- [ ] output in a tree format
 - [ ] tui
 - [ ] change the default hashing method to include the first & last page of a file (8K)
-- [ ] provide option to localize duplicate detection to arbitrary levels relative to current directory
-- [ ] bulk operations
-    - [ ] --keep-latest
-    - [ ] --keep-oldest
-    - [ ] --keep-last-modified
-    - [ ] --keep-first-modified
 
-- [ ] fix: partial hash collision - a file full of null bytes ("\0") and an empty file. This is a known trade off in gxhash.
-    - [ ] include initial pages and final pages of the file 
-    - [ ] append the offset between the last initial page hashed and the first final page hashed in the content passed to the hasher.
-
-## v0.3.1
-- [x] parallelization
-    - [x] (scanning + processing sw + processing hw) & formatting & printing
-- [x] remove formatting step and write directly to stdout
-- [x] simplify output to improve performance
-- [x] increase the number of pages hashed in partial hashing
-- [x] updated dependencies
-- [x] fix: full file hash collision between a file full of null bytes ("\0") and an empty file. This is a known trade off in gxhash.
-    - [x] appending the file size at the end of content before hashing.
-- [x] created an automated release pipeline
-
-## v0.3.0
+## v0.3
 - [x] parallelization
     - [x] (scanning) + (processing sw & processing hw & formatting & printing)
 - [x] reduce cloning values on the heap

rakelib/benchmark.rake

@@ -13,11 +13,9 @@ namespace :benchmark do
     # Range (min … max):     1.8 ms …   9.7 ms    1474 runs
 
     root = "bench_artifacts"
-    FileUtils.rm_rf(root)
     Dir.mkdir(root)
 
     # files with same size
-    puts "generating files of same size ..."
     2.times.map do |i|
       File.open(File.join(root, "file_#{i}_fwss.bin"), 'wb') do |f|
         f.write(SecureRandom.bytes(4096 * 100_000))
@@ -25,7 +23,6 @@ namespace :benchmark do
     end
 
     # files with different sizes
-    puts "generating files of different sizes ..."
     2.times.map do |i|
       File.open(File.join(root, "file_#{i}_fwds.bin"), 'wb') do |f|
         f.write(SecureRandom.bytes(4096 * (rand * 100_000).ceil))
@@ -33,7 +30,6 @@ namespace :benchmark do
     end
 
     # files with same content & size
-    puts "generating files of same content and sizes ..."
     2.times.each do |i|
       File.open(File.join(root, "file_#{i}_fwscas.bin"), 'wb') do |f|
         f.write("\0" * (4096 * 100_000))
@@ -41,7 +37,6 @@ namespace :benchmark do
     end
 
     # files with different content but same size
-    puts "generating files of different content but same sizes ..."
     2.times.each do |i|
       File.open(File.join(root, "file_#{i}_fwdcbss.bin"), 'wb') do |f|
         f.write(SecureRandom.bytes(4096 * 100_000))
@@ -63,7 +58,6 @@ namespace :benchmark do
     Dir.mkdir(root)
 
     # files with same size
-    puts "generating 1000 files of the same size ... "
     1000.times.each do |i|
       File.open(File.join(root, "file_#{i}_fwss.bin"), 'wb') do |f|
         f.write(SecureRandom.bytes(4096 * 1000))
@@ -71,7 +65,6 @@ namespace :benchmark do
     end
 
     # files with different sizes
-    puts "generating 1000 files of different sizes ... "
     1000.times.each do |i|
       File.open(File.join(root, "file_#{i}_fwds.bin"), 'wb') do |f|
         f.write(SecureRandom.bytes(4096 * (rand * 100).ceil))
@@ -79,7 +72,6 @@ namespace :benchmark do
     end
 
     # files with same content & size
-    puts "generating files of same content and sizes ..."
     1000.times.each do |i|
       File.open(File.join(root, "file_#{i}_fwscas.bin"), 'wb') do |f|
         f.write("\0" * (4096 * 1000))
@@ -87,7 +79,6 @@ namespace :benchmark do
     end
 
     # files with different content but same size
-    puts "generating files of different content but same sizes ..."
     1000.times.each do |i|
       File.open(File.join(root, "file_#{i}_fwdcbss.bin"), 'wb') do |f|
         f.write(SecureRandom.bytes(4096 * 1000))

src/fileinfo.rs

@@ -5,43 +5,30 @@ use std::{
     fs,
     io::Read,
     path::{Path, PathBuf},
-    sync::{Arc, Mutex},
     time::SystemTime,
 };
 
-#[derive(Debug, Clone, PartialEq)]
-pub enum FileState {
-    Unprocessed,
-    SwProcessed,
-}
-
 #[derive(Debug, Clone)]
 pub struct FileInfo {
     pub path: Box<Path>,
     pub size: u64,
     pub modified: SystemTime,
-    pub state: Arc<Mutex<FileState>>,
 }
 
 impl FileInfo {
     pub fn hash(&self, seed: i64) -> Result<u128> {
-        if self.size == 0 {
-            return Ok(0u128);
-        };
-
         let file = fs::File::open(&self.path)?;
         let mapper = unsafe { Mmap::map(&file)? };
-        let content_hash = mapper
-            .chunks(4096)
-            .fold(0u128, |acc, chunk: &[u8]| acc ^ gxhash128(chunk, seed));
+        let final_hash = mapper.chunks(4096).fold(0u128, |acc, chunk: &[u8]| {
+            acc.wrapping_add(gxhash128(chunk, seed))
+        });
 
-        // NOTE: avoids collision bw an empty file & a file full of null bytes.
-        Ok(content_hash ^ gxhash128(&self.size.to_ne_bytes(), seed))
+        Ok(final_hash)
     }
 
-    pub fn initpages_hash(&self, seed: i64) -> Result<u128> {
+    pub fn initial_page_hash(&self, seed: i64) -> Result<u128> {
         let mut file = fs::File::open(&self.path)?;
-        let mut buffer = [0; 16384];
+        let mut buffer = [0; 4096];
         let bytes_read = file.read(&mut buffer)?;
 
         Ok(gxhash128(&buffer[..bytes_read], seed))
@@ -53,52 +40,6 @@ impl FileInfo {
             path: path.into_boxed_path(),
             size: filemeta.len(),
             modified: filemeta.modified()?,
-            state: Arc::new(Mutex::new(FileState::Unprocessed)),
         })
     }
-
-    pub fn sw_processed(&self) {
-        let mut self_state = self.state.lock().unwrap();
-        *self_state = FileState::SwProcessed;
-    }
-
-    pub fn is_sw_processed(&self) -> bool {
-        let self_state = self.state.lock().unwrap();
-        *self_state == FileState::SwProcessed
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-    use tempfile::TempDir;
-    use std::fs::File;
-    use std::io::Write;
-    use anyhow::Result;
-
-    fn generate_null_bytes(size: usize) -> Vec<u8> {
-        (0..size).map(|_| 0).collect::<Vec<u8>>()
-    }
-
-    #[test]
-    fn hash_differentiates_between_a_file_of_null_bytes_vs_an_empty_file() -> Result<()> {
-        let root = TempDir::new()?;
-        let empty_file_name = root.path().join("empty_file.bin");
-
-        File::create_new(&empty_file_name)?;
-
-        let file_with_null_bytes_name = root.path().join("file_with_null_bytes.bin");
-        let mut file_with_null_bytes = File::create_new(&file_with_null_bytes_name)?;
-
-        file_with_null_bytes.write_all(&generate_null_bytes(1000 * 4096))?;
-
-        let empty_file_info = FileInfo::new(empty_file_name)?;
-        let file_with_empty_bytes_info = FileInfo::new(file_with_null_bytes_name)?;
-
-        let seed: i64 = 246910456374;
-
-        assert_ne!(empty_file_info.hash(seed)?, file_with_empty_bytes_info.hash(seed)?);
-
-        Ok(())
-    }
 }

src/formatter.rs

@@ -2,17 +2,20 @@ use crate::{fileinfo::FileInfo, params::Params};
 use anyhow::Result;
 use chrono::{DateTime, Utc};
 use dashmap::DashMap;
+use indicatif::{ParallelProgressIterator, ProgressBar, ProgressFinish, ProgressStyle};
 use pathdiff::diff_paths;
+use prettytable::{format, row, Row, Table};
 use rayon::prelude::*;
-use std::{path::PathBuf, sync::Arc};
-
-const YELLOW: &str = "\x1b[33m";
-const RESET: &str = "\x1b[0m";
+use std::{borrow::Cow, path::PathBuf, sync::Arc, time::Duration};
 
 pub struct Formatter;
 impl Formatter {
-    pub fn human_path(file: &FileInfo, aargs: &Params, min_path_length: usize) -> Result<String> {
-        let base_directory: PathBuf = aargs.get_directory()?;
+    pub fn human_path(
+        file: &FileInfo,
+        app_args: &Params,
+        min_path_length: usize,
+    ) -> Result<String> {
+        let base_directory: PathBuf = app_args.get_directory()?;
         let relative_path = diff_paths(&file.path, base_directory).unwrap_or_default();
 
         let formatted_path = format!(
@@ -33,32 +36,66 @@ impl Formatter {
         Ok(modified_time.format("%Y-%m-%d %H:%M:%S").to_string())
     }
 
-    pub fn print(raw: Arc<DashMap<u128, Vec<FileInfo>>>, max_path_len: u64, aargs: &Params) {
-        print!("{}", "\n".repeat(if aargs.progress { 2 } else { 1 })); // spacing
+    pub fn gen_sub_tbl(items: Vec<FileInfo>, app_args: &Params, max_path_len: u64) -> Table {
+        let mut inner_table = Table::new();
+        inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
+        items.iter().for_each(|file| {
+            inner_table.add_row(row![
+                Self::human_path(file, app_args, max_path_len as usize).unwrap_or_default(),
+                Self::human_filesize(file).unwrap_or_default(),
+                Self::human_mtime(file).unwrap_or_default()
+            ]);
+        });
+        inner_table
+    }
 
+    pub fn generate_table(
+        raw: Arc<DashMap<u128, Vec<FileInfo>>>,
+        mpath_len: u64,
+        args: &Params,
+    ) -> Result<Table> {
+        let progress_bar = match args.progress {
+            true => ProgressBar::new_spinner(),
+            false => ProgressBar::hidden(),
+        };
+
+        let progress_style = ProgressStyle::with_template("[{elapsed_precise}] {pos:>7} {msg}")?;
+        progress_bar.set_style(progress_style);
+        progress_bar.enable_steady_tick(Duration::from_millis(50));
+        progress_bar.set_message("generating output");
+
+        let rows = raw
+            .par_iter_mut()
+            .progress_with(progress_bar)
+            .with_finish(ProgressFinish::WithMessage(Cow::from("output generated")))
+            .filter(|i| i.value().len() > 1)
+            .map(|i| {
+                row![
+                    i.key(),
+                    Self::gen_sub_tbl(i.value().to_vec(), args, mpath_len)
+                ]
+            })
+            .collect::<Vec<Row>>();
+
+        let mut output_table = Table::new();
+        output_table.set_titles(row!["hash", "duplicates"]);
+        output_table.extend(rows);
+        Ok(output_table)
+    }
+
+    pub fn print(
+        raw: Arc<DashMap<u128, Vec<FileInfo>>>,
+        max_path_len: u64,
+        app_args: &Params,
+    ) -> Result<()> {
         if raw.is_empty() {
-            println!("No duplicates found matching your search criteria.");
-        } else {
-            raw.par_iter().for_each(|sref| {
-                let mut ostring = format!("{}{:32x}{}\n", YELLOW, sref.key(), RESET);
-                let subfields = sref
-                    .value()
-                    .par_iter()
-                    .map(|finfo| {
-                        format!(
-                            "├─ {}\t{}\t{}\n",
-                            Self::human_path(finfo, aargs, max_path_len as usize)
-                                .expect("path formatting failed."),
-                            Self::human_filesize(finfo).expect("filesize formatting failed."),
-                            Self::human_mtime(finfo).expect("modified time formatting failed.")
-                        )
-                    })
-                    .collect::<String>();
-
-                ostring.push_str(&subfields);
-
-                println!("{ostring}");
-            });
+            println!("\n\nNo duplicates found matching your search criteria.\n");
+            return Ok(());
         }
+
+        let output_table = Self::generate_table(raw, max_path_len, app_args)?;
+        output_table.printstd();
+
+        Ok(())
     }
 }

src/main.rs

@@ -24,7 +24,7 @@ fn main() -> Result<()> {
                 server.hw_duplicate_set,
                 server.max_file_path_len.load(Ordering::Acquire),
                 &app_args,
-            );
+            )?;
         }
         true => {
             Interactive::init(server.hw_duplicate_set, &app_args)?;

src/params.rs

@@ -2,14 +2,10 @@ use std::{fs, path::PathBuf};
 
 use anyhow::Result;
 use clap::{Parser, ValueHint};
-use std::collections::HashSet;
 
 #[derive(Parser, Debug, Default, Clone)]
 #[command(author, version, about, long_about = None)]
 pub struct Params {
-    /// Exclude Filetypes [default = none]
-    #[arg(short = 'T', long)]
-    pub exclude_types: Option<String>,
     /// Filetypes to deduplicate [default = all]
     #[arg(short, long)]
     pub types: Option<String>,
@@ -57,47 +53,7 @@ impl Params {
         Ok(dir)
     }
 
-    pub fn types_intersection(itypes: &str, xtypes: &str) -> String {
-        let iset = itypes
-            .split(",")
-            .map(String::from)
-            .collect::<HashSet<String>>();
-        let xset = xtypes
-            .split(",")
-            .map(String::from)
-            .collect::<HashSet<String>>();
-
-        iset.difference(&xset)
-            .cloned()
-            .collect::<Vec<String>>()
-            .join(",")
-    }
-
     pub fn get_types(&self) -> Option<String> {
-        match &self.types {
-            Some(itypes) => match &self.exclude_types {
-                Some(xtypes) => Some(Self::types_intersection(itypes, xtypes)),
-                None => Some(itypes.to_string()),
-            },
-            None => self.exclude_types.as_ref().map(|xtypes| xtypes.to_string()),
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn mixing_include_and_exclude_types_works_as_expected() {
-        let params = Params {
-            types: Some(String::from("js,xml,ts,pdf,tiff")),
-            exclude_types: Some(String::from("js,ts,xml")),
-            ..Default::default()
-        };
-
-        assert!(params
-            .get_types()
-            .is_some_and(|x| x == "pdf,tiff" || x == "tiff,pdf"))
+        self.types.clone()
     }
 }

src/processor.rs

@@ -1,8 +1,10 @@
 use anyhow::Result;
 use dashmap::DashMap;
-use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
-use rayon::iter::IntoParallelRefMutIterator;
+use indicatif::{
+    MultiProgress, ParallelProgressIterator, ProgressBar, ProgressFinish, ProgressStyle,
+};
 use rayon::prelude::{IntoParallelIterator, ParallelIterator};
+use std::borrow::Cow;
 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 use std::sync::{Arc, Mutex, TryLockError, TryLockResult};
 use std::time::Duration;
@@ -20,68 +22,57 @@ impl Processor {
         progress_bar_box: Arc<MultiProgress>,
         max_file_size: Arc<AtomicU64>,
         seed: i64,
-        sw_sorting_finished: Arc<AtomicBool>,
     ) -> Result<()> {
         let progress_bar = match app_args.progress {
             true => progress_bar_box.add(ProgressBar::new_spinner()),
             false => ProgressBar::hidden(),
         };
 
+        let keys: Vec<u64> = sw_store.clone().iter().map(|i| *i.key()).collect();
+
         let progress_style = ProgressStyle::with_template("[{elapsed_precise}] {pos:>7} {msg}")?;
         progress_bar.set_style(progress_style);
         progress_bar.enable_steady_tick(Duration::from_millis(50));
         progress_bar.set_message("files grouped by hash.");
 
-        loop {
-            let keys: Vec<u64> = sw_store
-                .clone()
-                .iter()
-                .filter(|i| !i.value().iter().all(|x| x.is_sw_processed()))
-                .filter(|i| i.value().len() > 1)
-                .map(|i| *i.key())
-                .collect();
+        keys.into_par_iter()
+            .progress_with(progress_bar)
+            .with_finish(ProgressFinish::WithMessage(Cow::from(
+                "files grouped by hash.",
+            )))
+            .for_each(|key| {
+                let group: Vec<FileInfo> = sw_store.get(&key).unwrap().to_vec();
+                if group.len() > 1 {
+                    group.into_par_iter().for_each(|file| {
+                        let fhash = if app_args.strict {
+                            file.hash(seed).expect("hashing file failed.")
+                        } else {
+                            file.initial_page_hash(seed).expect("hashing file failed.")
+                        };
 
-            if keys.is_empty() {
-                match sw_sorting_finished.load(std::sync::atomic::Ordering::Relaxed) {
-                    true => {
-                        progress_bar.finish_with_message("files grouped by hash.");
-                        break Ok(());
-                    }
-                    false => continue,
+                        Self::compare_and_update_max_path_len(
+                            max_file_size.clone(),
+                            file.path.to_string_lossy().len() as u64,
+                        )
+                        .unwrap();
+
+                        hw_store
+                            .entry(fhash)
+                            .and_modify(|fileset| fileset.push(file.clone()))
+                            .or_insert_with(|| vec![file]);
+                    });
                 }
-            } else {
-                keys.into_par_iter().for_each(|key| {
-                    let mut group: Vec<FileInfo> = sw_store.get(&key).unwrap().to_vec();
-                    if group.len() > 1 {
-                        group.par_iter_mut().for_each(|file| {
-                            progress_bar.inc(1);
-                            file.sw_processed();
+            });
 
-                            let fhash = match app_args.strict {
-                                true => file.hash(seed).expect("hashing file failed."),
-                                false => file.initpages_hash(seed).expect("hashing file failed."),
-                            };
-
-                            Self::compare_and_update_max_path_len(
-                                max_file_size.clone(),
-                                file.path.to_string_lossy().len() as u64,
-                            );
-
-                            hw_store
-                                .entry(fhash)
-                                .and_modify(|fileset| fileset.push(file.clone()))
-                                .or_insert_with(|| vec![file.clone()]);
-                        });
-                    };
-                });
-            }
-        }
+        Ok(())
     }
 
-    pub fn compare_and_update_max_path_len(current: Arc<AtomicU64>, next: u64) {
+    pub fn compare_and_update_max_path_len(current: Arc<AtomicU64>, next: u64) -> Result<()> {
         if current.load(Ordering::Relaxed) < next {
             current.store(next, Ordering::Release);
         }
+
+        Ok(())
     }
 
     pub fn sizewise(
@@ -153,9 +144,9 @@ mod tests {
     }
 
     #[test]
-    fn hashwise_sorting_two_files_with_identical_init_pages_only_strict_mode() -> Result<()> {
+    fn hashwise_sorting_two_files_with_identical_init_page_only_strict_mode() -> Result<()> {
         let root = TempDir::new()?;
-        let content = generate_bytes(16384);
+        let content = generate_bytes(4096);
 
         let mut content_x = content.clone();
         let mut content_y = content.clone();
@@ -202,7 +193,6 @@ mod tests {
             Arc::new(MultiProgress::new()),
             Arc::new(AtomicU64::new(32)),
             300,
-            Arc::new(AtomicBool::new(true)),
         )?;
 
         assert_eq!(hw_dupstore.len(), 2);
@@ -211,9 +201,9 @@ mod tests {
     }
 
     #[test]
-    fn hashwise_sorting_two_files_with_identical_init_pages_only_fast_mode() -> Result<()> {
+    fn hashwise_sorting_two_files_with_identical_init_page_only_fast_mode() -> Result<()> {
         let root = TempDir::new()?;
-        let content = generate_bytes(16384);
+        let content = generate_bytes(4096);
 
         let mut content_x = content.clone();
         let mut content_y = content.clone();
@@ -255,7 +245,6 @@ mod tests {
             Arc::new(MultiProgress::new()),
             Arc::new(AtomicU64::new(32)),
             300,
-            Arc::new(AtomicBool::new(true)),
         )?;
 
         assert_eq!(hw_dupstore.len(), 1);
@@ -301,7 +290,6 @@ mod tests {
             Arc::new(MultiProgress::new()),
             Arc::new(AtomicU64::new(32)),
             300,
-            Arc::new(AtomicBool::new(true)),
         )?;
 
         assert_eq!(hw_dupstore.len(), 1);

src/server.rs

@@ -43,61 +43,51 @@ impl Server {
         }
 
         let app_args_clone_for_sc = self.app_args.clone();
-        let app_args_clone_for_sw = self.app_args.clone();
-        let app_args_clone_for_hw = self.app_args.clone();
+        let app_args_clone_for_pr = self.app_args.clone();
         let file_queue_clone_sc = self.filequeue.clone();
         let file_queue_clone_pr = self.filequeue.clone();
         let scanner_finished = Arc::new(AtomicBool::new(false));
-        let sw_sort_finished = Arc::new(AtomicBool::new(false));
 
         let sfin_sc_tr_cl = scanner_finished.clone();
         let sfin_pr_tr_cl = scanner_finished.clone();
 
-        let swfin_pr_tr_sw = sw_sort_finished.clone();
-        let swfin_pr_tr_hw = sw_sort_finished.clone();
-
         let store_dupl_sw_for_sw = self.sw_duplicate_set.clone();
         let store_dupl_sw_for_hw = self.sw_duplicate_set.clone();
         let store_dupl_hw = self.hw_duplicate_set.clone();
         let max_file_path_len_clone = self.max_file_path_len.clone();
 
         let progbarbox_sc_clone = progbarbox.clone();
-        let progbarbox_pr_clone_for_sw = progbarbox.clone();
-        let progbarbox_pr_clone_for_hw = progbarbox.clone();
 
         self.threadpool.execute(move || {
             Scanner::new(app_args_clone_for_sc)
-                .expect("unable to initialize scanner.")
+                .unwrap()
                 .scan(file_queue_clone_sc, progbarbox_sc_clone)
-                .expect("scanner failed.");
+                .unwrap();
 
             sfin_sc_tr_cl.store(true, std::sync::atomic::Ordering::Relaxed);
         });
 
+        let progbarbox_pr_clone = progbarbox.clone();
+
         self.threadpool.execute(move || {
             Processor::sizewise(
-                app_args_clone_for_sw,
+                app_args_clone_for_pr.clone(),
                 sfin_pr_tr_cl,
                 store_dupl_sw_for_sw,
                 file_queue_clone_pr,
-                progbarbox_pr_clone_for_sw,
+                progbarbox_pr_clone.clone(),
             )
-            .expect("sizewise scanner failed.");
+            .unwrap();
 
-            swfin_pr_tr_sw.store(true, std::sync::atomic::Ordering::Relaxed);
-        });
-
-        self.threadpool.execute(move || {
             Processor::hashwise(
-                app_args_clone_for_hw,
+                app_args_clone_for_pr,
                 store_dupl_sw_for_hw,
                 store_dupl_hw,
-                progbarbox_pr_clone_for_hw,
+                progbarbox_pr_clone,
                 max_file_path_len_clone,
                 seed,
-                swfin_pr_tr_hw,
             )
-            .expect("sizewise scanner failed.");
+            .unwrap();
         });
 
         progbarbox.clear()?;