version 0.2.2 (#59)

* replaced fxhash with gxhash

.cargo/config.toml

@@ -0,0 +1,2 @@
+[build]
+rustflags = ["-C", "target-feature=+aes,+sse2"]

.github/workflows/release.yml

@@ -1,12 +1,12 @@
 # CI that:
 #
-# * checks for a Git Tag that looks like a release ("v1.2.0")
+# * checks for a Git Tag that looks like a release
-# * creates a Github Release™️
+# * creates a Github Release™ and fills in its text
-# * builds binaries/packages with cargo-dist
+# * builds artifacts with cargo-dist (executable-zips, installers)
-# * uploads those packages to the Github Release™️
+# * uploads those artifacts to the Github Release™
 #
-# Note that the Github Release™️ will be created before the packages,
+# Note that the Github Release™ will be created before the artifacts,
-# so there will be a few minutes where the release has no packages
+# so there will be a few minutes where the release has no artifacts
 # and then they will slowly trickle in, possibly failing. To make
 # this more pleasant we mark the release as a "draft" until all
 # artifacts have been successfully uploaded. This allows you to
@@ -17,114 +17,116 @@ name: Release
 permissions:
   contents: write
 
-# This task will run whenever you push a git tag that looks like
+# This task will run whenever you push a git tag that looks like a version
-# a version number. We just look for `v` followed by at least one number
+# like "v1", "v1.2.0", "v0.1.0-prerelease01", "my-app-v1.0.0", etc.
-# and then whatever. so `v1`, `v1.0.0`, and `v1.0.0-prerelease` all work.
+# The version will be roughly parsed as ({PACKAGE_NAME}-)?v{VERSION}, where
+# PACKAGE_NAME must be the name of a Cargo package in your workspace, and VERSION
+# must be a Cargo-style SemVer Version.
 #
-# If there's a prerelease-style suffix to the version then the Github Release™️
+# If PACKAGE_NAME is specified, then we will create a Github Release™ for that
-# will be marked as a prerelease (handled by taiki-e/create-gh-release-action).
+# package (erroring out if it doesn't have the given version or isn't cargo-dist-able).
 #
-# Note that when generating links to uploaded artifacts, cargo-dist will currently
+# If PACKAGE_NAME isn't specified, then we will create a Github Release™ for all
-# assume that your git tag is always v{VERSION} where VERSION is the version in
+# (cargo-dist-able) packages in the workspace with that version (this is mode is
-# the published package's Cargo.toml (this is the default behaviour of cargo-release).
+# intended for workspaces with only one dist-able package, or with all dist-able
-# In the future this may be made more robust/configurable.
+# packages versioned/released in lockstep).
+#
+# If you push multiple tags at once, separate instances of this workflow will
+# spin up, creating an independent Github Release™ for each one.
+#
+# If there's a prerelease-style suffix to the version then the Github Release™
+# will be marked as a prerelease.
 on:
   push:
     tags:
-      - v[0-9]+.*
+      - '*-?v[0-9]+*'
-
-env:
-  ALL_CARGO_DIST_TARGET_ARGS: --target=x86_64-unknown-linux-gnu --target=x86_64-apple-darwin --target=x86_64-pc-windows-msvc 
-  ALL_CARGO_DIST_INSTALLER_ARGS: 
 
 jobs:
-  # Create the Github Release™️ so the packages have something to be uploaded to
+  # Create the Github Release™ so the packages have something to be uploaded to
   create-release:
     runs-on: ubuntu-latest
     outputs:
-      tag: ${{ steps.create-gh-release.outputs.computed-prefix }}${{ steps.create-gh-release.outputs.version }}
+      has-releases: ${{ steps.create-release.outputs.has-releases }}
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - uses: actions/checkout@v3
-      - id: create-gh-release
+      - name: Install Rust
-        uses: taiki-e/create-gh-release-action@v1
+        run: rustup update 1.71.0 --no-self-update && rustup default 1.71.0
-        with:
+      - name: Install cargo-dist
-          draft: true
+        run: curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.0.7/cargo-dist-installer.sh | sh
-          # (required) GitHub token for creating GitHub Releases.
+      - id: create-release
-          token: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          cargo dist plan --tag=${{ github.ref_name }} --output-format=json > dist-manifest.json
+          echo "dist plan ran successfully"
+          cat dist-manifest.json
 
+          # Create the Github Release™ based on what cargo-dist thinks it should be
+          ANNOUNCEMENT_TITLE=$(jq --raw-output ".announcement_title" dist-manifest.json)
+          IS_PRERELEASE=$(jq --raw-output ".announcement_is_prerelease" dist-manifest.json)
+          jq --raw-output ".announcement_github_body" dist-manifest.json > new_dist_announcement.md
+          gh release create ${{ github.ref_name }} --draft --prerelease="$IS_PRERELEASE" --title="$ANNOUNCEMENT_TITLE" --notes-file=new_dist_announcement.md
+          echo "created announcement!"
+
+          # Upload the manifest to the Github Release™
+          gh release upload ${{ github.ref_name }} dist-manifest.json
+          echo "uploaded manifest!"
+
+          # Disable all the upload-artifacts tasks if we have no actual releases
+          HAS_RELEASES=$(jq --raw-output ".releases != null" dist-manifest.json)
+          echo "has-releases=$HAS_RELEASES" >> "$GITHUB_OUTPUT"
 
   # Build and packages all the things
   upload-artifacts:
+    # Let the initial task tell us to not run (currently very blunt)
     needs: create-release
+    if: ${{ needs.create-release.outputs.has-releases == 'true' }}
     strategy:
       matrix:
         # For these target platforms
         include:
-        - target: x86_64-unknown-linux-gnu
+        - os: macos-11
-          os: ubuntu-20.04
+          dist-args: --artifacts=local --target=aarch64-apple-darwin --target=x86_64-apple-darwin
-          install-dist: curl --proto '=https' --tlsv1.2 -L -sSf https://github.com/axodotdev/cargo-dist/releases/download/v0.0.2/installer.sh | sh
+          install-dist: curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.0.7/cargo-dist-installer.sh | sh
-        - target: x86_64-apple-darwin
+        - os: ubuntu-20.04
-          os: macos-11
+          dist-args: --artifacts=local --target=x86_64-unknown-linux-gnu
-          install-dist: curl --proto '=https' --tlsv1.2 -L -sSf https://github.com/axodotdev/cargo-dist/releases/download/v0.0.2/installer.sh | sh
+          install-dist: curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.0.7/cargo-dist-installer.sh | sh
-        - target: x86_64-pc-windows-msvc
+        - os: windows-2019
-          os: windows-2019
+          dist-args: --artifacts=local --target=x86_64-pc-windows-msvc
-          install-dist: irm 'https://github.com/axodotdev/cargo-dist/releases/download/v0.0.2/installer.ps1' | iex
+          install-dist: irm  https://github.com/axodotdev/cargo-dist/releases/download/v0.0.7/cargo-dist-installer.ps1 | iex
+
     runs-on: ${{ matrix.os }}
     env:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust
-        run: rustup update stable && rustup default stable
+        run: rustup update 1.71.0 --no-self-update && rustup default 1.71.0
       - name: Install cargo-dist
         run: ${{ matrix.install-dist }}
       - name: Run cargo-dist
         # This logic is a bit janky because it's trying to be a polyglot between
         # powershell and bash since this will run on windows, macos, and linux!
         # The two platforms don't agree on how to talk about env vars but they
-        # do agree on 'cat' and '$()' so we use that to marshal values between commmands.
+        # do agree on 'cat' and '$()' so we use that to marshal values between commands.
         run: |
           # Actually do builds and make zips and whatnot
-          cargo dist --target=${{ matrix.target }} --output-format=json > dist-manifest.json
+          cargo dist build --tag=${{ github.ref_name }} --output-format=json ${{ matrix.dist-args }} > dist-manifest.json
           echo "dist ran successfully"
           cat dist-manifest.json
-          # Parse out what we just built and upload it to the Github Release™️
+
-          cat dist-manifest.json | jq --raw-output ".releases[].artifacts[].path" > uploads.txt
+          # Parse out what we just built and upload it to the Github Release™
+          jq --raw-output ".artifacts[]?.path | select( . != null )" dist-manifest.json > uploads.txt
           echo "uploading..."
           cat uploads.txt
-          gh release upload ${{ needs.create-release.outputs.tag }} $(cat uploads.txt)
+          gh release upload ${{ github.ref_name }} $(cat uploads.txt)
           echo "uploaded!"
 
-  # Compute and upload the manifest for everything
+  # Mark the Github Release™ as a non-draft now that everything has succeeded!
-  upload-manifest:
-    needs: create-release
-    runs-on: ubuntu-latest
-    env:
-      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    steps:
-      - uses: actions/checkout@v3
-      - name: Install Rust
-        run: rustup update stable && rustup default stable
-      - name: Install cargo-dist
-        run: curl --proto '=https' --tlsv1.2 -L -sSf https://github.com/axodotdev/cargo-dist/releases/download/v0.0.2/installer.sh | sh
-      - name: Run cargo-dist manifest
-        run: |
-          # Generate a manifest describing everything
-          cargo dist manifest --no-local-paths --output-format=json $ALL_CARGO_DIST_TARGET_ARGS $ALL_CARGO_DIST_INSTALLER_ARGS > dist-manifest.json
-          echo "dist manifest ran successfully"
-          cat dist-manifest.json
-          # Upload the manifest to the Github Release™️
-          gh release upload ${{ needs.create-release.outputs.tag }} dist-manifest.json
-          echo "uploaded manifest!"
-          # Edit the Github Release™️ title/body to match what cargo-dist thinks it should be
-          CHANGELOG_TITLE=$(cat dist-manifest.json | jq --raw-output ".releases[].changelog_title")
-          cat dist-manifest.json | jq --raw-output ".releases[].changelog_body" > new_dist_changelog.md
-          gh release edit ${{ needs.create-release.outputs.tag }} --title="$CHANGELOG_TITLE" --notes-file=new_dist_changelog.md
-          echo "updated release notes!"
-
-  # Mark the Github Release™️ as a non-draft now that everything has succeeded!
   publish-release:
-    needs: [create-release, upload-artifacts, upload-manifest]
+    # Only run after all the other tasks, but it's ok if upload-artifacts was skipped
+    needs: [create-release, upload-artifacts]
+    if: ${{ always() && needs.create-release.result == 'success' && (needs.upload-artifacts.result == 'skipped' || needs.upload-artifacts.result == 'success') }}
     runs-on: ubuntu-latest
     env:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -132,5 +134,4 @@ jobs:
       - uses: actions/checkout@v3
       - name: mark release as non-draft
         run: |
-          gh release edit ${{ needs.create-release.outputs.tag }} --draft=false
+          gh release edit ${{ github.ref_name }} --draft=false
-

.gitignore

@@ -1,2 +1,3 @@
 /target
 /Cargo.lock
+/result-bin

Cargo.toml

@@ -1,8 +1,9 @@
 [package]
 name = "deduplicator"
-version = "0.2.0"
+version = "0.2.2"
 edition = "2021"
 description = "find,filter,delete Duplicates"
+repository = "https://github.com/sreedevk/deduplicator"
 license = "MIT"
 authors = [
     "Sreedev Kodichath <sreedevpadmakumar@gmail.com>",
@@ -19,18 +20,28 @@ chrono = "0.4.23"
 clap = { version = "4.0.32", features = ["derive"] }
 colored = "2.0.0"
 dashmap = { version = "5.4.0", features = ["rayon"] }
-fxhash = "0.2.1"
 globwalk = "0.8.1"
+gxhash = "3.4.1"
 indicatif = { version = "0.17.2", features = ["rayon"] }
 itertools = "0.10.5"
 memmap2 = "0.5.8"
 pathdiff = "0.2.1"
 prettytable-rs = "0.10.0"
 rayon = "1.6.1"
+serde = { version = "1.0.192", features = ["derive"] }
+serde_json = "1.0.108"
 unicode-segmentation = "1.10.0"
 
+[profile.release]
+strip = true
+
 # generated by 'cargo dist init'
 [profile.dist]
 inherits = "release"
-debug = true
+lto = "thin"
-split-debuginfo = "packed"
+
+[workspace.metadata.dist]
+rust-toolchain-version = "1.78.0"
+ci = ["github"]
+targets = ["x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc", "aarch64-apple-darwin"]
+cargo-dist-version = "0.0.7"

README.md

@@ -21,6 +21,7 @@ Options:
   -f, --follow-links           Follow links while scanning directories
   -h, --help                   Print help information
   -V, --version                Print version information
+      --json                    
 ```
 ### Examples
 
@@ -121,3 +122,8 @@ Memory: 31731MiB (~32GiB)
 ## Screenshots
 
 ![](https://user-images.githubusercontent.com/36154121/213618143-e5182e39-731e-4817-87dd-1a6a0f38a449.gif)
+
+## Roadmap
+    - Tree format output for duplicate file listing
+    - GUI
+    - Packages for different operating system repositories (currently only installable via cargo) 

src/fileinfo.rs

@@ -1,22 +1,25 @@
 use anyhow::Result;
+use gxhash::GxHasher;
 use memmap2::Mmap;
+use serde::Serialize;
 use std::fs;
 use std::hash::Hasher;
 use std::{fs::Metadata, path::PathBuf};
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize)]
 pub struct FileInfo {
     pub path: PathBuf,
-    pub filemeta: Metadata,
     pub hash: Option<String>,
     pub size: u64,
+    #[serde(skip)]
+    pub filemeta: Metadata,
 }
 
 impl FileInfo {
     pub fn hash(&self) -> Result<Self> {
         let file = fs::File::open(self.path.clone())?;
         let mapper = unsafe { Mmap::map(&file)? };
-        let mut primhasher = fxhash::FxHasher::default();
+        let mut primhasher = GxHasher::default();
 
         mapper
             .chunks(1_000_000)

src/formatter.rs

@@ -121,8 +121,13 @@ impl Formatter {
             return Ok(());
         }
 
-        let output_table = Self::generate_table(raw, app_args)?;
+        if app_args.json {
-        output_table.printstd();
+            let output_json = serde_json::to_string_pretty(&raw)?;
+            println!("{}", output_json);
+        } else {
+            let output_table = Self::generate_table(raw, app_args)?;
+            output_table.printstd();
+        }
 
         Ok(())
     }

src/main.rs

@@ -19,8 +19,12 @@ fn main() -> Result<()> {
     let results = processor.sizewise()?.hashwise()?;
 
     match app_args.interactive {
-        false => { Formatter::print(results.files, &app_args)?; }
+        false => {
-        true => { interactive::init(results.files, &app_args)?; }
+            Formatter::print(results.files, &app_args)?;
+        }
+        true => {
+            interactive::init(results.files, &app_args)?;
+        }
     }
 
     Ok(())

src/params.rs

@@ -27,6 +27,9 @@ pub struct Params {
     /// Follow links while scanning directories
     #[arg(long, short)]
     pub follow_links: bool,
+    /// print json output
+    #[arg(long)]
+    pub json: bool,
 }
 
 impl Params {