mirror of
https://github.com/sreedevk/deduplicator.git
synced 2026-08-26 10:05:32 +00:00
Compare commits
11 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
7bd88f1642 | ||
|
|
a65e22269c | ||
|
|
6e3fe4a37d | ||
|
|
eadffb5fea | ||
|
|
5532ded331 | ||
|
|
7119f019d3 | ||
|
|
493cac1762 | ||
|
|
f0dbf05705 | ||
|
|
ef1e9a1fce | ||
|
|
d06ca7897d | ||
|
|
b49940998b |
147
.github/workflows/release.yml
vendored
147
.github/workflows/release.yml
vendored
@@ -1,31 +1,136 @@
|
||||
# CI that:
|
||||
#
|
||||
# * checks for a Git Tag that looks like a release ("v1.2.0")
|
||||
# * creates a Github Release™️
|
||||
# * builds binaries/packages with cargo-dist
|
||||
# * uploads those packages to the Github Release™️
|
||||
#
|
||||
# Note that the Github Release™️ will be created before the packages,
|
||||
# so there will be a few minutes where the release has no packages
|
||||
# and then they will slowly trickle in, possibly failing. To make
|
||||
# this more pleasant we mark the release as a "draft" until all
|
||||
# artifacts have been successfully uploaded. This allows you to
|
||||
# choose what to do with partial successes and avoids spamming
|
||||
# anyone with notifications before the release is actually ready.
|
||||
name: Release
|
||||
|
||||
env:
|
||||
PROJECT_NAME: deduplicator
|
||||
PROJECT_DESC: "Filter, Sort & Delete Duplicate Files Recursively"
|
||||
PROJECT_AUTH: "sreedevk"
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
# This task will run whenever you push a git tag that looks like
|
||||
# a version number. We just look for `v` followed by at least one number
|
||||
# and then whatever. so `v1`, `v1.0.0`, and `v1.0.0-prerelease` all work.
|
||||
#
|
||||
# If there's a prerelease-style suffix to the version then the Github Release™️
|
||||
# will be marked as a prerelease (handled by taiki-e/create-gh-release-action).
|
||||
#
|
||||
# Note that when generating links to uploaded artifacts, cargo-dist will currently
|
||||
# assume that your git tag is always v{VERSION} where VERSION is the version in
|
||||
# the published package's Cargo.toml (this is the default behaviour of cargo-release).
|
||||
# In the future this may be made more robust/configurable.
|
||||
on:
|
||||
release:
|
||||
types:
|
||||
- created
|
||||
push:
|
||||
tags:
|
||||
- v[0-9]+.*
|
||||
|
||||
env:
|
||||
ALL_CARGO_DIST_TARGET_ARGS: --target=x86_64-unknown-linux-gnu --target=x86_64-apple-darwin --target=x86_64-pc-windows-msvc
|
||||
ALL_CARGO_DIST_INSTALLER_ARGS:
|
||||
|
||||
jobs:
|
||||
upload-assets:
|
||||
strategy:
|
||||
matrix:
|
||||
os:
|
||||
- ubuntu-latest
|
||||
- macos-latest
|
||||
- windows-latest
|
||||
runs-on: ${{ matrix.os }}
|
||||
# Create the Github Release™️ so the packages have something to be uploaded to
|
||||
create-release:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
tag: ${{ steps.create-gh-release.outputs.computed-prefix }}${{ steps.create-gh-release.outputs.version }}
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: taiki-e/upload-rust-binary-action@v1
|
||||
- id: create-gh-release
|
||||
uses: taiki-e/create-gh-release-action@v1
|
||||
with:
|
||||
bin: deduplicator
|
||||
tar: unix
|
||||
zip: windows
|
||||
draft: true
|
||||
# (required) GitHub token for creating GitHub Releases.
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
|
||||
# Build and packages all the things
|
||||
upload-artifacts:
|
||||
needs: create-release
|
||||
strategy:
|
||||
matrix:
|
||||
# For these target platforms
|
||||
include:
|
||||
- target: x86_64-unknown-linux-gnu
|
||||
os: ubuntu-20.04
|
||||
install-dist: curl --proto '=https' --tlsv1.2 -L -sSf https://github.com/axodotdev/cargo-dist/releases/download/v0.0.2/installer.sh | sh
|
||||
- target: x86_64-apple-darwin
|
||||
os: macos-11
|
||||
install-dist: curl --proto '=https' --tlsv1.2 -L -sSf https://github.com/axodotdev/cargo-dist/releases/download/v0.0.2/installer.sh | sh
|
||||
- target: x86_64-pc-windows-msvc
|
||||
os: windows-2019
|
||||
install-dist: irm 'https://github.com/axodotdev/cargo-dist/releases/download/v0.0.2/installer.ps1' | iex
|
||||
runs-on: ${{ matrix.os }}
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Install Rust
|
||||
run: rustup update stable && rustup default stable
|
||||
- name: Install cargo-dist
|
||||
run: ${{ matrix.install-dist }}
|
||||
- name: Run cargo-dist
|
||||
# This logic is a bit janky because it's trying to be a polyglot between
|
||||
# powershell and bash since this will run on windows, macos, and linux!
|
||||
# The two platforms don't agree on how to talk about env vars but they
|
||||
# do agree on 'cat' and '$()' so we use that to marshal values between commmands.
|
||||
run: |
|
||||
# Actually do builds and make zips and whatnot
|
||||
cargo dist --target=${{ matrix.target }} --output-format=json > dist-manifest.json
|
||||
echo "dist ran successfully"
|
||||
cat dist-manifest.json
|
||||
# Parse out what we just built and upload it to the Github Release™️
|
||||
cat dist-manifest.json | jq --raw-output ".releases[].artifacts[].path" > uploads.txt
|
||||
echo "uploading..."
|
||||
cat uploads.txt
|
||||
gh release upload ${{ needs.create-release.outputs.tag }} $(cat uploads.txt)
|
||||
echo "uploaded!"
|
||||
|
||||
# Compute and upload the manifest for everything
|
||||
upload-manifest:
|
||||
needs: create-release
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Install Rust
|
||||
run: rustup update stable && rustup default stable
|
||||
- name: Install cargo-dist
|
||||
run: curl --proto '=https' --tlsv1.2 -L -sSf https://github.com/axodotdev/cargo-dist/releases/download/v0.0.2/installer.sh | sh
|
||||
- name: Run cargo-dist manifest
|
||||
run: |
|
||||
# Generate a manifest describing everything
|
||||
cargo dist manifest --no-local-paths --output-format=json $ALL_CARGO_DIST_TARGET_ARGS $ALL_CARGO_DIST_INSTALLER_ARGS > dist-manifest.json
|
||||
echo "dist manifest ran successfully"
|
||||
cat dist-manifest.json
|
||||
# Upload the manifest to the Github Release™️
|
||||
gh release upload ${{ needs.create-release.outputs.tag }} dist-manifest.json
|
||||
echo "uploaded manifest!"
|
||||
# Edit the Github Release™️ title/body to match what cargo-dist thinks it should be
|
||||
CHANGELOG_TITLE=$(cat dist-manifest.json | jq --raw-output ".releases[].changelog_title")
|
||||
cat dist-manifest.json | jq --raw-output ".releases[].changelog_body" > new_dist_changelog.md
|
||||
gh release edit ${{ needs.create-release.outputs.tag }} --title="$CHANGELOG_TITLE" --notes-file=new_dist_changelog.md
|
||||
echo "updated release notes!"
|
||||
|
||||
# Mark the Github Release™️ as a non-draft now that everything has succeeded!
|
||||
publish-release:
|
||||
needs: [create-release, upload-artifacts, upload-manifest]
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: mark release as non-draft
|
||||
run: |
|
||||
gh release edit ${{ needs.create-release.outputs.tag }} --draft=false
|
||||
|
||||
|
||||
185
Cargo.lock
generated
185
Cargo.lock
generated
@@ -73,9 +73,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "3.11.1"
|
||||
version = "3.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba"
|
||||
checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535"
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
@@ -83,12 +83,6 @@ version = "1.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c"
|
||||
|
||||
[[package]]
|
||||
name = "bytesize"
|
||||
version = "1.1.0"
|
||||
@@ -124,9 +118,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.0.32"
|
||||
version = "4.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a7db700bc935f9e43e88d00b0850dae18a63773cfbec6d8e070fccf7fef89a39"
|
||||
checksum = "f13b9c79b5d1dd500d20ef541215a6423c75829ef43117e1b4d17fd8af0b5d76"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"clap_derive",
|
||||
@@ -139,9 +133,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "clap_derive"
|
||||
version = "4.0.21"
|
||||
version = "4.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0177313f9f02afc995627906bbd8967e2be069f5261954222dac78290c2b9014"
|
||||
checksum = "684a277d672e91966334af371f1a7b5833f9aa00b07c84e92fbce95e00208ce8"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro-error",
|
||||
@@ -152,9 +146,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "clap_lex"
|
||||
version = "0.3.0"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d4198f73e42b4936b35b5bb248d81d2b595ecb170da0bac7655c54eedfa8da8"
|
||||
checksum = "783fe232adfca04f90f56201b26d79682d4cd2625e0bc7290b95123afe558ade"
|
||||
dependencies = [
|
||||
"os_str_bytes",
|
||||
]
|
||||
@@ -182,9 +176,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "console"
|
||||
version = "0.15.4"
|
||||
version = "0.15.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c9b6515d269224923b26b5febea2ed42b2d5f2ce37284a4dd670fedd6cb8347a"
|
||||
checksum = "c3d79fbe8970a77e3e34151cc13d3b3e248aa0faaecb9f6091fa07ebefe5ad60"
|
||||
dependencies = [
|
||||
"encode_unicode 0.3.6",
|
||||
"lazy_static",
|
||||
@@ -266,9 +260,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cxx"
|
||||
version = "1.0.85"
|
||||
version = "1.0.87"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5add3fc1717409d029b20c5b6903fc0c0b02fa6741d820054f4a2efa5e5816fd"
|
||||
checksum = "b61a7545f753a88bcbe0a70de1fcc0221e10bfc752f576754fa91e663db1622e"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cxxbridge-flags",
|
||||
@@ -278,9 +272,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cxx-build"
|
||||
version = "1.0.85"
|
||||
version = "1.0.87"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b4c87959ba14bc6fbc61df77c3fcfe180fc32b93538c4f1031dd802ccb5f2ff0"
|
||||
checksum = "f464457d494b5ed6905c63b0c4704842aba319084a0a3561cdc1359536b53200"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"codespan-reporting",
|
||||
@@ -293,15 +287,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cxxbridge-flags"
|
||||
version = "1.0.85"
|
||||
version = "1.0.87"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "69a3e162fde4e594ed2b07d0f83c6c67b745e7f28ce58c6df5e6b6bef99dfb59"
|
||||
checksum = "43c7119ce3a3701ed81aca8410b9acf6fc399d2629d057b87e2efa4e63a3aaea"
|
||||
|
||||
[[package]]
|
||||
name = "cxxbridge-macro"
|
||||
version = "1.0.85"
|
||||
version = "1.0.87"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3e7e2adeb6a0d4a282e581096b06e1791532b7d576dcde5ccd9382acf55db8e6"
|
||||
checksum = "65e07508b90551e610910fa648a1878991d367064997a596135b86df30daf07e"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -324,7 +318,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "deduplicator"
|
||||
version = "0.1.5"
|
||||
version = "0.1.6"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bytesize",
|
||||
@@ -339,8 +333,6 @@ dependencies = [
|
||||
"memmap2",
|
||||
"prettytable-rs",
|
||||
"rayon",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"unicode-segmentation",
|
||||
]
|
||||
|
||||
@@ -527,23 +519,22 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "indicatif"
|
||||
version = "0.17.2"
|
||||
version = "0.17.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4295cbb7573c16d310e99e713cf9e75101eb190ab31fccd35f2d2691b4352b19"
|
||||
checksum = "cef509aa9bc73864d6756f0d34d35504af3cf0844373afe9b8669a5b8005a729"
|
||||
dependencies = [
|
||||
"console",
|
||||
"number_prefix",
|
||||
"portable-atomic",
|
||||
"rayon",
|
||||
"tokio",
|
||||
"unicode-width",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "io-lifetimes"
|
||||
version = "1.0.3"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c"
|
||||
checksum = "e7d6c6f8c91b4b9ed43484ad1a938e393caf35960fce7f82a040497207bd8e9e"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys",
|
||||
@@ -655,18 +646,6 @@ dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mio"
|
||||
version = "0.8.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"log",
|
||||
"wasi 0.11.0+wasi-snapshot-preview1",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-integer"
|
||||
version = "0.1.45"
|
||||
@@ -704,9 +683,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.16.0"
|
||||
version = "1.17.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860"
|
||||
checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66"
|
||||
|
||||
[[package]]
|
||||
name = "os_str_bytes"
|
||||
@@ -714,21 +693,11 @@ version = "6.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee"
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
|
||||
dependencies = [
|
||||
"lock_api",
|
||||
"parking_lot_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot_core"
|
||||
version = "0.9.5"
|
||||
version = "0.9.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7ff9f3fef3968a3ec5945535ed654cb38ff72d7495a25619e2247fb15a2ed9ba"
|
||||
checksum = "ba1ef8814b5c993410bb3adfad7a5ed269563e4a2f90c41f5d85be7fb47133bf"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
@@ -737,12 +706,6 @@ dependencies = [
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.2.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
|
||||
|
||||
[[package]]
|
||||
name = "portable-atomic"
|
||||
version = "0.3.19"
|
||||
@@ -789,9 +752,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.49"
|
||||
version = "1.0.50"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5"
|
||||
checksum = "6ef7d57beacfaf2d8aee5937dab7b7f28de3cb8b1828479bb5de2a7106f2bae2"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
@@ -817,9 +780,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rayon-core"
|
||||
version = "1.10.1"
|
||||
version = "1.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3"
|
||||
checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b"
|
||||
dependencies = [
|
||||
"crossbeam-channel",
|
||||
"crossbeam-deque",
|
||||
@@ -872,9 +835,9 @@ checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "0.36.5"
|
||||
version = "0.36.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a3807b5d10909833d3e9acd1eb5fb988f79376ff10fce42937de71a449c4c588"
|
||||
checksum = "d4fdebc4b395b7fbb9ab11e462e20ed9051e7b16e42d24042c776eca0ac81b03"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"errno",
|
||||
@@ -923,31 +886,12 @@ version = "1.0.152"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb"
|
||||
|
||||
[[package]]
|
||||
name = "signal-hook-registry"
|
||||
version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
|
||||
|
||||
[[package]]
|
||||
name = "socket2"
|
||||
version = "0.4.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.10.0"
|
||||
@@ -978,9 +922,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "termcolor"
|
||||
version = "1.1.3"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755"
|
||||
checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6"
|
||||
dependencies = [
|
||||
"winapi-util",
|
||||
]
|
||||
@@ -1025,37 +969,6 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio"
|
||||
version = "1.23.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "38a54aca0c15d014013256222ba0ebed095673f89345dd79119d912eb561b7a8"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"bytes",
|
||||
"libc",
|
||||
"memchr",
|
||||
"mio",
|
||||
"num_cpus",
|
||||
"parking_lot",
|
||||
"pin-project-lite",
|
||||
"signal-hook-registry",
|
||||
"socket2",
|
||||
"tokio-macros",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-macros"
|
||||
version = "1.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.6"
|
||||
@@ -1205,42 +1118,42 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_gnullvm"
|
||||
version = "0.42.0"
|
||||
version = "0.42.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e"
|
||||
checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_msvc"
|
||||
version = "0.42.0"
|
||||
version = "0.42.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4"
|
||||
checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
version = "0.42.0"
|
||||
version = "0.42.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7"
|
||||
checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_msvc"
|
||||
version = "0.42.0"
|
||||
version = "0.42.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246"
|
||||
checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnu"
|
||||
version = "0.42.0"
|
||||
version = "0.42.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed"
|
||||
checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnullvm"
|
||||
version = "0.42.0"
|
||||
version = "0.42.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028"
|
||||
checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
version = "0.42.0"
|
||||
version = "0.42.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5"
|
||||
checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd"
|
||||
|
||||
13
Cargo.toml
13
Cargo.toml
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "deduplicator"
|
||||
version = "0.1.5"
|
||||
version = "0.1.6"
|
||||
edition = "2021"
|
||||
description = "find,filter,delete Duplicates"
|
||||
license = "MIT"
|
||||
@@ -21,11 +21,16 @@ colored = "2.0.0"
|
||||
dashmap = { version = "5.4.0", features = ["rayon"] }
|
||||
fxhash = "0.2.1"
|
||||
globwalk = "0.8.1"
|
||||
indicatif = { version = "0.17.2", features = ["rayon", "tokio"] }
|
||||
indicatif = { version = "0.17.2", features = ["rayon"] }
|
||||
itertools = "0.10.5"
|
||||
memmap2 = "0.5.8"
|
||||
prettytable-rs = "0.10.0"
|
||||
rayon = "1.6.1"
|
||||
thiserror = "1.0.38"
|
||||
tokio = { version = "1.23.1", features = ["full"] }
|
||||
unicode-segmentation = "1.10.0"
|
||||
|
||||
# generated by 'cargo dist init'
|
||||
[profile.dist]
|
||||
inherits = "release"
|
||||
debug = true
|
||||
split-debuginfo = "packed"
|
||||
|
||||
|
||||
12
README.md
12
README.md
@@ -94,12 +94,12 @@ Deduplicator uses size comparison and fxhash (a non non-cryptographic hashing al
|
||||
|
||||
## benchmarks
|
||||
|
||||
| Command | Dirsize | Mean [ms] | Min [ms] | Max [ms] | Relative |
|
||||
|:---|:---|---:|---:|---:|---:|
|
||||
| `deduplicator --dir ~/Data/tmp` | (~120G) | 27.5 ± 1.0 | 26.0 | 32.1 | 1.70 ± 0.09 |
|
||||
| `deduplicator --dir ~/Data/books` | (~8.6G) | 21.8 ± 0.7 | 20.5 | 24.4 | 1.35 ± 0.07 |
|
||||
| `deduplicator --dir ~/Data/books --min-size 10M` | (~8.6G) | 16.1 ± 0.6 | 14.9 | 18.8 | 1.00 |
|
||||
| `deduplicator --dir ~/Data/ --types pdf,jpg,png,jpeg` | (~290G) | 1857.4 ± 24.5 | 1817.0 | 1895.5 | 115.07 ± 4.64 |
|
||||
| Command | Dirsize | Filecount | Mean [ms] | Min [ms] | Max [ms] | Relative |
|
||||
|:---|:---|---:|---:|---:|---:|---:|
|
||||
| `deduplicator ~/Data/tmp` | (~120G) | 721 files | 33.5 ± 28.6 | 25.3 | 151.5 | 1.87 ± 1.60 |
|
||||
| `deduplicator ~/Data/books` | (~8.6G) | 1419 files | 24.5 ± 1.0 | 22.9 | 28.1 | 1.37 ± 0.08 |
|
||||
| `deduplicator ~/Data/books --min-size 10M` | (~8.6G) | 1419 files | 17.9 ± 0.7 | 16.8 | 20.0 | 1.00 |
|
||||
| `deduplicator ~/Data/ --types pdf,jpg,png,jpeg` | (~290G) | 104222 files | 1207.2 ± 37.0 | 1172.2 | 1287.7 | 67.27 ± 3.33 |
|
||||
|
||||
* The last entry is lower because of the number of files deduplicator had to go through (~660895 Files). The average size of the files rarely affect the performance of deduplicator.
|
||||
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
use anyhow::Result;
|
||||
use colored::Colorize;
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct File {
|
||||
pub path: String,
|
||||
pub path: PathBuf,
|
||||
pub size: Option<u64>,
|
||||
pub hash: Option<String>,
|
||||
}
|
||||
@@ -11,8 +12,8 @@ pub struct File {
|
||||
pub fn delete_files(files: Vec<File>) -> Result<()> {
|
||||
files.into_iter().for_each(|file| {
|
||||
match std::fs::remove_file(file.path.clone()) {
|
||||
Ok(_) => println!("{}: {}", "DELETED".green(), file.path),
|
||||
Err(_) => println!("{}: {}", "FAILED".red(), file.path)
|
||||
Ok(_) => println!("{}: {}", "DELETED".green(), file.path.display()),
|
||||
Err(_) => println!("{}: {}", "FAILED".red(), file.path.display())
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -9,7 +9,6 @@ use anyhow::Result;
|
||||
use app::App;
|
||||
use clap::Parser;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
fn main() -> Result<()> {
|
||||
App::init(¶ms::Params::parse())
|
||||
}
|
||||
|
||||
@@ -9,11 +9,15 @@ use indicatif::{ProgressBar, ProgressIterator, ProgressStyle};
|
||||
use itertools::Itertools;
|
||||
use prettytable::{format, row, Table};
|
||||
use std::io::Write;
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
use std::{fs, io};
|
||||
use unicode_segmentation::UnicodeSegmentation;
|
||||
|
||||
fn format_path(path: &str, opts: &Params) -> Result<String> {
|
||||
let display_path = path.replace(opts.get_directory()?.to_string_lossy().as_ref(), "");
|
||||
fn format_path(path: &Path, opts: &Params) -> Result<String> {
|
||||
let display_path = path
|
||||
.to_string_lossy()
|
||||
.replace(opts.get_directory()?.to_string_lossy().as_ref(), "");
|
||||
let display_range = if display_path.chars().count() > 32 {
|
||||
display_path
|
||||
.graphemes(true)
|
||||
@@ -34,7 +38,7 @@ fn file_size(file: &File) -> Result<String> {
|
||||
Ok(format!("{:>12}", bytesize::ByteSize::b(file.size.unwrap())))
|
||||
}
|
||||
|
||||
fn modified_time(path: &String) -> Result<String> {
|
||||
fn modified_time(path: &Path) -> Result<String> {
|
||||
let mdata = fs::metadata(path)?;
|
||||
let modified_time: DateTime<Utc> = mdata.modified()?.into();
|
||||
|
||||
@@ -74,7 +78,7 @@ fn process_group_action(duplicates: &Vec<File>, dup_index: usize, dup_size: usiz
|
||||
.split(',')
|
||||
.filter(|element| !element.is_empty())
|
||||
.map(|index| index.parse::<usize>().unwrap_or_default())
|
||||
.collect_vec();
|
||||
.collect::<Vec<usize>>();
|
||||
|
||||
if parsed_file_indices
|
||||
.clone()
|
||||
@@ -100,12 +104,12 @@ fn process_group_action(duplicates: &Vec<File>, dup_index: usize, dup_size: usiz
|
||||
.clone()
|
||||
.enumerate()
|
||||
.for_each(|(index, file)| {
|
||||
println!("{}: {}", index.to_string().blue(), file.path);
|
||||
println!("{}: {}", index.to_string().blue(), file.path.display());
|
||||
});
|
||||
|
||||
match scan_group_confirmation().unwrap() {
|
||||
true => {
|
||||
file_manager::delete_files(files_to_delete.collect_vec()).ok();
|
||||
file_manager::delete_files(files_to_delete.collect::<Vec<File>>()).ok();
|
||||
}
|
||||
false => println!("{}", "\nCancelled Delete Operation.".red()),
|
||||
}
|
||||
@@ -155,6 +159,7 @@ pub fn print(duplicates: DashMap<String, Vec<File>>, opts: &Params) {
|
||||
|
||||
let mut output_table = Table::new();
|
||||
let progress_bar = ProgressBar::new(duplicates.len() as u64);
|
||||
progress_bar.enable_steady_tick(Duration::from_millis(50));
|
||||
let progress_style = ProgressStyle::default_bar()
|
||||
.template("{spinner:.green} [generating output] [{wide_bar:.cyan/blue}] {pos}/{len} files")
|
||||
.unwrap();
|
||||
@@ -181,3 +186,32 @@ pub fn print(duplicates: DashMap<String, Vec<File>>, opts: &Params) {
|
||||
|
||||
output_table.printstd();
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
pub fn raw(duplicates: DashMap<String, Vec<File>>, opts: &Params) -> Result<()> {
|
||||
if duplicates.is_empty() {
|
||||
println!(
|
||||
"\n{}",
|
||||
"No duplicates found matching your search criteria.".green()
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
duplicates
|
||||
.into_iter()
|
||||
.sorted_unstable_by_key(|(_, f)| f.first().and_then(|ff| ff.size).unwrap_or_default())
|
||||
.for_each(|(_hash, group)| {
|
||||
group.iter().for_each(|file| {
|
||||
println!(
|
||||
"{}\t{}\t{}",
|
||||
format_path(&file.path, opts).unwrap_or_default().blue(),
|
||||
file_size(file).unwrap_or_default().red(),
|
||||
modified_time(&file.path).unwrap_or_default().yellow()
|
||||
)
|
||||
});
|
||||
|
||||
println!("---");
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -7,7 +7,10 @@ use memmap2::Mmap;
|
||||
use rayon::prelude::*;
|
||||
use std::hash::Hasher;
|
||||
use std::time::Duration;
|
||||
use std::{fs, path::PathBuf};
|
||||
use std::{
|
||||
fs,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum IndexCritera {
|
||||
@@ -45,28 +48,38 @@ fn scan(app_opts: &Params) -> Result<Vec<File>> {
|
||||
let progress_style =
|
||||
ProgressStyle::with_template("{spinner:.green} [mapping paths] {pos} paths")?;
|
||||
progress.set_style(progress_style);
|
||||
progress.enable_steady_tick(Duration::from_millis(100));
|
||||
progress.enable_steady_tick(Duration::from_millis(50));
|
||||
|
||||
let files = walker
|
||||
.progress_with(progress)
|
||||
.filter_map(Result::ok)
|
||||
.map(|file| file.into_path())
|
||||
.filter(|fpath| fpath.is_file())
|
||||
.collect::<Vec<PathBuf>>()
|
||||
.collect::<Vec<PathBuf>>();
|
||||
|
||||
let scan_progress = ProgressBar::new(files.len() as u64);
|
||||
let scan_progress_style = ProgressStyle::with_template(
|
||||
"{spinner:.green} [processing mapped paths] [{wide_bar:.cyan/blue}] {pos}/{len} files",
|
||||
)?;
|
||||
scan_progress.set_style(scan_progress_style);
|
||||
scan_progress.enable_steady_tick(Duration::from_millis(50));
|
||||
|
||||
let scan_results = files
|
||||
.into_par_iter()
|
||||
.progress_with_style(ProgressStyle::with_template(
|
||||
"{spinner:.green} [processing mapped paths] [{wide_bar:.cyan/blue}] {pos}/{len} files",
|
||||
)?)
|
||||
.map(|fpath| fpath.display().to_string())
|
||||
.progress_with(scan_progress)
|
||||
.map(|fpath| File {
|
||||
path: fpath.clone(),
|
||||
hash: None,
|
||||
size: Some(fs::metadata(fpath).unwrap().len()),
|
||||
size: Some(
|
||||
fs::metadata(fpath)
|
||||
.map(|metadata| metadata.len())
|
||||
.unwrap_or_default(),
|
||||
),
|
||||
})
|
||||
.filter(|file| filters::is_file_gt_min_size(app_opts, file))
|
||||
.collect();
|
||||
|
||||
Ok(files)
|
||||
Ok(scan_results)
|
||||
}
|
||||
|
||||
fn process_file_index(
|
||||
@@ -84,7 +97,7 @@ fn process_file_index(
|
||||
IndexCritera::Hash => {
|
||||
file.hash = Some(hash_file(&file.path).unwrap_or_default());
|
||||
store
|
||||
.entry(file.clone().hash.unwrap())
|
||||
.entry(file.clone().hash.unwrap_or_default())
|
||||
.and_modify(|fileset| fileset.push(file.clone()))
|
||||
.or_insert_with(|| vec![file]);
|
||||
}
|
||||
@@ -96,17 +109,22 @@ fn index_files(
|
||||
index_criteria: IndexCritera,
|
||||
) -> Result<DashMap<String, Vec<File>>> {
|
||||
let store: DashMap<String, Vec<File>> = DashMap::new();
|
||||
let index_progress = ProgressBar::new(files.len() as u64);
|
||||
let index_progress_style = ProgressStyle::with_template(
|
||||
"{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files",
|
||||
)?;
|
||||
index_progress.set_style(index_progress_style);
|
||||
index_progress.enable_steady_tick(Duration::from_millis(50));
|
||||
|
||||
files
|
||||
.into_par_iter()
|
||||
.progress_with_style(ProgressStyle::with_template(
|
||||
"{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files",
|
||||
)?)
|
||||
.progress_with(index_progress)
|
||||
.for_each(|file| process_file_index(file, &store, index_criteria));
|
||||
|
||||
Ok(store)
|
||||
}
|
||||
|
||||
fn incremental_hashing(filepath: &str) -> Result<String> {
|
||||
fn incremental_hashing(filepath: &Path) -> Result<String> {
|
||||
let file = fs::File::open(filepath)?;
|
||||
let fmap = unsafe { Mmap::map(&file)? };
|
||||
let mut inchasher = fxhash::FxHasher::default();
|
||||
@@ -117,12 +135,12 @@ fn incremental_hashing(filepath: &str) -> Result<String> {
|
||||
Ok(format!("{}", inchasher.finish()))
|
||||
}
|
||||
|
||||
fn standard_hashing(filepath: &str) -> Result<String> {
|
||||
fn standard_hashing(filepath: &Path) -> Result<String> {
|
||||
let file = fs::read(filepath)?;
|
||||
Ok(hasher(&*file).to_string())
|
||||
}
|
||||
|
||||
fn hash_file(filepath: &str) -> Result<String> {
|
||||
fn hash_file(filepath: &Path) -> Result<String> {
|
||||
let filemeta = fs::metadata(filepath)?;
|
||||
|
||||
// NOTE: USE INCREMENTAL HASHING ONLY FOR FILES > 100MB
|
||||
|
||||
Reference in New Issue
Block a user