mirror of
https://github.com/sreedevk/deduplicator.git
synced 2026-08-31 12:35:34 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
028b868ea9 |
2
.cargo/config.toml
Normal file
2
.cargo/config.toml
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
[build]
|
||||||
|
rustflags = ["-C", "target-feature=+aes,+sse2"]
|
||||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -1,2 +1,3 @@
|
|||||||
/target
|
/target
|
||||||
/Cargo.lock
|
/Cargo.lock
|
||||||
|
/result-bin
|
||||||
|
|||||||
672
Cargo.lock
generated
672
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "deduplicator"
|
name = "deduplicator"
|
||||||
version = "0.2.1"
|
version = "0.2.2"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
description = "find,filter,delete Duplicates"
|
description = "find,filter,delete Duplicates"
|
||||||
repository = "https://github.com/sreedevk/deduplicator"
|
repository = "https://github.com/sreedevk/deduplicator"
|
||||||
@@ -20,8 +20,8 @@ chrono = "0.4.23"
|
|||||||
clap = { version = "4.0.32", features = ["derive"] }
|
clap = { version = "4.0.32", features = ["derive"] }
|
||||||
colored = "2.0.0"
|
colored = "2.0.0"
|
||||||
dashmap = { version = "5.4.0", features = ["rayon"] }
|
dashmap = { version = "5.4.0", features = ["rayon"] }
|
||||||
fxhash = "0.2.1"
|
|
||||||
globwalk = "0.8.1"
|
globwalk = "0.8.1"
|
||||||
|
gxhash = "3.4.1"
|
||||||
indicatif = { version = "0.17.2", features = ["rayon"] }
|
indicatif = { version = "0.17.2", features = ["rayon"] }
|
||||||
itertools = "0.10.5"
|
itertools = "0.10.5"
|
||||||
memmap2 = "0.5.8"
|
memmap2 = "0.5.8"
|
||||||
@@ -41,7 +41,7 @@ inherits = "release"
|
|||||||
lto = "thin"
|
lto = "thin"
|
||||||
|
|
||||||
[workspace.metadata.dist]
|
[workspace.metadata.dist]
|
||||||
rust-toolchain-version = "1.71.0"
|
rust-toolchain-version = "1.78.0"
|
||||||
ci = ["github"]
|
ci = ["github"]
|
||||||
targets = ["x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc", "aarch64-apple-darwin"]
|
targets = ["x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc", "aarch64-apple-darwin"]
|
||||||
cargo-dist-version = "0.0.7"
|
cargo-dist-version = "0.0.7"
|
||||||
|
|||||||
@@ -1,9 +1,10 @@
|
|||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
|
use gxhash::GxHasher;
|
||||||
use memmap2::Mmap;
|
use memmap2::Mmap;
|
||||||
|
use serde::Serialize;
|
||||||
use std::fs;
|
use std::fs;
|
||||||
use std::hash::Hasher;
|
use std::hash::Hasher;
|
||||||
use std::{fs::Metadata, path::PathBuf};
|
use std::{fs::Metadata, path::PathBuf};
|
||||||
use serde::Serialize;
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize)]
|
#[derive(Debug, Clone, Serialize)]
|
||||||
pub struct FileInfo {
|
pub struct FileInfo {
|
||||||
@@ -18,7 +19,7 @@ impl FileInfo {
|
|||||||
pub fn hash(&self) -> Result<Self> {
|
pub fn hash(&self) -> Result<Self> {
|
||||||
let file = fs::File::open(self.path.clone())?;
|
let file = fs::File::open(self.path.clone())?;
|
||||||
let mapper = unsafe { Mmap::map(&file)? };
|
let mapper = unsafe { Mmap::map(&file)? };
|
||||||
let mut primhasher = fxhash::FxHasher::default();
|
let mut primhasher = GxHasher::default();
|
||||||
|
|
||||||
mapper
|
mapper
|
||||||
.chunks(1_000_000)
|
.chunks(1_000_000)
|
||||||
|
|||||||
Reference in New Issue
Block a user