smolkik_adm/deduplicator
1
0
Fork 0
mirror of https://github.com/sreedevk/deduplicator.git synced 2026-09-07 02:24:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

faster hashing algorithm

Browse Source
This commit is contained in:
sreedev
2022-12-30 00:25:07 -05:00
parent 64e122b6ce
commit 2cfb298468
5 changed files with 20 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
Cargo.lock generated
View File

@@ -46,6 +46,12 @@ version = "3.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba"
[[package]]
name = "byteorder"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
[[package]] [[package]]
name = "bytes" name = "bytes"
version = "1.3.0" version = "1.3.0"
@@ -238,6 +244,7 @@ dependencies = [
"chrono", "chrono",
"clap", "clap",
"colored", "colored",
"fxhash",
"glob", "glob",
"humansize", "humansize",
"itertools", "itertools",
@@ -275,6 +282,15 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "fxhash"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
dependencies = [
"byteorder",
]
[[package]] [[package]]
name = "glob" name = "glob"
version = "0.3.0" version = "0.3.0"

2
Cargo.toml
View File

@@ -10,6 +10,7 @@ anyhow = "1.0.68"
chrono = "0.4.23" chrono = "0.4.23"
clap = { version = "4.0.32", features = ["derive"] } clap = { version = "4.0.32", features = ["derive"] }
colored = "2.0.0" colored = "2.0.0"
fxhash = "0.2.1"
glob = "0.3.0" glob = "0.3.0"
humansize = "2.1.2" humansize = "2.1.2"
itertools = "0.10.5" itertools = "0.10.5"
@@ -17,4 +18,3 @@ rayon = "1.6.1"
sqlite = "0.30.3" sqlite = "0.30.3"
thiserror = "1.0.38" thiserror = "1.0.38"
tokio = { version = "1.23.0", features = ["full"] } tokio = { version = "1.23.0", features = ["full"] }
xxhash-rust = { version = "0.8.6", features = ["xxh3"] }

1
src/database.rs
View File

@@ -67,6 +67,7 @@ pub fn duplicate_hashes(connection: &sqlite::Connection, path: &String) -> Resul
ORDER BY a.file_identifier ORDER BY a.file_identifier
", path ", path
); );
let result: Vec<File> = connection let result: Vec<File> = connection
.prepare(query)? .prepare(query)?
.into_iter() .into_iter()

2
src/output.rs
View File

@@ -48,7 +48,7 @@ pub fn print(duplicates: Vec<File>) {
group.into_iter().for_each(|file| { group.into_iter().for_each(|file| {
println!( println!(
"| {0: <16} | {1: <35} | {2: <16} | {3: <32} |", "| {0: <16} | {1: <35} | {2: <16} | {3: <32} |",
&file.hash[0..16].red(), file.hash.red(),
format_path(&file.path).yellow(), format_path(&file.path).yellow(),
file_size(&file.path).blue(), file_size(&file.path).blue(),
modified_time(&file.path).blue() modified_time(&file.path).blue()

2
src/scanner.rs
View File

@@ -6,7 +6,7 @@ use itertools::Itertools;
use rayon::prelude::*; use rayon::prelude::*;
use std::fs; use std::fs;
use std::path::PathBuf; use std::path::PathBuf;
use xxhash_rust::xxh3::xxh3_64 as hasher; use fxhash::hash32 as hasher;
pub fn duplicates(app_opts: &App, connection: &sqlite::Connection) -> Result<Vec<File>> { pub fn duplicates(app_opts: &App, connection: &sqlite::Connection) -> Result<Vec<File>> {
let scan_results = scan(app_opts, connection)?; let scan_results = scan(app_opts, connection)?;