From 68286afab32fe5df9662dc14f1ce98862948b3ea Mon Sep 17 00:00:00 2001 From: sreedev Date: Fri, 30 Dec 2022 00:30:52 -0500 Subject: [PATCH] removed old hashing lib --- Cargo.lock | 7 ------- README.md | 9 +++++++++ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4574b86..59f1ff8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -252,7 +252,6 @@ dependencies = [ "sqlite", "thiserror", "tokio", - "xxhash-rust", ] [[package]] @@ -962,9 +961,3 @@ name = "windows_x86_64_msvc" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" - -[[package]] -name = "xxhash-rust" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "735a71d46c4d68d71d4b24d03fdc2b98e38cea81730595801db779c04fe80d70" diff --git a/README.md b/README.md index 1c834c6..40d5040 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,15 @@ Options: -h, --help Print help information -V, --version Print version information ``` +

Performance

+ +

+ Deduplicator uses fxhash (a non-cryptographic hashing algorithm) which is extremely fast. As a result, deduplicator is able to process huge amounts of data in a couple of seconds.

+ +

+ While testing, Deduplicator was able to go through 8.6GB of pdf files and detect duplicates in 2.9 seconds +

+
5.99s user 6.16s system 411% cpu 2.950 total

Screenshots