mirror of
https://github.com/sreedevk/deduplicator.git
synced 2026-08-26 10:05:32 +00:00
Compare commits
25 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
01dd93a0ea | ||
|
|
8ac78fb856 | ||
|
|
37a4c4d52d | ||
|
|
1ea5705474 | ||
|
|
ea748c60d8 | ||
|
|
51f5f8e61a | ||
|
|
175c7579c4 | ||
|
|
81c96ce3b8 | ||
|
|
0031891b8b | ||
|
|
ae87e4e830 | ||
|
|
284e168453 | ||
|
|
a6511f2cf3 | ||
|
|
64d0106765 | ||
|
|
c75b2eb1c9 | ||
|
|
b736853dfe | ||
|
|
92290480a8 | ||
|
|
b7f775e04c | ||
|
|
12295d7847 | ||
|
|
d81f499db3 | ||
|
|
9e1360aeb4 | ||
|
|
38ea37711f | ||
|
|
2294471b50 | ||
|
|
dbf504fc17 | ||
|
|
06efb2ed6f | ||
|
|
95af6c4a70 |
30
.github/ISSUE_TEMPLATE/bug_report.md
vendored
Normal file
30
.github/ISSUE_TEMPLATE/bug_report.md
vendored
Normal file
@@ -0,0 +1,30 @@
|
||||
---
|
||||
name: Bug report
|
||||
about: Create a report to help us improve
|
||||
title: "[Bug] Title"
|
||||
labels: ''
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Describe the bug**
|
||||
A clear and concise description of what the bug is.
|
||||
|
||||
** Runtime Info **
|
||||
App Arguments: [e.g. `-i --nocache`]
|
||||
Install Type: [e.g. `cargo install`]
|
||||
App Version: [e.g. v0.0.7]
|
||||
|
||||
**Expected behavior**
|
||||
A clear and concise description of what you expected to happen.
|
||||
|
||||
**Screenshots**
|
||||
If applicable, add screenshots to help explain your problem.
|
||||
|
||||
**Platform Details (please complete the following information):**
|
||||
- OS: [e.g. Arch Linux]
|
||||
- Terminal Emulator: [e.g Alacritty]
|
||||
- Shell [e.g. Zshell]
|
||||
|
||||
**Additional context**
|
||||
Add any other context about the problem here.
|
||||
20
.github/ISSUE_TEMPLATE/feature-request.md
vendored
Normal file
20
.github/ISSUE_TEMPLATE/feature-request.md
vendored
Normal file
@@ -0,0 +1,20 @@
|
||||
---
|
||||
name: Feature request
|
||||
about: Suggest an idea for this project
|
||||
title: "[Feature] Title"
|
||||
labels: ''
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Is your feature request related to a problem? Please describe.**
|
||||
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
|
||||
|
||||
**Describe the solution you'd like**
|
||||
A clear and concise description of what you want to happen.
|
||||
|
||||
**Describe alternatives you've considered**
|
||||
A clear and concise description of any alternative solutions or features you've considered.
|
||||
|
||||
**Additional context**
|
||||
Add any other context or screenshots about the feature request here.
|
||||
20
.github/workflows/rust.yml
vendored
Normal file
20
.github/workflows/rust.yml
vendored
Normal file
@@ -0,0 +1,20 @@
|
||||
name: Rust
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "main" ]
|
||||
pull_request:
|
||||
branches: [ "main" ]
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Build
|
||||
run: cargo build --verbose
|
||||
- name: Run tests
|
||||
run: cargo test --verbose
|
||||
28
CONTRIBUTING.md
Normal file
28
CONTRIBUTING.md
Normal file
@@ -0,0 +1,28 @@
|
||||
## How to contribute to Deduplicator
|
||||
|
||||
#### **Did you find a bug?**
|
||||
|
||||
* **Ensure the bug was not already reported** by searching on GitHub under [Issues](https://github.com/sreedevk/deduplicator/issues).
|
||||
|
||||
* If you're unable to find an open issue addressing the problem, [open a new one](https://github.com/sreedevk/deduplicator/issues/new). Be sure to include a **title and clear description**, as much relevant information as possible, and a **code sample** or an **executable test case** demonstrating the expected behavior that is not occurring.
|
||||
|
||||
* If possible, use the [bug report template](https://github.com/sreedevk/deduplicator/blob/main/.github/ISSUE_TEMPLATE/bug_report.md) to create the issue.
|
||||
|
||||
#### **Would you like to write a fix for the bug?**
|
||||
* Assign the Issue to yourself (if unassigned) before you start working in order to avoid any conficts.
|
||||
* Open a new GitHub pull request with the patch.
|
||||
* Ensure the PR description clearly describes the problem and solution. Include the relevant issue number.
|
||||
* Make sure that the PR points to the development branch.
|
||||
|
||||
#### **Did you fix whitespace, format code, or make a purely cosmetic patch?**
|
||||
|
||||
Changes that are cosmetic in nature and do not add anything substantial to the stability, functionality, or testability of Deduplicator will generally not be accepted/
|
||||
|
||||
#### **Do you intend to add a new feature or change an existing one?**
|
||||
|
||||
* First open an issue with the sugggestion using the [feature request template](https://github.com/sreedevk/deduplicator/blob/main/.github/ISSUE_TEMPLATE/feature-request.md)
|
||||
* Do not create a PR before one of the core contributors has conveyed acceptance for a feature request.
|
||||
|
||||
#### **Do you have questions about the source code?**
|
||||
|
||||
* If you have a question, raise an issue in the repository with a "question" label.
|
||||
180
Cargo.lock
generated
180
Cargo.lock
generated
@@ -70,12 +70,6 @@ version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c"
|
||||
|
||||
[[package]]
|
||||
name = "cassowary"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.0.78"
|
||||
@@ -161,6 +155,19 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "console"
|
||||
version = "0.15.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c9b6515d269224923b26b5febea2ed42b2d5f2ce37284a4dd670fedd6cb8347a"
|
||||
dependencies = [
|
||||
"encode_unicode 0.3.6",
|
||||
"lazy_static",
|
||||
"libc",
|
||||
"unicode-width",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "core-foundation-sys"
|
||||
version = "0.8.3"
|
||||
@@ -210,31 +217,6 @@ dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossterm"
|
||||
version = "0.25.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e64e6c0fbe2c17357405f7c758c1ef960fce08bdfb2c03d88d2a18d7e09c4b67"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"crossterm_winapi",
|
||||
"libc",
|
||||
"mio",
|
||||
"parking_lot",
|
||||
"signal-hook",
|
||||
"signal-hook-mio",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossterm_winapi"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2ae1b35a484aa10e07fe0638d02301c5ad24de82d310ccbd2f3693da5f09bf1c"
|
||||
dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "csv"
|
||||
version = "1.1.6"
|
||||
@@ -301,25 +283,40 @@ dependencies = [
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dashmap"
|
||||
version = "5.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"hashbrown",
|
||||
"lock_api",
|
||||
"once_cell",
|
||||
"parking_lot_core",
|
||||
"rayon",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "deduplicator"
|
||||
version = "0.0.7"
|
||||
version = "0.1.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"chrono",
|
||||
"clap",
|
||||
"colored",
|
||||
"crossterm",
|
||||
"dashmap",
|
||||
"fxhash",
|
||||
"glob",
|
||||
"humansize",
|
||||
"indicatif",
|
||||
"itertools",
|
||||
"memmap2",
|
||||
"prettytable-rs",
|
||||
"rayon",
|
||||
"sqlite",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tui",
|
||||
"unicode-segmentation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -349,6 +346,12 @@ version = "1.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797"
|
||||
|
||||
[[package]]
|
||||
name = "encode_unicode"
|
||||
version = "0.3.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
|
||||
|
||||
[[package]]
|
||||
name = "encode_unicode"
|
||||
version = "1.0.0"
|
||||
@@ -402,6 +405,12 @@ version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.12.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.4.0"
|
||||
@@ -459,6 +468,20 @@ dependencies = [
|
||||
"cxx-build",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "indicatif"
|
||||
version = "0.17.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4295cbb7573c16d310e99e713cf9e75101eb190ab31fccd35f2d2691b4352b19"
|
||||
dependencies = [
|
||||
"console",
|
||||
"number_prefix",
|
||||
"portable-atomic",
|
||||
"rayon",
|
||||
"tokio",
|
||||
"unicode-width",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "io-lifetimes"
|
||||
version = "1.0.3"
|
||||
@@ -563,6 +586,15 @@ version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
|
||||
|
||||
[[package]]
|
||||
name = "memmap2"
|
||||
version = "0.5.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4b182332558b18d807c4ce1ca8ca983b34c3ee32765e47b3f0f69b90355cc1dc"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memoffset"
|
||||
version = "0.7.1"
|
||||
@@ -613,6 +645,12 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "number_prefix"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.16.0"
|
||||
@@ -655,10 +693,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
|
||||
|
||||
[[package]]
|
||||
name = "pkg-config"
|
||||
version = "0.3.26"
|
||||
name = "portable-atomic"
|
||||
version = "0.3.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
|
||||
checksum = "26f6a7b87c2e435a3241addceeeff740ff8b7e76b74c13bf9acb17fa454ea00b"
|
||||
|
||||
[[package]]
|
||||
name = "prettytable-rs"
|
||||
@@ -667,7 +705,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eea25e07510aa6ab6547308ebe3c036016d162b8da920dbb079e3ba8acf3d95a"
|
||||
dependencies = [
|
||||
"csv",
|
||||
"encode_unicode",
|
||||
"encode_unicode 1.0.0",
|
||||
"is-terminal",
|
||||
"lazy_static",
|
||||
"term",
|
||||
@@ -808,27 +846,6 @@ version = "1.0.152"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb"
|
||||
|
||||
[[package]]
|
||||
name = "signal-hook"
|
||||
version = "0.3.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a253b5e89e2698464fc26b545c9edceb338e18a89effeeecfea192c3025be29d"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"signal-hook-registry",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "signal-hook-mio"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"mio",
|
||||
"signal-hook",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "signal-hook-registry"
|
||||
version = "1.4.0"
|
||||
@@ -854,36 +871,6 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sqlite"
|
||||
version = "0.30.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "12e072cb5fb89b3fe5e9c9584676348feb503f9fb3ae829d9868171bc5372d48"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"sqlite3-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sqlite3-src"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d1815a7a02c996eb8e5c64f61fcb6fd9b12e593ce265c512c5853b2513635691"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sqlite3-sys"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d47c99824fc55360ba00caf28de0b8a0458369b832e016a64c13af0ad9fbb9ee"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"sqlite3-src",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.10.0"
|
||||
@@ -983,19 +970,6 @@ dependencies = [
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tui"
|
||||
version = "0.19.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ccdd26cbd674007e649a272da4475fb666d3aa0ad0531da7136db6fab0e5bad1"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"cassowary",
|
||||
"crossterm",
|
||||
"unicode-segmentation",
|
||||
"unicode-width",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.6"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "deduplicator"
|
||||
version = "0.0.7"
|
||||
version = "0.1.1"
|
||||
edition = "2021"
|
||||
description = "find,filter,delete Duplicates"
|
||||
license = "MIT"
|
||||
@@ -13,14 +13,15 @@ anyhow = "1.0.68"
|
||||
chrono = "0.4.23"
|
||||
clap = { version = "4.0.32", features = ["derive"] }
|
||||
colored = "2.0.0"
|
||||
crossterm = "0.25.0"
|
||||
dashmap = { version = "5.4.0", features = ["rayon"] }
|
||||
fxhash = "0.2.1"
|
||||
glob = "0.3.0"
|
||||
humansize = "2.1.2"
|
||||
indicatif = { version = "0.17.2", features = ["rayon", "tokio"] }
|
||||
itertools = "0.10.5"
|
||||
memmap2 = "0.5.8"
|
||||
prettytable-rs = "0.10.0"
|
||||
rayon = "1.6.1"
|
||||
sqlite = "0.30.3"
|
||||
thiserror = "1.0.38"
|
||||
tokio = { version = "1.23.0", features = ["full"] }
|
||||
tui = "0.19.0"
|
||||
unicode-segmentation = "1.10.0"
|
||||
|
||||
15
README.md
15
README.md
@@ -16,7 +16,7 @@ Usage: deduplicator [OPTIONS]
|
||||
Options:
|
||||
-t, --types <TYPES> Filetypes to deduplicate (default = all)
|
||||
--dir <DIR> Run Deduplicator on dir different from pwd
|
||||
-n, --nocache Don't use cache for indexing files (default = true)
|
||||
-i, --interactive Delete files interactively
|
||||
-h, --help Print help information
|
||||
-V, --version Print version information
|
||||
```
|
||||
@@ -37,12 +37,13 @@ cargo install deduplicator
|
||||
<p align="center">
|
||||
Deduplicator uses fxhash (a non-cryptographic hashing algorithm) which is extremely fast. As a result, deduplicator is able to process huge amounts of data in a couple of seconds.</p>
|
||||
|
||||
<p align="center">
|
||||
While testing, Deduplicator was able to go through 8.6GB of pdf files and detect duplicates in 2.9 seconds
|
||||
</p>
|
||||
<h2 align="center">Screenshots</h2>
|
||||
|
||||
<p align="center">
|
||||
<img align="center" src="https://user-images.githubusercontent.com/36154121/211458077-90092aa3-496c-492f-a061-618059890d5f.png" width="500" height="400" />
|
||||
While testing, Deduplicator was able to go through 8.6GB of pdf files and detect duplicates in 2.9 seconds
|
||||
</p>
|
||||
|
||||
<h2 align="center">Screenshots</h2>
|
||||
|
||||
|
||||
<img src="https://user-images.githubusercontent.com/36154121/211948081-63c12b94-6251-487b-a49f-ac5418169d5a.gif" />
|
||||
<img src="https://user-images.githubusercontent.com/36154121/211458077-90092aa3-496c-492f-a061-618059890d5f.png" />
|
||||
|
||||
|
||||
18
src/app.rs
Normal file
18
src/app.rs
Normal file
@@ -0,0 +1,18 @@
|
||||
use crate::output;
|
||||
use crate::params::Params;
|
||||
use crate::scanner;
|
||||
use anyhow::Result;
|
||||
|
||||
pub struct App;
|
||||
|
||||
impl App {
|
||||
pub fn init(app_args: &Params) -> Result<()> {
|
||||
let duplicates = scanner::duplicates(app_args)?;
|
||||
match app_args.interactive {
|
||||
true => output::interactive(duplicates, app_args),
|
||||
false => output::print(duplicates, app_args),
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::Result;
|
||||
use crossterm::event::{self, KeyCode, KeyEvent};
|
||||
|
||||
use super::events;
|
||||
|
||||
pub struct EventHandler;
|
||||
|
||||
impl EventHandler {
|
||||
pub fn init() -> Result<events::Event> {
|
||||
if crossterm::event::poll(Duration::from_millis(10))? {
|
||||
match event::read()? {
|
||||
event::Event::Key(keycode) => Self::handle_keypress(keycode),
|
||||
_ => Ok(events::Event::Noop),
|
||||
}
|
||||
} else {
|
||||
Ok(events::Event::Noop)
|
||||
}
|
||||
}
|
||||
|
||||
fn handle_keypress(keyevent: KeyEvent) -> Result<events::Event> {
|
||||
match keyevent.code {
|
||||
KeyCode::Char('q') => Ok(events::Event::Exit),
|
||||
_ => Ok(events::Event::Noop),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
pub enum Event {
|
||||
Exit,
|
||||
Noop,
|
||||
}
|
||||
@@ -1,89 +0,0 @@
|
||||
#![allow(unused)]
|
||||
|
||||
mod event_handler;
|
||||
mod events;
|
||||
mod formatter;
|
||||
mod ui;
|
||||
pub mod file_manager;
|
||||
|
||||
use std::{io, thread, time::Duration};
|
||||
|
||||
use anyhow::{anyhow, Result};
|
||||
use crossterm::{event, execute, terminal};
|
||||
use event_handler::EventHandler;
|
||||
use tui::{
|
||||
backend::CrosstermBackend,
|
||||
widgets::{Block, Borders, Widget},
|
||||
Terminal,
|
||||
};
|
||||
use ui::Ui;
|
||||
|
||||
use crate::database;
|
||||
use crate::output;
|
||||
use crate::params::Params;
|
||||
use crate::scanner;
|
||||
|
||||
pub struct App;
|
||||
|
||||
impl App {
|
||||
pub fn init(app_args: &Params) -> Result<()> {
|
||||
// let mut term = Self::init_terminal()?;
|
||||
|
||||
let connection = database::get_connection(app_args)?;
|
||||
let duplicates = scanner::duplicates(app_args, &connection)?;
|
||||
|
||||
// Self::init_render_loop(&mut term)?;
|
||||
// Self::cleanup(&mut term)?;
|
||||
|
||||
match app_args.interactive {
|
||||
true => output::interactive(duplicates, app_args),
|
||||
false => output::print(duplicates, app_args) /* TODO: APP TUI INIT FUNCTION */
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cleanup(term: &mut Terminal<CrosstermBackend<io::Stdout>>) -> Result<()> {
|
||||
terminal::disable_raw_mode()?;
|
||||
execute!(
|
||||
term.backend_mut(),
|
||||
terminal::LeaveAlternateScreen,
|
||||
event::DisableMouseCapture
|
||||
)?;
|
||||
|
||||
term.show_cursor()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn render_cycle(term: &mut Terminal<CrosstermBackend<io::Stdout>>) -> Result<()> {
|
||||
match EventHandler::init()? {
|
||||
events::Event::Noop => Ui::render_frame(term),
|
||||
events::Event::Exit => Err(anyhow!("Exit")),
|
||||
}
|
||||
}
|
||||
|
||||
fn init_render_loop(term: &mut Terminal<CrosstermBackend<io::Stdout>>) -> Result<()> {
|
||||
// this could be simplified with a `while Self::render_cycle(term).is_ok() {}` in the current state, but maybe
|
||||
// it's good to keep it to handle errors in the future
|
||||
loop {
|
||||
match Self::render_cycle(term) {
|
||||
Ok(_) => continue,
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn init_terminal() -> Result<Terminal<CrosstermBackend<io::Stdout>>> {
|
||||
terminal::enable_raw_mode()?;
|
||||
let mut stdout = io::stdout();
|
||||
execute!(
|
||||
stdout,
|
||||
terminal::EnterAlternateScreen,
|
||||
event::EnableMouseCapture
|
||||
)?;
|
||||
let backend = CrosstermBackend::new(stdout);
|
||||
Ok(Terminal::new(backend)?)
|
||||
}
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
use std::io;
|
||||
|
||||
use anyhow::Result;
|
||||
use tui::{
|
||||
backend::{Backend, CrosstermBackend},
|
||||
layout::{Constraint, Direction, Layout, Rect},
|
||||
style::{Modifier, Style},
|
||||
text::{Span, Spans},
|
||||
widgets::{Block, Borders, List, ListItem, Widget},
|
||||
Frame, Terminal,
|
||||
};
|
||||
|
||||
pub struct Ui;
|
||||
|
||||
impl Ui {
|
||||
fn generate_file_list() -> impl Widget {
|
||||
let tasks: Vec<ListItem> = vec!["Sreedev"; 100]
|
||||
.into_iter()
|
||||
.map(|item| ListItem::new(vec![Spans::from(Span::raw(item))]))
|
||||
.collect();
|
||||
|
||||
List::new(tasks)
|
||||
.block(Block::default().borders(Borders::ALL).title("List"))
|
||||
.highlight_style(Style::default().add_modifier(Modifier::BOLD))
|
||||
.highlight_symbol("> ")
|
||||
}
|
||||
|
||||
fn generate_info_bar() -> impl Widget {
|
||||
Block::default().title("Description").borders(Borders::ALL)
|
||||
}
|
||||
|
||||
fn generate_file_desc() -> impl Widget {
|
||||
Block::default().title("Description").borders(Borders::ALL)
|
||||
}
|
||||
|
||||
pub fn render_frame(term: &mut Terminal<CrosstermBackend<io::Stdout>>) -> Result<()> {
|
||||
term.draw(|f| {
|
||||
let windows = Layout::default()
|
||||
.direction(Direction::Vertical)
|
||||
.constraints([Constraint::Ratio(2, 16), Constraint::Ratio(14, 16)].as_ref())
|
||||
.split(f.size());
|
||||
|
||||
let subwindows = Layout::default()
|
||||
.direction(Direction::Horizontal)
|
||||
.constraints([Constraint::Ratio(1, 4), Constraint::Ratio(3, 4)].as_ref())
|
||||
.split(windows[1]);
|
||||
|
||||
f.render_widget(Self::generate_info_bar(), windows[0]);
|
||||
f.render_widget(Self::generate_file_list(), subwindows[0]);
|
||||
f.render_widget(Self::generate_file_desc(), subwindows[1]);
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -1,89 +0,0 @@
|
||||
use std::env::temp_dir;
|
||||
|
||||
use anyhow::Result;
|
||||
|
||||
use crate::params::Params;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct File {
|
||||
pub path: String,
|
||||
pub hash: String,
|
||||
}
|
||||
|
||||
fn db_connection_url(args: &Params) -> String {
|
||||
match args.nocache {
|
||||
true => String::from(":memory:"),
|
||||
false => {
|
||||
let temp_dir_path = temp_dir();
|
||||
format!("{}/deduplicator.db", temp_dir_path.display())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_connection(args: &Params) -> Result<sqlite::Connection, sqlite::Error> {
|
||||
sqlite::open(db_connection_url(args)).and_then(|conn| {
|
||||
setup(&conn).ok();
|
||||
Ok(conn)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn setup(connection: &sqlite::Connection) -> Result<()> {
|
||||
let query = "CREATE TABLE files (file_identifier STRING, hash STRING)";
|
||||
connection.execute(query).ok();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn put(file: &File, connection: &sqlite::Connection) -> Result<()> {
|
||||
let query = format!(
|
||||
"INSERT INTO files (file_identifier, hash) VALUES (\"{}\", \"{}\")",
|
||||
file.path, file.hash
|
||||
);
|
||||
connection.execute(query)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn indexed_paths(connection: &sqlite::Connection) -> Result<Vec<File>> {
|
||||
let query = "SELECT * FROM files";
|
||||
|
||||
let result: Vec<File> = connection
|
||||
.prepare(query)?
|
||||
.into_iter()
|
||||
.filter_map(|row_result| row_result.ok())
|
||||
.map(|row| {
|
||||
let path = row.read::<&str, _>("file_identifier").to_string();
|
||||
let hash = row.read::<i64, _>("hash").to_string();
|
||||
File { path, hash }
|
||||
})
|
||||
.collect();
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub fn duplicate_hashes(connection: &sqlite::Connection, path: &str) -> Result<Vec<File>> {
|
||||
let query = format!(
|
||||
"
|
||||
SELECT a.* FROM files a
|
||||
JOIN (SELECT file_identifier, hash, COUNT(*)
|
||||
FROM files
|
||||
GROUP BY hash
|
||||
HAVING count(*) > 1 ) b
|
||||
ON a.hash = b.hash
|
||||
WHERE a.file_identifier LIKE \"{}%\"
|
||||
ORDER BY a.file_identifier
|
||||
",
|
||||
path
|
||||
);
|
||||
|
||||
let result: Vec<File> = connection
|
||||
.prepare(query)?
|
||||
.into_iter()
|
||||
.filter_map(|row_result| row_result.ok())
|
||||
.map(|row| {
|
||||
let path = row.read::<&str, _>("file_identifier").to_string();
|
||||
let hash = row.read::<i64, _>("hash").to_string();
|
||||
File { path, hash }
|
||||
})
|
||||
.collect();
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
@@ -1,12 +1,18 @@
|
||||
use crate::database::File;
|
||||
use anyhow::Result;
|
||||
use colored::Colorize;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct File {
|
||||
pub path: String,
|
||||
pub size: Option<u64>,
|
||||
pub hash: Option<String>,
|
||||
}
|
||||
|
||||
pub fn delete_files(files: Vec<File>) -> Result<()> {
|
||||
files.into_iter().for_each(|file| {
|
||||
match std::fs::remove_file(file.path.clone()) {
|
||||
Ok(_) => println!("{}: {}", "DELETED".green(), file.path),
|
||||
Err(e) => println!("{}: {}", "FAILED".red(), file.path)
|
||||
Err(_) => println!("{}: {}", "FAILED".red(), file.path)
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
#![allow(unused)] // TODO: remove this once TUI is implemented
|
||||
mod app;
|
||||
mod database;
|
||||
mod file_manager;
|
||||
mod output;
|
||||
mod params;
|
||||
mod scanner;
|
||||
|
||||
113
src/output.rs
113
src/output.rs
@@ -1,26 +1,28 @@
|
||||
use std::{collections::HashMap, fs, io};
|
||||
use std::io::Write;
|
||||
|
||||
use crate::file_manager::{self, File};
|
||||
use crate::params::Params;
|
||||
use anyhow::Result;
|
||||
use chrono::offset::Utc;
|
||||
use chrono::DateTime;
|
||||
use colored::Colorize;
|
||||
use dashmap::DashMap;
|
||||
use humansize::{format_size, DECIMAL};
|
||||
use itertools::Itertools;
|
||||
|
||||
use crate::app::file_manager;
|
||||
use crate::database::File;
|
||||
use crate::params::Params;
|
||||
use prettytable::{format, row, Cell, Row, Table};
|
||||
use prettytable::{format, row, Table};
|
||||
use std::io::Write;
|
||||
use std::{fs, io};
|
||||
use unicode_segmentation::UnicodeSegmentation;
|
||||
|
||||
fn format_path(path: &str, opts: &Params) -> Result<String> {
|
||||
let display_path = path.replace(&opts.get_directory()?, "");
|
||||
let text_vec = display_path.chars().collect::<Vec<_>>();
|
||||
|
||||
let display_range = if text_vec.len() > 32 {
|
||||
text_vec[(display_path.len() - 32)..]
|
||||
.iter()
|
||||
.collect::<String>()
|
||||
let display_range = if display_path.chars().count() > 32 {
|
||||
display_path
|
||||
.graphemes(true)
|
||||
.collect::<Vec<&str>>()
|
||||
.into_iter()
|
||||
.rev()
|
||||
.take(32)
|
||||
.rev()
|
||||
.collect()
|
||||
} else {
|
||||
display_path
|
||||
};
|
||||
@@ -41,19 +43,7 @@ fn modified_time(path: &String) -> Result<String> {
|
||||
Ok(modified_time.format("%Y-%m-%d %H:%M:%S").to_string())
|
||||
}
|
||||
|
||||
fn group_duplicates(duplicates: Vec<File>) -> HashMap<String, Vec<File>> {
|
||||
let mut duplicate_mapper: HashMap<String, Vec<File>> = HashMap::new();
|
||||
duplicates.into_iter().for_each(|file| {
|
||||
duplicate_mapper
|
||||
.entry(file.hash.clone())
|
||||
.and_modify(|value| value.push(file.clone()))
|
||||
.or_insert_with(|| vec![file]);
|
||||
});
|
||||
|
||||
duplicate_mapper
|
||||
}
|
||||
|
||||
fn print_meta_info(duplicates: &Vec<File>, opts: &Params) {
|
||||
fn print_meta_info() {
|
||||
println!("Deduplicator v{}", std::env!("CARGO_PKG_VERSION"));
|
||||
}
|
||||
|
||||
@@ -70,14 +60,14 @@ fn scan_group_instruction() -> Result<String> {
|
||||
}
|
||||
|
||||
fn scan_group_confirmation() -> Result<bool> {
|
||||
print!("\nconfirm? [Y/n]: ");
|
||||
print!("\nconfirm? [y/N]: ");
|
||||
std::io::stdout().flush()?;
|
||||
let mut user_input = String::new();
|
||||
io::stdin().read_line(&mut user_input)?;
|
||||
|
||||
match user_input.trim() {
|
||||
"Y" | "y" => Ok(true),
|
||||
_ => Ok(false)
|
||||
_ => Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -103,52 +93,59 @@ fn process_group_action(duplicates: &Vec<File>, dup_index: usize, dup_size: usiz
|
||||
|
||||
print!("{esc}[2J{esc}[1;1H", esc = 27 as char);
|
||||
|
||||
if parsed_file_indices.is_empty() { return }
|
||||
if parsed_file_indices.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let files_to_delete = parsed_file_indices
|
||||
.into_iter()
|
||||
.map(|index| duplicates[index].clone());
|
||||
|
||||
println!("\n{}", "The following files will be deleted:".red());
|
||||
files_to_delete.clone().enumerate().for_each(|(index, file)| {
|
||||
println!("{}: {}", index.to_string().blue(), file.path);
|
||||
});
|
||||
|
||||
files_to_delete
|
||||
.clone()
|
||||
.enumerate()
|
||||
.for_each(|(index, file)| {
|
||||
println!("{}: {}", index.to_string().blue(), file.path);
|
||||
});
|
||||
|
||||
match scan_group_confirmation().unwrap() {
|
||||
true => { file_manager::delete_files(files_to_delete.collect_vec()); },
|
||||
false => println!("{}", "\nCancelled Delete Operation.".red())
|
||||
true => {
|
||||
file_manager::delete_files(files_to_delete.collect_vec()).ok();
|
||||
}
|
||||
false => println!("{}", "\nCancelled Delete Operation.".red()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn interactive(duplicates: Vec<File>, opts: &Params) {
|
||||
print_meta_info(&duplicates, opts);
|
||||
let grouped_duplicates = group_duplicates(duplicates);
|
||||
pub fn interactive(duplicates: DashMap<String, Vec<File>>, opts: &Params) {
|
||||
print_meta_info();
|
||||
duplicates
|
||||
.clone()
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.for_each(|(gindex, (_, group))| {
|
||||
let mut itable = Table::new();
|
||||
itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
|
||||
itable.set_titles(row!["index", "filename", "size", "updated_at"]);
|
||||
group.iter().enumerate().for_each(|(index, file)| {
|
||||
itable.add_row(row![
|
||||
index,
|
||||
format_path(&file.path, opts).unwrap_or_default().blue(),
|
||||
file_size(&file.path).unwrap_or_default().red(),
|
||||
modified_time(&file.path).unwrap_or_default().yellow()
|
||||
]);
|
||||
});
|
||||
|
||||
grouped_duplicates.iter().enumerate().for_each(|(gindex, (hash, group))| {
|
||||
let mut itable = Table::new();
|
||||
itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
|
||||
itable.set_titles(row!["index", "filename", "size", "updated_at"]);
|
||||
group.iter().enumerate().for_each(|(index, file)| {
|
||||
itable.add_row(row![
|
||||
index,
|
||||
format_path(&file.path, opts).unwrap_or_default().blue(),
|
||||
file_size(&file.path).unwrap_or_default().red(),
|
||||
modified_time(&file.path).unwrap_or_default().yellow()
|
||||
]);
|
||||
process_group_action(&group, gindex, duplicates.len(), itable);
|
||||
});
|
||||
|
||||
process_group_action(group, gindex, grouped_duplicates.len(), itable);
|
||||
});
|
||||
}
|
||||
|
||||
pub fn print(duplicates: Vec<File>, opts: &Params) {
|
||||
print_meta_info(&duplicates, opts);
|
||||
pub fn print(duplicates: DashMap<String, Vec<File>>, opts: &Params) {
|
||||
print_meta_info();
|
||||
|
||||
let mut output_table = Table::new();
|
||||
let grouped_duplicates: HashMap<String, Vec<File>> = group_duplicates(duplicates);
|
||||
|
||||
output_table.set_titles(row!["hash", "duplicates"]);
|
||||
grouped_duplicates.iter().for_each(|(hash, group)| {
|
||||
duplicates.into_iter().for_each(|(hash, group)| {
|
||||
let mut inner_table = Table::new();
|
||||
inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
|
||||
group.iter().for_each(|file| {
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
use std::{fs, path::PathBuf};
|
||||
|
||||
use anyhow::{anyhow, Result};
|
||||
use clap::Parser;
|
||||
use std::{fs, path::PathBuf};
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(author, version, about, long_about = None)]
|
||||
@@ -12,12 +11,9 @@ pub struct Params {
|
||||
/// Run Deduplicator on dir different from pwd
|
||||
#[arg(long)]
|
||||
pub dir: Option<PathBuf>,
|
||||
/// Don't use cache for indexing files (default = false)
|
||||
#[arg(long, short)]
|
||||
pub nocache: bool,
|
||||
/// Delete files interactively
|
||||
#[arg(long, short)]
|
||||
pub interactive: bool
|
||||
pub interactive: bool,
|
||||
}
|
||||
|
||||
impl Params {
|
||||
@@ -37,4 +33,18 @@ impl Params {
|
||||
|
||||
Ok(dir)
|
||||
}
|
||||
|
||||
pub fn get_glob_patterns(&self) -> Vec<PathBuf> {
|
||||
self.types
|
||||
.clone()
|
||||
.unwrap_or_else(|| String::from("*"))
|
||||
.split(',')
|
||||
.map(|filetype| format!("*.{}", filetype))
|
||||
.map(|filetype| {
|
||||
vec![self.get_directory().unwrap(), String::from("**"), filetype]
|
||||
.iter()
|
||||
.collect()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
174
src/scanner.rs
174
src/scanner.rs
@@ -1,56 +1,61 @@
|
||||
use anyhow::Result;
|
||||
use dashmap::DashMap;
|
||||
use fxhash::hash64 as hasher;
|
||||
use glob::glob;
|
||||
use indicatif::{ParallelProgressIterator, ProgressStyle};
|
||||
use memmap2::Mmap;
|
||||
use rayon::prelude::*;
|
||||
use std::hash::Hasher;
|
||||
use std::{fs, path::PathBuf};
|
||||
|
||||
use anyhow::Result;
|
||||
use fxhash::hash32 as hasher;
|
||||
use glob::glob;
|
||||
use itertools::Itertools;
|
||||
use rayon::prelude::*;
|
||||
use crate::{file_manager::File, params::Params};
|
||||
|
||||
use crate::{
|
||||
database::{self, File},
|
||||
params::Params,
|
||||
};
|
||||
|
||||
pub fn duplicates(app_opts: &Params, connection: &sqlite::Connection) -> Result<Vec<File>> {
|
||||
let scan_results = scan(app_opts, connection)?;
|
||||
let base_path = app_opts.get_directory()?;
|
||||
|
||||
index_files(scan_results, connection)?;
|
||||
database::duplicate_hashes(connection, &base_path)
|
||||
#[derive(Clone, Copy)]
|
||||
enum IndexCritera {
|
||||
Size,
|
||||
Hash,
|
||||
}
|
||||
|
||||
fn get_glob_patterns(opts: &Params, directory: &str) -> Vec<PathBuf> {
|
||||
opts.types
|
||||
.clone()
|
||||
.unwrap_or_else(|| String::from("*"))
|
||||
.split(',')
|
||||
.map(|filetype| format!("*.{}", filetype))
|
||||
.map(|filetype| {
|
||||
vec![directory.to_owned(), String::from("**"), filetype]
|
||||
.iter()
|
||||
.collect()
|
||||
})
|
||||
.collect()
|
||||
pub fn duplicates(app_opts: &Params) -> Result<DashMap<String, Vec<File>>> {
|
||||
let scan_results = scan(app_opts)?;
|
||||
let size_index_store = index_files(scan_results, IndexCritera::Size)?;
|
||||
|
||||
let sizewize_duplicate_files = size_index_store
|
||||
.into_par_iter()
|
||||
.filter(|(_, files)| files.len() > 1)
|
||||
.map(|(_, files)| files)
|
||||
.flatten()
|
||||
.collect::<Vec<File>>();
|
||||
|
||||
if sizewize_duplicate_files.len() > 1 {
|
||||
let size_wise_duplicate_paths = sizewize_duplicate_files
|
||||
.into_par_iter()
|
||||
.map(|file| file.path)
|
||||
.collect::<Vec<String>>();
|
||||
|
||||
let hash_index_store = index_files(size_wise_duplicate_paths, IndexCritera::Hash)?;
|
||||
let duplicate_files = hash_index_store
|
||||
.into_par_iter()
|
||||
.filter(|(_, files)| files.len() > 1)
|
||||
.collect();
|
||||
|
||||
Ok(duplicate_files)
|
||||
} else {
|
||||
Ok(DashMap::new())
|
||||
}
|
||||
}
|
||||
|
||||
fn is_indexed_file(path: impl Into<String>, indexed: &[File]) -> bool {
|
||||
indexed
|
||||
.iter()
|
||||
.map(|file| file.path.clone())
|
||||
.contains(&path.into())
|
||||
}
|
||||
|
||||
fn scan(app_opts: &Params, connection: &sqlite::Connection) -> Result<Vec<String>> {
|
||||
let directory = app_opts.get_directory()?;
|
||||
let glob_patterns: Vec<PathBuf> = get_glob_patterns(app_opts, &directory);
|
||||
let indexed_paths = database::indexed_paths(connection)?;
|
||||
fn scan(app_opts: &Params) -> Result<Vec<String>> {
|
||||
let glob_patterns: Vec<PathBuf> = app_opts.get_glob_patterns();
|
||||
let files: Vec<String> = glob_patterns
|
||||
.par_iter()
|
||||
.progress_with_style(ProgressStyle::with_template(
|
||||
"{spinner:.green} [scanning files] [{wide_bar:.cyan/blue}] {pos}/{len} files",
|
||||
)?)
|
||||
.filter_map(|glob_pattern| glob(glob_pattern.as_os_str().to_str()?).ok())
|
||||
.flat_map(|file_vec| {
|
||||
file_vec
|
||||
.filter_map(|x| Some(x.ok()?.as_os_str().to_str()?.to_string()))
|
||||
.filter(|fpath| !is_indexed_file(fpath, &indexed_paths))
|
||||
.filter(|glob_result| {
|
||||
fs::metadata(glob_result)
|
||||
.map(|f| f.is_file())
|
||||
@@ -63,23 +68,84 @@ fn scan(app_opts: &Params, connection: &sqlite::Connection) -> Result<Vec<String
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
fn index_files(files: Vec<String>, connection: &sqlite::Connection) -> Result<()> {
|
||||
let hashed: Vec<File> = files
|
||||
.into_par_iter()
|
||||
.filter_map(|file| {
|
||||
let hash = hash_file(&file).ok()?;
|
||||
Some(database::File { path: file, hash })
|
||||
})
|
||||
.collect();
|
||||
fn process_file_size_index(fpath: String) -> Result<File> {
|
||||
Ok(File {
|
||||
path: fpath.clone(),
|
||||
size: Some(fs::metadata(fpath)?.len()),
|
||||
hash: None,
|
||||
})
|
||||
}
|
||||
|
||||
hashed
|
||||
.iter()
|
||||
.try_for_each(|file| database::put(file, connection))
|
||||
fn process_file_hash_index(fpath: String) -> Result<File> {
|
||||
Ok(File {
|
||||
path: fpath.clone(),
|
||||
size: None,
|
||||
hash: Some(hash_file(&fpath).unwrap_or_default()),
|
||||
})
|
||||
}
|
||||
|
||||
fn process_file_index(
|
||||
fpath: String,
|
||||
store: &DashMap<String, Vec<File>>,
|
||||
index_criteria: IndexCritera,
|
||||
) {
|
||||
match index_criteria {
|
||||
IndexCritera::Size => {
|
||||
let processed_file = process_file_size_index(fpath).unwrap();
|
||||
store
|
||||
.entry(processed_file.size.unwrap_or_default().to_string())
|
||||
.and_modify(|fileset| fileset.push(processed_file.clone()))
|
||||
.or_insert_with(|| vec![processed_file]);
|
||||
}
|
||||
IndexCritera::Hash => {
|
||||
let processed_file = process_file_hash_index(fpath).unwrap();
|
||||
let indexhash = processed_file.clone().hash.unwrap_or_default();
|
||||
|
||||
store
|
||||
.entry(indexhash)
|
||||
.and_modify(|fileset| fileset.push(processed_file.clone()))
|
||||
.or_insert_with(|| vec![processed_file]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn index_files(
|
||||
files: Vec<String>,
|
||||
index_criteria: IndexCritera,
|
||||
) -> Result<DashMap<String, Vec<File>>> {
|
||||
let store: DashMap<String, Vec<File>> = DashMap::new();
|
||||
files
|
||||
.into_par_iter()
|
||||
.progress_with_style(ProgressStyle::with_template(
|
||||
"{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files",
|
||||
)?)
|
||||
.for_each(|file| process_file_index(file, &store, index_criteria));
|
||||
|
||||
Ok(store)
|
||||
}
|
||||
|
||||
pub fn incremental_hashing(filepath: &str) -> Result<String> {
|
||||
let file = fs::File::open(filepath)?;
|
||||
let fmap = unsafe { Mmap::map(&file)? };
|
||||
let mut inchasher = fxhash::FxHasher::default();
|
||||
|
||||
fmap.chunks(1_000_000)
|
||||
.for_each(|mega| inchasher.write(mega));
|
||||
|
||||
Ok(format!("{}", inchasher.finish()))
|
||||
}
|
||||
|
||||
pub fn standard_hashing(filepath: &str) -> Result<String> {
|
||||
let file = fs::read(filepath)?;
|
||||
Ok(hasher(&*file).to_string())
|
||||
}
|
||||
|
||||
pub fn hash_file(filepath: &str) -> Result<String> {
|
||||
let file = fs::read(filepath)?;
|
||||
let hash = hasher(&*file).to_string();
|
||||
let filemeta = fs::metadata(filepath)?;
|
||||
|
||||
Ok(hash)
|
||||
// NOTE: USE INCREMENTAL HASHING ONLY FOR FILES > 100MB
|
||||
match filemeta.len() < 100_000_000 {
|
||||
true => standard_hashing(filepath),
|
||||
false => incremental_hashing(filepath),
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user