smolkik_adm/deduplicator
1
0
Fork 0
mirror of https://github.com/sreedevk/deduplicator.git synced 2026-09-13 09:06:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

added nocache option

Browse Source
This commit is contained in:
sreedev
2022-12-29 23:48:34 -05:00
parent 61df73e146
commit 57083a7346
6 changed files with 35 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

93
Cargo.lock generated
View File

@@ -40,15 +40,6 @@ version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "block-buffer"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4"
dependencies = [
"generic-array",
]
[[package]]
name = "bumpalo"
version = "3.11.1"
@@ -152,15 +143,6 @@ version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
[[package]]
name = "cpufeatures"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320"
dependencies = [
"libc",
]
[[package]]
name = "crossbeam-channel"
version = "0.5.6"
@@ -259,21 +241,11 @@ dependencies = [
"glob",
"humansize",
"itertools",
"md5",
"rayon",
"sha256",
"sqlite",
"thiserror",
"tokio",
]
[[package]]
name = "digest"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066"
dependencies = [
"generic-array",
"xxhash-rust",
]
[[package]]
@@ -303,16 +275,6 @@ dependencies = [
"libc",
]
[[package]]
name = "generic-array"
version = "0.14.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9"
dependencies = [
"typenum",
"version_check",
]
[[package]]
name = "glob"
version = "0.3.0"
@@ -343,12 +305,6 @@ dependencies = [
"libc",
]
[[package]]
name = "hex"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
[[package]]
name = "humansize"
version = "2.1.2"
@@ -474,12 +430,6 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "md5"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
[[package]]
name = "memchr"
version = "2.5.0"
@@ -542,12 +492,6 @@ version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860"
[[package]]
name = "opaque-debug"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
[[package]]
name = "os_str_bytes"
version = "6.4.1"
@@ -688,29 +632,6 @@ version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2"
[[package]]
name = "sha2"
version = "0.9.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800"
dependencies = [
"block-buffer",
"cfg-if",
"cpufeatures",
"digest",
"opaque-debug",
]
[[package]]
name = "sha256"
version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e334db67871c14c18fc066ad14af13f9fdf5f9a91c61af432d1e3a39c8c6a141"
dependencies = [
"hex",
"sha2",
]
[[package]]
name = "signal-hook-registry"
version = "1.4.0"
@@ -854,12 +775,6 @@ dependencies = [
"syn",
]
[[package]]
name = "typenum"
version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"
[[package]]
name = "unicode-ident"
version = "1.0.6"
@@ -1031,3 +946,9 @@ name = "windows_x86_64_msvc"
version = "0.42.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5"
[[package]]
name = "xxhash-rust"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "735a71d46c4d68d71d4b24d03fdc2b98e38cea81730595801db779c04fe80d70"

3
Cargo.toml
View File

@@ -13,9 +13,8 @@ colored = "2.0.0"
glob = "0.3.0"
humansize = "2.1.2"
itertools = "0.10.5"
md5 = "0.7.0"
rayon = "1.6.1"
sha256 = "1.1.1"
sqlite = "0.30.3"
thiserror = "1.0.38"
tokio = { version = "1.23.0", features = ["full"] }
xxhash-rust = { version = "0.8.6", features = ["xxh3"] }

5
src/cli.rs
View File

@@ -12,5 +12,8 @@ pub struct App {
pub delete: Option<String>,
/// Run Deduplicator on dir different from pwd
#[arg(long)]
pub dir: Option<PathBuf>
pub dir: Option<PathBuf>,
/// Don't use cache for indexing files (default = true)
#[arg(long, short)]
pub nocache: bool
}

17
src/database.rs
View File

@@ -1,4 +1,5 @@
use anyhow::Result;
use crate::cli::App;
#[derive(Debug, Clone)]
pub struct File {
@@ -6,6 +7,18 @@ pub struct File {
pub hash: String,
}
pub fn get_connection(args: &App) -> Result<sqlite::Connection, sqlite::Error> {
let connection_url = match args.nocache {
false => "/tmp/deduplicator.db",
true => ":memory:"
};
sqlite::open(connection_url).and_then(|conn| {
setup(&conn).ok();
Ok(conn)
})
}
pub fn setup(connection: &sqlite::Connection) -> Result<()> {
let query = "CREATE TABLE files (file_identifier STRING, hash STRING)";
connection.execute(query).ok();
@@ -33,7 +46,7 @@ pub fn indexed_paths(connection: &sqlite::Connection) -> Result<Vec<File>> {
.map(|row_result| row_result.unwrap())
.map(|row| {
let path = row.read::<&str, _>("file_identifier").to_string();
let hash = row.read::<&str, _>("hash").to_string();
let hash = row.read::<i64, _>("hash").to_string();
File { path, hash }
})
.collect();
@@ -60,7 +73,7 @@ pub fn duplicate_hashes(connection: &sqlite::Connection, path: &String) -> Resul
.map(|row_result| row_result.unwrap())
.map(|row| {
let path = row.read::<&str, _>("file_identifier").to_string();
let hash = row.read::<&str, _>("hash").to_string();
let hash = row.read::<i64, _>("hash").to_string();
File { path, hash }
})
.collect();

9
src/main.rs
View File

@@ -8,13 +8,10 @@ use clap::Parser;
#[tokio::main]
async fn main() -> Result<()> {
let connection = sqlite::open("/tmp/deduplicator.db").and_then(|conn| {
database::setup(&conn).ok();
Ok(conn)
})?;
let app_args = cli::App::parse();
let connection = database::get_connection(&app_args)?;
let duplicates = scanner::duplicates(&app_args, &connection)?;
let duplicates = scanner::duplicates(cli::App::parse(), &connection)?;
output::print(duplicates);
Ok(())
}

9
src/scanner.rs
View File

@@ -6,10 +6,11 @@ use itertools::Itertools;
use rayon::prelude::*;
use std::fs;
use std::path::PathBuf;
use xxhash_rust::xxh3::xxh3_64 as hasher;
pub fn duplicates(app_opts: App, connection: &sqlite::Connection) -> Result<Vec<File>> {
let scan_results = scan(&app_opts, connection)?;
let base_path = get_directory(&app_opts)?;
pub fn duplicates(app_opts: &App, connection: &sqlite::Connection) -> Result<Vec<File>> {
let scan_results = scan(app_opts, connection)?;
let base_path = get_directory(app_opts)?;
index_files(scan_results, connection);
database::duplicate_hashes(connection, &base_path)
@@ -92,7 +93,7 @@ fn index_files(files: Vec<String>, connection: &sqlite::Connection) {
pub fn hash_file(filepath: &str) -> Result<String> {
let file = fs::read(filepath)?;
let hash = sha256::digest(&*file);
let hash = hasher(&*file).to_string();
Ok(hash)
}