smolkik_adm/deduplicator
1
0
Fork 0
mirror of https://github.com/sreedevk/deduplicator.git synced 2026-08-26 02:04:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

implemented deduplicator with file identifiers

Browse Source
This commit is contained in:
sreedev
2022-12-24 19:16:50 -05:00
parent 3b06c0851f
commit b822aba661
3 changed files with 15 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/database.rs
View File

@@ -6,8 +6,7 @@ pub struct File {
pub hash: String,
}
pub fn setup() -> Result<()> {
let connection = sqlite::open(":memory:")?;
pub fn setup(connection: &sqlite::Connection) -> Result<()> {
let query = "CREATE TABLE files (file_identifier STRING, hash STRING)";
match connection.execute(query) {
Ok(_) => Ok(()),
@@ -15,10 +14,9 @@ pub fn setup() -> Result<()> {
}
}
pub fn put(file: File) -> Result<()> {
let connection = sqlite::open(":memory:")?;
pub fn put(file: &File, connection: &sqlite::Connection) -> Result<()> {
let query = format!(
"INSERT INTO files (file_identifier, hash) VALUES ({}, {})",
"INSERT INTO files (file_identifier, hash) VALUES (\"{}\", \"{}\")",
file.path, file.hash
);
let result = connection.execute(query)?;
@@ -26,8 +24,7 @@ pub fn put(file: File) -> Result<()> {
Ok(result)
}
pub fn duplicate_hashes() -> Result<Vec<File>> {
let connection = sqlite::open(":memory:")?;
pub fn duplicate_hashes(connection: &sqlite::Connection) -> Result<Vec<File>> {
let query = format!(
"
SELECT a.* FROM files a
@@ -35,7 +32,7 @@ pub fn duplicate_hashes() -> Result<Vec<File>> {
FROM files
GROUP BY hash
HAVING count(*) > 1 ) b
ON a.file_identifier = b.file_identifier
ON a.hash = b.hash
ORDER BY a.file_identifier
"
);

7
src/main.rs
View File

@@ -7,8 +7,11 @@ use anyhow::Result;
#[tokio::main]
async fn main() -> Result<()> {
database::setup()?;
let duplicates = scanner::duplicates(cli::App::parse())?;
let connection = sqlite::open(":memory:")?;
database::setup(&connection)?;
let duplicates = scanner::duplicates(cli::App::parse(), &connection)?;
dbg!(duplicates);
Ok(())

10
src/scanner.rs
View File

@@ -6,9 +6,9 @@ use anyhow::Result;
use glob::glob;
use std::{fs, io};
pub fn duplicates(app_opts: App) -> Result<Vec<File>> {
index_files(scan(app_opts)?);
database::duplicate_hashes()
pub fn duplicates(app_opts: App, connection: &sqlite::Connection) -> Result<Vec<File>> {
index_files(scan(app_opts)?, connection);
database::duplicate_hashes(connection)
}
fn scan(app_opts: App) -> Result<Vec<String>> {
@@ -46,7 +46,7 @@ fn scan(app_opts: App) -> Result<Vec<String>> {
Ok(files)
}
fn index_files(files: Vec<String>) {
fn index_files(files: Vec<String>, connection: &sqlite::Connection) {
files
.into_iter()
.map(|file| {
@@ -54,7 +54,7 @@ fn index_files(files: Vec<String>) {
database::File { path: file, hash }
})
.for_each(|file| {
database::put(file).unwrap();
database::put(&file, connection).unwrap();
});
}