diff --git a/src/database.rs b/src/database.rs index 9b6bce4..e7fb28a 100644 --- a/src/database.rs +++ b/src/database.rs @@ -1,4 +1,4 @@ -use anyhow::{anyhow, Result}; +use anyhow::Result; #[derive(Debug)] pub struct File { @@ -8,10 +8,8 @@ pub struct File { pub fn setup(connection: &sqlite::Connection) -> Result<()> { let query = "CREATE TABLE files (file_identifier STRING, hash STRING)"; - match connection.execute(query) { - Ok(_) => Ok(()), - Err(e) => Err(anyhow!("Database Setup Failed! {}", e)), - } + connection.execute(query).ok(); + Ok(()) } pub fn put(file: &File, connection: &sqlite::Connection) -> Result<()> { @@ -24,6 +22,25 @@ pub fn put(file: &File, connection: &sqlite::Connection) -> Result<()> { Ok(result) } +pub fn indexed_paths(connection: &sqlite::Connection) -> Result> { + let query = format!( + "SELECT * FROM files" + ); + + let result: Vec = connection + .prepare(query)? + .into_iter() + .map(|row_result| row_result.unwrap()) + .map(|row| { + let path = row.read::<&str, _>("file_identifier").to_string(); + let hash = row.read::<&str, _>("hash").to_string(); + File { path, hash } + }) + .collect(); + + Ok(result) +} + pub fn duplicate_hashes(connection: &sqlite::Connection) -> Result> { let query = format!( " diff --git a/src/main.rs b/src/main.rs index 7a37c97..df01841 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,7 +8,7 @@ use anyhow::Result; #[tokio::main] async fn main() -> Result<()> { - let connection = sqlite::open(":memory:")?; + let connection = sqlite::open("/tmp/deduplicator.db")?; database::setup(&connection)?; let duplicates = scanner::duplicates(cli::App::parse(), &connection)?; diff --git a/src/scanner.rs b/src/scanner.rs index 3305f93..5c6d167 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -1,28 +1,42 @@ use std::path::PathBuf; -use crate::{cli::App, database::File}; use crate::database; +use crate::{cli::App, database::File}; use anyhow::Result; use glob::glob; -use std::fs; +use itertools::Itertools; use rayon::prelude::*; +use std::fs; pub fn duplicates(app_opts: App, connection: &sqlite::Connection) -> Result> { - index_files(scan(app_opts)?, connection); + let scan_results = scan(app_opts, connection)?; + index_files(scan_results, connection); database::duplicate_hashes(connection) } -fn scan(app_opts: App) -> Result> { - let directory: String = app_opts +fn get_directory(opts: &App) -> Result { + let dir_string: String = opts .dir + .clone() .unwrap_or(std::env::current_dir()?) .as_os_str() .to_str() .unwrap() .to_string(); - let glob_patterns: Vec = app_opts - .filetypes + let dir_pathbuf = PathBuf::from(&dir_string); + let dir = fs::canonicalize(&dir_pathbuf)? + .as_os_str() + .to_str() + .unwrap() + .to_string(); + + Ok(dir) +} + +fn get_glob_patterns(opts: &App, directory: &String) -> Vec { + opts.filetypes + .clone() .unwrap_or(String::from("*")) .split(",") .map(|filetype| format!("*.{}", filetype)) @@ -31,8 +45,20 @@ fn scan(app_opts: App) -> Result> { .iter() .collect() }) - .collect(); + .collect() +} +fn is_indexed_file(path: &String, indexed: &Vec) -> bool { + indexed + .into_iter() + .map(|file| file.path.clone()) + .contains(path) +} + +fn scan(app_opts: App, connection: &sqlite::Connection) -> Result> { + let directory = get_directory(&app_opts)?; + let glob_patterns: Vec = get_glob_patterns(&app_opts, &directory); + let indexed_paths = database::indexed_paths(connection)?; let files: Vec = glob_patterns .into_par_iter() .map(|glob_pattern| glob(&glob_pattern.as_os_str().to_str().unwrap())) @@ -40,7 +66,8 @@ fn scan(app_opts: App) -> Result> { .flat_map(|file_vec| { file_vec .map(|x| x.unwrap().as_os_str().to_str().unwrap().to_string()) - .filter(|glob_result| fs::metadata(glob_result).unwrap().is_file() ) + .filter(|fpath| !is_indexed_file(fpath, &indexed_paths)) + .filter(|glob_result| fs::metadata(glob_result).unwrap().is_file()) .collect::>() }) .collect(); diff --git a/test_data/potato_us_style.jpg b/test_data/potato_us_style.jpg new file mode 100644 index 0000000..a59598c Binary files /dev/null and b/test_data/potato_us_style.jpg differ diff --git a/test_data/potato_us_style.png b/test_data/potato_us_style.png new file mode 100644 index 0000000..a59598c Binary files /dev/null and b/test_data/potato_us_style.png differ