added caching

src/database.rs

@@ -1,4 +1,4 @@
-use anyhow::{anyhow, Result};
+use anyhow::Result;
 
 #[derive(Debug)]
 pub struct File {
@@ -8,10 +8,8 @@ pub struct File {
 
 pub fn setup(connection: &sqlite::Connection) -> Result<()> {
     let query = "CREATE TABLE files (file_identifier STRING, hash STRING)";
-    match connection.execute(query) {
-        Ok(_) => Ok(()),
-        Err(e) => Err(anyhow!("Database Setup Failed! {}", e)),
-    }
+    connection.execute(query).ok();
+    Ok(())
 }
 
 pub fn put(file: &File, connection: &sqlite::Connection) -> Result<()> {
@@ -24,6 +22,25 @@ pub fn put(file: &File, connection: &sqlite::Connection) -> Result<()> {
     Ok(result)
 }
 
+pub fn indexed_paths(connection: &sqlite::Connection) -> Result<Vec<File>> {
+    let query = format!(
+        "SELECT * FROM files"
+        );
+
+    let result: Vec<File> = connection
+        .prepare(query)?
+        .into_iter()
+        .map(|row_result| row_result.unwrap())
+        .map(|row| {
+            let path = row.read::<&str, _>("file_identifier").to_string();
+            let hash = row.read::<&str, _>("hash").to_string();
+            File { path, hash }
+        })
+    .collect();
+
+    Ok(result)
+}
+
 pub fn duplicate_hashes(connection: &sqlite::Connection) -> Result<Vec<File>> {
     let query = format!(
         " 

src/main.rs

@@ -8,7 +8,7 @@ use anyhow::Result;
 
 #[tokio::main]
 async fn main() -> Result<()> {
-    let connection = sqlite::open(":memory:")?;
+    let connection = sqlite::open("/tmp/deduplicator.db")?;
     database::setup(&connection)?;
 
     let duplicates = scanner::duplicates(cli::App::parse(), &connection)?;

src/scanner.rs

@@ -1,28 +1,42 @@
 use std::path::PathBuf;
 
-use crate::{cli::App, database::File};
 use crate::database;
+use crate::{cli::App, database::File};
 use anyhow::Result;
 use glob::glob;
-use std::fs;
+use itertools::Itertools;
 use rayon::prelude::*;
+use std::fs;
 
 pub fn duplicates(app_opts: App, connection: &sqlite::Connection) -> Result<Vec<File>> {
-    index_files(scan(app_opts)?, connection);
+    let scan_results = scan(app_opts, connection)?;
+    index_files(scan_results, connection);
     database::duplicate_hashes(connection)
 }
 
-fn scan(app_opts: App) -> Result<Vec<String>> {
-    let directory: String = app_opts
+fn get_directory(opts: &App) -> Result<String> {
+    let dir_string: String = opts
         .dir
+        .clone()
         .unwrap_or(std::env::current_dir()?)
         .as_os_str()
         .to_str()
         .unwrap()
         .to_string();
 
-    let glob_patterns: Vec<PathBuf> = app_opts
-        .filetypes
+    let dir_pathbuf = PathBuf::from(&dir_string);
+    let dir = fs::canonicalize(&dir_pathbuf)?
+        .as_os_str()
+        .to_str()
+        .unwrap()
+        .to_string();
+
+    Ok(dir)
+}
+
+fn get_glob_patterns(opts: &App, directory: &String) -> Vec<PathBuf> {
+    opts.filetypes
+        .clone()
         .unwrap_or(String::from("*"))
         .split(",")
         .map(|filetype| format!("*.{}", filetype))
@@ -31,8 +45,20 @@ fn scan(app_opts: App) -> Result<Vec<String>> {
                 .iter()
                 .collect()
         })
-        .collect();
+        .collect()
+}
 
+fn is_indexed_file(path: &String, indexed: &Vec<File>) -> bool {
+    indexed
+        .into_iter()
+        .map(|file| file.path.clone())
+        .contains(path)
+}
+
+fn scan(app_opts: App, connection: &sqlite::Connection) -> Result<Vec<String>> {
+    let directory = get_directory(&app_opts)?;
+    let glob_patterns: Vec<PathBuf> = get_glob_patterns(&app_opts, &directory);
+    let indexed_paths = database::indexed_paths(connection)?;
     let files: Vec<String> = glob_patterns
         .into_par_iter()
         .map(|glob_pattern| glob(&glob_pattern.as_os_str().to_str().unwrap()))
@@ -40,7 +66,8 @@ fn scan(app_opts: App) -> Result<Vec<String>> {
         .flat_map(|file_vec| {
             file_vec
                 .map(|x| x.unwrap().as_os_str().to_str().unwrap().to_string())
-                .filter(|glob_result| fs::metadata(glob_result).unwrap().is_file() )
+                .filter(|fpath| !is_indexed_file(fpath, &indexed_paths))
+                .filter(|glob_result| fs::metadata(glob_result).unwrap().is_file())
                 .collect::<Vec<String>>()
         })
         .collect();