add more tests

Cargo.toml

@@ -11,6 +11,10 @@ authors = [
   "Dhruva Sagar <dhruva.sagar@gmail.com>",
 ]
 
+[[bin]]
+name = "deduplicator"
+path = "src/main.rs"
+
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 [dependencies]
 anyhow = "1.0.68"
@@ -21,7 +25,6 @@ dashmap = { version = "5.4.0", features = ["rayon"] }
 globwalk = "0.8.1"
 gxhash = { version = "3.4.1", default-features = false }
 indicatif = { version = "0.17.2", features = ["rayon"] }
-itertools = "0.10.5"
 memmap2 = "0.5.8"
 pathdiff = "0.2.1"
 prettytable-rs = "0.10.0"
@@ -29,15 +32,17 @@ rand = "0.9.1"
 rayon = "1.6.1"
 tempfile = "3.20.0"
 threadpool = "1.8.1"
-unicode-segmentation = "1.10.0"
 
 [profile.release]
 strip = true
+opt-level = 3
+lto = "thin"
+debug = false
+codegen-units = 1
 
 # generated by 'cargo dist init'
 [profile.dist]
 inherits = "release"
-lto = "thin"
 
 [workspace.metadata.dist]
 rust-toolchain-version = "1.87.0"
@@ -49,3 +54,6 @@ targets = [
   "aarch64-apple-darwin",
 ]
 cargo-dist-version = "0.0.7"
+
+[dev-dependencies]
+assert_cmd = "2.0.0"