diff --git a/tools/Cargo.lock b/tools/Cargo.lock index fcc58884..e6e4bd09 100644 --- a/tools/Cargo.lock +++ b/tools/Cargo.lock @@ -53,10 +53,36 @@ dependencies = [ ] [[package]] -name = "anyhow" -version = "1.0.104" +name = "bitflags" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core", +] [[package]] name = "clap" @@ -104,6 +130,113 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "rand_core", +] + [[package]] name = "heck" version = "0.5.0" @@ -116,6 +249,36 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + [[package]] name = "once_cell_polyfill" version = "1.70.2" @@ -140,6 +303,62 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + [[package]] name = "same-file" version = "1.0.6" @@ -149,6 +368,60 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + [[package]] name = "strsim" version = "0.11.1" @@ -177,6 +450,34 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "twox-hash" +version = "2.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5283634e518fe9e82c7b20520bb4bc209009fd16c82077c802f8111ecbb0117a" +dependencies = [ + "rand", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unicode-ident" version = "1.0.26" @@ -189,6 +490,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "walkdir" version = "2.5.0" @@ -224,12 +531,23 @@ dependencies = [ ] [[package]] -name = "xray-clusters" +name = "xray-clones" version = "0.1.0" dependencies = [ - "anyhow", "clap", "proc-macro2", + "rayon", + "serde", + "serde_json", + "sha2", "syn 2.0.119", + "tempfile", + "twox-hash", "walkdir", ] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/tools/Cargo.toml b/tools/Cargo.toml index ca4592ff..e520828c 100644 --- a/tools/Cargo.toml +++ b/tools/Cargo.toml @@ -9,7 +9,7 @@ # cargo clippy --manifest-path tools/Cargo.toml --all-targets -- -D warnings [workspace] resolver = "3" -members = ["xray", "xray-clusters"] +members = ["xray", "xray-clones", "xray-clusters"] [workspace.package] version = "0.1.0" diff --git a/tools/xray-clones/Cargo.toml b/tools/xray-clones/Cargo.toml new file mode 100644 index 00000000..b5eb1a25 --- /dev/null +++ b/tools/xray-clones/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "xray-clones" +description = "Finds near-duplicate functions across the didbot workspace." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +publish.workspace = true + +[dependencies] +# `full` parses whole files including function bodies; `visit` generates the +# borrowing visitor the normaliser implements. +syn = { version = "2", features = ["full", "visit", "parsing", "printing", "clone-impls"] } +# `span-locations` is what turns a parsed span back into a line number. +proc-macro2 = { version = "1", features = ["span-locations"] } +clap = { version = "4", features = ["derive"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +walkdir = "2" +rayon = "1" +# A fingerprint is written into allow.toml by hand, so it has to stay the same +# across compiler and crate versions. sha2 and xxhash are both fixed by their +# specifications; std's hasher is not. +sha2 = "0.10" +twox-hash = "2" + +[dev-dependencies] +tempfile = "3" diff --git a/tools/xray-clones/src/collect.rs b/tools/xray-clones/src/collect.rs new file mode 100644 index 00000000..6c3b18a1 --- /dev/null +++ b/tools/xray-clones/src/collect.rs @@ -0,0 +1,333 @@ +//! Finds the items worth fingerprinting, and drops the ones whose repetition +//! is the language rather than a mistake. + +use crate::normalize::{shape_fn, shape_impl, Shape}; +use std::path::{Path, PathBuf}; +use syn::spanned::Spanned; + +#[derive(Clone, Copy, PartialEq, Eq, Debug, serde::Serialize)] +#[serde(rename_all = "lowercase")] +pub enum Kind { + Fn, + Method, + Impl, +} + +pub struct Item { + pub krate: String, + pub file: PathBuf, + pub path: String, + pub kind: Kind, + pub start: usize, + pub end: usize, + pub fingerprint: String, + pub shingles: Vec, + pub tokens: usize, +} + +impl Item { + pub fn lines(&self) -> usize { + self.end.saturating_sub(self.start) + 1 + } +} + +/// Traits whose implementations are shaped by the trait, not by the author. +/// Two `Display` bodies looking alike is the point of `Display`. +const BORING_TRAITS: &[&str] = &["Display", "Debug", "From", "TryFrom"]; + +pub struct Options { + pub min_lines: usize, + pub min_stmts: usize, + pub shingle: usize, + pub include_tests: bool, + pub include_impl_blocks: bool, +} + +pub struct Collected { + pub items: Vec, + pub skipped: Skipped, +} + +#[derive(Default, Clone, Copy)] +pub struct Skipped { + pub tests: usize, + pub boring_traits: usize, + pub derived: usize, + pub trivial: usize, + pub too_small: usize, +} + +impl Skipped { + pub fn merge(&mut self, o: Skipped) { + self.tests += o.tests; + self.boring_traits += o.boring_traits; + self.derived += o.derived; + self.trivial += o.trivial; + self.too_small += o.too_small; + } +} + +/// Parse one file and collect its candidate items. Returns `None` if the file +/// is machine-written or does not parse. +pub fn collect_file(path: &Path, krate: &str, opts: &Options) -> Option { + let text = std::fs::read_to_string(path).ok()?; + if is_generated(&text) { + return None; + } + let file = syn::parse_file(&text).ok()?; + let mut c = Walker { + krate: krate.to_string(), + file: path.to_path_buf(), + opts, + scope: Vec::new(), + out: Vec::new(), + skipped: Skipped::default(), + }; + c.items(&file.items); + Some(Collected { + items: c.out, + skipped: c.skipped, + }) +} + +fn is_generated(text: &str) -> bool { + text.lines() + .take(5) + .any(|l| l.contains("@generated") || l.contains("DO NOT EDIT")) +} + +struct Walker<'a> { + krate: String, + file: PathBuf, + opts: &'a Options, + scope: Vec, + out: Vec, + skipped: Skipped, +} + +impl Walker<'_> { + fn items(&mut self, items: &[syn::Item]) { + for item in items { + match item { + syn::Item::Mod(m) => { + if let Some((_, inner)) = &m.content { + self.scope.push(m.ident.to_string()); + self.items(inner); + self.scope.pop(); + } + } + syn::Item::Fn(f) => { + if is_test(&f.attrs) && !self.opts.include_tests { + self.skipped.tests += 1; + continue; + } + let name = f.sig.ident.to_string(); + self.push(&name, Kind::Fn, item.span(), f.block.stmts.len(), || { + shape_fn(&f.sig, &f.block) + }); + } + syn::Item::Impl(i) => self.item_impl(i), + syn::Item::Trait(t) => { + self.scope.push(t.ident.to_string()); + for ti in &t.items { + if let syn::TraitItem::Fn(f) = ti { + if let Some(block) = &f.default { + let name = f.sig.ident.to_string(); + let n = block.stmts.len(); + self.push(&name, Kind::Method, ti.span(), n, || { + shape_fn(&f.sig, block) + }); + } + } + } + self.scope.pop(); + } + _ => {} + } + } + } + + fn item_impl(&mut self, i: &syn::ItemImpl) { + if i.attrs + .iter() + .any(|a| a.path().is_ident("automatically_derived")) + { + self.skipped.derived += 1; + return; + } + let trait_name = i + .trait_ + .as_ref() + .and_then(|(_, p, _)| p.segments.last()) + .map(|s| s.ident.to_string()); + if trait_name + .as_deref() + .is_some_and(|t| BORING_TRAITS.contains(&t)) + { + self.skipped.boring_traits += 1; + return; + } + let self_name = type_name(&i.self_ty).unwrap_or_else(|| "_".into()); + let label = match &trait_name { + Some(t) => format!("<{self_name} as {t}>"), + None => self_name, + }; + + if self.opts.include_impl_blocks { + let n = i.items.len(); + self.push(&label, Kind::Impl, i.span(), n, || shape_impl(i)); + } + + self.scope.push(label); + for ii in &i.items { + if let syn::ImplItem::Fn(f) = ii { + if is_test(&f.attrs) && !self.opts.include_tests { + self.skipped.tests += 1; + continue; + } + let name = f.sig.ident.to_string(); + let n = f.block.stmts.len(); + self.push(&name, Kind::Method, ii.span(), n, || { + shape_fn(&f.sig, &f.block) + }); + } + } + self.scope.pop(); + } + + fn push( + &mut self, + name: &str, + kind: Kind, + span: proc_macro2::Span, + stmts: usize, + shape: impl FnOnce() -> Shape, + ) { + let start = span.start().line; + let end = span.end().line; + let lines = end.saturating_sub(start) + 1; + if lines < self.opts.min_lines && stmts < self.opts.min_stmts { + self.skipped.too_small += 1; + return; + } + let shape = shape(); + // A body that is one short expression -- a getter, a delegation -- is + // the same shape as every other getter, and says nothing. + if stmts <= 1 && shape.len() < 24 { + self.skipped.trivial += 1; + return; + } + let mut path = self.scope.clone(); + path.push(name.to_string()); + self.out.push(Item { + krate: self.krate.clone(), + file: self.file.clone(), + path: path.join("::"), + kind, + start, + end, + fingerprint: shape.fingerprint(), + shingles: crate::similar::shingles(shape.tokens(), self.opts.shingle), + tokens: shape.len(), + }); + } +} + +fn is_test(attrs: &[syn::Attribute]) -> bool { + attrs.iter().any(|a| { + a.path() + .segments + .last() + .is_some_and(|s| s.ident == "test" || s.ident == "bench") + }) +} + +fn type_name(ty: &syn::Type) -> Option { + match ty { + syn::Type::Path(p) => Some(p.path.segments.last()?.ident.to_string()), + syn::Type::Reference(r) => type_name(&r.elem), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn opts() -> Options { + Options { + min_lines: 4, + min_stmts: 2, + shingle: 5, + include_tests: false, + include_impl_blocks: true, + } + } + + fn names(src: &str, opts: &Options) -> Vec { + let dir = tempfile::tempdir().expect("a temp dir"); + let path = dir.path().join("lib.rs"); + std::fs::write(&path, src).expect("write"); + collect_file(&path, "fixture", opts) + .expect("parses") + .items + .into_iter() + .map(|i| i.path) + .collect() + } + + const BODY: &str = "let a = one(); let b = two(a); let c = three(b); four(c)"; + + #[test] + fn a_derived_impl_is_not_a_candidate() { + let src = format!( + "#[automatically_derived] + impl Clone for Thing {{ fn clone(&self) -> Self {{ {BODY} }} }} + impl Thing {{ fn kept(&self) -> u32 {{ {BODY} }} }}" + ); + assert_eq!(names(&src, &opts()), vec!["Thing::kept"]); + } + + #[test] + fn a_display_impl_is_not_a_candidate() { + let src = format!( + "impl Display for Thing {{ fn fmt(&self, f: &mut Formatter) -> Result {{ {BODY} }} }} + impl Thing {{ fn kept(&self) -> u32 {{ {BODY} }} }}" + ); + assert_eq!(names(&src, &opts()), vec!["Thing::kept"]); + } + + #[test] + fn a_test_function_is_a_candidate_only_when_asked_for() { + let src = format!("#[test] fn checks() {{ {BODY} }} fn kept() {{ {BODY} }}"); + assert_eq!(names(&src, &opts()), vec!["kept"]); + let wanted = Options { + include_tests: true, + ..opts() + }; + assert_eq!(names(&src, &wanted), vec!["checks", "kept"]); + } + + #[test] + fn a_getter_is_not_a_candidate() { + let src = format!( + "impl Thing {{ + fn name(&self) + -> &str + {{ &self.name }} + fn kept(&self) -> u32 {{ {BODY} }} + }}" + ); + let found = names(&src, &opts()); + assert!(!found.contains(&"Thing::name".to_string()), "{found:?}"); + assert!(found.contains(&"Thing::kept".to_string()), "{found:?}"); + } + + #[test] + fn a_machine_written_file_is_skipped() { + let dir = tempfile::tempdir().expect("a temp dir"); + let path = dir.path().join("lib.rs"); + std::fs::write(&path, format!("// @generated\nfn f() {{ {BODY} }}")).expect("write"); + assert!(collect_file(&path, "fixture", &opts()).is_none()); + } +} diff --git a/tools/xray-clones/src/main.rs b/tools/xray-clones/src/main.rs new file mode 100644 index 00000000..fa7548a6 --- /dev/null +++ b/tools/xray-clones/src/main.rs @@ -0,0 +1,406 @@ +//! Finds near-duplicate functions across the workspace. +//! +//! See `README.md` for what the measure catches and what it does not. + +mod collect; +mod normalize; +mod similar; + +use clap::Parser; +use collect::{Item, Kind, Options, Skipped}; +use rayon::prelude::*; +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; + +#[derive(Parser)] +#[command( + name = "xray-clones", + about = "Find near-duplicate functions across the didbot workspace." +)] +struct Args { + /// Repository root. `crates/*/src` and `crates/*/tests` under it are + /// scanned. + #[arg(long, default_value = ".")] + root: PathBuf, + + /// An extra directory to scan, repeatable. Use it to compare the + /// workspace against a dependency's sources. + #[arg(long = "scan", value_name = "DIR")] + scan: Vec, + + /// Ignore items shorter than this, unless they clear --min-stmts. + #[arg(long, default_value_t = 25)] + min_lines: usize, + + /// Ignore items with fewer statements than this, unless they clear + /// --min-lines. + #[arg(long, default_value_t = 12)] + min_stmts: usize, + + /// Jaccard similarity a pair must reach to be grouped. 0.80 is where the + /// two `create_dir` implementations in didbot-pds and didbot-tls appear; + /// above it the report is shorter but loses that kind of find. + #[arg(long, default_value_t = 0.80)] + threshold: f64, + + /// Length of the token k-grams the similarity is measured over. Lower + /// finds looser matches; higher demands longer identical runs. + #[arg(long, default_value_t = 5)] + shingle: usize, + + /// Only report groups whose members all live in one crate. + #[arg(long, conflicts_with = "cross_crate_only")] + same_crate_only: bool, + + /// Only report groups that span more than one crate. + #[arg(long)] + cross_crate_only: bool, + + /// Report `#[test]` functions too. They repeat setup shapes on purpose. + #[arg(long)] + include_tests: bool, + + /// Do not fingerprint whole `impl` blocks, only the functions in them. + #[arg(long)] + no_impl_blocks: bool, + + /// Print at most this many groups. + #[arg(long, default_value_t = 20)] + top: usize, + + /// Print machine-readable JSON instead of a report. + #[arg(long)] + json: bool, + + /// Instead of grouping, print the nearest neighbours of every item whose + /// path or file contains this string, with their similarity, whatever + /// the threshold. Use it to ask "is this function like anything else?". + #[arg(long, value_name = "SUBSTRING")] + focus: Option, + + /// How many neighbours --focus prints per item. + #[arg(long, default_value_t = 5)] + focus_top: usize, +} + +fn main() -> std::process::ExitCode { + let args = Args::parse(); + match run(&args) { + Ok(found) => { + if found { + std::process::ExitCode::from(1) + } else { + std::process::ExitCode::SUCCESS + } + } + Err(e) => { + eprintln!("xray-clones: {e}"); + std::process::ExitCode::from(2) + } + } +} + +fn run(args: &Args) -> Result { + let opts = Options { + min_lines: args.min_lines, + min_stmts: args.min_stmts, + shingle: args.shingle, + include_tests: args.include_tests, + include_impl_blocks: !args.no_impl_blocks, + }; + + let files = discover(&args.root, &args.scan); + if files.is_empty() { + return Err(format!("no .rs files under {}", args.root.display())); + } + let collected: Vec<_> = files + .par_iter() + .filter_map(|(path, krate)| collect::collect_file(path, krate, &opts)) + .collect(); + + let mut items: Vec = Vec::new(); + let mut skipped = Skipped::default(); + for c in collected { + skipped.merge(c.skipped); + items.extend(c.items); + } + items.sort_by(|a, b| (&a.file, a.start).cmp(&(&b.file, b.start))); + + let sets: Vec> = items.iter().map(|i| i.shingles.clone()).collect(); + if let Some(needle) = &args.focus { + return Ok(focus(needle, args.focus_top, &items, &sets)); + } + let groups = similar::group(&sets, args.threshold); + + let mut reports: Vec = groups + .into_iter() + .filter_map(|g| Report::build(g, &items, args)) + .collect(); + reports.sort_by(|a, b| { + b.score + .cmp(&a.score) + .then(a.members[0].file.cmp(&b.members[0].file)) + }); + let reports = drop_subsumed(reports); + + let summary = Summary { + files: files.len(), + items: items.len(), + groups: reports.len(), + threshold: args.threshold, + shingle: args.shingle, + skipped_tests: skipped.tests, + skipped_boring_traits: skipped.boring_traits, + skipped_derived: skipped.derived, + skipped_trivial: skipped.trivial, + skipped_too_small: skipped.too_small, + }; + + let shown: Vec<&Report> = reports.iter().take(args.top).collect(); + if args.json { + let doc = serde_json::json!({ "summary": summary, "groups": shown }); + println!( + "{}", + serde_json::to_string_pretty(&doc).map_err(|e| e.to_string())? + ); + } else { + print_report(&summary, &shown, reports.len()); + } + Ok(!reports.is_empty()) +} + +/// Every item matching `needle`, with its nearest neighbours by exact +/// Jaccard. No threshold and no grouping: this answers a question about one +/// function rather than surveying the tree. +fn focus(needle: &str, top: usize, items: &[Item], sets: &[Vec]) -> bool { + let mut any = false; + for (i, item) in items.iter().enumerate() { + if !item.path.contains(needle) && !item.file.to_string_lossy().contains(needle) { + continue; + } + any = true; + println!( + "{}:{}-{} {} ({})", + item.file.display(), + item.start, + item.end, + item.path, + item.fingerprint + ); + let mut near: Vec<(f64, usize)> = (0..items.len()) + .filter(|&j| j != i) + .map(|j| (similar::jaccard(&sets[i], &sets[j]), j)) + .collect(); + near.sort_by(|a, b| b.0.total_cmp(&a.0)); + for (score, j) in near.into_iter().take(top) { + let o = &items[j]; + println!( + " {score:.3} {}:{}-{} {}", + o.file.display(), + o.start, + o.end, + o.path + ); + } + } + if !any { + println!("nothing matches {needle:?}"); + } + false +} + +/// Every `.rs` file under `crates/*/src` and `crates/*/tests`, plus any +/// directory named with `--scan`, paired with the crate it belongs to. +fn discover(root: &Path, extra: &[PathBuf]) -> Vec<(PathBuf, String)> { + let mut out = Vec::new(); + let crates = root.join("crates"); + if let Ok(entries) = std::fs::read_dir(&crates) { + let mut dirs: Vec<_> = entries.flatten().map(|e| e.path()).collect(); + dirs.sort(); + for dir in dirs { + let Some(name) = dir.file_name().and_then(|n| n.to_str()) else { + continue; + }; + let name = name.to_string(); + for sub in ["src", "tests"] { + walk_rs(&dir.join(sub), &name, &mut out); + } + } + } + for dir in extra { + let name = dir + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("scan") + .to_string(); + walk_rs(dir, &name, &mut out); + } + out +} + +fn walk_rs(dir: &Path, krate: &str, out: &mut Vec<(PathBuf, String)>) { + if !dir.is_dir() { + return; + } + for entry in walkdir::WalkDir::new(dir).into_iter().flatten() { + let p = entry.path(); + if p.extension().is_some_and(|e| e == "rs") && p.is_file() { + out.push((p.to_path_buf(), krate.to_string())); + } + } +} + +#[derive(serde::Serialize)] +struct Summary { + files: usize, + items: usize, + groups: usize, + threshold: f64, + shingle: usize, + skipped_tests: usize, + skipped_boring_traits: usize, + skipped_derived: usize, + skipped_trivial: usize, + skipped_too_small: usize, +} + +#[derive(serde::Serialize)] +struct Member { + file: String, + start: usize, + end: usize, + lines: usize, + path: String, + kind: Kind, + krate: String, + /// Shape tokens in the normalised stream, a size measure that ignores + /// how the source was laid out. + tokens: usize, + fingerprint: String, +} + +#[derive(serde::Serialize)] +struct Report { + score: usize, + min_similarity: f64, + exact: bool, + crates: Vec, + members: Vec, +} + +impl Report { + fn build(g: similar::Group, items: &[Item], args: &Args) -> Option { + let mut members: Vec<&Item> = g.members.iter().map(|&i| &items[i]).collect(); + // An `impl` block and a method inside it overlap in the source, so + // their shapes match for a reason that is not duplication. Keep the + // inner item and drop whatever encloses it. + members.retain(|m| { + !g.members.iter().map(|&i| &items[i]).any(|other| { + !std::ptr::eq(*m, other) + && other.file == m.file + && other.start >= m.start + && other.end <= m.end + }) + }); + if members.len() < 2 { + return None; + } + let crates: BTreeSet<&str> = members.iter().map(|m| m.krate.as_str()).collect(); + if args.cross_crate_only && crates.len() < 2 { + return None; + } + if args.same_crate_only && crates.len() > 1 { + return None; + } + let total: usize = members.iter().map(|m| m.lines()).sum(); + let mean = total / members.len(); + let first = members[0].fingerprint.clone(); + Some(Report { + score: members.len() * mean, + min_similarity: g.min_similarity, + exact: members.iter().all(|m| m.fingerprint == first), + crates: crates.into_iter().map(str::to_string).collect(), + members: members + .iter() + .map(|m| Member { + file: m.file.display().to_string(), + start: m.start, + end: m.end, + lines: m.lines(), + path: m.path.clone(), + kind: m.kind, + krate: m.krate.clone(), + tokens: m.tokens, + fingerprint: m.fingerprint.clone(), + }) + .collect(), + }) + } + + /// True when every member of this group wraps a member of `inner`. An + /// `impl` block reported beside the methods inside it says nothing new. + fn wraps(&self, inner: &Report) -> bool { + self.members.len() == inner.members.len() + && self.members.iter().all(|outer| { + inner + .members + .iter() + .any(|i| i.file == outer.file && i.start >= outer.start && i.end <= outer.end) + }) + } +} + +fn drop_subsumed(reports: Vec) -> Vec { + let mut kept: Vec = Vec::new(); + for r in reports { + let redundant = + r.members.iter().all(|m| m.kind == Kind::Impl) && kept.iter().any(|k| r.wraps(k)); + if !redundant { + kept.push(r); + } + } + kept +} + +fn print_report(s: &Summary, groups: &[&Report], total: usize) { + println!( + "xray-clones: {} items in {} files; {total} groups at threshold {:.2} (k={})", + s.items, s.files, s.threshold, s.shingle + ); + println!( + " skipped: {} tests, {} Display/Debug/From/TryFrom impls, {} derived, \ +{} trivial, {} under the size floor", + s.skipped_tests, + s.skipped_boring_traits, + s.skipped_derived, + s.skipped_trivial, + s.skipped_too_small + ); + if groups.is_empty() { + println!("\nno groups."); + return; + } + for (n, g) in groups.iter().enumerate() { + let kind = if g.exact { "identical" } else { "similar" }; + println!( + "\n#{} score {} -- {} members, {kind}, min similarity {:.2}, crates: {}", + n + 1, + g.score, + g.members.len(), + g.min_similarity, + g.crates.join(", ") + ); + for m in &g.members { + println!( + " {}:{}-{} {} ({} lines, {})", + m.file, m.start, m.end, m.path, m.lines, m.fingerprint + ); + } + } + if total > groups.len() { + println!( + "\n{} more groups; raise --top to see them.", + total - groups.len() + ); + } +} diff --git a/tools/xray-clones/src/normalize.rs b/tools/xray-clones/src/normalize.rs new file mode 100644 index 00000000..aff842f9 --- /dev/null +++ b/tools/xray-clones/src/normalize.rs @@ -0,0 +1,381 @@ +//! Turns a piece of Rust syntax into a stream of shape tokens. +//! +//! The stream keeps control flow, call arity, operator structure, match arms +//! and the shape of types. It drops identifiers, literal values, lifetimes, +//! attributes (doc comments included), spans and parentheses. Two functions +//! that differ only in what things are called therefore produce the same +//! stream, and two that differ in how they branch do not. + +use sha2::{Digest, Sha256}; +use syn::visit::{self, Visit}; +use twox_hash::XxHash64; + +/// A tag's numeric form. xxHash64 is a fixed specification, so a fingerprint +/// written into `allow.toml` today still matches next year. +fn tag(name: &str) -> u64 { + XxHash64::oneshot(0, name.as_bytes()) +} + +/// The shape tokens of one item, in visit order. +#[derive(Default)] +pub struct Shape { + toks: Vec, + scratch: String, +} + +impl Shape { + pub fn len(&self) -> usize { + self.toks.len() + } + + pub fn tokens(&self) -> &[u64] { + &self.toks + } + + /// The exact-shape fingerprint, as the first 16 hex digits of a SHA-256 + /// over the token stream. This is the key an `allow.toml` entry names. + pub fn fingerprint(&self) -> String { + let mut h = Sha256::new(); + for t in &self.toks { + h.update(t.to_le_bytes()); + } + let digest = h.finalize(); + digest[..8].iter().map(|b| format!("{b:02x}")).collect() + } + + fn t(&mut self, name: &str) { + self.toks.push(tag(name)); + } + + /// A tag carrying a small count, such as a call's arity. + fn tn(&mut self, name: &str, n: usize) { + use std::fmt::Write as _; + self.scratch.clear(); + let _ = write!(self.scratch, "{name}{n}"); + let t = tag(&self.scratch); + self.toks.push(t); + } + + fn close(&mut self) { + self.t(")"); + } +} + +/// Normalise a function signature and body together. +pub fn shape_fn(sig: &syn::Signature, block: &syn::Block) -> Shape { + let mut s = Shape::default(); + s.visit_signature(sig); + s.visit_block(block); + s +} + +/// Normalise a whole `impl` block, including every item in it. +pub fn shape_impl(item: &syn::ItemImpl) -> Shape { + let mut s = Shape::default(); + s.visit_item_impl(item); + s +} + +impl<'ast> Visit<'ast> for Shape { + // --- erasures ------------------------------------------------------- + + fn visit_ident(&mut self, _: &'ast proc_macro2::Ident) { + self.t("i"); + } + + fn visit_lifetime(&mut self, _: &'ast syn::Lifetime) {} + + fn visit_attribute(&mut self, _: &'ast syn::Attribute) {} + + fn visit_visibility(&mut self, _: &'ast syn::Visibility) {} + + /// A field name and a tuple index become the same thing, so `.0` and + /// `.name` do not disagree. + fn visit_member(&mut self, _: &'ast syn::Member) { + self.t("m"); + } + + fn visit_label(&mut self, _: &'ast syn::Label) {} + + /// A path contributes that it is a path and the shape of its generic + /// arguments. Its segment names and their number do not survive, so one + /// copy importing a name and another spelling it out still match. + fn visit_path(&mut self, p: &'ast syn::Path) { + self.t("p"); + for seg in &p.segments { + self.visit_path_arguments(&seg.arguments); + } + } + + // --- structure ------------------------------------------------------ + + fn visit_expr(&mut self, e: &'ast syn::Expr) { + match expr_tag(e) { + Tag::Plain(name) => self.t(name), + Tag::Counted(name, n) => self.tn(name, n), + Tag::Transparent => { + // Parentheses and invisible groups are not shape. + visit::visit_expr(self, e); + return; + } + } + visit::visit_expr(self, e); + self.close(); + } + + fn visit_stmt(&mut self, s: &'ast syn::Stmt) { + self.t(match s { + syn::Stmt::Local(_) => "s.let", + syn::Stmt::Item(_) => "s.item", + syn::Stmt::Expr(_, Some(_)) => "s.semi", + syn::Stmt::Expr(_, None) => "s.tail", + syn::Stmt::Macro(_) => "s.mac", + }); + visit::visit_stmt(self, s); + self.close(); + } + + fn visit_pat(&mut self, p: &'ast syn::Pat) { + self.t(match p { + syn::Pat::Const(_) => "q.const", + syn::Pat::Ident(_) => "q.id", + syn::Pat::Lit(_) => "q.lit", + syn::Pat::Macro(_) => "q.mac", + syn::Pat::Or(_) => "q.or", + syn::Pat::Paren(_) => "q.paren", + syn::Pat::Path(_) => "q.path", + syn::Pat::Range(_) => "q.range", + syn::Pat::Reference(_) => "q.ref", + syn::Pat::Rest(_) => "q.rest", + syn::Pat::Slice(_) => "q.slice", + syn::Pat::Struct(_) => "q.struct", + syn::Pat::Tuple(_) => "q.tuple", + syn::Pat::TupleStruct(_) => "q.tuplestruct", + syn::Pat::Type(_) => "q.type", + syn::Pat::Wild(_) => "q.wild", + _ => "q.other", + }); + visit::visit_pat(self, p); + self.close(); + } + + fn visit_type(&mut self, t: &'ast syn::Type) { + match t { + // A parenthesised type is the type. + syn::Type::Paren(_) | syn::Type::Group(_) => { + visit::visit_type(self, t); + return; + } + _ => self.t(match t { + syn::Type::Array(_) => "t.array", + syn::Type::BareFn(_) => "t.fn", + syn::Type::ImplTrait(_) => "t.impl", + syn::Type::Infer(_) => "t.infer", + syn::Type::Macro(_) => "t.mac", + syn::Type::Never(_) => "t.never", + syn::Type::Path(_) => "t.path", + syn::Type::Ptr(_) => "t.ptr", + syn::Type::Reference(_) => "t.ref", + syn::Type::Slice(_) => "t.slice", + syn::Type::TraitObject(_) => "t.dyn", + syn::Type::Tuple(_) => "t.tuple", + _ => "t.other", + }), + } + visit::visit_type(self, t); + self.close(); + } + + fn visit_bin_op(&mut self, op: &'ast syn::BinOp) { + self.t(match op { + syn::BinOp::Add(_) => "o+", + syn::BinOp::Sub(_) => "o-", + syn::BinOp::Mul(_) => "o*", + syn::BinOp::Div(_) => "o/", + syn::BinOp::Rem(_) => "o%", + syn::BinOp::And(_) => "o&&", + syn::BinOp::Or(_) => "o||", + syn::BinOp::BitXor(_) => "o^", + syn::BinOp::BitAnd(_) => "o&", + syn::BinOp::BitOr(_) => "o|", + syn::BinOp::Shl(_) => "o<<", + syn::BinOp::Shr(_) => "o>>", + syn::BinOp::Eq(_) => "o==", + syn::BinOp::Lt(_) => "o<", + syn::BinOp::Le(_) => "o<=", + syn::BinOp::Ne(_) => "o!=", + syn::BinOp::Ge(_) => "o>=", + syn::BinOp::Gt(_) => "o>", + _ => "o=assign", + }); + } + + fn visit_un_op(&mut self, op: &'ast syn::UnOp) { + self.t(match op { + syn::UnOp::Deref(_) => "u*", + syn::UnOp::Not(_) => "u!", + syn::UnOp::Neg(_) => "u-", + _ => "u?", + }); + } + + /// A macro's body is not Rust until something expands it. Where the body + /// does parse as a comma-separated expression list -- `vec!`, `format!`, + /// `assert_eq!` and most others -- normalise those expressions, so that a + /// table of addresses inside a `vec!` is still shape. Where it does not, + /// only the fact of a macro call survives. + fn visit_macro(&mut self, m: &'ast syn::Macro) { + self.t("mac"); + type Args = syn::punctuated::Punctuated; + match m.parse_body_with(Args::parse_terminated) { + Ok(args) => { + for a in &args { + self.visit_expr(a); + } + } + Err(_) => self.t("mac.opaque"), + } + self.close(); + } +} + +enum Tag { + Plain(&'static str), + Counted(&'static str, usize), + Transparent, +} + +fn expr_tag(e: &syn::Expr) -> Tag { + use syn::Expr as E; + match e { + E::Paren(_) | E::Group(_) => Tag::Transparent, + E::Call(c) => Tag::Counted("e.call", c.args.len()), + E::MethodCall(c) => Tag::Counted("e.mcall", c.args.len()), + E::Tuple(t) => Tag::Counted("e.tuple", t.elems.len()), + E::Array(a) => Tag::Counted("e.array", a.elems.len()), + E::Match(_) => Tag::Plain("e.match"), + E::If(_) => Tag::Plain("e.if"), + E::ForLoop(_) => Tag::Plain("e.for"), + E::While(_) => Tag::Plain("e.while"), + E::Loop(_) => Tag::Plain("e.loop"), + E::Return(_) => Tag::Plain("e.return"), + E::Break(_) => Tag::Plain("e.break"), + E::Continue(_) => Tag::Plain("e.continue"), + E::Try(_) => Tag::Plain("e.try"), + E::Await(_) => Tag::Plain("e.await"), + E::Async(_) => Tag::Plain("e.async"), + E::Closure(_) => Tag::Plain("e.closure"), + E::Binary(_) => Tag::Plain("e.bin"), + E::Unary(_) => Tag::Plain("e.un"), + E::Assign(_) => Tag::Plain("e.assign"), + E::Cast(_) => Tag::Plain("e.cast"), + E::Field(_) => Tag::Plain("e.field"), + E::Index(_) => Tag::Plain("e.index"), + E::Lit(_) => Tag::Plain("e.lit"), + E::Let(_) => Tag::Plain("e.letif"), + E::Macro(_) => Tag::Plain("e.mac"), + E::Path(_) => Tag::Plain("e.path"), + E::Range(_) => Tag::Plain("e.range"), + E::Reference(_) => Tag::Plain("e.ref"), + E::Repeat(_) => Tag::Plain("e.repeat"), + E::Struct(_) => Tag::Plain("e.struct"), + E::Block(_) => Tag::Plain("e.block"), + E::Unsafe(_) => Tag::Plain("e.unsafe"), + E::TryBlock(_) => Tag::Plain("e.tryblock"), + E::Const(_) => Tag::Plain("e.const"), + E::Yield(_) => Tag::Plain("e.yield"), + E::Infer(_) => Tag::Plain("e.infer"), + _ => Tag::Plain("e.other"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn shape(src: &str) -> Shape { + let f: syn::ItemFn = syn::parse_str(src).expect("a function"); + shape_fn(&f.sig, &f.block) + } + + fn print(src: &str) -> String { + shape(src).fingerprint() + } + + #[test] + fn renaming_everything_leaves_the_shape_alone() { + let a = print( + r#"fn refuse(address: IpAddr, limit: usize) -> bool { + let mut count = 0; + for block in REFUSED.iter() { + if block.contains(&address) { count += 1; } + } + count > limit + }"#, + ); + let b = print( + r#"fn permit(host: SocketAddr, ceiling: u32) -> bool { + let mut total = 99; + for range in ALLOWED.iter() { + if range.holds(&host) { total += 7; } + } + total > ceiling + }"#, + ); + assert_eq!(a, b); + } + + #[test] + fn changing_the_control_flow_changes_the_shape() { + let straight = print( + r#"fn f(x: u32) -> u32 { + let y = g(x); + h(y) + }"#, + ); + let branched = print( + r#"fn f(x: u32) -> u32 { + let y = g(x); + if y > 0 { h(y) } else { 0 } + }"#, + ); + assert_ne!(straight, branched); + } + + #[test] + fn a_doc_comment_is_not_shape() { + let bare = print("fn f(x: u32) -> u32 { let y = g(x); h(y) }"); + let documented = print( + r#"/// Turns an x into a y. + /// + /// At length, and with a second paragraph. + fn f(x: u32) -> u32 { + // An ordinary comment too. + let y = g(x); + h(y) + }"#, + ); + assert_eq!(bare, documented); + } + + #[test] + fn a_macro_body_that_parses_as_expressions_is_shape() { + let two = print("fn f() { let v = vec![a, b]; use_it(v); }"); + let three = print("fn f() { let v = vec![a, b, c]; use_it(v); }"); + assert_ne!(two, three); + } + + #[test] + fn a_swapped_operator_changes_the_shape() { + let and = print("fn f(a: bool, b: bool) -> bool { let c = a && b; c }"); + let or = print("fn f(a: bool, b: bool) -> bool { let c = a || b; c }"); + assert_ne!(and, or); + } + + #[test] + fn a_fingerprint_is_stable_and_short() { + let p = print("fn f(x: u32) -> u32 { let y = g(x); h(y) }"); + assert_eq!(p.len(), 16); + assert!(p.chars().all(|c| c.is_ascii_hexdigit())); + } +} diff --git a/tools/xray-clones/src/similar.rs b/tools/xray-clones/src/similar.rs new file mode 100644 index 00000000..fca7c9be --- /dev/null +++ b/tools/xray-clones/src/similar.rs @@ -0,0 +1,205 @@ +//! Groups items whose shapes are close. +//! +//! The measure is Jaccard similarity over the set of k-grams ("shingles") of +//! the normalised token stream: the size of the shared k-gram set divided by +//! the size of the combined one. Identical shapes score 1.0. Candidate pairs +//! come from banded MinHash, which finds pairs likely to be above the +//! threshold without comparing every item to every other; each candidate is +//! then scored exactly, so the threshold means what it says. + +use std::collections::HashMap; +use twox_hash::XxHash64; + +/// Enough permutations that the banding is not noisy, few enough that +/// signing every item stays under a second on this workspace. +const HASHES: usize = 128; + +/// The k-gram set of a token stream, sorted and deduplicated. +pub fn shingles(tokens: &[u64], k: usize) -> Vec { + let k = k.max(1); + let mut out: Vec = if tokens.len() < k { + vec![hash_window(tokens)] + } else { + tokens.windows(k).map(hash_window).collect() + }; + out.sort_unstable(); + out.dedup(); + out +} + +fn hash_window(w: &[u64]) -> u64 { + let mut bytes = Vec::with_capacity(w.len() * 8); + for t in w { + bytes.extend_from_slice(&t.to_le_bytes()); + } + XxHash64::oneshot(0, &bytes) +} + +/// Jaccard similarity of two sorted, deduplicated shingle sets. +pub fn jaccard(a: &[u64], b: &[u64]) -> f64 { + if a.is_empty() || b.is_empty() { + return 0.0; + } + let (mut i, mut j, mut shared) = (0, 0, 0usize); + while i < a.len() && j < b.len() { + match a[i].cmp(&b[j]) { + std::cmp::Ordering::Less => i += 1, + std::cmp::Ordering::Greater => j += 1, + std::cmp::Ordering::Equal => { + shared += 1; + i += 1; + j += 1; + } + } + } + let union = a.len() + b.len() - shared; + shared as f64 / union as f64 +} + +fn signature(shingles: &[u64]) -> [u64; HASHES] { + let mut sig = [u64::MAX; HASHES]; + for &s in shingles { + let bytes = s.to_le_bytes(); + for (seed, slot) in sig.iter_mut().enumerate() { + let h = XxHash64::oneshot(seed as u64, &bytes); + if h < *slot { + *slot = h; + } + } + } + sig +} + +/// Split the signature into `bands` bands of `rows` each. +/// +/// Banding is a filter, not the answer: every pair it proposes is then scored +/// exactly, so a band that proposes too many pairs costs only time, while one +/// that proposes too few silently loses a real duplicate. So pick the widest +/// bands that still catch a pair sitting exactly on the threshold at least +/// [`RECALL`] of the time. Above the threshold the odds only improve. +fn banding(threshold: f64) -> (usize, usize) { + let t = threshold.clamp(0.01, 0.999); + let mut choice = (HASHES, 1); + for rows in [1usize, 2, 4, 8, 16, 32, 64, 128] { + let bands = HASHES / rows; + let per_band = t.powi(rows as i32); + let recall = 1.0 - (1.0 - per_band).powi(bands as i32); + if recall >= RECALL { + choice = (bands, rows); + } + } + choice +} + +/// How often a pair sitting exactly on the threshold must reach the exact +/// comparison. The remainder is this tool's false-negative rate, on top of +/// whatever the measure itself misses. +const RECALL: f64 = 0.98; + +/// A cluster of items joined by at least one above-threshold pair. Similarity +/// is not transitive, so a group can hold two members that are not themselves +/// above the threshold; the reported `min_similarity` says when that happened. +pub struct Group { + pub members: Vec, + pub min_similarity: f64, +} + +pub fn group(sets: &[Vec], threshold: f64) -> Vec { + let sigs: Vec<[u64; HASHES]> = sets.iter().map(|s| signature(s)).collect(); + let (bands, rows) = banding(threshold); + + let mut uf = Union::new(sets.len()); + let mut scores: HashMap<(usize, usize), f64> = HashMap::new(); + let mut buckets: HashMap<(usize, u64), Vec> = HashMap::new(); + for (idx, sig) in sigs.iter().enumerate() { + for b in 0..bands { + let row = &sig[b * rows..(b + 1) * rows]; + buckets.entry((b, hash_window(row))).or_default().push(idx); + } + } + + for members in buckets.values() { + // A very wide bucket is one shape repeated many times. Chaining to the + // first member joins them all without paying for every pair. + let pairs: Vec<(usize, usize)> = if members.len() > 64 { + members[1..].iter().map(|&b| (members[0], b)).collect() + } else { + let mut v = Vec::new(); + for (i, &a) in members.iter().enumerate() { + for &b in &members[i + 1..] { + v.push((a, b)); + } + } + v + }; + for (a, b) in pairs { + let key = (a.min(b), a.max(b)); + if scores.contains_key(&key) { + continue; + } + let s = jaccard(&sets[a], &sets[b]); + scores.insert(key, s); + if s >= threshold { + uf.join(a, b); + } + } + } + + let mut by_root: HashMap> = HashMap::new(); + for i in 0..sets.len() { + by_root.entry(uf.find(i)).or_default().push(i); + } + let mut out: Vec = by_root + .into_values() + .filter(|m| m.len() > 1) + .map(|mut members| { + members.sort_unstable(); + let min_similarity = min_pairwise(&members, sets); + Group { + members, + min_similarity, + } + }) + .collect(); + out.sort_by_key(|g| g.members[0]); + out +} + +fn min_pairwise(members: &[usize], sets: &[Vec]) -> f64 { + // Comparing every pair inside a huge group is not worth it; the first + // member against the rest is a fair enough floor. + let mut lo = 1.0f64; + if members.len() > 32 { + for &b in &members[1..] { + lo = lo.min(jaccard(&sets[members[0]], &sets[b])); + } + return lo; + } + for (i, &a) in members.iter().enumerate() { + for &b in &members[i + 1..] { + lo = lo.min(jaccard(&sets[a], &sets[b])); + } + } + lo +} + +struct Union(Vec); + +impl Union { + fn new(n: usize) -> Self { + Union((0..n).collect()) + } + fn find(&mut self, mut x: usize) -> usize { + while self.0[x] != x { + self.0[x] = self.0[self.0[x]]; + x = self.0[x]; + } + x + } + fn join(&mut self, a: usize, b: usize) { + let (ra, rb) = (self.find(a), self.find(b)); + if ra != rb { + self.0[ra] = rb; + } + } +}