From fa7dd1e2fad36107e0d60cdcd330fde7a210b784 Mon Sep 17 00:00:00 2001 From: "@permadeath.com" Date: Thu, 17 Sep 2026 13:50:59 -0400 Subject: [PATCH] feat(tools): index every item and reference in the workspace xray joins rustdoc's JSON inventory to rust-analyzer's SCIP reference edges in one SQLite file, so "is this dead", "who calls this" and "does this dependency carry anything" have compiler-accurate answers. Two sources because neither is enough alone. SCIP has the edges and nothing else resolves a name through re-exports, trait impls and macros; rustdoc has kind, visibility, module path and span, which SCIP does not carry. SCIP rather than the LSIF the same binary writes: 85s and 32 MB against 127s and 134 MB, and only `scip` takes a --config-path, which the dump needs to be taken with all features on. A default-features LSIF called `CertificateFleet::bootstrap` unreferenced while a `route53` call site sat in the tree. tools/ is a workspace of its own, excluded from the root manifest, so none of this enters a product build. Co-Authored-By: Claude Opus 5 Change-Id: I542a6369fed19929f33bc58fa7e3b75b72022550 (cherry picked from commit 6da5f462dce89bfe84b761d0ce7a2319a3b68cf9) --- .gitignore | 5 + Cargo.toml | 3 + tools/Cargo.lock | 430 +++++++++++++++++++++++++++++++++++ tools/Cargo.toml | 20 ++ tools/README.md | 86 +++++++ tools/xray/Cargo.toml | 22 ++ tools/xray/src/layout.rs | 125 ++++++++++ tools/xray/src/main.rs | 95 ++++++++ tools/xray/src/query.rs | 204 +++++++++++++++++ tools/xray/src/rustdoc.rs | 168 ++++++++++++++ tools/xray/src/scip_index.rs | 173 ++++++++++++++ tools/xray/src/store.rs | 236 +++++++++++++++++++ 12 files changed, 1567 insertions(+) create mode 100644 tools/Cargo.lock create mode 100644 tools/Cargo.toml create mode 100644 tools/README.md create mode 100644 tools/xray/Cargo.toml create mode 100644 tools/xray/src/layout.rs create mode 100644 tools/xray/src/main.rs create mode 100644 tools/xray/src/query.rs create mode 100644 tools/xray/src/rustdoc.rs create mode 100644 tools/xray/src/scip_index.rs create mode 100644 tools/xray/src/store.rs diff --git a/.gitignore b/.gitignore index f8ab5eb1..30e82aa1 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,11 @@ Thumbs.db # is not here. /.cache +# The reference index tools/xray builds, and the reports run out of it. Both +# are derived from a single revision of the tree and go stale the moment it +# moves, so they are rebuilt rather than committed. +/.xray + # Scratch output: hand-run git format-patch, redirected --debug logs *.patch *.diff diff --git a/Cargo.toml b/Cargo.toml index 9a3ab336..b9694cad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,9 @@ [workspace] resolver = "3" members = ["crates/*"] +# tools/ is a workspace of its own: developer tooling that must never enter a +# product build or `cargo test --workspace`. See tools/README.md. +exclude = ["tools"] [workspace.package] version = "0.1.0" diff --git a/tools/Cargo.lock b/tools/Cargo.lock new file mode 100644 index 00000000..a7de6cff --- /dev/null +++ b/tools/Cargo.lock @@ -0,0 +1,430 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "cc" +version = "1.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3eb0f42d6c360dc3f8a821f6bf2fdea7f72bfd36b3076eb0e6d1e9e0752fff4" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "find-msvc-tools" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libsqlite3-sys" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "133c182a6a2c87864fe97778797e46c7e999672690dc9fa3ee8e241aa4a9c13f" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "protobuf" +version = "3.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4" +dependencies = [ + "once_cell", + "protobuf-support", + "thiserror", +] + +[[package]] +name = "protobuf-support" +version = "3.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6" +dependencies = [ + "thiserror", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rusqlite" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "165ca6e57b20e1351573e3729b958bc62f0e48025386970b6e4d29e7a7e71f3f" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + +[[package]] +name = "scip" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7f651fbd0f98742a47b58623a7bcfd8fad14455912689e4867a1f43ee7e02c6" +dependencies = [ + "protobuf", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "smallvec" +version = "1.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "xray" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "protobuf", + "rusqlite", + "scip", + "serde_json", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/tools/Cargo.toml b/tools/Cargo.toml new file mode 100644 index 00000000..14ce1031 --- /dev/null +++ b/tools/Cargo.toml @@ -0,0 +1,20 @@ +# The developer tools workspace, kept separate from the product on purpose. +# +# The root manifest lists `tools` under `exclude`, so nothing here is reachable +# from `cargo build --workspace`, `cargo test --workspace` or the clippy run in +# scripts/lint.sh. A tool may take a dependency the product would never ship, +# and may lag or lead the product's own lints, without any of that showing up +# in a release build. Lint this workspace on its own: +# +# cargo clippy --manifest-path tools/Cargo.toml --all-targets -- -D warnings +[workspace] +resolver = "3" +members = ["xray"] + +[workspace.package] +version = "0.1.0" +edition = "2021" +rust-version = "1.90" +license = "MIT" +repository = "https://tangled.org/permadeath.com/didbot" +publish = false diff --git a/tools/README.md b/tools/README.md new file mode 100644 index 00000000..91707000 --- /dev/null +++ b/tools/README.md @@ -0,0 +1,86 @@ +# tools/ + +Developer tooling. Its own cargo workspace, listed under `exclude` in the root +manifest, so nothing here is built by `cargo build --workspace`, tested by +`cargo test --workspace` or linted by `scripts/lint.sh`. Lint it on its own: + + cargo clippy --manifest-path tools/Cargo.toml --all-targets -- -D warnings + +## xray — the reference index + +`git grep` is a text search over a language with modules, `use` re-exports, +trait impls, macros and `cfg`. It cannot tell a definition from a word in a +comment, it cannot follow a re-export, and it cannot tell two methods named +`cursor` apart. `xray` builds the index a compiler-accurate answer needs, and +answers four questions from it: + +| Query | Answer | +| --- | --- | +| `dead-pub` | Public items that nothing outside their crate and no test reaches, each marked `unreferenced`, `file-local` or `crate-local`. This is what `dead_code` will not tell you about a library crate. | +| `refs ` | Every reference to an item, with file and line, the crate and target each comes from, and the item it sits inside. | +| `crate-edges` | Every declared workspace dependency with the number of item-level references crossing it. A dependency at zero is one nothing uses. | +| `file-items ` | Every item in a file with its span, kind, visibility and reference count. | + +Paths are the ones rustdoc reports: `didbot_name::lists::bundled_names`, +`didbot_name::fragments::Template::slots`. `refs` also takes any suffix of one. + +### Running it + + scripts/xray.sh dead-pub + scripts/xray.sh refs didbot_name::lists::bundled_names + scripts/xray.sh crate-edges + scripts/xray.sh file-items crates/didbot-serve/src/routes.rs + scripts/xray.sh build # rebuild and stop + +The index lands in `.xray/index.db`, which is gitignored along with the two +dumps it is built from. A rebuild takes about two minutes and 4 GB. + +**An index is only as current as the revision it was built from.** +`scripts/xray.sh` rebuilds when `HEAD` has moved since the last build, and +`scripts/xray.sh info` prints the revision the current index holds. It does not +rebuild for uncommitted edits: after editing, run `scripts/xray.sh build`. + +### Where the data comes from + +Two sources, because neither knows everything: + +- `rust-analyzer scip` for the reference edges. It resolves names the way the + compiler does. SCIP rather than the LSIF the same binary also writes, because + `scip` takes `--config-path` and `lsif` takes nothing: the dump has to be + taken with all cargo features on, or everything behind an off feature reads + as unreferenced. It is also faster and a quarter the size. +- `cargo doc --output-format json` for the item inventory — kind, visibility, + module path and source span, none of which SCIP carries. Nightly only, and + run once for libs and once for bins, because rustdoc names its output after + the crate and five packages here have a bin sharing their lib's name. + +`xray build` joins them: a SCIP definition sits on the first line of a rustdoc +span, and an item with no source position of its own is matched by the path its +SCIP symbol spells. + +### Schema + +`sqlite3 .xray/index.db` if a question needs SQL rather than a subcommand. + + item(id, crate, module_path, name, kind, visibility, + file, line_start, line_end, target_kind, exported) + ref(from_item, to_item, file, line, crate, target_kind, cfg_gated) + dep(from_crate, to_crate, kind) + meta(key, value) + +`target_kind` is lib, bin, test, bench, example or build. `exported` marks an +item every module between it and the crate root is `pub`. `cfg_gated` marks a +reference from code rustdoc does not see — a `#[cfg(test)]` module, or a +feature that is off — which `dead-pub` counts as a use. + +### What it cannot see + +- Code in doc comments. A doctest is the one caller a `dead-pub` hit may still + have. +- Callers outside this repository. Everything here is `publish = false`, so in + practice that means vibescrobble.com and anything built against a running + server rather than against the crates. +- A trait impl reached only through `dyn` dispatch is recorded against the + trait method, not the impl's. +- Items generated by a macro carry the span of the macro call, so + `file-items` reports them where they are invoked. diff --git a/tools/xray/Cargo.toml b/tools/xray/Cargo.toml new file mode 100644 index 00000000..feeb6290 --- /dev/null +++ b/tools/xray/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "xray" +description = "A compiler-accurate reference index over the didbot workspace." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +publish.workspace = true + +[dependencies] +anyhow = "1" +clap = { version = "4", features = ["derive"] } +serde_json = "1" +# `bundled` compiles SQLite from source rather than linking whatever the host +# has. The index is the tool's whole output, so the file format it writes must +# not depend on which libsqlite3 a machine happens to carry. +rusqlite = { version = "0.37", features = ["bundled"] } +# The SCIP index rust-analyzer writes is protobuf; this is the schema crate +# Sourcegraph generates it from. +scip = "0.10" +protobuf = "3" diff --git a/tools/xray/src/layout.rs b/tools/xray/src/layout.rs new file mode 100644 index 00000000..75ded3ab --- /dev/null +++ b/tools/xray/src/layout.rs @@ -0,0 +1,125 @@ +//! Which crate and which build target a workspace file belongs to. +//! +//! `cargo metadata` is the only thing that knows this. A path alone does not: +//! `crates/didbot-name/src/tests.rs` is a unit-test module inside the lib +//! target, while `crates/didbot-tls/tests/rotation.rs` is a target of its own, +//! and the reference index has to tell those apart to answer `dead-pub`. + +use anyhow::{Context, Result}; +use serde_json::Value; +use std::collections::BTreeMap; +use std::path::Path; +use std::process::Command; + +/// A declared dependency between two workspace crates. +pub struct Dep { + pub from: String, + pub to: String, + pub kind: String, +} + +pub struct Layout { + /// (manifest directory relative to the workspace root, package name), + /// longest directory first so a prefix match lands on the inner package. + packages: Vec<(String, String)>, + has_lib: BTreeMap, + pub deps: Vec, +} + +impl Layout { + /// Reads the workspace layout with `cargo metadata --no-deps`. + pub fn probe(root: &Path) -> Result { + let out = Command::new("cargo") + .args(["metadata", "--no-deps", "--format-version", "1"]) + .current_dir(root) + .output() + .context("running cargo metadata")?; + if !out.status.success() { + anyhow::bail!( + "cargo metadata failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + } + let meta: Value = serde_json::from_slice(&out.stdout)?; + let root_str = root.to_string_lossy().to_string(); + let mut packages = Vec::new(); + let mut has_lib = BTreeMap::new(); + let mut names = Vec::new(); + for pkg in meta["packages"].as_array().unwrap_or(&Vec::new()) { + let name = pkg["name"].as_str().unwrap_or_default().to_string(); + let manifest = pkg["manifest_path"].as_str().unwrap_or_default(); + let dir = Path::new(manifest).parent().unwrap_or(Path::new("")); + let rel = dir + .to_string_lossy() + .strip_prefix(&root_str) + .unwrap_or_default() + .trim_start_matches('/') + .to_string(); + let lib = pkg["targets"] + .as_array() + .map(|ts| { + ts.iter().any(|t| { + t["kind"] + .as_array() + .is_some_and(|ks| ks.iter().any(|k| k == "lib" || k == "proc-macro")) + }) + }) + .unwrap_or(false); + has_lib.insert(name.clone(), lib); + names.push(name.clone()); + packages.push((rel, name)); + } + packages.sort_by_key(|(dir, _)| std::cmp::Reverse(dir.len())); + + let mut deps = Vec::new(); + for pkg in meta["packages"].as_array().unwrap_or(&Vec::new()) { + let from = pkg["name"].as_str().unwrap_or_default().to_string(); + for dep in pkg["dependencies"].as_array().unwrap_or(&Vec::new()) { + let to = dep["name"].as_str().unwrap_or_default().to_string(); + if !names.contains(&to) { + continue; + } + deps.push(Dep { + from: from.clone(), + to, + kind: dep["kind"].as_str().unwrap_or("normal").to_string(), + }); + } + } + Ok(Layout { + packages, + has_lib, + deps, + }) + } + + /// The crate a workspace-relative file belongs to, and the kind of build + /// target it is compiled into. + pub fn locate(&self, file: &str) -> Option<(&str, &'static str)> { + let (dir, name) = self + .packages + .iter() + .find(|(dir, _)| dir.is_empty() || file.starts_with(&format!("{dir}/")))?; + let rel = if dir.is_empty() { + file + } else { + &file[dir.len() + 1..] + }; + let kind = if rel.starts_with("tests/") { + "test" + } else if rel.starts_with("benches/") { + "bench" + } else if rel.starts_with("examples/") { + "example" + } else if rel == "build.rs" { + "build" + } else if rel.starts_with("src/bin/") || rel == "src/main.rs" { + "bin" + } else if self.has_lib.get(name).copied().unwrap_or(false) { + "lib" + } else { + "bin" + }; + Some((name.as_str(), kind)) + } +} diff --git a/tools/xray/src/main.rs b/tools/xray/src/main.rs new file mode 100644 index 00000000..35cf177c --- /dev/null +++ b/tools/xray/src/main.rs @@ -0,0 +1,95 @@ +//! A reference index over the didbot workspace, and the queries it answers. +//! +//! `git grep` is a text search over a language that has modules, `use` +//! re-exports, trait impls, macros and `cfg`. It cannot tell a definition from +//! a mention in a comment, and it cannot follow a re-export. This builds the +//! index a compiler-accurate answer needs instead: the item inventory from +//! rustdoc's JSON, the reference edges from `rust-analyzer scip`, joined into +//! one SQLite file. +//! +//! scripts/xray.sh produces both inputs and calls this. + +mod layout; +mod query; +mod rustdoc; +mod scip_index; +mod store; + +use anyhow::Result; +use clap::{Parser, Subcommand}; +use std::path::PathBuf; + +#[derive(Parser)] +#[command(name = "xray", about = "A reference index over a Rust workspace.")] +struct Cli { + /// The index to read or write. + #[arg(long, default_value = ".xray/index.db", global = true)] + db: PathBuf, + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// Loads a rustdoc JSON directory and a SCIP index into the index. + Build { + /// The workspace root the two dumps were taken in. + #[arg(long, default_value = ".")] + workspace: PathBuf, + /// The `rust-analyzer scip` output. + #[arg(long)] + scip: PathBuf, + /// A directory `cargo doc --output-format json` wrote. Repeatable: + /// the lib and bin runs go to directories of their own. + #[arg(long, required = true)] + docs: Vec, + /// The revision the sources were at. An index is only as current as + /// this. + #[arg(long, default_value = "unknown")] + rev: String, + }, + /// What the index was built from. + Info, + /// Public items nothing outside their own crate, and no test, reaches. + DeadPub { + /// Only this crate. + #[arg(long = "crate")] + krate: Option, + /// Include `pub` items that no public path reaches, which cannot be + /// used from another crate whatever they are marked. + #[arg(long)] + all: bool, + }, + /// Every reference to an item, with file and line. + Refs { path: String }, + /// Declared workspace dependencies and the references crossing them. + CrateEdges, + /// Every item in a file, with its span. + FileItems { file: String }, +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + match cli.command { + Command::Build { + workspace, + scip, + docs, + rev, + } => { + let stats = store::build(&cli.db, &workspace, &scip, &docs, &rev)?; + println!( + "{} items, {} references, {} definitions rustdoc does not know", + stats.items, stats.refs, stats.unresolved + ); + } + Command::Info => query::info(&query::open(&cli.db)?)?, + Command::DeadPub { krate, all } => { + query::dead_pub(&query::open(&cli.db)?, krate.as_deref(), all)? + } + Command::Refs { path } => query::refs(&query::open(&cli.db)?, &path)?, + Command::CrateEdges => query::crate_edges(&query::open(&cli.db)?)?, + Command::FileItems { file } => query::file_items(&query::open(&cli.db)?, &file)?, + } + Ok(()) +} diff --git a/tools/xray/src/query.rs b/tools/xray/src/query.rs new file mode 100644 index 00000000..f6a32504 --- /dev/null +++ b/tools/xray/src/query.rs @@ -0,0 +1,204 @@ +//! The questions the index exists to answer. + +use anyhow::{Context, Result}; +use rusqlite::{params, Connection}; +use std::path::Path; + +pub fn open(db_path: &Path) -> Result { + let conn = + Connection::open(db_path).with_context(|| format!("opening {}", db_path.display()))?; + conn.query_row("SELECT value FROM meta WHERE key = 'revision'", [], |r| { + r.get::<_, String>(0) + }) + .with_context(|| format!("{} is not an xray index", db_path.display()))?; + Ok(conn) +} + +/// A reference that counts as a use. A `pub use` re-export names an item +/// without using it, so a crate that only re-exports something has not kept +/// it alive. +const LIVE_REF: &str = " + SELECT r.* FROM ref r + LEFT JOIN item f ON f.id = r.from_item + WHERE f.kind IS NULL OR f.kind <> 'use' +"; + +pub fn info(conn: &Connection) -> Result<()> { + let mut stmt = conn.prepare("SELECT key, value FROM meta ORDER BY key")?; + let rows = stmt.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))?; + for row in rows { + let (k, v) = row?; + println!("{k}\t{v}"); + } + for (label, sql) in [ + ("items", "SELECT COUNT(*) FROM item"), + ("refs", "SELECT COUNT(*) FROM ref"), + ("crates", "SELECT COUNT(DISTINCT crate) FROM item"), + ] { + let n: i64 = conn.query_row(sql, [], |r| r.get(0))?; + println!("{label}\t{n}"); + } + Ok(()) +} + +/// Public items whose `pub` earns nothing: nothing outside the crate reaches +/// them, and no test does either. +pub fn dead_pub(conn: &Connection, krate: Option<&str>, all: bool) -> Result<()> { + let sql = format!( + " + SELECT i.module_path || '::' || i.name AS path, i.kind, i.file, i.line_start, + (SELECT COUNT(*) FROM ({LIVE_REF}) l WHERE l.to_item = i.id) AS n, + (SELECT COUNT(DISTINCT l.file) FROM ({LIVE_REF}) l + WHERE l.to_item = i.id AND l.file <> i.file) AS elsewhere + FROM item i + WHERE i.visibility = 'public' + AND i.target_kind = 'lib' + AND i.kind NOT IN ('use', 'module', 'extern_crate') + AND (?1 OR i.exported = 1) + AND (?2 IS NULL OR i.crate = ?2) + AND NOT EXISTS ( + SELECT 1 FROM ({LIVE_REF}) l + WHERE l.to_item = i.id + AND (l.crate <> i.crate OR l.target_kind NOT IN ('lib', 'bin') + OR l.cfg_gated = 1) + ) + ORDER BY i.crate, i.file, i.line_start + " + ); + let mut stmt = conn.prepare(&sql)?; + let rows = stmt.query_map(params![all, krate], |r| { + Ok(( + r.get::<_, String>(0)?, + r.get::<_, String>(1)?, + r.get::<_, String>(2)?, + r.get::<_, i64>(3)?, + r.get::<_, i64>(4)?, + r.get::<_, i64>(5)?, + )) + })?; + let mut total = 0; + for row in rows { + let (path, kind, file, line, n, elsewhere) = row?; + let verdict = match (n, elsewhere) { + (0, _) => "unreferenced", + (_, 0) => "file-local", + _ => "crate-local", + }; + println!("{file}:{line}\t{kind}\t{verdict}\t{n} refs\t{path}"); + total += 1; + } + eprintln!("{total} public items reach nothing outside their own crate"); + Ok(()) +} + +/// Every reference to an item, by full path or by any suffix of one. +pub fn refs(conn: &Connection, path: &str) -> Result<()> { + let mut stmt = conn.prepare( + "SELECT id, module_path || '::' || name, kind, file, line_start FROM item + WHERE module_path || '::' || name = ?1 + OR module_path || '::' || name LIKE '%::' || ?1 + ORDER BY file, line_start", + )?; + let found = stmt + .query_map(params![path], |r| { + Ok(( + r.get::<_, i64>(0)?, + r.get::<_, String>(1)?, + r.get::<_, String>(2)?, + r.get::<_, String>(3)?, + r.get::<_, i64>(4)?, + )) + })? + .collect::>>()?; + if found.is_empty() { + anyhow::bail!("no item matches {path}"); + } + for (id, full, kind, file, line) in found { + println!("{kind} {full}\n defined at {file}:{line}"); + let mut stmt = conn.prepare( + "SELECT r.file, r.line, r.crate, r.target_kind, r.cfg_gated, + CASE WHEN f.kind = 'use' THEN '(re-export)' + ELSE COALESCE(f.module_path || '::' || f.name, '') END + FROM ref r LEFT JOIN item f ON f.id = r.from_item + WHERE r.to_item = ?1 ORDER BY r.file, r.line", + )?; + let rows = stmt.query_map(params![id], |r| { + Ok(( + r.get::<_, String>(0)?, + r.get::<_, i64>(1)?, + r.get::<_, String>(2)?, + r.get::<_, String>(3)?, + r.get::<_, i64>(4)?, + r.get::<_, String>(5)?, + )) + })?; + let mut n = 0; + for row in rows { + let (file, line, krate, target, gated, from) = row?; + let tag = if gated == 1 { " [cfg-gated]" } else { "" }; + println!(" {file}:{line}\t{krate} {target}{tag}\t{from}"); + n += 1; + } + println!(" {n} references"); + } + Ok(()) +} + +/// Declared workspace dependencies, each with the number of item-level +/// references that actually cross that edge. +pub fn crate_edges(conn: &Connection) -> Result<()> { + let mut stmt = conn.prepare( + " + SELECT d.from_crate, d.to_crate, d.kind, + (SELECT COUNT(*) FROM ref r JOIN item i ON i.id = r.to_item + WHERE r.crate = d.from_crate AND i.crate = d.to_crate) AS n + FROM dep d ORDER BY d.from_crate, d.to_crate + ", + )?; + let rows = stmt.query_map([], |r| { + Ok(( + r.get::<_, String>(0)?, + r.get::<_, String>(1)?, + r.get::<_, String>(2)?, + r.get::<_, i64>(3)?, + )) + })?; + let mut unused = 0; + for row in rows { + let (from, to, kind, n) = row?; + if n == 0 { + unused += 1; + } + println!("{from}\t{to}\t{kind}\t{n}"); + } + eprintln!("{unused} declared dependencies carry no item-level reference"); + Ok(()) +} + +/// Every item in a file, with the span it occupies. +pub fn file_items(conn: &Connection, file: &str) -> Result<()> { + let mut stmt = conn.prepare( + "SELECT line_start, line_end, kind, visibility, module_path || '::' || name, + (SELECT COUNT(*) FROM ref r WHERE r.to_item = item.id) + FROM item WHERE file = ?1 OR file LIKE '%' || ?1 + ORDER BY line_start", + )?; + let rows = stmt.query_map(params![file], |r| { + Ok(( + r.get::<_, i64>(0)?, + r.get::<_, i64>(1)?, + r.get::<_, String>(2)?, + r.get::<_, String>(3)?, + r.get::<_, String>(4)?, + r.get::<_, i64>(5)?, + )) + })?; + let mut n = 0; + for row in rows { + let (start, end, kind, vis, path, refs) = row?; + println!("{start}-{end}\t{vis} {kind}\t{refs} refs\t{path}"); + n += 1; + } + eprintln!("{n} items"); + Ok(()) +} diff --git a/tools/xray/src/rustdoc.rs b/tools/xray/src/rustdoc.rs new file mode 100644 index 00000000..7d82791d --- /dev/null +++ b/tools/xray/src/rustdoc.rs @@ -0,0 +1,168 @@ +//! The item inventory, read out of rustdoc's JSON output. +//! +//! rustdoc knows each item's kind, its visibility, the module path it is +//! reached by and the source span it occupies. LSIF knows none of that well, +//! which is why the index is built from both. + +use crate::layout::Layout; +use anyhow::{Context, Result}; +use serde_json::Value; +use std::collections::HashSet; +use std::path::PathBuf; + +pub struct Item { + pub krate: String, + pub module_path: String, + pub name: String, + pub kind: String, + pub visibility: String, + pub file: String, + pub line_start: u32, + pub line_end: u32, + pub target_kind: String, + /// Every module between the crate root and this item is `pub`, so the + /// item is part of the crate's published API rather than merely spelled + /// `pub` inside a private module. + pub exported: bool, +} + +/// Reads every `*.json` in each rustdoc JSON output directory. +/// +/// More than one directory, because rustdoc names its output after the crate +/// and a package's lib and its bin often share a name: documenting the two +/// into one directory loses whichever ran first. +pub fn load(dirs: &[PathBuf], layout: &Layout) -> Result> { + let mut items = Vec::new(); + let mut entries: Vec<_> = Vec::new(); + for dir in dirs { + entries.extend( + std::fs::read_dir(dir) + .with_context(|| format!("reading {}", dir.display()))? + .filter_map(|e| e.ok()) + .map(|e| e.path()) + .filter(|p| p.extension().is_some_and(|e| e == "json")), + ); + } + entries.sort(); + for path in entries { + let text = std::fs::read_to_string(&path) + .with_context(|| format!("reading {}", path.display()))?; + let doc: Value = + serde_json::from_str(&text).with_context(|| format!("parsing {}", path.display()))?; + walk_crate(&doc, layout, &mut items); + } + items.sort_by(|a, b| (&a.file, a.line_start, &a.name).cmp(&(&b.file, b.line_start, &b.name))); + items.dedup_by(|a, b| (&a.file, a.line_start, &a.name) == (&b.file, b.line_start, &b.name)); + Ok(items) +} + +fn walk_crate(doc: &Value, layout: &Layout, out: &mut Vec) { + let index = match doc["index"].as_object() { + Some(i) => i, + None => return, + }; + let root = doc["root"].to_string().trim_matches('"').to_string(); + let mut seen = HashSet::new(); + let mut stack = vec![(root, Vec::::new(), true)]; + while let Some((id, prefix, public_chain)) = stack.pop() { + if !seen.insert(id.clone()) { + continue; + } + let item = match index.get(&id) { + Some(i) => i, + None => continue, + }; + let kind = kind_of(item); + // A `use` carries no name of its own; the name it binds sits in the + // re-export body, and its span is exactly that name's spelling. + let name = item["name"] + .as_str() + .or_else(|| item["inner"]["use"]["name"].as_str()) + .unwrap_or_default() + .to_string(); + let visibility = visibility_of(item); + let public = matches!(visibility.as_str(), "public" | "default"); + + if let Some(span) = item["span"].as_object() { + let file = span["filename"].as_str().unwrap_or_default(); + if file.starts_with("crates/") && kind != "impl" && !name.is_empty() { + if let Some((krate, target_kind)) = layout.locate(file) { + out.push(Item { + krate: krate.to_string(), + module_path: prefix.join("::"), + name: name.clone(), + kind: kind.clone(), + visibility: visibility.clone(), + file: file.to_string(), + line_start: span["begin"][0].as_u64().unwrap_or(0) as u32, + line_end: span["end"][0].as_u64().unwrap_or(0) as u32, + target_kind: target_kind.to_string(), + exported: public_chain && visibility == "public", + }); + } + } + } + + // An impl block is already filed under the type it is written for, so + // it adds nothing to the path; everything else that can hold items + // does. A `use` binds a name without opening a scope. + let child_prefix = if kind == "impl" || kind == "use" || name.is_empty() { + prefix + } else { + let mut p = prefix; + p.push(name); + p + }; + let child_chain = public_chain && public; + for child in children(item) { + stack.push((child, child_prefix.clone(), child_chain)); + } + } +} + +fn kind_of(item: &Value) -> String { + item["inner"] + .as_object() + .and_then(|o| o.keys().next()) + .cloned() + .unwrap_or_else(|| "unknown".to_string()) +} + +fn visibility_of(item: &Value) -> String { + match &item["visibility"] { + Value::String(s) => s.clone(), + Value::Object(_) => "restricted".to_string(), + _ => "default".to_string(), + } +} + +/// Every child id an item can carry, whatever shape rustdoc gives it. +fn children(item: &Value) -> Vec { + let mut out = Vec::new(); + let Some(inner) = item["inner"].as_object() else { + return out; + }; + for body in inner.values() { + for key in ["items", "impls", "variants", "fields"] { + push_ids(&body[key], &mut out); + } + // Struct and enum-variant payloads nest their field list one level + // further, under the shape of the thing. + if let Some(shape) = body["kind"].as_object() { + for form in shape.values() { + push_ids(&form["fields"], &mut out); + } + } + } + out +} + +fn push_ids(value: &Value, out: &mut Vec) { + if let Some(list) = value.as_array() { + for id in list { + if let Some(n) = id.as_u64() { + out.push(n.to_string()); + } + } + } +} diff --git a/tools/xray/src/scip_index.rs b/tools/xray/src/scip_index.rs new file mode 100644 index 00000000..82b1b792 --- /dev/null +++ b/tools/xray/src/scip_index.rs @@ -0,0 +1,173 @@ +//! The reference edges, read out of `rust-analyzer scip`. +//! +//! rust-analyzer resolves names the way the compiler does: through modules, +//! `use` re-exports, trait impls, macros and `cfg`. Its SCIP index is the only +//! thing in reach that answers "who calls this" without guessing. +//! +//! SCIP rather than the LSIF dump the same binary can also write, for one +//! reason that matters and two that are pleasant. `rust-analyzer scip` takes +//! `--config-path`, so the dump can be taken with every cargo feature on; +//! `rust-analyzer lsif` has no such flag, and a default-features dump reports +//! anything a feature gates as unreferenced. It is also about a third faster +//! and a quarter the size. + +use anyhow::{Context, Result}; +use protobuf::Message; +use scip::types::{Index, SymbolRole, TextEncoding}; +use std::collections::HashMap; +use std::fs::File; +use std::io::BufReader; +use std::path::Path; + +/// One definition and every place that reaches it. +pub struct DefGroup { + /// Where the definition's name is written: file, line, identifier. + /// Absent for an item a macro generated, which rust-analyzer names but + /// gives no source position of its own. + pub def: Option<(String, u32, String)>, + /// The path the SCIP symbol spells, in the form rustdoc uses. + pub path: Option, + pub sites: Vec<(String, u32)>, +} + +/// Reads a whole SCIP index. `root` is the workspace the dump was taken in, +/// which the index names its files relative to. +pub fn load(path: &Path, root: &Path) -> Result> { + let file = File::open(path).with_context(|| format!("reading {}", path.display()))?; + let index = Index::parse_from_reader(&mut BufReader::with_capacity(1 << 20, file)) + .with_context(|| format!("parsing {}", path.display()))?; + let utf16 = index + .metadata + .as_ref() + .map(|m| m.text_document_encoding.enum_value_or_default() == TextEncoding::UTF16) + .unwrap_or(false); + + let mut defs: HashMap = HashMap::new(); + let mut sites: HashMap> = HashMap::new(); + for doc in &index.documents { + let file = doc.relative_path.clone(); + for occ in &doc.occurrences { + // A `local 3` symbol is a binding inside one body. It is never an + // item, and its name collides with every other body's. + if occ.symbol.starts_with("local ") || occ.symbol.is_empty() { + continue; + } + let Some((line, start, end)) = span(&occ.range) else { + continue; + }; + if occ.symbol_roles & SymbolRole::Definition as i32 != 0 { + defs.entry(occ.symbol.clone()) + .or_insert((file.clone(), line, start, end)); + } else { + sites + .entry(occ.symbol.clone()) + .or_default() + .push((file.clone(), line + 1)); + } + } + } + + let mut source = SourceCache::new(root, utf16); + let mut out = Vec::with_capacity(defs.len() + sites.len()); + for (symbol, (file, line, start, end)) in defs { + let def = source + .slice(&file, line, start, end) + .map(|name| (file, line + 1, name)); + out.push(DefGroup { + path: symbol_path(&symbol), + def, + sites: sites.remove(&symbol).unwrap_or_default(), + }); + } + // What is left is referenced but never defined at a source position: an + // item some macro generated, or one belonging to a dependency. The symbol + // still says which, so the path is all there is to go on. + for (symbol, sites) in sites { + if let Some(path) = symbol_path(&symbol) { + out.push(DefGroup { + path: Some(path), + def: None, + sites, + }); + } + } + Ok(out) +} + +/// The path a SCIP symbol spells, in the form rustdoc reports. +/// +/// `rust-analyzer cargo didbot-lexicon 0.1.0 lexicon/EMBEDDED.` is +/// `didbot_lexicon::lexicon::EMBEDDED`. Descriptors carrying an impl, a +/// parameter or a quoted name are left alone: those always have a source +/// position, so nothing needs this. +fn symbol_path(symbol: &str) -> Option { + let mut fields = symbol.splitn(5, ' '); + let package = fields.nth(2)?; + let descriptors = fields.nth(1)?; + if descriptors.contains(['[', ']', '(', ')', '`']) { + return None; + } + let trimmed = descriptors.trim_end_matches(['.', '#', '/', '!', ':']); + if trimmed.is_empty() || trimmed == "crate" { + return None; + } + let mut path = package.replace('-', "_"); + for segment in trimmed.split('/') { + path.push_str("::"); + path.push_str(segment); + } + Some(path) +} + +/// A SCIP range is `[line, start, end]` on one line, or `[line, start, line, +/// end]` across several. A definition's name never spans lines. +fn span(range: &[i32]) -> Option<(u32, u32, u32)> { + match range { + [line, start, end] => Some((*line as u32, *start as u32, *end as u32)), + [line, start, end_line, end] if line == end_line => { + Some((*line as u32, *start as u32, *end as u32)) + } + _ => None, + } +} + +/// Source files, read once and kept as lines. SCIP columns count code units of +/// whatever encoding the index declares, which is what the slicing here undoes. +struct SourceCache { + root: std::path::PathBuf, + utf16: bool, + files: HashMap>>, +} + +impl SourceCache { + fn new(root: &Path, utf16: bool) -> Self { + SourceCache { + root: root.to_path_buf(), + utf16, + files: HashMap::new(), + } + } + + fn slice(&mut self, file: &str, line: u32, start: u32, end: u32) -> Option { + let lines = self + .files + .entry(file.to_string()) + .or_insert_with(|| { + std::fs::read_to_string(self.root.join(file)) + .ok() + .map(|t| t.lines().map(str::to_string).collect()) + }) + .as_ref()?; + let text = lines.get(line as usize)?; + let (start, end) = (start as usize, end as usize); + if start >= end { + return None; + } + if self.utf16 { + let units: Vec = text.encode_utf16().collect(); + String::from_utf16(units.get(start..end.min(units.len()))?).ok() + } else { + text.get(start..end.min(text.len())).map(str::to_string) + } + } +} diff --git a/tools/xray/src/store.rs b/tools/xray/src/store.rs new file mode 100644 index 00000000..7a07e837 --- /dev/null +++ b/tools/xray/src/store.rs @@ -0,0 +1,236 @@ +//! The SQLite index: the schema, and the join that fills it. + +use crate::layout::Layout; +use crate::rustdoc; +use anyhow::{Context, Result}; +use rusqlite::{params, Connection}; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +pub const SCHEMA: &str = " +CREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL); + +CREATE TABLE item( + id INTEGER PRIMARY KEY, + crate TEXT NOT NULL, + module_path TEXT NOT NULL, + name TEXT NOT NULL, + kind TEXT NOT NULL, + visibility TEXT NOT NULL, + file TEXT NOT NULL, + line_start INTEGER NOT NULL, + line_end INTEGER NOT NULL, + target_kind TEXT NOT NULL, + exported INTEGER NOT NULL +); + +CREATE TABLE ref( + from_item INTEGER REFERENCES item(id), + to_item INTEGER NOT NULL REFERENCES item(id), + file TEXT NOT NULL, + line INTEGER NOT NULL, + crate TEXT NOT NULL, + target_kind TEXT NOT NULL, + cfg_gated INTEGER NOT NULL +); + +CREATE TABLE dep( + from_crate TEXT NOT NULL, + to_crate TEXT NOT NULL, + kind TEXT NOT NULL +); + +CREATE INDEX item_path ON item(module_path, name); +CREATE INDEX item_file ON item(file, line_start); +CREATE INDEX ref_to ON ref(to_item); +CREATE INDEX ref_file ON ref(file, line); +"; + +pub struct Stats { + pub items: usize, + pub refs: usize, + pub unresolved: usize, +} + +pub fn build( + db_path: &Path, + root: &Path, + scip_path: &Path, + docs_dirs: &[PathBuf], + rev: &str, +) -> Result { + // Source files are read back to check what a definition is named, so the + // root has to be the path rust-analyzer was pointed at. + let root = &root + .canonicalize() + .with_context(|| format!("resolving {}", root.display()))?; + let layout = Layout::probe(root)?; + let items = rustdoc::load(docs_dirs, &layout)?; + let groups = crate::scip_index::load(scip_path, root)?; + + // Items grouped by file, so a source position can be turned into the item + // that owns it, and by path, for the ones that have no position. + let mut by_file: HashMap<&str, Vec> = HashMap::new(); + let mut by_path: HashMap> = HashMap::new(); + for (i, item) in items.iter().enumerate() { + by_file.entry(item.file.as_str()).or_default().push(i); + by_path + .entry(format!("{}::{}", item.module_path, item.name)) + .and_modify(|slot| *slot = None) + .or_insert(Some(i)); + } + + if let Some(parent) = db_path.parent() { + std::fs::create_dir_all(parent)?; + } + let _ = std::fs::remove_file(db_path); + let mut conn = + Connection::open(db_path).with_context(|| format!("opening {}", db_path.display()))?; + conn.execute_batch(SCHEMA)?; + let tx = conn.transaction()?; + + for (i, item) in items.iter().enumerate() { + tx.execute( + "INSERT INTO item VALUES (?,?,?,?,?,?,?,?,?,?,?)", + params![ + i as i64 + 1, + item.krate, + item.module_path, + item.name, + item.kind, + item.visibility, + item.file, + item.line_start, + item.line_end, + item.target_kind, + item.exported as i64, + ], + )?; + } + for dep in &layout.deps { + tx.execute( + "INSERT INTO dep VALUES (?,?,?)", + params![dep.from, dep.to, dep.kind], + )?; + } + + let mut refs = 0usize; + let mut unresolved = 0usize; + { + let mut insert = tx.prepare("INSERT INTO ref VALUES (?,?,?,?,?,?,?)")?; + for group in &groups { + let Some(to) = resolve_def(&items, &by_file, &by_path, group) else { + unresolved += 1; + continue; + }; + for (file, line) in &group.sites { + let Some((krate, target_kind)) = layout.locate(file) else { + continue; + }; + let from = enclosing(&items, &by_file, file, *line); + // rustdoc never sees `#[cfg(test)]` modules or a feature that + // is off, so a reference from a line no item covers is a + // reference from cfg-gated code. That still counts as a use. + let cfg_gated = from.is_none() && matches!(target_kind, "lib" | "bin"); + insert.execute(params![ + from.map(|i| i as i64 + 1), + to as i64 + 1, + file, + line, + krate, + target_kind, + cfg_gated as i64, + ])?; + refs += 1; + } + } + } + + for (key, value) in [ + ("revision", rev), + ("workspace", &root.to_string_lossy()), + ("built_at", &now()), + ("schema", "1"), + ] { + tx.execute("INSERT INTO meta VALUES (?,?)", params![key, value])?; + } + tx.commit()?; + Ok(Stats { + items: items.len(), + refs, + unresolved, + }) +} + +/// The item an LSIF definition range names. +/// +/// rustdoc's span for an item starts at the item itself, attributes and doc +/// comments excluded, so the name sits on the span's first line. Matching on +/// that line plus the identifier text keeps a local binding from being +/// mistaken for the item it is written inside. +fn resolve_def( + items: &[rustdoc::Item], + by_file: &HashMap<&str, Vec>, + by_path: &HashMap>, + group: &crate::scip_index::DefGroup, +) -> Option { + let by_symbol = || { + group + .path + .as_ref() + .and_then(|path| by_path.get(path)) + .copied() + .flatten() + }; + let Some((def_file, def_line, def_name)) = &group.def else { + return by_symbol(); + }; + let (def_line, def_name) = (*def_line, def_name.as_str()); + let candidates = by_file.get(def_file.as_str())?; + candidates + .iter() + .copied() + .filter(|&i| items[i].name == def_name && items[i].line_start == def_line) + .min_by_key(|&i| items[i].line_end - items[i].line_start) + .or_else(|| { + candidates + .iter() + .copied() + .filter(|&i| { + items[i].name == def_name + && items[i].line_start <= def_line + && def_line <= items[i].line_end + }) + .max_by_key(|&i| items[i].line_start) + }) + .or_else(by_symbol) +} + +/// The innermost item whose span covers a line. +fn enclosing( + items: &[rustdoc::Item], + by_file: &HashMap<&str, Vec>, + file: &str, + line: u32, +) -> Option { + by_file + .get(file)? + .iter() + .copied() + .filter(|&i| { + items[i].kind != "module" && items[i].line_start <= line && line <= items[i].line_end + }) + .min_by_key(|&i| { + ( + std::cmp::Reverse(items[i].line_start), + items[i].line_end - items[i].line_start, + ) + }) +} + +fn now() -> String { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs().to_string()) + .unwrap_or_default() +} -- 2.51.2