From 63a31c83a87a38a7118057b0100c0ba53fd1c925 Mon Sep 17 00:00:00 2001 From: zzstoatzz Date: Sat, 29 Aug 2026 00:15:17 -0500 Subject: [PATCH 01/17] strata: worker + D1 schema for per-segment collection counts of the stream archive Ingest endpoint (bearer) upserts segment rows and replaces their collection rows; read endpoints aggregate by segment bucket. Bound to strata.zat.dev. The flow that feeds it lives in my-prefect-server. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01B927dNwNKNYsbQdUoNJwHS --- .gitignore | 3 + README.md | 7 + docs/how-it-works.md | 42 + package-lock.json | 2082 ++++++++++++++++++++++++++++++++++++++++++ package.json | 18 + schema.sql | 22 + src/index.ts | 173 ++++ src/ingest.ts | 97 ++ tests/ingest.test.ts | 45 + tsconfig.json | 15 + wrangler.jsonc | 16 + 11 files changed, 2520 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 docs/how-it-works.md create mode 100644 package-lock.json create mode 100644 package.json create mode 100644 schema.sql create mode 100644 src/index.ts create mode 100644 src/ingest.ts create mode 100644 tests/ingest.test.ts create mode 100644 tsconfig.json create mode 100644 wrangler.jsonc diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a933f10 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +node_modules/ +.wrangler/ +.dev.vars diff --git a/README.md b/README.md new file mode 100644 index 0000000..3932838 --- /dev/null +++ b/README.md @@ -0,0 +1,7 @@ +# strata + +the shape of the [stream.waow.tech](https://stream.waow.tech) archive, read +from outside: per-segment, per-collection event counts, stored in D1 and +served by a Worker. the archive process is not involved. + +see [docs/how-it-works.md](docs/how-it-works.md). diff --git a/docs/how-it-works.md b/docs/how-it-works.md new file mode 100644 index 0000000..b752e1f --- /dev/null +++ b/docs/how-it-works.md @@ -0,0 +1,42 @@ +# how it works + +## the read + +every sealed `.jss` segment carries a footer whose layout is fixed by +[jss-format-v1](https://tangled.org/zat.dev/stream/blob/main/docs/jss-format-v1.md). +The 256-byte header at offset 0 holds `collection_index_offset`; the +collection block index at that offset (to EOF) is a 16-byte header plus one +zstd frame containing `(len u8, count u32, nsid)` per collection. Both are +reachable with HTTP `Range` requests against `getSegment`, so a segment's +per-collection counts cost about 1.3 KB and two requests, not its 270 MB body. + +Measured 2026-08-29 on segment 6000: header 256 B (0.56 s), collection index +1,033 B (0.46 s), 28 collections whose counts sum to the segment's event count +minus its `$account/$identity/$sync` marker events, as the format specifies. + +## who does what + +- **the flow** (`flows/strata.py` in my-prefect-server, scheduled there — the + only cron): asks this worker for its progress, lists segments past it via + `listSegments`, reads header + collection index per new segment with a + bearer archive key rate-limited to 8 Mbps, and POSTs batches to `/ingest`. + stateless; reruns are idempotent (segment rows are upserted, collection + rows replaced). +- **this worker**: `/ingest` (bearer `STRATA_INGEST_TOKEN`) writes D1; + `/api/progress`, `/api/heat?step=N`, `/api/collections` read it. no + schedule, no archive access. + +## the axis caveat + +`min_witnessed_at`/`max_witnessed_at` are when *stream* ingested the events, +not when they happened. the ~6,750 bootstrap segments were all witnessed inside +the six-day backfill from 2026-07-28; only later segments are live-era +(≈2 minutes each). true record time lives inside event payloads, which this +project does not decode. plot by `idx` or `seq`; treat witnessed time as an +ingest-era marker. + +## secrets + +`STRATA_INGEST_TOKEN` and the archive key `stream-archive-key-strata` live in +the sops store and reach the flow as prefect Secret blocks; the worker's copy +of the ingest token is a wrangler secret set from the same store value. diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..2c76033 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,2082 @@ +{ + "name": "strata", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "strata", + "devDependencies": { + "@cloudflare/workers-types": "5.20260829.1", + "@types/node": "22.20.1", + "tsx": "4.23.12", + "typescript": "5.9.3", + "wrangler": "4.127.1" + } + }, + "node_modules/@cloudflare/kv-asset-handler": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@cloudflare/kv-asset-handler/-/kv-asset-handler-0.5.0.tgz", + "integrity": "sha512-jxQYkj8dSIzc0cD6cMMNdOc1UVjqSqu8BZdor5s8cGjW2I8BjODt/kWPVdY+u9zj3ms75Q5qaZgnxUad83+eAg==", + "dev": true, + "license": "MIT OR Apache-2.0", + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@cloudflare/unenv-preset": { + "version": "2.16.1", + "resolved": "https://registry.npmjs.org/@cloudflare/unenv-preset/-/unenv-preset-2.16.1.tgz", + "integrity": "sha512-ECxObrMfyTl5bhQf/lZCXwo5G6xX9IAUo+nDMKK4SZ8m4Jvvxp52vilxyySSWh2YTZz8+HQ07qGH/2rEom1vDw==", + "dev": true, + "license": "MIT OR Apache-2.0", + "peerDependencies": { + "unenv": "2.0.0-rc.24", + "workerd": ">1.20260305.0 <2.0.0-0" + }, + "peerDependenciesMeta": { + "workerd": { + "optional": true + } + } + }, + "node_modules/@cloudflare/workerd-darwin-64": { + "version": "1.20260828.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20260828.1.tgz", + "integrity": "sha512-CVd+xPhqUESg8Xhq09TZx0wl4FSirfJGOzvbPz2yHhBIvmNHFFQkSN3rkd7wEwnhQQk37Xi0/aD6ykPLJbmGiQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workerd-darwin-arm64": { + "version": "1.20260828.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20260828.1.tgz", + "integrity": "sha512-5HDPXRM152vU5JveByGFk34X57TVyIsfp4cabepAf45DC0MKvm52ucJqAjW1h8bvW4X+zRw9GU35OHF9FEC9Ww==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workerd-linux-64": { + "version": "1.20260828.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20260828.1.tgz", + "integrity": "sha512-MQ1Ll9P7F72HHUKizbb7BlDfbY8fRoNMpbIpZoU6uKsSkneFICWSKv6UlgU9EQZ+w0i7TMa12iUgJ8l29eRI9A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workerd-linux-arm64": { + "version": "1.20260828.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20260828.1.tgz", + "integrity": "sha512-FBTaUQ1xcU9jcp4OyBPcH8x0QiFvc1iuZL2GkD8zp2q1WyTVHYOptRDQUU+cuHjt0rQ2EIKVPBjahPxfa0joBw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workerd-windows-64": { + "version": "1.20260828.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20260828.1.tgz", + "integrity": "sha512-yvr77hC7dUbvK5K+SCg062kkPq3sx+drV1PcgHslzHDYcJBtT0V3X80qLE49LW1vq2svaeNmsVQS+vHsqWu8cQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workers-types": { + "version": "5.20260829.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-5.20260829.1.tgz", + "integrity": "sha512-SsH4G9+JePvrBmz+b5Dl67LQ9w2/ycoIpAPcAkfg2aEs2SEisQHk7/AY0TymSXEBbDbWgcBqoLe4hNcuY3CxWA==", + "dev": true, + "license": "MIT OR Apache-2.0" + }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.11.3", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", + "integrity": "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", + "integrity": "sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.2.tgz", + "integrity": "sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.2.tgz", + "integrity": "sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.2.tgz", + "integrity": "sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.2.tgz", + "integrity": "sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.2.tgz", + "integrity": "sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.2.tgz", + "integrity": "sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.2.tgz", + "integrity": "sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.2.tgz", + "integrity": "sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.2.tgz", + "integrity": "sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.2.tgz", + "integrity": "sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.2.tgz", + "integrity": "sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.2.tgz", + "integrity": "sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.2.tgz", + "integrity": "sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.2.tgz", + "integrity": "sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.2.tgz", + "integrity": "sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.2.tgz", + "integrity": "sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.2.tgz", + "integrity": "sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.2.tgz", + "integrity": "sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.2.tgz", + "integrity": "sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.2.tgz", + "integrity": "sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.2.tgz", + "integrity": "sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.2.tgz", + "integrity": "sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.2.tgz", + "integrity": "sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.2.tgz", + "integrity": "sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.2.tgz", + "integrity": "sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/colour": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", + "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.35.2.tgz", + "integrity": "sha512-eEieHsMksAW4IiO5NzauESRl2D2qz3J/kwUxUrSfV06A93eEaRfMpHXyUb1mAqrR7i8U9A0GRqE9pjn6u1Jjpg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.3.1" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.35.2.tgz", + "integrity": "sha512-BaktuGPCeHJMARpodR8jK4uKiZrPAy9WrfQW0sdI37clracq8Bp01AYS3SZgi5FS/y5twa9t4+LIuuxQjqRrWw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.3.1" + } + }, + "node_modules/@img/sharp-freebsd-wasm32": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-freebsd-wasm32/-/sharp-freebsd-wasm32-0.35.2.tgz", + "integrity": "sha512-YoAxdnd8hPUkvLHd3bWY+YA8nw3xM/RyRopYucNsWHVSan8NLVM3X2volsfoRDcXdUJPg6tXahSd7HXPK7lRnw==", + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "dependencies": { + "@img/sharp-wasm32": "0.35.2" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.3.1.tgz", + "integrity": "sha512-4V/M3roRMTYjiwZY9IOVQOE8OyeCxFAkYmyZDrZl51uOKjibm3oeEJ4WAmLxutAfzFbC9jqUiPs2gbnGflH+7g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.3.1.tgz", + "integrity": "sha512-c0/DxItpJv2+dGhgycJBBgotdqruGYDvA79drdh0MD1dFpy7JzJ/PlXwi1H4rFf0eTy8tgbI91aHDnZIceY3jQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.3.1.tgz", + "integrity": "sha512-aGGy9aWzXgHBG7HNyQPWorZthlp7+x6fDRoPAQbGO3ThcttuTyKIx3NuSHb6zb4gBNq6/yNn9f1cy9nFKS/Vmg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.3.1.tgz", + "integrity": "sha512-JznefmcK9j1JKPz8AkQDh89kjojubyfOasWBPKfzMIhPwsgDy9evpE/naJTXXXmghS1iFwR8u/kTwh/I2/+GCw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.3.1.tgz", + "integrity": "sha512-1EkwGNCZk6iWNCMWqrvdJ+r1j0PT1zIz60CNPhYnJlK/zyeWqlsPZIe+ocBVqPF8k/Ssee/NCk+tE9Ryrko6ng==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.3.1.tgz", + "integrity": "sha512-Ilays+w2bXdnxzxtQdmXR62u8o8GYa3eL4+Gr+1KiE4xperMZUslRaVPJwwPkzlHEjGfXAfRVAa/7CYCtSqsBw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.3.1.tgz", + "integrity": "sha512-VfBwVHQTbRoj4XlpA/KLZ7ltgMpz+4WSejFzQ+GnoImjo1PtEJ59QB2qR1xQEeRPYIkNrPIm2L4cICMvz4C2ew==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.3.1.tgz", + "integrity": "sha512-+c8ukgwU62DS54nCAjw7keOfHUkmr0B5QHEdcOqRnodF/MNXJbVI8Eopoj4B/0H8Asr65I+A4Amrn7a85/md6A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.3.1.tgz", + "integrity": "sha512-qlKb/pwbkAi1WMsJrYHk7CuDrd12s27U2QnRhFYUoJNrRCmkosMTttuRFat/DDB3IlDm5qE1TJgZ4JDnHX8Ldw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.3.1.tgz", + "integrity": "sha512-yO21HwoUVLN8Qa+/SBjQLMYwBWAVJjeGPNe+hc0OUeMeifEtJqu5a1c4HayE1nNpDih9y3/KkoltfkDodmKAlg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.35.2.tgz", + "integrity": "sha512-SE4kzF2mepn6z+6E7L6lsV8FzuLL6IPQdyX8ZiwROAG/G8td+hP/m7FsFPwidtrF19gvajuC9l6TxAVcsA4S7A==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.3.1" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.35.2.tgz", + "integrity": "sha512-af12Pnd0ZGu2HfP8NayB0kk6eC/lrfbQE6HlR4jD+34wdJ1Vw9TF6TMn6ZvffT+WgqVsl0hRbmNvz2u/23VmwA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.3.1" + } + }, + "node_modules/@img/sharp-linux-ppc64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.35.2.tgz", + "integrity": "sha512-hYSBm7zcNtDCozCxQHYZJiu63b/bXsgRZuOxCIBZsStMM9Vap47iFHdbX4kCvQsblPB/k+clhELpdQJHQLSHvg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.3.1" + } + }, + "node_modules/@img/sharp-linux-riscv64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.35.2.tgz", + "integrity": "sha512-qQt0Kc13+Hoan/Awq/qMSQw3L+RI1NCRPgD5cUJ/1WSSmIoysLOc72jlRM3E0OHN9Yr313jgeQ2T+zW+F03QFA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.3.1" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.35.2.tgz", + "integrity": "sha512-E4fLLfRPzDLlEeDaTzI98OFLcv++WL5ChLLMwPoVd0CIoZQqupBSNbOisPL5am9XsbQ9T84+iiMpUvbFtkunbA==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.3.1" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.35.2.tgz", + "integrity": "sha512-gi0zFJJRLswfCZmHtJdikXPOc5u7qamSOS3NHedLqLd4W8Q0NqjdBr6TTRIgsfFjqfTsHFgdfvJ9LwqSgcHiAA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.3.1" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.35.2.tgz", + "integrity": "sha512-siWbOW1u6HFnFLrp0waKyW7VEf7jYvcDWdrXEFa8AkdAQgEvuu5Fz8/Y70w9EeqAdwDtfU012BhEHHaDqvQNzg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.3.1" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.35.2.tgz", + "integrity": "sha512-YBqMMcjDi4QGYiSn4vNOYBhmlC4z5AXqkOUUqI2e0AFA4urNv4ESgOgwNl3K+4etQhha0twXlzeF20bbULm9Yg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.3.1" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.35.2.tgz", + "integrity": "sha512-Mrv4JQNYVQ94xH+jzZ9r+gowleN8mv2FTgKT+PI6bx5C0G8TdNYndu161pg2i7uoBwxy2ImPMHrJOM2LZef7Bw==", + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.11.1" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-webcontainers-wasm32": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-webcontainers-wasm32/-/sharp-webcontainers-wasm32-0.35.2.tgz", + "integrity": "sha512-QNV27pxs9wpApEiCfvHM1RDoP1w1+2KrUWWDPEhEwg+latvOrfuhWrHWZKwdSFwU6jh3myjw/yOCRsUIuOft3g==", + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@img/sharp-wasm32": "0.35.2" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.35.2.tgz", + "integrity": "sha512-BiVRYc/t6/Vl3e1hBx0hugG4oN9Pydf4fgMSpxTQJmwGUg/YoXTWHiFeRymHfCZzifxu4F4rpk/I67D0LQ20wQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.35.2.tgz", + "integrity": "sha512-YYEhx9PImCC7T0tI8JDMi4DB9LwLCXCU5OWNYEXAxh5Q1ShKkyC6byxzoBJ3gEFDnH2lQckWuDe70G7mB2XJog==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.35.2.tgz", + "integrity": "sha512-imoOyBcoM/iiUr4J6VPpCNjPnjvP/Gks95898yB8YqoGGYmHYbOyCuNv9FMhFgtaiHFGbHW8bxKqRV6VjtXThQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "node_modules/@poppinss/colors": { + "version": "4.1.6", + "resolved": "https://registry.npmjs.org/@poppinss/colors/-/colors-4.1.6.tgz", + "integrity": "sha512-H9xkIdFswbS8n1d6vmRd8+c10t2Qe+rZITbbDHHkQixH5+2x1FDGmi/0K+WgWiqQFKPSlIYB7jlH6Kpfn6Fleg==", + "dev": true, + "license": "MIT", + "dependencies": { + "kleur": "^4.1.5" + } + }, + "node_modules/@poppinss/dumper": { + "version": "0.6.5", + "resolved": "https://registry.npmjs.org/@poppinss/dumper/-/dumper-0.6.5.tgz", + "integrity": "sha512-NBdYIb90J7LfOI32dOewKI1r7wnkiH6m920puQ3qHUeZkxNkQiFnXVWoE6YtFSv6QOiPPf7ys6i+HWWecDz7sw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@poppinss/colors": "^4.1.5", + "@sindresorhus/is": "^7.0.2", + "supports-color": "^10.0.0" + } + }, + "node_modules/@poppinss/exception": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@poppinss/exception/-/exception-1.2.3.tgz", + "integrity": "sha512-dCED+QRChTVatE9ibtoaxc+WkdzOSjYTKi/+uacHWIsfodVfpsueo3+DKpgU5Px8qXjgmXkSvhXvSCz3fnP9lw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@sindresorhus/is": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-7.2.0.tgz", + "integrity": "sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sindresorhus/is?sponsor=1" + } + }, + "node_modules/@speed-highlight/core": { + "version": "1.2.24", + "resolved": "https://registry.npmjs.org/@speed-highlight/core/-/core-1.2.24.tgz", + "integrity": "sha512-qeW2e1l78afw8VhRPfPQ1Gjj+KU5XFQ/OFV5ti6eTa9bruO7mJyZtA4vw0ofqmA3tKCkROE9xLk3VZoeRc98nw==", + "dev": true, + "license": "CC0-1.0" + }, + "node_modules/@types/node": { + "version": "22.20.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.1.tgz", + "integrity": "sha512-EANqOCF9QFyra+4pfxUcX9STKJpCLjMbObVzljIJomAWSnuSIEAvyzEU53GaajbXJEgdh0iEcPL+DGvpUd4k1Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/blake3-wasm": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/blake3-wasm/-/blake3-wasm-2.1.5.tgz", + "integrity": "sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==", + "dev": true, + "license": "MIT" + }, + "node_modules/cookie": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", + "integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/error-stack-parser-es": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/error-stack-parser-es/-/error-stack-parser-es-1.0.5.tgz", + "integrity": "sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/esbuild": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", + "integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.2", + "@esbuild/android-arm": "0.28.2", + "@esbuild/android-arm64": "0.28.2", + "@esbuild/android-x64": "0.28.2", + "@esbuild/darwin-arm64": "0.28.2", + "@esbuild/darwin-x64": "0.28.2", + "@esbuild/freebsd-arm64": "0.28.2", + "@esbuild/freebsd-x64": "0.28.2", + "@esbuild/linux-arm": "0.28.2", + "@esbuild/linux-arm64": "0.28.2", + "@esbuild/linux-ia32": "0.28.2", + "@esbuild/linux-loong64": "0.28.2", + "@esbuild/linux-mips64el": "0.28.2", + "@esbuild/linux-ppc64": "0.28.2", + "@esbuild/linux-riscv64": "0.28.2", + "@esbuild/linux-s390x": "0.28.2", + "@esbuild/linux-x64": "0.28.2", + "@esbuild/netbsd-arm64": "0.28.2", + "@esbuild/netbsd-x64": "0.28.2", + "@esbuild/openbsd-arm64": "0.28.2", + "@esbuild/openbsd-x64": "0.28.2", + "@esbuild/openharmony-arm64": "0.28.2", + "@esbuild/sunos-x64": "0.28.2", + "@esbuild/win32-arm64": "0.28.2", + "@esbuild/win32-ia32": "0.28.2", + "@esbuild/win32-x64": "0.28.2" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/kleur": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", + "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/miniflare": { + "version": "5.20260828.0-alpha", + "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-5.20260828.0-alpha.tgz", + "integrity": "sha512-6nbxhZEcz/UET3Y1OnYPsrAUjUmuFoib3ynUqteRdn1YnDxsLg8cwgZJZCk9QmtOmGzXwzXzgE/d/C0dJAPtVw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cspotcode/source-map-support": "0.8.1", + "sharp": "0.35.2", + "undici": "7.29.0", + "workerd": "1.20260828.1", + "ws": "8.21.0", + "youch": "4.1.0-beta.10" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/path-to-regexp": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", + "integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "dev": true, + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/sharp": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.2.tgz", + "integrity": "sha512-FVtFjtBCMiJS6yb5CX7Sop45WFMpeGw6oRKuJnXYgf/f1ms/D7LE/ZUSNxnW7rZ/dbslQWYkoqFHGPaDBtaK4w==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@img/colour": "^1.1.0", + "detect-libc": "^2.1.2", + "semver": "^7.8.4" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.35.2", + "@img/sharp-darwin-x64": "0.35.2", + "@img/sharp-freebsd-wasm32": "0.35.2", + "@img/sharp-libvips-darwin-arm64": "1.3.1", + "@img/sharp-libvips-darwin-x64": "1.3.1", + "@img/sharp-libvips-linux-arm": "1.3.1", + "@img/sharp-libvips-linux-arm64": "1.3.1", + "@img/sharp-libvips-linux-ppc64": "1.3.1", + "@img/sharp-libvips-linux-riscv64": "1.3.1", + "@img/sharp-libvips-linux-s390x": "1.3.1", + "@img/sharp-libvips-linux-x64": "1.3.1", + "@img/sharp-libvips-linuxmusl-arm64": "1.3.1", + "@img/sharp-libvips-linuxmusl-x64": "1.3.1", + "@img/sharp-linux-arm": "0.35.2", + "@img/sharp-linux-arm64": "0.35.2", + "@img/sharp-linux-ppc64": "0.35.2", + "@img/sharp-linux-riscv64": "0.35.2", + "@img/sharp-linux-s390x": "0.35.2", + "@img/sharp-linux-x64": "0.35.2", + "@img/sharp-linuxmusl-arm64": "0.35.2", + "@img/sharp-linuxmusl-x64": "0.35.2", + "@img/sharp-webcontainers-wasm32": "0.35.2", + "@img/sharp-win32-arm64": "0.35.2", + "@img/sharp-win32-ia32": "0.35.2", + "@img/sharp-win32-x64": "0.35.2" + } + }, + "node_modules/supports-color": { + "version": "10.2.2", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-10.2.2.tgz", + "integrity": "sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD", + "optional": true + }, + "node_modules/tsx": { + "version": "4.23.12", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.12.tgz", + "integrity": "sha512-FDf4L4sYzKtzWYhU/Xm0AQFdTjdIxNo9ElTf2mxXM6k8YMHXzYUe4yODVaXP4V9uMFbVg8c0qyBccK2OOxb45Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.28.0" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz", + "integrity": "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/unenv": { + "version": "2.0.0-rc.24", + "resolved": "https://registry.npmjs.org/unenv/-/unenv-2.0.0-rc.24.tgz", + "integrity": "sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "pathe": "^2.0.3" + } + }, + "node_modules/workerd": { + "version": "1.20260828.1", + "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260828.1.tgz", + "integrity": "sha512-pB9yvt0kkwZDAGZHmpY59r0o3hM0DzdW6BJERqwZOhunZ3ssOyDSgQxOQer2cSZW4YCFeOTIQYN1qwhK5wv/Cw==", + "dev": true, + "hasInstallScript": true, + "license": "Apache-2.0", + "bin": { + "workerd": "bin/workerd" + }, + "engines": { + "node": ">=16" + }, + "optionalDependencies": { + "@cloudflare/workerd-darwin-64": "1.20260828.1", + "@cloudflare/workerd-darwin-arm64": "1.20260828.1", + "@cloudflare/workerd-linux-64": "1.20260828.1", + "@cloudflare/workerd-linux-arm64": "1.20260828.1", + "@cloudflare/workerd-windows-64": "1.20260828.1" + } + }, + "node_modules/wrangler": { + "version": "4.127.1", + "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-4.127.1.tgz", + "integrity": "sha512-OzsiNgaI8i681L/+KnAKc+uEZ5D57xK5JuNvCOpRKICF4/5Q3Cu1oTGuUiT/f3GDUqQb3gzXNT0tfOHGMEtknw==", + "dev": true, + "license": "MIT OR Apache-2.0", + "dependencies": { + "@cloudflare/kv-asset-handler": "0.5.0", + "@cloudflare/unenv-preset": "2.16.1", + "blake3-wasm": "2.1.5", + "esbuild": "0.28.1", + "miniflare": "5.20260828.0-alpha", + "path-to-regexp": "6.3.0", + "unenv": "2.0.0-rc.24", + "workerd": "1.20260828.1" + }, + "bin": { + "cf-wrangler": "bin/cf-wrangler.js", + "wrangler": "bin/wrangler.js", + "wrangler2": "bin/wrangler.js" + }, + "engines": { + "node": ">=22.0.0" + }, + "optionalDependencies": { + "fsevents": "2.3.3" + }, + "peerDependencies": { + "@cloudflare/workers-types": "^5.20260828.1" + }, + "peerDependenciesMeta": { + "@cloudflare/workers-types": { + "optional": true + } + } + }, + "node_modules/wrangler/node_modules/@esbuild/aix-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", + "integrity": "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/wrangler/node_modules/@esbuild/android-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.1.tgz", + "integrity": "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/wrangler/node_modules/@esbuild/android-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.1.tgz", + "integrity": "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/wrangler/node_modules/@esbuild/android-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.1.tgz", + "integrity": "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/wrangler/node_modules/@esbuild/darwin-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.1.tgz", + "integrity": "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/wrangler/node_modules/@esbuild/darwin-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.1.tgz", + "integrity": "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/wrangler/node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.1.tgz", + "integrity": "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/wrangler/node_modules/@esbuild/freebsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.1.tgz", + "integrity": "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/wrangler/node_modules/@esbuild/linux-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.1.tgz", + "integrity": "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/wrangler/node_modules/@esbuild/linux-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.1.tgz", + "integrity": "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/wrangler/node_modules/@esbuild/linux-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.1.tgz", + "integrity": "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/wrangler/node_modules/@esbuild/linux-loong64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.1.tgz", + "integrity": "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/wrangler/node_modules/@esbuild/linux-mips64el": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.1.tgz", + "integrity": "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/wrangler/node_modules/@esbuild/linux-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.1.tgz", + "integrity": "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/wrangler/node_modules/@esbuild/linux-riscv64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.1.tgz", + "integrity": "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/wrangler/node_modules/@esbuild/linux-s390x": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.1.tgz", + "integrity": "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/wrangler/node_modules/@esbuild/linux-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.1.tgz", + "integrity": "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/wrangler/node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.1.tgz", + "integrity": "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/wrangler/node_modules/@esbuild/netbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.1.tgz", + "integrity": "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/wrangler/node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.1.tgz", + "integrity": "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/wrangler/node_modules/@esbuild/openbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.1.tgz", + "integrity": "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/wrangler/node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.1.tgz", + "integrity": "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/wrangler/node_modules/@esbuild/sunos-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.1.tgz", + "integrity": "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/wrangler/node_modules/@esbuild/win32-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.1.tgz", + "integrity": "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/wrangler/node_modules/@esbuild/win32-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.1.tgz", + "integrity": "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/wrangler/node_modules/@esbuild/win32-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.1.tgz", + "integrity": "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/wrangler/node_modules/esbuild": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", + "integrity": "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.1", + "@esbuild/android-arm": "0.28.1", + "@esbuild/android-arm64": "0.28.1", + "@esbuild/android-x64": "0.28.1", + "@esbuild/darwin-arm64": "0.28.1", + "@esbuild/darwin-x64": "0.28.1", + "@esbuild/freebsd-arm64": "0.28.1", + "@esbuild/freebsd-x64": "0.28.1", + "@esbuild/linux-arm": "0.28.1", + "@esbuild/linux-arm64": "0.28.1", + "@esbuild/linux-ia32": "0.28.1", + "@esbuild/linux-loong64": "0.28.1", + "@esbuild/linux-mips64el": "0.28.1", + "@esbuild/linux-ppc64": "0.28.1", + "@esbuild/linux-riscv64": "0.28.1", + "@esbuild/linux-s390x": "0.28.1", + "@esbuild/linux-x64": "0.28.1", + "@esbuild/netbsd-arm64": "0.28.1", + "@esbuild/netbsd-x64": "0.28.1", + "@esbuild/openbsd-arm64": "0.28.1", + "@esbuild/openbsd-x64": "0.28.1", + "@esbuild/openharmony-arm64": "0.28.1", + "@esbuild/sunos-x64": "0.28.1", + "@esbuild/win32-arm64": "0.28.1", + "@esbuild/win32-ia32": "0.28.1", + "@esbuild/win32-x64": "0.28.1" + } + }, + "node_modules/ws": { + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/youch": { + "version": "4.1.0-beta.10", + "resolved": "https://registry.npmjs.org/youch/-/youch-4.1.0-beta.10.tgz", + "integrity": "sha512-rLfVLB4FgQneDr0dv1oddCVZmKjcJ6yX6mS4pU82Mq/Dt9a3cLZQ62pDBL4AUO+uVrCvtWz3ZFUL2HFAFJ/BXQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@poppinss/colors": "^4.1.5", + "@poppinss/dumper": "^0.6.4", + "@speed-highlight/core": "^1.2.7", + "cookie": "^1.0.2", + "youch-core": "^0.3.3" + } + }, + "node_modules/youch-core": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/youch-core/-/youch-core-0.3.3.tgz", + "integrity": "sha512-ho7XuGjLaJ2hWHoK8yFnsUGy2Y5uDpqSTq1FkHLK4/oqKtyUU1AFbOOxY4IpC9f0fTLjwYbslUz0Po5BpD1wrA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@poppinss/exception": "^1.2.2", + "error-stack-parser-es": "^1.0.5" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..2f54ff2 --- /dev/null +++ b/package.json @@ -0,0 +1,18 @@ +{ + "name": "strata", + "private": true, + "type": "module", + "scripts": { + "typecheck": "tsc --noEmit", + "test": "node --test --import tsx tests/*.test.ts", + "deploy": "wrangler deploy", + "migrate": "wrangler d1 execute strata --remote --file schema.sql" + }, + "devDependencies": { + "@cloudflare/workers-types": "5.20260829.1", + "@types/node": "22.20.1", + "tsx": "4.23.12", + "typescript": "5.9.3", + "wrangler": "4.127.1" + } +} diff --git a/schema.sql b/schema.sql new file mode 100644 index 0000000..49a11f4 --- /dev/null +++ b/schema.sql @@ -0,0 +1,22 @@ +CREATE TABLE IF NOT EXISTS segments ( + idx INTEGER PRIMARY KEY, + name TEXT NOT NULL, + size_bytes INTEGER NOT NULL, + checksum TEXT NOT NULL, + event_count INTEGER NOT NULL, + block_count INTEGER NOT NULL, + min_seq INTEGER NOT NULL, + max_seq INTEGER NOT NULL, + min_witnessed_at INTEGER NOT NULL, + max_witnessed_at INTEGER NOT NULL, + ingested_at INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS segment_collections ( + idx INTEGER NOT NULL REFERENCES segments(idx), + nsid TEXT NOT NULL, + count INTEGER NOT NULL, + PRIMARY KEY (idx, nsid) +); + +CREATE INDEX IF NOT EXISTS segment_collections_nsid ON segment_collections (nsid, idx); diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..519937f --- /dev/null +++ b/src/index.ts @@ -0,0 +1,173 @@ +/** strata — the shape of the stream.waow.tech archive, read from outside. + * + * a prefect flow walks the archive's sealed segments, decodes each footer's + * collection index, and POSTs the result here. this worker only stores and + * aggregates; it holds no schedule and never talks to the archive itself. */ + +import { IngestParseError, parseIngestBatch, type SegmentRecord } from "./ingest"; + +interface Env { + readonly DB: D1Database; + readonly STRATA_INGEST_TOKEN: string; +} + +const MAX_BODY_BYTES = 8 << 20; +const MAX_HEAT_BUCKETS = 2000; + +function json(body: unknown, status = 200, headers: Record = {}): Response { + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json", "access-control-allow-origin": "*", ...headers }, + }); +} + +function timingSafeEqual(a: string, b: string): boolean { + const enc = new TextEncoder(); + const ab = enc.encode(a); + const bb = enc.encode(b); + if (ab.byteLength !== bb.byteLength) return false; + return crypto.subtle.timingSafeEqual(ab, bb); +} + +function authorized(request: Request, env: Env): boolean { + const header = request.headers.get("authorization") ?? ""; + const token = header.startsWith("Bearer ") ? header.slice("Bearer ".length) : ""; + return token.length > 0 && timingSafeEqual(token, env.STRATA_INGEST_TOKEN); +} + +function segmentStatements(db: D1Database, segment: SegmentRecord, ingestedAt: number): D1PreparedStatement[] { + const upsertSegment = db + .prepare( + `INSERT INTO segments (idx, name, size_bytes, checksum, event_count, block_count, min_seq, max_seq, + min_witnessed_at, max_witnessed_at, ingested_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11) + ON CONFLICT(idx) DO UPDATE SET name = excluded.name, size_bytes = excluded.size_bytes, + checksum = excluded.checksum, event_count = excluded.event_count, block_count = excluded.block_count, + min_seq = excluded.min_seq, max_seq = excluded.max_seq, min_witnessed_at = excluded.min_witnessed_at, + max_witnessed_at = excluded.max_witnessed_at, ingested_at = excluded.ingested_at`, + ) + .bind( + segment.idx, + segment.name, + segment.sizeBytes, + segment.checksum, + segment.eventCount, + segment.blockCount, + segment.minSeq, + segment.maxSeq, + segment.minWitnessedAt, + segment.maxWitnessedAt, + ingestedAt, + ); + const clear = db.prepare("DELETE FROM segment_collections WHERE idx = ?1").bind(segment.idx); + const upsertCollection = db.prepare("INSERT INTO segment_collections (idx, nsid, count) VALUES (?1, ?2, ?3)"); + return [upsertSegment, clear, ...segment.collections.map((c) => upsertCollection.bind(segment.idx, c.nsid, c.count))]; +} + +async function ingest(request: Request, env: Env): Promise { + if (!authorized(request, env)) return json({ error: "unauthorized" }, 401); + const length = Number(request.headers.get("content-length") ?? "0"); + if (!Number.isFinite(length) || length > MAX_BODY_BYTES) return json({ error: "body too large" }, 413); + let batch; + try { + batch = parseIngestBatch(await request.json()); + } catch (err) { + if (err instanceof IngestParseError) return json({ error: err.message }, 400); + return json({ error: "invalid json" }, 400); + } + const ingestedAt = Date.now(); + for (const segment of batch.segments) { + await env.DB.batch(segmentStatements(env.DB, segment, ingestedAt)); + } + return json({ ingested: batch.segments.length }); +} + +interface ProgressRow { + readonly segments: number; + readonly max_idx: number | null; + readonly max_seq: number | null; +} + +async function progress(env: Env): Promise { + const row = await env.DB.prepare( + "SELECT COUNT(*) AS segments, MAX(idx) AS max_idx, MAX(max_seq) AS max_seq FROM segments", + ).first(); + return json({ segments: row?.segments ?? 0, maxIdx: row?.max_idx ?? null, maxSeq: row?.max_seq ?? null }); +} + +interface HeatRow { + readonly bucket: number; + readonly nsid: string; + readonly count: number; +} + +interface BucketRow { + readonly bucket: number; + readonly first_idx: number; + readonly last_idx: number; + readonly min_seq: number; + readonly max_seq: number; + readonly min_witnessed_at: number; + readonly max_witnessed_at: number; + readonly event_count: number; +} + +async function heat(url: URL, env: Env): Promise { + const step = Number(url.searchParams.get("step") ?? "10"); + if (!Number.isSafeInteger(step) || step < 1) return json({ error: "step must be a positive integer" }, 400); + const progressRow = await env.DB.prepare("SELECT MAX(idx) AS max_idx FROM segments").first<{ max_idx: number | null }>(); + const maxIdx = progressRow?.max_idx ?? -1; + if (Math.floor(maxIdx / step) + 1 > MAX_HEAT_BUCKETS) { + return json({ error: `step ${step} yields more than ${MAX_HEAT_BUCKETS} buckets` }, 400); + } + const results = await env.DB.batch([ + env.DB.prepare( + `SELECT idx / ?1 AS bucket, MIN(idx) AS first_idx, MAX(idx) AS last_idx, MIN(min_seq) AS min_seq, + MAX(max_seq) AS max_seq, MIN(min_witnessed_at) AS min_witnessed_at, MAX(max_witnessed_at) AS max_witnessed_at, + SUM(event_count) AS event_count + FROM segments GROUP BY bucket ORDER BY bucket`, + ).bind(step), + env.DB.prepare( + `SELECT idx / ?1 AS bucket, nsid, SUM(count) AS count + FROM segment_collections GROUP BY bucket, nsid ORDER BY bucket, nsid`, + ).bind(step), + ]); + const [buckets, cells] = results; + if (buckets === undefined || cells === undefined) return json({ error: "batch returned fewer results than statements" }, 500); + // SAFETY: the two statements above select exactly the columns of BucketRow and HeatRow. + const bucketRows = buckets.results as BucketRow[]; + const heatRows = cells.results as HeatRow[]; + return json({ step, buckets: bucketRows, cells: heatRows }, 200, { "cache-control": "public, max-age=300" }); +} + +async function collections(env: Env): Promise { + const { results } = await env.DB.prepare( + "SELECT nsid, SUM(count) AS count, COUNT(*) AS segments FROM segment_collections GROUP BY nsid ORDER BY count DESC", + ).all<{ nsid: string; count: number; segments: number }>(); + return json({ collections: results }, 200, { "cache-control": "public, max-age=300" }); +} + +export default { + async fetch(request: Request, env: Env): Promise { + const url = new URL(request.url); + if (request.method === "POST" && url.pathname === "/ingest") return ingest(request, env); + if (request.method !== "GET") return json({ error: "method not allowed" }, 405); + switch (url.pathname) { + case "/api/progress": + return progress(env); + case "/api/heat": + return heat(url, env); + case "/api/collections": + return collections(env); + case "/": + return json({ + name: "strata", + what: "per-segment, per-collection event counts of the stream.waow.tech archive", + endpoints: ["/api/progress", "/api/heat?step=N", "/api/collections"], + source: "https://tangled.org/zat.dev/strata", + }); + default: + return json({ error: "not found" }, 404); + } + }, +} satisfies ExportedHandler; diff --git a/src/ingest.ts b/src/ingest.ts new file mode 100644 index 0000000..cd63937 --- /dev/null +++ b/src/ingest.ts @@ -0,0 +1,97 @@ +/** the record contract between the strata flow and this worker. + * + * one row per sealed stream segment, with the per-collection event counts + * decoded from the segment's collection block index. everything here is + * boundary input from an HTTP body and is parsed before it reaches D1. */ + +export interface CollectionCount { + readonly nsid: string; + readonly count: number; +} + +export interface SegmentRecord { + readonly idx: number; + readonly name: string; + readonly sizeBytes: number; + readonly checksum: string; + readonly eventCount: number; + readonly blockCount: number; + readonly minSeq: number; + readonly maxSeq: number; + readonly minWitnessedAt: number; + readonly maxWitnessedAt: number; + readonly collections: readonly CollectionCount[]; +} + +export interface IngestBatch { + readonly segments: readonly SegmentRecord[]; +} + +export class IngestParseError extends Error {} + +const MAX_SEGMENTS_PER_BATCH = 200; +const MAX_COLLECTIONS_PER_SEGMENT = 1 << 20; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function integer(source: Record, key: string, where: string): number { + const value = source[key]; + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) { + throw new IngestParseError(`${where}.${key}: expected a non-negative integer`); + } + return value; +} + +function text(source: Record, key: string, where: string, maxLength: number): string { + const value = source[key]; + if (typeof value !== "string" || value.length === 0 || value.length > maxLength) { + throw new IngestParseError(`${where}.${key}: expected a string of 1..${maxLength} chars`); + } + return value; +} + +function parseCollection(value: unknown, where: string): CollectionCount { + if (!isRecord(value)) throw new IngestParseError(`${where}: expected an object`); + return { nsid: text(value, "nsid", where, 255), count: integer(value, "count", where) }; +} + +function parseSegment(value: unknown, where: string): SegmentRecord { + if (!isRecord(value)) throw new IngestParseError(`${where}: expected an object`); + const rawCollections = value["collections"]; + if (!Array.isArray(rawCollections) || rawCollections.length > MAX_COLLECTIONS_PER_SEGMENT) { + throw new IngestParseError(`${where}.collections: expected an array`); + } + const collections = rawCollections.map((c, i) => parseCollection(c, `${where}.collections[${i}]`)); + const seen = new Set(); + for (const c of collections) { + if (seen.has(c.nsid)) throw new IngestParseError(`${where}.collections: duplicate nsid ${c.nsid}`); + seen.add(c.nsid); + } + const minSeq = integer(value, "minSeq", where); + const maxSeq = integer(value, "maxSeq", where); + if (maxSeq < minSeq) throw new IngestParseError(`${where}: maxSeq < minSeq`); + return { + idx: integer(value, "idx", where), + name: text(value, "name", where, 64), + sizeBytes: integer(value, "sizeBytes", where), + checksum: text(value, "checksum", where, 32), + eventCount: integer(value, "eventCount", where), + blockCount: integer(value, "blockCount", where), + minSeq, + maxSeq, + minWitnessedAt: integer(value, "minWitnessedAt", where), + maxWitnessedAt: integer(value, "maxWitnessedAt", where), + collections, + }; +} + +export function parseIngestBatch(value: unknown): IngestBatch { + if (!isRecord(value)) throw new IngestParseError("body: expected an object"); + const raw = value["segments"]; + if (!Array.isArray(raw) || raw.length === 0 || raw.length > MAX_SEGMENTS_PER_BATCH) { + throw new IngestParseError(`body.segments: expected 1..${MAX_SEGMENTS_PER_BATCH} segments`); + } + return { segments: raw.map((s, i) => parseSegment(s, `segments[${i}]`)) }; +} diff --git a/tests/ingest.test.ts b/tests/ingest.test.ts new file mode 100644 index 0000000..c17253d --- /dev/null +++ b/tests/ingest.test.ts @@ -0,0 +1,45 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { IngestParseError, parseIngestBatch } from "../src/ingest"; + +const segment = { + idx: 6000, + name: "seg_00000004mo.jss", + sizeBytes: 268937693, + checksum: "0a1b2c3d4e5f6071", + eventCount: 3178197, + blockCount: 818, + minSeq: 21322223343, + maxSeq: 21325402013, + minWitnessedAt: 1785768533809776, + maxWitnessedAt: 1785768618450665, + collections: [ + { nsid: "app.bsky.feed.like", count: 1885670 }, + { nsid: "app.bsky.feed.post", count: 620964 }, + ], +} as const; + +test("accepts a well-formed batch and preserves values", () => { + const batch = parseIngestBatch({ segments: [segment] }); + assert.equal(batch.segments.length, 1); + assert.deepEqual(batch.segments[0], segment); +}); + +test("rejects an empty batch", () => { + assert.throws(() => parseIngestBatch({ segments: [] }), IngestParseError); +}); + +test("rejects duplicate collections within a segment", () => { + const dup = { ...segment, collections: [...segment.collections, { nsid: "app.bsky.feed.like", count: 1 }] }; + assert.throws(() => parseIngestBatch({ segments: [dup] }), /duplicate nsid/); +}); + +test("rejects seq ranges that run backwards", () => { + assert.throws(() => parseIngestBatch({ segments: [{ ...segment, minSeq: 5, maxSeq: 4 }] }), /maxSeq < minSeq/); +}); + +test("rejects non-integer counts", () => { + const bad = { ...segment, collections: [{ nsid: "app.bsky.feed.like", count: "many" }] }; + assert.throws(() => parseIngestBatch({ segments: [bad] }), /count: expected a non-negative integer/); +}); diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..5340fd7 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "es2022", + "module": "es2022", + "moduleResolution": "bundler", + "lib": ["es2022"], + "types": ["@cloudflare/workers-types", "node"], + "strict": true, + "noUncheckedIndexedAccess": true, + "exactOptionalPropertyTypes": true, + "noEmit": true, + "skipLibCheck": true + }, + "include": ["src", "tests"] +} diff --git a/wrangler.jsonc b/wrangler.jsonc new file mode 100644 index 0000000..eb82fa1 --- /dev/null +++ b/wrangler.jsonc @@ -0,0 +1,16 @@ +{ + "$schema": "node_modules/wrangler/config-schema.json", + "name": "strata", + "main": "src/index.ts", + "compatibility_date": "2026-08-01", + "observability": { "enabled": true }, + "workers_dev": false, + "routes": [{ "pattern": "strata.zat.dev", "custom_domain": true }], + "d1_databases": [ + { + "binding": "DB", + "database_name": "strata", + "database_id": "3585ad5d-a112-4781-9e0c-aeea8ab4cb3c" + } + ] +} -- 2.51.2 From dec7d472304ac1b5a731c9feaed9098499def32f Mon Sep 17 00:00:00 2001 From: zzstoatzz Date: Sat, 29 Aug 2026 00:39:41 -0500 Subject: [PATCH 02/17] page: heat map of collections across segments at /; /api/heat integer buckets and top-N fold Canvas heat map with per-row or log color, hover tooltip, table view, light and dark palettes from the sequential blue ramp. /api/heat now casts the step to integer (the bound number was dividing as a real) and folds collections outside top=N into 'other'. JSON index moved to /api. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01B927dNwNKNYsbQdUoNJwHS --- src/index.ts | 43 +++++- src/page.html | 342 ++++++++++++++++++++++++++++++++++++++++++ src/text-modules.d.ts | 4 + wrangler.jsonc | 1 + 4 files changed, 382 insertions(+), 8 deletions(-) create mode 100644 src/page.html create mode 100644 src/text-modules.d.ts diff --git a/src/index.ts b/src/index.ts index 519937f..76935f1 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,6 +5,7 @@ * aggregates; it holds no schedule and never talks to the archive itself. */ import { IngestParseError, parseIngestBatch, type SegmentRecord } from "./ingest"; +import page from "./page.html"; interface Env { readonly DB: D1Database; @@ -101,6 +102,11 @@ interface HeatRow { readonly count: number; } +interface RankRow { + readonly nsid: string; + readonly count: number; +} + interface BucketRow { readonly bucket: number; readonly first_idx: number; @@ -112,9 +118,13 @@ interface BucketRow { readonly event_count: number; } +const MAX_TOP = 40; + async function heat(url: URL, env: Env): Promise { const step = Number(url.searchParams.get("step") ?? "10"); + const top = Number(url.searchParams.get("top") ?? "12"); if (!Number.isSafeInteger(step) || step < 1) return json({ error: "step must be a positive integer" }, 400); + if (!Number.isSafeInteger(top) || top < 1 || top > MAX_TOP) return json({ error: `top must be 1..${MAX_TOP}` }, 400); const progressRow = await env.DB.prepare("SELECT MAX(idx) AS max_idx FROM segments").first<{ max_idx: number | null }>(); const maxIdx = progressRow?.max_idx ?? -1; if (Math.floor(maxIdx / step) + 1 > MAX_HEAT_BUCKETS) { @@ -122,22 +132,37 @@ async function heat(url: URL, env: Env): Promise { } const results = await env.DB.batch([ env.DB.prepare( - `SELECT idx / ?1 AS bucket, MIN(idx) AS first_idx, MAX(idx) AS last_idx, MIN(min_seq) AS min_seq, + `SELECT idx / CAST(?1 AS INTEGER) AS bucket, MIN(idx) AS first_idx, MAX(idx) AS last_idx, MIN(min_seq) AS min_seq, MAX(max_seq) AS max_seq, MIN(min_witnessed_at) AS min_witnessed_at, MAX(max_witnessed_at) AS max_witnessed_at, SUM(event_count) AS event_count FROM segments GROUP BY bucket ORDER BY bucket`, ).bind(step), env.DB.prepare( - `SELECT idx / ?1 AS bucket, nsid, SUM(count) AS count - FROM segment_collections GROUP BY bucket, nsid ORDER BY bucket, nsid`, - ).bind(step), + `WITH ranked AS ( + SELECT nsid FROM segment_collections GROUP BY nsid ORDER BY SUM(count) DESC LIMIT CAST(?2 AS INTEGER) + ) + SELECT idx / CAST(?1 AS INTEGER) AS bucket, + CASE WHEN nsid IN (SELECT nsid FROM ranked) THEN nsid ELSE 'other' END AS nsid, + SUM(count) AS count + FROM segment_collections GROUP BY bucket, 2 ORDER BY bucket, 2`, + ).bind(step, top), + env.DB.prepare( + "SELECT nsid, SUM(count) AS count FROM segment_collections GROUP BY nsid ORDER BY count DESC LIMIT CAST(?1 AS INTEGER)", + ).bind(top), ]); - const [buckets, cells] = results; - if (buckets === undefined || cells === undefined) return json({ error: "batch returned fewer results than statements" }, 500); - // SAFETY: the two statements above select exactly the columns of BucketRow and HeatRow. + const [buckets, cells, ranked] = results; + if (buckets === undefined || cells === undefined || ranked === undefined) { + return json({ error: "batch returned fewer results than statements" }, 500); + } + // SAFETY: the statements above select exactly the columns of BucketRow, HeatRow, and RankRow. const bucketRows = buckets.results as BucketRow[]; const heatRows = cells.results as HeatRow[]; - return json({ step, buckets: bucketRows, cells: heatRows }, 200, { "cache-control": "public, max-age=300" }); + const rankRows = ranked.results as RankRow[]; + return json( + { step, top, buckets: bucketRows, cells: heatRows, collections: rankRows }, + 200, + { "cache-control": "public, max-age=300" }, + ); } async function collections(env: Env): Promise { @@ -160,6 +185,8 @@ export default { case "/api/collections": return collections(env); case "/": + return new Response(page, { headers: { "content-type": "text/html; charset=utf-8", "cache-control": "public, max-age=300" } }); + case "/api": return json({ name: "strata", what: "per-segment, per-collection event counts of the stream.waow.tech archive", diff --git a/src/page.html b/src/page.html new file mode 100644 index 0000000..a8e7d95 --- /dev/null +++ b/src/page.html @@ -0,0 +1,342 @@ + + + + + +strata + + + + + +
+

strata

+

the shape of the stream.waow.tech archive, by lexicon — read from the sealed segments, not the process

+
+ +
loading…
+ +
+ + + + +
+ +
+
+ + +
+
fewermore
+
+ x is archive position (segment index; each segment ≈ 3.2M events, ~270 MB). the dates under it are when + stream witnessed the events, not when they happened: the bootstrap era replayed years of history + inside a six-day window from 2026-07-28, so those columns are dense; segments after it are live-era, about + two minutes each. counts come from each segment's collection index; account/identity/sync markers carry no + collection and are excluded. "other" folds every collection outside the top rows. +
+
+ + + +
+ fed hourly by a prefect flow that range-reads segment footers · + api · + source +
+ + + + diff --git a/src/text-modules.d.ts b/src/text-modules.d.ts new file mode 100644 index 0000000..9b4de38 --- /dev/null +++ b/src/text-modules.d.ts @@ -0,0 +1,4 @@ +declare module "*.html" { + const text: string; + export default text; +} diff --git a/wrangler.jsonc b/wrangler.jsonc index eb82fa1..6d8d04a 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -2,6 +2,7 @@ "$schema": "node_modules/wrangler/config-schema.json", "name": "strata", "main": "src/index.ts", + "rules": [{ "type": "Text", "globs": ["**/*.html"], "fallthrough": true }], "compatibility_date": "2026-08-01", "observability": { "enabled": true }, "workers_dev": false, -- 2.51.2 From 772ce7baaefeefceb6f40e47661409213ccb2a9b Mon Sep 17 00:00:00 2001 From: zzstoatzz Date: Sat, 29 Aug 2026 00:40:12 -0500 Subject: [PATCH 03/17] page: prefer dropping whole nsid parts over trimming characters in row labels --- src/page.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/page.html b/src/page.html index a8e7d95..e7798fe 100644 --- a/src/page.html +++ b/src/page.html @@ -266,7 +266,7 @@ function fitLabel(name, maxW) { if (ctx.measureText(name).width <= maxW) return name; const parts = name.split("."); - while (parts.length > 2 && ctx.measureText("…" + parts.join(".")).width > maxW) parts.shift(); + while (parts.length > 1 && ctx.measureText("…" + parts.join(".")).width > maxW) parts.shift(); let text = "…" + parts.join("."); while (text.length > 2 && ctx.measureText(text).width > maxW) text = "…" + text.slice(2); return text; -- 2.51.2 From ae663c8da4371b533fb3a2fadbe1bbf4c953f2ea Mon Sep 17 00:00:00 2001 From: zzstoatzz Date: Sat, 29 Aug 2026 01:12:47 -0500 Subject: [PATCH 04/17] drain: segment rows to namespace-partitioned parquet in R2 via PUT /archive Worker binds strata-archive (public) and strata-archive-private; app.bsky raw rows are the only private partition. drain.sh runs on the stream box, resumable, 16-17 s/segment measured. Also: ns column + namespace/collection heat modes, /api/namespaces, richer /api/progress. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01B927dNwNKNYsbQdUoNJwHS --- docs/how-it-works.md | 19 ++++ drain/drain.sh | 57 +++++++++++ drain/segment.sql | 15 +++ migrations/0001-ns.sql | 6 ++ schema.sql | 2 + src/index.ts | 212 ++++++++++++++++++++++++++++++++--------- src/ingest.ts | 6 ++ tests/ingest.test.ts | 8 +- wrangler.jsonc | 4 + 9 files changed, 285 insertions(+), 44 deletions(-) create mode 100755 drain/drain.sh create mode 100644 drain/segment.sql create mode 100644 migrations/0001-ns.sql diff --git a/docs/how-it-works.md b/docs/how-it-works.md index b752e1f..2e80c26 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -40,3 +40,22 @@ ingest-era marker. `STRATA_INGEST_TOKEN` and the archive key `stream-archive-key-strata` live in the sops store and reach the flow as prefect Secret blocks; the worker's copy of the ingest token is a wrangler secret set from the same store value. + +## the drain (decided 2026-08-29) + +the long tail needs rows, not footer counts. `drain/drain.sh` runs on the +stream box (`/opt/strata`, with the duckdb CLI and the jetstream SDK's +`segment-export` example): for every sealed segment it decodes rows from +local disk (`seq, time_us, did, kind, op, collection, rkey` — witnessed +time, no record bodies), writes parquet partitioned by namespace with +duckdb, and PUTs each partition to this worker's `/archive/` with the +ingest bearer. keys: `events/ns=/seg=.parquet`, +`agg/seg=.parquet` (per collection/kind/op: events, distinct dids, seq +and time bounds), `manifest/seg=.json`. resumable via `done.txt`; +the active segment is never read. measured: 16–17 s per segment at nice 19. + +placement policy: `events/ns=app.bsky/*` goes to the private bucket +(`strata-archive-private`); every other partition, all aggregates and +manifests are public at `pub-735e1688181b45e49c925eed81fe2da7.r2.dev` +(bucket `strata-archive`, CORS open for range reads). the tail is what the +page queries live with duckdb-wasm; `app.bsky` is served from aggregates. diff --git a/drain/drain.sh b/drain/drain.sh new file mode 100755 index 0000000..23ef9d6 --- /dev/null +++ b/drain/drain.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# drain: every sealed segment on this box → parquet partitioned by namespace → strata via the Worker. +# +# state is done.txt (one segment index per line). resumable; idempotent per segment (PUTs overwrite). +# the active (highest-index) segment is never touched. everything runs at nice 19 / idle IO. +# +# env: STRATA_URL (default https://strata.zat.dev), STRATA_INGEST_TOKEN (required) +set -euo pipefail +cd "$(dirname "$0")" +: "${STRATA_INGEST_TOKEN:?set STRATA_INGEST_TOKEN}" +STRATA_URL="${STRATA_URL:-https://strata.zat.dev}" +SEGMENTS=/data/stream/segments +WORK=./work +LIMIT="${1:-0}" # max segments this run; 0 = all +touch done.txt + +idx_of() { python3 -c 'import sys;print(int(sys.argv[1][4:14],36))' "$(basename "$1" .jss)"; } + +put() { # key file + local code + code=$(curl -sS -m 600 -o /tmp/drain.put.json -w "%{http_code}" -X PUT "$STRATA_URL/archive/$1" \ + -H "Authorization: Bearer $STRATA_INGEST_TOKEN" -H "Content-Type: application/octet-stream" \ + -A "strata-drain/1" --data-binary "@$2") + [ "$code" = "200" ] || { echo "PUT $1 -> $code $(cat /tmp/drain.put.json)" >&2; return 1; } +} + +mapfile -t all < <(ls "$SEGMENTS"/seg_*.jss | sort) +unset 'all[-1]' # the active segment +n=0 +for seg in "${all[@]}"; do + idx=$(idx_of "$seg") + grep -qx "$idx" done.txt && continue + [ "$LIMIT" -gt 0 ] && [ "$n" -ge "$LIMIT" ] && break + rm -rf "$WORK"; mkdir -p "$WORK" + t0=$(date +%s) + nice -n 19 ionice -c 3 ./segment-export "$seg" > "$WORK/rows.tsv" 2> "$WORK/export.log" + nice -n 19 ionice -c 3 ./duckdb < <(sed "s#__WORK__#$WORK#g; s#__IDX__#$idx#g" segment.sql) > "$WORK/duckdb.log" 2>&1 + rm -f "$WORK/rows.tsv" + for part in "$WORK"/events/ns=*/; do + ns=$(basename "$part"); ns=${ns#ns=} + put "events/ns=$ns/seg=$idx.parquet" "$part/data_0.parquet" + done + put "agg/seg=$idx.parquet" "$WORK/agg.parquet" + python3 - "$idx" "$seg" "$WORK" "$t0" > "$WORK/manifest.json" <<'EOF' +import json, os, sys, time, glob +idx, seg, work, t0 = int(sys.argv[1]), sys.argv[2], sys.argv[3], int(sys.argv[4]) +parts = {os.path.basename(p)[3:]: os.path.getsize(os.path.join(p, "data_0.parquet")) for p in glob.glob(f"{work}/events/ns=*")} +print(json.dumps({"segment": idx, "name": os.path.basename(seg), "segment_bytes": os.path.getsize(seg), + "rows": int(next(l for l in open(f"{work}/export.log") if " rows from " in l).split()[0]), "partitions": parts, + "agg_bytes": os.path.getsize(f"{work}/agg.parquet"), "seconds": int(time.time()) - t0, "drained_at": int(time.time())})) +EOF + put "manifest/seg=$idx.json" "$WORK/manifest.json" + echo "$idx" >> done.txt + n=$((n + 1)) + echo "$(date -u +%FT%TZ) seg $idx done in $(( $(date +%s) - t0 ))s: $(python3 -c 'import json,sys;m=json.load(open(sys.argv[1]));print(m["rows"],"rows,",len(m["partitions"]),"partitions,",sum(m["partitions"].values())//1024,"KB events,",m["agg_bytes"]//1024,"KB agg")' "$WORK/manifest.json")" +done +echo "drained $n segment(s); $(wc -l < done.txt) total done; $(( ${#all[@]} - $(wc -l < done.txt) )) remaining" diff --git a/drain/segment.sql b/drain/segment.sql new file mode 100644 index 0000000..9887ce2 --- /dev/null +++ b/drain/segment.sql @@ -0,0 +1,15 @@ +CREATE TABLE ev AS + SELECT *, CASE WHEN kind = 'commit' THEN regexp_extract(collection, '^([^.]+\.[^.]+)', 1) ELSE '_' || kind END AS ns + FROM read_csv('__WORK__/rows.tsv', delim = '\t', header = false, + columns = {seq: 'BIGINT', time_us: 'BIGINT', did: 'VARCHAR', kind: 'VARCHAR', op: 'VARCHAR', collection: 'VARCHAR', rkey: 'VARCHAR'}, + quote = '', escape = '', null_padding = true); + +COPY (SELECT seq, time_us, did, kind, op, collection, rkey, ns FROM ev ORDER BY seq) + TO '__WORK__/events' (FORMAT parquet, PARTITION_BY (ns), COMPRESSION zstd); + +COPY ( + SELECT __IDX__ AS segment, ns, collection, kind, op, + count(*) AS events, count(DISTINCT did) AS dids, + min(seq) AS min_seq, max(seq) AS max_seq, min(time_us) AS min_time_us, max(time_us) AS max_time_us + FROM ev GROUP BY ALL ORDER BY ns, collection, kind, op +) TO '__WORK__/agg.parquet' (FORMAT parquet, COMPRESSION zstd); diff --git a/migrations/0001-ns.sql b/migrations/0001-ns.sql new file mode 100644 index 0000000..c099f70 --- /dev/null +++ b/migrations/0001-ns.sql @@ -0,0 +1,6 @@ +ALTER TABLE segment_collections ADD COLUMN ns TEXT; +UPDATE segment_collections SET ns = CASE + WHEN instr(substr(nsid, instr(nsid, '.') + 1), '.') = 0 THEN nsid + ELSE substr(nsid, 1, instr(nsid, '.') + instr(substr(nsid, instr(nsid, '.') + 1), '.') - 1) +END WHERE ns IS NULL; +CREATE INDEX IF NOT EXISTS segment_collections_ns ON segment_collections (ns, idx); diff --git a/schema.sql b/schema.sql index 49a11f4..43bc147 100644 --- a/schema.sql +++ b/schema.sql @@ -15,8 +15,10 @@ CREATE TABLE IF NOT EXISTS segments ( CREATE TABLE IF NOT EXISTS segment_collections ( idx INTEGER NOT NULL REFERENCES segments(idx), nsid TEXT NOT NULL, + ns TEXT NOT NULL, count INTEGER NOT NULL, PRIMARY KEY (idx, nsid) ); CREATE INDEX IF NOT EXISTS segment_collections_nsid ON segment_collections (nsid, idx); +CREATE INDEX IF NOT EXISTS segment_collections_ns ON segment_collections (ns, idx); diff --git a/src/index.ts b/src/index.ts index 76935f1..9902e0a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,14 +4,38 @@ * collection index, and POSTs the result here. this worker only stores and * aggregates; it holds no schedule and never talks to the archive itself. */ -import { IngestParseError, parseIngestBatch, type SegmentRecord } from "./ingest"; +import { IngestParseError, namespaceOf, parseIngestBatch, type SegmentRecord } from "./ingest"; import page from "./page.html"; interface Env { readonly DB: D1Database; + readonly PUBLIC: R2Bucket; + readonly PRIVATE: R2Bucket; readonly STRATA_INGEST_TOKEN: string; } +const MAX_ARCHIVE_OBJECT_BYTES = 200 << 20; +const ARCHIVE_KEY = /^(events\/ns=[a-z0-9._-]+\/seg=\d+\.parquet|agg\/seg=\d+\.parquet|manifest\/seg=\d+\.json)$/; + +/** the drain uploads one object per (segment, partition). app.bsky raw rows are the only private thing: + * everything else — tail namespaces, marker rows, per-segment aggregates — is public by decision (2026-08-29). */ +function bucketFor(key: string, env: Env): R2Bucket { + return key.startsWith("events/ns=app.bsky/") ? env.PRIVATE : env.PUBLIC; +} + +async function putArchive(request: Request, key: string, env: Env): Promise { + if (!authorized(request, env)) return json({ error: "unauthorized" }, 401); + if (!ARCHIVE_KEY.test(key)) return json({ error: "key must be events/ns=/seg=.parquet, agg/seg=.parquet, or manifest/seg=.json" }, 400); + const length = Number(request.headers.get("content-length") ?? "0"); + if (!Number.isFinite(length) || length <= 0 || length > MAX_ARCHIVE_OBJECT_BYTES) return json({ error: "content-length required, ≤ 200 MiB" }, 413); + if (request.body === null) return json({ error: "empty body" }, 400); + const bucket = bucketFor(key, env); + const object = await bucket.put(key, request.body, { + httpMetadata: { contentType: key.endsWith(".json") ? "application/json" : "application/vnd.apache.parquet" }, + }); + return json({ key, size: object.size, etag: object.httpEtag, bucket: bucket === env.PRIVATE ? "private" : "public" }); +} + const MAX_BODY_BYTES = 8 << 20; const MAX_HEAT_BUCKETS = 2000; @@ -61,8 +85,12 @@ function segmentStatements(db: D1Database, segment: SegmentRecord, ingestedAt: n ingestedAt, ); const clear = db.prepare("DELETE FROM segment_collections WHERE idx = ?1").bind(segment.idx); - const upsertCollection = db.prepare("INSERT INTO segment_collections (idx, nsid, count) VALUES (?1, ?2, ?3)"); - return [upsertSegment, clear, ...segment.collections.map((c) => upsertCollection.bind(segment.idx, c.nsid, c.count))]; + const upsertCollection = db.prepare("INSERT INTO segment_collections (idx, nsid, ns, count) VALUES (?1, ?2, ?3, ?4)"); + return [ + upsertSegment, + clear, + ...segment.collections.map((c) => upsertCollection.bind(segment.idx, c.nsid, namespaceOf(c.nsid), c.count)), + ]; } async function ingest(request: Request, env: Env): Promise { @@ -87,23 +115,34 @@ interface ProgressRow { readonly segments: number; readonly max_idx: number | null; readonly max_seq: number | null; + readonly events: number | null; + readonly updated_at: number | null; } async function progress(env: Env): Promise { - const row = await env.DB.prepare( - "SELECT COUNT(*) AS segments, MAX(idx) AS max_idx, MAX(max_seq) AS max_seq FROM segments", - ).first(); - return json({ segments: row?.segments ?? 0, maxIdx: row?.max_idx ?? null, maxSeq: row?.max_seq ?? null }); + const [row, kinds] = await env.DB.batch([ + env.DB.prepare( + "SELECT COUNT(*) AS segments, MAX(idx) AS max_idx, MAX(max_seq) AS max_seq, SUM(event_count) AS events, MAX(ingested_at) AS updated_at FROM segments", + ), + env.DB.prepare("SELECT COUNT(DISTINCT nsid) AS collections, COUNT(DISTINCT ns) AS namespaces FROM segment_collections"), + ]); + // SAFETY: the two statements select exactly these columns. + const p = row?.results[0] as ProgressRow | undefined; + const k = kinds?.results[0] as { collections: number; namespaces: number } | undefined; + return json({ + segments: p?.segments ?? 0, + maxIdx: p?.max_idx ?? null, + maxSeq: p?.max_seq ?? null, + events: p?.events ?? 0, + updatedAt: p?.updated_at ?? null, + collections: k?.collections ?? 0, + namespaces: k?.namespaces ?? 0, + }); } interface HeatRow { readonly bucket: number; - readonly nsid: string; - readonly count: number; -} - -interface RankRow { - readonly nsid: string; + readonly key: string; readonly count: number; } @@ -118,17 +157,101 @@ interface BucketRow { readonly event_count: number; } -const MAX_TOP = 40; +const MAX_TOP = 60; -async function heat(url: URL, env: Env): Promise { +interface HeatQuery { + readonly step: number; + readonly top: number; + readonly scope: { readonly kind: "namespaces" } | { readonly kind: "namespace"; readonly ns: string } | { readonly kind: "collections" }; +} + +function parseHeatQuery(url: URL): HeatQuery | string { const step = Number(url.searchParams.get("step") ?? "10"); const top = Number(url.searchParams.get("top") ?? "12"); - if (!Number.isSafeInteger(step) || step < 1) return json({ error: "step must be a positive integer" }, 400); - if (!Number.isSafeInteger(top) || top < 1 || top > MAX_TOP) return json({ error: `top must be 1..${MAX_TOP}` }, 400); + if (!Number.isSafeInteger(step) || step < 1) return "step must be a positive integer"; + if (!Number.isSafeInteger(top) || top < 1 || top > MAX_TOP) return `top must be 1..${MAX_TOP}`; + const ns = url.searchParams.get("ns"); + const group = url.searchParams.get("group") ?? "namespaces"; + if (ns !== null) { + if (!/^[a-z0-9-]+\.[a-z0-9-]+$/i.test(ns)) return "ns must be an authority like fm.teal"; + return { step, top, scope: { kind: "namespace", ns } }; + } + if (group === "collections") return { step, top, scope: { kind: "collections" } }; + if (group === "namespaces") return { step, top, scope: { kind: "namespaces" } }; + return "group must be namespaces or collections"; +} + +/** rows of the heat map: (bucket, key, count) where key is a namespace, or an nsid inside one namespace, or a + * top-N nsid; everything outside the top N folds into "other" so the response stays small at any archive size. */ +function heatRowsStatement(db: D1Database, q: HeatQuery): D1PreparedStatement { + switch (q.scope.kind) { + case "namespaces": + return db + .prepare( + `WITH ranked AS (SELECT ns FROM segment_collections GROUP BY ns ORDER BY SUM(count) DESC LIMIT CAST(?2 AS INTEGER)) + SELECT idx / CAST(?1 AS INTEGER) AS bucket, + CASE WHEN ns IN (SELECT ns FROM ranked) THEN ns ELSE 'other' END AS key, SUM(count) AS count + FROM segment_collections GROUP BY bucket, 2 ORDER BY bucket, 2`, + ) + .bind(q.step, q.top); + case "namespace": + return db + .prepare( + `SELECT idx / CAST(?1 AS INTEGER) AS bucket, nsid AS key, SUM(count) AS count + FROM segment_collections WHERE ns = ?2 GROUP BY bucket, nsid ORDER BY bucket, nsid`, + ) + .bind(q.step, q.scope.ns); + case "collections": + return db + .prepare( + `WITH ranked AS (SELECT nsid FROM segment_collections GROUP BY nsid ORDER BY SUM(count) DESC LIMIT CAST(?2 AS INTEGER)) + SELECT idx / CAST(?1 AS INTEGER) AS bucket, + CASE WHEN nsid IN (SELECT nsid FROM ranked) THEN nsid ELSE 'other' END AS key, SUM(count) AS count + FROM segment_collections GROUP BY bucket, 2 ORDER BY bucket, 2`, + ) + .bind(q.step, q.top); + } +} + +function heatKeysStatement(db: D1Database, q: HeatQuery): D1PreparedStatement { + switch (q.scope.kind) { + case "namespaces": + return db + .prepare( + `SELECT ns AS key, SUM(count) AS count, COUNT(DISTINCT nsid) AS members + FROM segment_collections GROUP BY ns ORDER BY count DESC LIMIT CAST(?1 AS INTEGER)`, + ) + .bind(q.top); + case "namespace": + return db + .prepare( + `SELECT nsid AS key, SUM(count) AS count, 1 AS members FROM segment_collections WHERE ns = ?1 + GROUP BY nsid ORDER BY count DESC`, + ) + .bind(q.scope.ns); + case "collections": + return db + .prepare( + `SELECT nsid AS key, SUM(count) AS count, 1 AS members FROM segment_collections + GROUP BY nsid ORDER BY count DESC LIMIT CAST(?1 AS INTEGER)`, + ) + .bind(q.top); + } +} + +interface KeyRow { + readonly key: string; + readonly count: number; + readonly members: number; +} + +async function heat(url: URL, env: Env): Promise { + const q = parseHeatQuery(url); + if (typeof q === "string") return json({ error: q }, 400); const progressRow = await env.DB.prepare("SELECT MAX(idx) AS max_idx FROM segments").first<{ max_idx: number | null }>(); const maxIdx = progressRow?.max_idx ?? -1; - if (Math.floor(maxIdx / step) + 1 > MAX_HEAT_BUCKETS) { - return json({ error: `step ${step} yields more than ${MAX_HEAT_BUCKETS} buckets` }, 400); + if (Math.floor(maxIdx / q.step) + 1 > MAX_HEAT_BUCKETS) { + return json({ error: `step ${q.step} yields more than ${MAX_HEAT_BUCKETS} buckets` }, 400); } const results = await env.DB.batch([ env.DB.prepare( @@ -136,46 +259,47 @@ async function heat(url: URL, env: Env): Promise { MAX(max_seq) AS max_seq, MIN(min_witnessed_at) AS min_witnessed_at, MAX(max_witnessed_at) AS max_witnessed_at, SUM(event_count) AS event_count FROM segments GROUP BY bucket ORDER BY bucket`, - ).bind(step), - env.DB.prepare( - `WITH ranked AS ( - SELECT nsid FROM segment_collections GROUP BY nsid ORDER BY SUM(count) DESC LIMIT CAST(?2 AS INTEGER) - ) - SELECT idx / CAST(?1 AS INTEGER) AS bucket, - CASE WHEN nsid IN (SELECT nsid FROM ranked) THEN nsid ELSE 'other' END AS nsid, - SUM(count) AS count - FROM segment_collections GROUP BY bucket, 2 ORDER BY bucket, 2`, - ).bind(step, top), - env.DB.prepare( - "SELECT nsid, SUM(count) AS count FROM segment_collections GROUP BY nsid ORDER BY count DESC LIMIT CAST(?1 AS INTEGER)", - ).bind(top), + ).bind(q.step), + heatRowsStatement(env.DB, q), + heatKeysStatement(env.DB, q), ]); - const [buckets, cells, ranked] = results; - if (buckets === undefined || cells === undefined || ranked === undefined) { + const [buckets, cells, keys] = results; + if (buckets === undefined || cells === undefined || keys === undefined) { return json({ error: "batch returned fewer results than statements" }, 500); } - // SAFETY: the statements above select exactly the columns of BucketRow, HeatRow, and RankRow. + // SAFETY: the statements above select exactly the columns of BucketRow, HeatRow, and KeyRow. const bucketRows = buckets.results as BucketRow[]; const heatRows = cells.results as HeatRow[]; - const rankRows = ranked.results as RankRow[]; + const keyRows = keys.results as KeyRow[]; return json( - { step, top, buckets: bucketRows, cells: heatRows, collections: rankRows }, + { step: q.step, top: q.top, scope: q.scope, buckets: bucketRows, cells: heatRows, keys: keyRows }, 200, { "cache-control": "public, max-age=300" }, ); } -async function collections(env: Env): Promise { - const { results } = await env.DB.prepare( - "SELECT nsid, SUM(count) AS count, COUNT(*) AS segments FROM segment_collections GROUP BY nsid ORDER BY count DESC", - ).all<{ nsid: string; count: number; segments: number }>(); +async function collections(url: URL, env: Env): Promise { + const ns = url.searchParams.get("ns"); + if (ns !== null && !/^[a-z0-9-]+\.[a-z0-9-]+$/i.test(ns)) return json({ error: "ns must be an authority like fm.teal" }, 400); + const stmt = ns === null + ? env.DB.prepare("SELECT nsid, ns, SUM(count) AS count, COUNT(*) AS segments FROM segment_collections GROUP BY nsid ORDER BY count DESC") + : env.DB.prepare("SELECT nsid, ns, SUM(count) AS count, COUNT(*) AS segments FROM segment_collections WHERE ns = ?1 GROUP BY nsid ORDER BY count DESC").bind(ns); + const { results } = await stmt.all<{ nsid: string; ns: string; count: number; segments: number }>(); return json({ collections: results }, 200, { "cache-control": "public, max-age=300" }); } +async function namespaces(env: Env): Promise { + const { results } = await env.DB.prepare( + "SELECT ns, SUM(count) AS count, COUNT(DISTINCT nsid) AS members FROM segment_collections GROUP BY ns ORDER BY count DESC", + ).all<{ ns: string; count: number; members: number }>(); + return json({ namespaces: results }, 200, { "cache-control": "public, max-age=300" }); +} + export default { async fetch(request: Request, env: Env): Promise { const url = new URL(request.url); if (request.method === "POST" && url.pathname === "/ingest") return ingest(request, env); + if (request.method === "PUT" && url.pathname.startsWith("/archive/")) return putArchive(request, url.pathname.slice("/archive/".length), env); if (request.method !== "GET") return json({ error: "method not allowed" }, 405); switch (url.pathname) { case "/api/progress": @@ -183,14 +307,16 @@ export default { case "/api/heat": return heat(url, env); case "/api/collections": - return collections(env); + return collections(url, env); + case "/api/namespaces": + return namespaces(env); case "/": return new Response(page, { headers: { "content-type": "text/html; charset=utf-8", "cache-control": "public, max-age=300" } }); case "/api": return json({ name: "strata", what: "per-segment, per-collection event counts of the stream.waow.tech archive", - endpoints: ["/api/progress", "/api/heat?step=N", "/api/collections"], + endpoints: ["/api/progress", "/api/heat?step=N&top=N&group=namespaces|collections", "/api/heat?step=N&ns=", "/api/namespaces", "/api/collections?ns="], source: "https://tangled.org/zat.dev/strata", }); default: diff --git a/src/ingest.ts b/src/ingest.ts index cd63937..43ae8e5 100644 --- a/src/ingest.ts +++ b/src/ingest.ts @@ -9,6 +9,12 @@ export interface CollectionCount { readonly count: number; } +/** the authority prefix of an nsid: its first two labels (`app.bsky.feed.like` → `app.bsky`). */ +export function namespaceOf(nsid: string): string { + const parts = nsid.split("."); + return parts.length <= 2 ? nsid : `${parts[0]}.${parts[1]}`; +} + export interface SegmentRecord { readonly idx: number; readonly name: string; diff --git a/tests/ingest.test.ts b/tests/ingest.test.ts index c17253d..6090fdb 100644 --- a/tests/ingest.test.ts +++ b/tests/ingest.test.ts @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { IngestParseError, parseIngestBatch } from "../src/ingest"; +import { IngestParseError, namespaceOf, parseIngestBatch } from "../src/ingest"; const segment = { idx: 6000, @@ -43,3 +43,9 @@ test("rejects non-integer counts", () => { const bad = { ...segment, collections: [{ nsid: "app.bsky.feed.like", count: "many" }] }; assert.throws(() => parseIngestBatch({ segments: [bad] }), /count: expected a non-negative integer/); }); + +test("namespaceOf takes the first two labels", () => { + assert.equal(namespaceOf("app.bsky.feed.like"), "app.bsky"); + assert.equal(namespaceOf("net.anisota.beta.game.log"), "net.anisota"); + assert.equal(namespaceOf("sh.tangled"), "sh.tangled"); +}); diff --git a/wrangler.jsonc b/wrangler.jsonc index 6d8d04a..2ed24c5 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -7,6 +7,10 @@ "observability": { "enabled": true }, "workers_dev": false, "routes": [{ "pattern": "strata.zat.dev", "custom_domain": true }], + "r2_buckets": [ + { "binding": "PUBLIC", "bucket_name": "strata-archive" }, + { "binding": "PRIVATE", "bucket_name": "strata-archive-private" } + ], "d1_databases": [ { "binding": "DB", -- 2.51.2 From 33bb33b7cf2556aff448997345fbea8a03420324 Mon Sep 17 00:00:00 2001 From: zzstoatzz Date: Sat, 29 Aug 2026 01:17:52 -0500 Subject: [PATCH 05/17] =?UTF-8?q?page:=20strata=20redesign=20=E2=80=94=20n?= =?UTF-8?q?amespace=20layers,=20reader=20line,=20ask-the-tail=20duckdb-was?= =?UTF-8?q?m=20panel?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rows are namespaces whose height follows log(events); click splits one into its collections (fetched on demand); find pins any authority; more layers grows the top-N. The tail panel lists a namespace's public parquet partitions via /api/partitions and runs SQL over them in the browser. Era bands keyed on the bootstrap end date, not witnessed spans. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01B927dNwNKNYsbQdUoNJwHS --- src/index.ts | 18 +- src/page.html | 602 +++++++++++++++++++++++++++----------------------- 2 files changed, 347 insertions(+), 273 deletions(-) diff --git a/src/index.ts b/src/index.ts index 9902e0a..339c4f9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -288,6 +288,20 @@ async function collections(url: URL, env: Env): Promise { return json({ collections: results }, 200, { "cache-control": "public, max-age=300" }); } +const PUBLIC_BASE = "https://pub-735e1688181b45e49c925eed81fe2da7.r2.dev"; + +/** which drained segments hold rows for a namespace, as public parquet URLs the page can hand to duckdb-wasm. + * app.bsky is private by policy, so it is refused here. */ +async function partitions(url: URL, env: Env): Promise { + const ns = url.searchParams.get("ns"); + if (ns === null || !/^[a-z0-9-]+\.[a-z0-9-]+$/i.test(ns)) return json({ error: "ns must be an authority like fm.teal" }, 400); + if (ns === "app.bsky") return json({ error: "app.bsky rows are not public; use the aggregates" }, 403); + const listed = await env.PUBLIC.list({ prefix: `events/ns=${ns}/`, limit: 1000 }); + const urls = listed.objects.map((o) => `${PUBLIC_BASE}/${o.key}`); + const bytes = listed.objects.reduce((a, o) => a + o.size, 0); + return json({ ns, files: urls, bytes, truncated: listed.truncated }, 200, { "cache-control": "public, max-age=120" }); +} + async function namespaces(env: Env): Promise { const { results } = await env.DB.prepare( "SELECT ns, SUM(count) AS count, COUNT(DISTINCT nsid) AS members FROM segment_collections GROUP BY ns ORDER BY count DESC", @@ -310,13 +324,15 @@ export default { return collections(url, env); case "/api/namespaces": return namespaces(env); + case "/api/partitions": + return partitions(url, env); case "/": return new Response(page, { headers: { "content-type": "text/html; charset=utf-8", "cache-control": "public, max-age=300" } }); case "/api": return json({ name: "strata", what: "per-segment, per-collection event counts of the stream.waow.tech archive", - endpoints: ["/api/progress", "/api/heat?step=N&top=N&group=namespaces|collections", "/api/heat?step=N&ns=", "/api/namespaces", "/api/collections?ns="], + endpoints: ["/api/progress", "/api/heat?step=N&top=N&group=namespaces|collections", "/api/heat?step=N&ns=", "/api/namespaces", "/api/collections?ns=", "/api/partitions?ns="], source: "https://tangled.org/zat.dev/strata", }); default: diff --git a/src/page.html b/src/page.html index e7798fe..67015a0 100644 --- a/src/page.html +++ b/src/page.html @@ -4,263 +4,285 @@ strata - - + + + + + +
-

strata

-

the shape of the stream.waow.tech archive, by lexicon — read from the sealed segments, not the process

+ ~  s t r a t a  ~ + · reading the archive +

the stream archive, cut across. every lexicon on the network is a layer: thickness is how much of it there is, shade is how it comes and goes.

+
+
+
—segments
+
—events
+
—namespaces
+
—collections
+
-
loading…
+
+ + + + + +
-
- - - - -
+
+ +
hover a layer to read it · click a namespace to split it into its collections
+
-
-
- - +
+ nothing in that column + fewermore +
-
fewermore
-
- x is archive position (segment index; each segment ≈ 3.2M events, ~270 MB). the dates under it are when - stream witnessed the events, not when they happened: the bootstrap era replayed years of history - inside a six-day window from 2026-07-28, so those columns are dense; segments after it are live-era, about - two minutes each. counts come from each segment's collection index; account/identity/sync markers carry no - collection and are excluded. "other" folds every collection outside the top rows. -
-
- +

columns are archive position: sealed segments in order, ≈3.2M events each. the dates are when stream witnessed the events, not when they happened — the bootstrap band replayed years of history inside six days from 2026-07-28, so it is dense; the live band is about two minutes per segment. account, identity and sync markers carry no collection and are not counted. "other" is every namespace below the ones shown.

+ + +
ask the tail
+
+
+ + pick one + +
+
+ +
every row this namespace ever wrote — rows(seq, time_us, did, kind, op, collection, rkey) — queried in your browser with duckdb, straight from the public archive. app.bsky is aggregates-only by policy.
+
+
+
+
-- 2.51.2 From 3e2afdf554df1b232898cf1db4889a21698722a6 Mon Sep 17 00:00:00 2001 From: zzstoatzz Date: Sat, 29 Aug 2026 01:26:41 -0500 Subject: [PATCH 06/17] page: plain-language copy; relative urls so it mounts at stream.waow.tech/strata/ apps / record types / archive chunks instead of namespaces / collections / segments; reader line in sentences; era bands named 'history, replayed' and 'as it happened'; the SQL box hides behind 'edit the query'. Brand links back to stream. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01B927dNwNKNYsbQdUoNJwHS --- src/page.html | 181 ++++++++++++++++++++++++++------------------------ 1 file changed, 96 insertions(+), 85 deletions(-) diff --git a/src/page.html b/src/page.html index 67015a0..88fb292 100644 --- a/src/page.html +++ b/src/page.html @@ -3,11 +3,11 @@ -strata - +strata · stream + - +
- ~  s t r a t a  ~ + ~  s t r e a m  ~ · reading the archive -

the stream archive, cut across. every lexicon on the network is a layer: thickness is how much of it there is, shade is how it comes and goes.

+

strata — what's in the archive. every app on the network writes its own kinds of records; this is how much of each there is, and when it arrived.

-
—segments
-
—events
-
—namespaces
-
—collections
+
—records
+
—apps
+
—record types
+
—archive chunks
+

each row is an app, named by the identifier its records carry (app.bsky is Bluesky, sh.tangled is Tangled). a taller row has written more records overall. a brighter cell means that app was busier in that slice of the archive. ▸ opens an app to show its record types.

+
- - + - - - + + +
- -
hover a layer to read it · click a namespace to split it into its collections
+ +
move over the picture to read a cell · click an app's name to open it
- nothing in that column - fewermore + nothing from that app in this slice + quieterbusier
-

columns are archive position: sealed segments in order, ≈3.2M events each. the dates are when stream witnessed the events, not when they happened — the bootstrap band replayed years of history inside six days from 2026-07-28, so it is dense; the live band is about two minutes per segment. account, identity and sync markers carry no collection and are not counted. "other" is every namespace below the ones shown.

+

the archive is stored in chunks of about 3 million records; the columns are those chunks in order, oldest on the left. the dates are when the archive received the records, not when people wrote them: in its first six days (from 2026-07-28) it replayed the network's whole history, so those columns are packed — history, replayed. after that each chunk is a few minutes of live traffic — as it happened. account and identity changes belong to no app and aren't counted. everything else bundles the apps not shown.

-
ask the tail
+
look inside an app
+

pick an app and every record it has ever written is searched right here, in your browser, from the public archive. Bluesky itself is too big for that and is shown from summaries only.

- + pick one - + +
- -
every row this namespace ever wrote — rows(seq, time_us, did, kind, op, collection, rkey) — queried in your browser with duckdb, straight from the public archive. app.bsky is aggregates-only by policy.
+ +
@@ -140,9 +148,11 @@ footer a:hover{color:var(--hot);border-bottom-color:var(--water)} const css = (n) => getComputedStyle(document.documentElement).getPropertyValue(n).trim(); const STEPS = [1, 2, 5, 10, 25, 50, 100]; const FLOOR = 0.14; + const OTHER = "other", OTHER_LABEL = "everything else"; const esc = (s) => String(s).replace(/[&<>"]/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """ }[c])); + const name = (key) => key === OTHER ? OTHER_LABEL : key; - // one hue, dark → light, with a floor so a seam with anything in it never fades into the well + // one hue, dark → light, with a floor so a cell with anything in it never fades into the empty tone const shade = (t) => { const u = FLOOR + (1 - FLOOR) * Math.max(0, Math.min(1, t)); return `oklch(${(0.27 + 0.66 * u).toFixed(3)} ${(0.045 + 0.08 * u * (1 - u)).toFixed(3)} 222)`; @@ -151,28 +161,27 @@ footer a:hover{color:var(--hot);border-bottom-color:var(--water)} const view = { top: 12, expanded: new Map(), pinned: [], namespaces: [] }; let data = null, layout = null, hover = null, progress = null, stepChosen = false; const stepSelect = $("step"); - for (const s of STEPS) { const o = document.createElement("option"); o.value = s; o.textContent = s === 1 ? "1 segment each" : `${s} segments each`; stepSelect.append(o); } + for (const s of STEPS) { const o = document.createElement("option"); o.value = s; o.textContent = s === 1 ? "1 chunk per column" : `${s} chunks per column`; stepSelect.append(o); } async function api(path) { const r = await fetch(path); return r.json(); } async function loadProgress() { - progress = await api("/api/progress"); + progress = await api("api/progress"); $("s-segments").textContent = fmt.format(progress.segments); $("s-events").textContent = compact.format(progress.events); $("s-namespaces").textContent = fmt.format(progress.namespaces); $("s-collections").textContent = fmt.format(progress.collections); if (progress.updatedAt) { const d = new Date(progress.updatedAt); - $("live").innerHTML = `· ${fmt.format(progress.segments)} segments · through seq ${fmt.format(progress.maxSeq)}`; + $("live").innerHTML = `· ${fmt.format(progress.segments)} chunks read · ${compact.format(progress.events)} records`; $("updated").textContent = `updated ${dayFmt.format(d)} ${timeFmt.format(d)} utc`; } if (!stepChosen) stepSelect.value = String(STEPS.find((s) => Math.ceil(progress.segments / s) <= 160) ?? STEPS[STEPS.length - 1]); if (!view.namespaces.length) { - const { namespaces } = await api("/api/namespaces"); + const { namespaces } = await api("api/namespaces"); view.namespaces = namespaces; - $("ns-list").innerHTML = namespaces.map((n) => ``).join(""); - const askNs = $("ask-ns"); - askNs.innerHTML = `` + namespaces.filter((n) => n.ns !== "app.bsky").map((n) => ``).join(""); + $("ns-list").innerHTML = namespaces.map((n) => ``).join(""); + $("ask-ns").innerHTML = `` + namespaces.filter((n) => n.ns !== "app.bsky").map((n) => ``).join(""); } } @@ -182,21 +191,21 @@ footer a:hover{color:var(--hot);border-bottom-color:var(--water)} async function load() { await loadProgress(); const step = Number(stepSelect.value); - const heat = await api(`/api/heat?step=${step}&top=${view.top}`); + const heat = await api(`api/heat?step=${step}&top=${view.top}`); if (heat.error) { reader.innerHTML = `${esc(heat.error)}`; return; } const buckets = heat.buckets, bucketIndex = new Map(buckets.map((b, i) => [b.bucket, i])); const keys = new Map(heat.keys.map((k) => [k.key, k])); const order = heat.keys.map((k) => k.key); const extra = view.pinned.filter((p) => !keys.has(p.ns)); - const extraHeat = await Promise.all(extra.map((p) => api(`/api/heat?step=${step}&ns=${p.ns}`))); + const extraHeat = await Promise.all(extra.map((p) => api(`api/heat?step=${step}&ns=${p.ns}`))); extra.forEach((p, i) => { keys.set(p.ns, { key: p.ns, count: p.count, members: p.members, cells: extraHeat[i].cells }); order.push(p.ns); }); - if (heat.cells.some((c) => c.key === "other")) { order.push("other"); keys.set("other", { key: "other", count: 0, members: 0 }); } + if (heat.cells.some((c) => c.key === OTHER)) { order.push(OTHER); keys.set(OTHER, { key: OTHER, count: 0, members: 0 }); } const rows = []; for (const key of order) { const k = keys.get(key); const values = k.cells ? (() => { const a = new Float64Array(buckets.length); for (const c of k.cells) { const i = bucketIndex.get(c.bucket); if (i !== undefined) a[i] += c.count; } return a; })() : series(heat.cells, key, bucketIndex, buckets.length); - const total = key === "other" ? values.reduce((a, b) => a + b, 0) : k.count; - rows.push({ key, depth: 0, total, members: k.members, values, expandable: key !== "other" && k.members > 1 }); + const total = key === OTHER ? values.reduce((a, b) => a + b, 0) : k.count; + rows.push({ key, depth: 0, total, members: k.members, values, expandable: key !== OTHER && k.members > 1 }); const sub = view.expanded.get(key); if (sub && sub.step === step) rows.push(...sub.rows); } @@ -207,7 +216,7 @@ footer a:hover{color:var(--hot);border-bottom-color:var(--water)} async function expand(key) { const step = Number(stepSelect.value); - const sub = await api(`/api/heat?step=${step}&ns=${key}`); + const sub = await api(`api/heat?step=${step}&ns=${key}`); const bucketIndex = new Map(sub.buckets.map((b, i) => [b.bucket, i])); const rows = sub.keys.map((k) => ({ key: k.key, depth: 1, total: k.count, members: 1, values: series(sub.cells, k.key, bucketIndex, sub.buckets.length), expandable: false })); view.expanded.set(key, { step, rows }); @@ -221,9 +230,9 @@ footer a:hover{color:var(--hot);border-bottom-color:var(--water)} } function collapse(key) { view.expanded.delete(key); if (data) collapseRows(key); draw(); renderTable(); } - // the whole-network bootstrap ran 2026-07-28 → 2026-08-03 ~07:00 utc (stream /status: "backfill took 6d 2h"); - // witnessed spans inside bootstrap segments are not a reliable tell, the date is + // the whole-network bootstrap ran 2026-07-28 → 2026-08-03 ~07:00 utc (stream /status: "backfill took 6d 2h") const BOOTSTRAP_END_US = 1785740400000000; + const ERA = { bootstrap: "history, replayed", live: "as it happened" }; const eras = (buckets) => buckets.map((b) => b.min_witnessed_at < BOOTSTRAP_END_US ? "bootstrap" : "live"); function draw() { @@ -232,7 +241,7 @@ footer a:hover{color:var(--hot);border-bottom-color:var(--water)} const dpr = Math.min(window.devicePixelRatio || 1, 2); const width = canvas.parentElement.clientWidth, narrow = width < 640; ctx.font = "12px ui-monospace, Menlo, monospace"; - const labelW = narrow ? Math.round(width * 0.36) : Math.min(300, Math.max(...rows.map((r) => ctx.measureText(r.key).width + (r.depth ? 26 : 16))) + 10); + const labelW = narrow ? Math.round(width * 0.36) : Math.min(300, Math.max(...rows.map((r) => ctx.measureText(name(r.key)).width + (r.depth ? 26 : 16))) + 10); const countW = narrow ? 0 : 68; const top = 6, axisH = 44; const maxTotal = Math.max(...rows.filter((r) => r.depth === 0).map((r) => r.total)); @@ -248,7 +257,7 @@ footer a:hover{color:var(--hot);border-bottom-color:var(--water)} ctx.font = "12px ui-monospace, Menlo, monospace"; ctx.textBaseline = "middle"; rows.forEach((r, ri) => { const y0 = offsets[ri], h = heights[ri]; - const label = r.depth ? r.key.slice(r.key.indexOf(".", r.key.indexOf(".") + 1) + 1) : r.key; + const label = r.depth ? r.key.slice(r.key.indexOf(".", r.key.indexOf(".") + 1) + 1) : name(r.key); ctx.textAlign = "right"; ctx.fillStyle = hover && hover.ri === ri ? hot : r.depth ? dim : ink; const glyph = r.expandable ? (view.expanded.has(r.key) ? "▾ " : "▸ ") : (r.depth ? "· " : ""); ctx.fillText(fitLabel(glyph + label, labelW - 12), labelW - 10, y0 + h / 2); @@ -267,14 +276,14 @@ footer a:hover{color:var(--hot);border-bottom-color:var(--water)} const x0 = plotX + start * cellW, x1 = plotX + bi * cellW; ctx.fillStyle = era[start] === "live" ? water : faint; ctx.fillRect(x0, axisY, Math.max(x1 - x0 - 1, 1), 2); ctx.fillStyle = era[start] === "live" ? water : dim; - if (x1 - x0 > 60) ctx.fillText(era[start], x0, axisY + 6); + if (x1 - x0 > 120) ctx.fillText(ERA[era[start]], x0, axisY + 6); start = bi; } } ctx.fillStyle = dim; let lastDay = "", lastX = -Infinity; for (let bi = 0; bi < buckets.length; bi++) { const b = buckets[bi], day = dayFmt.format(new Date(b.min_witnessed_at / 1000)), x = plotX + bi * cellW; - if (day !== lastDay && x - lastX > 96) { ctx.fillText(`${day} · #${b.first_idx}`, x, axisY + 22); lastDay = day; lastX = x; } + if (day !== lastDay && x - lastX > 96) { ctx.fillText(day, x, axisY + 22); lastDay = day; lastX = x; } } if (hover) { const { bi, ri } = hover; @@ -282,16 +291,16 @@ footer a:hover{color:var(--hot);border-bottom-color:var(--water)} ctx.strokeStyle = hot; ctx.lineWidth = 1.5; ctx.strokeRect(plotX + bi * cellW - 0.5, offsets[ri] - 0.5, Math.max(cellW - gap, 0.6) + 1, heights[ri] + 1); } $("legend-bar").style.background = `linear-gradient(90deg, ${[0, .25, .5, .75, 1].map(shade).join(",")})`; - $("legend-note").textContent = scale === "row" ? "· each layer shaded to its own peak" : "· log events, one scale for every layer"; + $("legend-note").textContent = scale === "row" ? "· brightest = that app's busiest column" : "· same scale for every app (log)"; } - function fitLabel(name, maxW) { - if (ctx.measureText(name).width <= maxW) return name; - const parts = name.split("."); + function fitLabel(text, maxW) { + if (ctx.measureText(text).width <= maxW) return text; + const parts = text.split("."); while (parts.length > 1 && ctx.measureText("…" + parts.join(".")).width > maxW) parts.shift(); - let text = "…" + parts.join("."); - while (text.length > 2 && ctx.measureText(text).width > maxW) text = "…" + text.slice(2); - return text; + let out = "…" + parts.join("."); + while (out.length > 2 && ctx.measureText(out).width > maxW) out = "…" + out.slice(2); + return out; } function cellAt(ev) { if (!layout || !data) return null; @@ -303,12 +312,13 @@ footer a:hover{color:var(--hot);border-bottom-color:var(--water)} } function read(c) { const r = data.rows[c.ri], b = c.bi >= 0 ? data.buckets[c.bi] : null; - const kind = r.depth ? "collection" : r.key === "other" ? "everything below the layers shown" : "namespace"; - if (!b) { reader.innerHTML = `${esc(r.key)} · ${kind} · ${fmt.format(r.total)} events${r.expandable ? ` across ${r.members} collections · click to split` : ""}`; return; } + const what = r.depth ? "record type" : r.key === OTHER ? "the apps not shown" : "app"; + if (!b) { reader.innerHTML = `${esc(name(r.key))} · ${what} · ${fmt.format(r.total)} records in the archive${r.expandable ? ` across ${r.members} record types · click to open` : ""}`; return; } const v = r.values[c.bi], share = b.event_count ? 100 * v / b.event_count : 0; const from = dayFmt.format(new Date(b.min_witnessed_at / 1000)), to = dayFmt.format(new Date(b.max_witnessed_at / 1000)); - reader.innerHTML = `${esc(r.key)} · ${fmt.format(v)} events in this column · ${share < 0.01 && v ? "<0.01" : share.toFixed(2)}% of its ${compact.format(b.event_count)}
` + - `segments #${b.first_idx}${b.last_idx !== b.first_idx ? `–${b.last_idx}` : ""} · seq ${fmt.format(b.min_seq)} – ${fmt.format(b.max_seq)} · witnessed ${from}${to !== from ? ` → ${to}` : ""} utc`; + const chunk = b.last_idx !== b.first_idx ? `chunks #${b.first_idx}–${b.last_idx}` : `chunk #${b.first_idx}`; + reader.innerHTML = `${esc(name(r.key))} wrote ${fmt.format(v)} records in this slice — ${share < 0.01 && v ? "under 0.01" : share.toFixed(2)}% of the ${compact.format(b.event_count)} in it
` + + `archive ${chunk} · received ${from}${to !== from ? ` → ${to}` : ""} utc · record numbers ${fmt.format(b.min_seq)} – ${fmt.format(b.max_seq)}`; } canvas.addEventListener("pointermove", (ev) => { const c = cellAt(ev); if (!c) { clearHover(); return; } @@ -317,15 +327,15 @@ footer a:hover{color:var(--hot);border-bottom-color:var(--water)} canvas.style.cursor = c.onLabel && data.rows[c.ri].expandable ? "pointer" : "crosshair"; read(c); }); - function clearHover() { if (hover) { hover = null; draw(); } reader.innerHTML = 'hover a layer to read it · click a namespace to split it into its collections'; } + function clearHover() { if (hover) { hover = null; draw(); } reader.innerHTML = 'move over the picture to read a cell · click an app\'s name to open it'; } canvas.addEventListener("pointerleave", clearHover); canvas.addEventListener("click", (ev) => { const c = cellAt(ev); if (!c) return; const r = data.rows[c.ri]; if (!r.expandable) return; view.expanded.has(r.key) ? collapse(r.key) : expand(r.key); }); function renderTable() { const wrap = $("table"); if (!data || wrap.hidden) return; const { rows, buckets } = data; - wrap.innerHTML = `${buckets.map((b) => ``).join("")}` + - rows.map((r) => `${Array.from(r.values, (v) => ``).join("")}`).join("") + `
layertotal#${b.first_idx}
${r.depth ? "  · " : ""}${esc(r.key)}${fmt.format(r.total)}${v ? fmt.format(v) : ""}
`; + wrap.innerHTML = `${buckets.map((b) => ``).join("")}` + + rows.map((r) => `${Array.from(r.values, (v) => ``).join("")}`).join("") + `
app / record typerecords#${b.first_idx}
${r.depth ? "  · " : ""}${esc(name(r.key))}${fmt.format(r.total)}${v ? fmt.format(v) : ""}
`; } $("table-toggle").addEventListener("click", (ev) => { const w = $("table"); w.hidden = !w.hidden; ev.currentTarget.setAttribute("aria-pressed", String(!w.hidden)); renderTable(); }); $("more").addEventListener("click", () => { view.top = Math.min(60, view.top + 12); $("more").disabled = view.top >= 60; load(); }); @@ -333,27 +343,28 @@ footer a:hover{color:var(--hot);border-bottom-color:var(--water)} $("scale").addEventListener("change", draw); $("find").addEventListener("change", (ev) => { const ns = ev.target.value.trim().toLowerCase(), hit = view.namespaces.find((n) => n.ns === ns); - if (!hit) { reader.innerHTML = `no namespace called ${esc(ns)} in the archive yet`; return; } + if (!hit) { reader.innerHTML = `no app called ${esc(ns)} has written to the archive yet`; return; } if (!view.pinned.some((p) => p.ns === ns)) view.pinned.push(hit); ev.target.value = ""; load(); }); window.addEventListener("resize", draw); - // ---- ask the tail: duckdb-wasm over the namespace's public parquet partitions ---- + // ---- look inside an app: duckdb-wasm over the app's public parquet files ---- const PROMPTS = [ - ["by collection", "select collection, op, count(*) as events, count(distinct did) as accounts\nfrom rows group by all order by events desc"], - ["adoption by day", "select strftime(make_timestamp(time_us), '%Y-%m-%d') as day, count(distinct did) as accounts, count(*) as events\nfrom rows where kind = 'commit' group by 1 order by 1"], - ["first seen", "select collection, min(seq) as first_seq, strftime(make_timestamp(min(time_us)), '%Y-%m-%d %H:%M') as first_witnessed\nfrom rows where kind = 'commit' group by 1 order by 2"], - ["busiest accounts", "select did, count(*) as events, count(distinct collection) as collections\nfrom rows where kind = 'commit' group by 1 order by 2 desc limit 20"], - ["op mix", "select op, count(*) as events from rows where kind = 'commit' group by 1 order by 2 desc"], + ["what it writes", "select collection as record_type, op as action, count(*) as records, count(distinct did) as accounts\nfrom rows group by all order by records desc"], + ["people per day", "select strftime(make_timestamp(time_us), '%Y-%m-%d') as day, count(distinct did) as accounts, count(*) as records\nfrom rows where kind = 'commit' group by 1 order by 1"], + ["when each type first appeared", "select collection as record_type, strftime(make_timestamp(min(time_us)), '%Y-%m-%d %H:%M') as first_seen, count(*) as records\nfrom rows where kind = 'commit' group by 1 order by min(time_us)"], + ["most active accounts", "select did as account, count(*) as records, count(distinct collection) as record_types\nfrom rows where kind = 'commit' group by 1 order by 2 desc limit 20"], + ["creates vs deletes", "select op as action, count(*) as records from rows where kind = 'commit' group by 1 order by 2 desc"], ]; const askNs = $("ask-ns"), askSql = $("ask-sql"), askOut = $("ask-out"), askRun = $("ask-run"), askTable = $("ask-table"); - for (const [name, sql] of PROMPTS) { const b = document.createElement("button"); b.type = "button"; b.textContent = name; b.addEventListener("click", () => { askSql.value = sql; if (askNs.value) runAsk(); }); $("prompts").append(b); } + for (const [label, sql] of PROMPTS) { const b = document.createElement("button"); b.type = "button"; b.textContent = label; b.addEventListener("click", () => { askSql.value = sql; if (askNs.value) runAsk(); }); $("prompts").append(b); } askSql.value = PROMPTS[0][1]; + $("ask-edit").addEventListener("click", (ev) => { askSql.hidden = !askSql.hidden; ev.currentTarget.setAttribute("aria-pressed", String(!askSql.hidden)); if (!askSql.hidden) askSql.focus(); }); let duck = null, partition = null; async function duckdb() { if (duck) return duck; - askOut.innerHTML = "loading duckdb-wasm…"; + askOut.innerHTML = "loading the query engine…"; const mod = await import("https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm@1.29.0/+esm"); const bundle = await mod.selectBundle(mod.getJsDelivrBundles()); const worker = new Worker(URL.createObjectURL(new Blob([`importScripts("${bundle.mainWorker}");`], { type: "text/javascript" }))); @@ -363,13 +374,14 @@ footer a:hover{color:var(--hot);border-bottom-color:var(--water)} return duck; } askNs.addEventListener("change", async () => { - partition = null; askRun.disabled = true; askTable.innerHTML = ""; + partition = null; askRun.disabled = true; askTable.innerHTML = ""; askOut.innerHTML = ""; if (!askNs.value) { $("ask-size").textContent = "pick one"; return; } - $("ask-size").textContent = "listing…"; - const p = await api(`/api/partitions?ns=${askNs.value}`); + $("ask-size").textContent = "checking…"; + const p = await api(`api/partitions?ns=${askNs.value}`); if (p.error) { $("ask-size").textContent = p.error; return; } partition = p; - $("ask-size").textContent = p.files.length ? `${p.files.length} segment file${p.files.length === 1 ? "" : "s"} · ${p.bytes < 1 << 20 ? `${Math.round(p.bytes / 1024)} KB` : `${(p.bytes / (1 << 20)).toFixed(1)} MB`} to scan` : "not drained yet"; + const size = p.bytes < 1 << 20 ? `${Math.round(p.bytes / 1024)} KB` : `${(p.bytes / (1 << 20)).toFixed(1)} MB`; + $("ask-size").textContent = p.files.length ? `${size} to read from ${p.files.length} chunk${p.files.length === 1 ? "" : "s"}` : "not copied out of the archive yet"; askRun.disabled = !p.files.length; }); async function runAsk() { @@ -377,15 +389,14 @@ footer a:hover{color:var(--hot);border-bottom-color:var(--water)} askRun.disabled = true; try { const conn = await duckdb(); - const list = partition.files.map((f) => `'${f}'`).join(", "); - await conn.query(`create or replace view rows as select * from read_parquet([${list}])`); + await conn.query(`create or replace view rows as select * from read_parquet([${partition.files.map((f) => `'${f}'`).join(", ")}])`); const t = performance.now(); const res = await conn.query(askSql.value); const cols = res.schema.fields.map((f) => f.name); const out = res.toArray().map((r) => r.toJSON()); - askOut.innerHTML = `${fmt.format(out.length)} row${out.length === 1 ? "" : "s"} in ${(performance.now() - t).toFixed(0)} ms over ${partition.files.length} file${partition.files.length === 1 ? "" : "s"}`; - const cell = (v) => v === null || v === undefined ? "" : typeof v === "bigint" ? fmt.format(v) : typeof v === "number" ? fmt.format(v) : esc(v); - askTable.innerHTML = `${cols.map((c) => ``).join("")}${out.slice(0, 500).map((r) => `${cols.map((c) => ``).join("")}`).join("")}
${esc(c)}
${cell(r[c])}
` + (out.length > 500 ? `

showing 500 of ${fmt.format(out.length)}

` : ""); + askOut.innerHTML = `${fmt.format(out.length)} row${out.length === 1 ? "" : "s"} in ${(performance.now() - t).toFixed(0)} ms`; + const cell = (v) => v === null || v === undefined ? "" : typeof v === "bigint" || typeof v === "number" ? fmt.format(v) : esc(v); + askTable.innerHTML = `${cols.map((c) => ``).join("")}${out.slice(0, 500).map((r) => `${cols.map((c) => ``).join("")}`).join("")}
${esc(c.replaceAll("_", " "))}
${cell(r[c])}
` + (out.length > 500 ? `

showing 500 of ${fmt.format(out.length)}

` : ""); } catch (err) { askOut.innerHTML = `${esc(err && err.message || err)}`; } finally { askRun.disabled = false; } -- 2.51.2 From 6afe3614bf65ca96c9fffd87f634cfd0fc8a277c Mon Sep 17 00:00:00 2001 From: zzstoatzz Date: Sat, 29 Aug 2026 01:49:42 -0500 Subject: [PATCH 07/17] page: DOM lanes with a fixed name column; anti-slop lint; schema-parsed ingest; hashed script url The picture is one grid row per app (name | strip | total) so opening an app never moves the strips; phones stack name over strip. Copy trimmed for non-experts. Worker: @atcute/lexicons validators at the ingest boundary, anti-slop oxlint clean, ui.js served by content hash so html/js cannot skew. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01B927dNwNKNYsbQdUoNJwHS --- .oxlintrc.json | 21 + package-lock.json | 435 +++++++++++++++ package.json | 9 +- src/index.ts | 91 ++-- src/ingest.ts | 145 ++--- src/page.html | 366 +++---------- src/text-modules.d.ts | 4 + src/ui.js | 428 +++++++++++++++ tests/ingest.test.ts | 34 +- tools/oxlint/anti-slop/index.ts | 41 ++ .../rules/no-chained-type-assertions.ts | 77 +++ .../no-conditional-empty-object-spread.ts | 49 ++ .../rules/no-known-value-widening.ts | 247 +++++++++ .../anti-slop/rules/no-module-mocking.ts | 91 ++++ .../anti-slop/rules/no-object-parameters.ts | 126 +++++ .../anti-slop/rules/no-reflect-apply.ts | 28 + .../oxlint/anti-slop/rules/no-reflect-get.ts | 28 + .../anti-slop/rules/no-runtime-typeof.ts | 67 +++ .../rules/no-shape-in-symbol-names.ts | 39 ++ .../anti-slop/rules/no-unknown-parameters.ts | 83 +++ .../anti-slop/rules/no-unknown-returns.ts | 115 ++++ .../rules/no-unknown-type-aliases.ts | 70 +++ .../rules/no-unsafe-dictionary-type.ts | 134 +++++ .../anti-slop/rules/no-widen-then-assert.ts | 366 +++++++++++++ ...quire-safety-comment-for-type-assertion.ts | 62 +++ .../anti-slop/shared/dictionary-types.ts | 502 ++++++++++++++++++ .../shared/lexical-type-parameters.ts | 61 +++ .../oxlint/anti-slop/shared/reflect-method.ts | 35 ++ wrangler.jsonc | 2 +- 29 files changed, 3321 insertions(+), 435 deletions(-) create mode 100644 .oxlintrc.json create mode 100644 src/ui.js create mode 100644 tools/oxlint/anti-slop/index.ts create mode 100644 tools/oxlint/anti-slop/rules/no-chained-type-assertions.ts create mode 100644 tools/oxlint/anti-slop/rules/no-conditional-empty-object-spread.ts create mode 100644 tools/oxlint/anti-slop/rules/no-known-value-widening.ts create mode 100644 tools/oxlint/anti-slop/rules/no-module-mocking.ts create mode 100644 tools/oxlint/anti-slop/rules/no-object-parameters.ts create mode 100644 tools/oxlint/anti-slop/rules/no-reflect-apply.ts create mode 100644 tools/oxlint/anti-slop/rules/no-reflect-get.ts create mode 100644 tools/oxlint/anti-slop/rules/no-runtime-typeof.ts create mode 100644 tools/oxlint/anti-slop/rules/no-shape-in-symbol-names.ts create mode 100644 tools/oxlint/anti-slop/rules/no-unknown-parameters.ts create mode 100644 tools/oxlint/anti-slop/rules/no-unknown-returns.ts create mode 100644 tools/oxlint/anti-slop/rules/no-unknown-type-aliases.ts create mode 100644 tools/oxlint/anti-slop/rules/no-unsafe-dictionary-type.ts create mode 100644 tools/oxlint/anti-slop/rules/no-widen-then-assert.ts create mode 100644 tools/oxlint/anti-slop/rules/require-safety-comment-for-type-assertion.ts create mode 100644 tools/oxlint/anti-slop/shared/dictionary-types.ts create mode 100644 tools/oxlint/anti-slop/shared/lexical-type-parameters.ts create mode 100644 tools/oxlint/anti-slop/shared/reflect-method.ts diff --git a/.oxlintrc.json b/.oxlintrc.json new file mode 100644 index 0000000..bdf4c5d --- /dev/null +++ b/.oxlintrc.json @@ -0,0 +1,21 @@ +{ + "ignorePatterns": [".claude/**", "node_modules/**", "tools/oxlint/anti-slop/**"], + "jsPlugins": [{ "name": "anti-slop", "specifier": "./tools/oxlint/anti-slop/index.ts" }], + "rules": { + "anti-slop/no-chained-type-assertions": "error", + "anti-slop/no-conditional-empty-object-spread": "error", + "anti-slop/no-known-value-widening": "error", + "anti-slop/no-module-mocking": "error", + "anti-slop/no-object-parameters": "error", + "anti-slop/no-reflect-apply": "error", + "anti-slop/no-reflect-get": "error", + "anti-slop/no-runtime-typeof": "error", + "anti-slop/no-shape-in-symbol-names": "error", + "anti-slop/no-unknown-parameters": "error", + "anti-slop/no-unknown-returns": "error", + "anti-slop/no-unknown-type-aliases": "error", + "anti-slop/no-unsafe-dictionary-type": "error", + "anti-slop/no-widen-then-assert": "error", + "anti-slop/require-safety-comment-for-type-assertion": "error" + } +} diff --git a/package-lock.json b/package-lock.json index 2c76033..dac4249 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,14 +5,46 @@ "packages": { "": { "name": "strata", + "dependencies": { + "@atcute/lexicons": "2.1.0" + }, "devDependencies": { "@cloudflare/workers-types": "5.20260829.1", + "@oxlint/plugins": "1.80.0", "@types/node": "22.20.1", + "oxlint": "1.80.0", "tsx": "4.23.12", "typescript": "5.9.3", "wrangler": "4.127.1" } }, + "node_modules/@atcute/lexicons": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@atcute/lexicons/-/lexicons-2.1.0.tgz", + "integrity": "sha512-cqbtPfFctoddrSFBt65xyD7st01d8rNntyinRfRhlRPGlqEg8FWacgc7piBw0SAs1SG+5BoV3d3mhzWfQHcByg==", + "license": "0BSD", + "dependencies": { + "@atcute/uint8array": "^1.1.5", + "@atcute/util-text": "^1.3.4", + "@standard-schema/spec": "^1.1.0", + "esm-env": "^1.2.2" + } + }, + "node_modules/@atcute/uint8array": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@atcute/uint8array/-/uint8array-1.1.5.tgz", + "integrity": "sha512-1SFCXOtjE3ismP92CqzbOfYSwJmPPPFbj798wLXSpLv2hSd5OIaUrTh6EqRVR1VZ3ZhR+vaBmd3kxAN0wkw+gQ==", + "license": "0BSD" + }, + "node_modules/@atcute/util-text": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@atcute/util-text/-/util-text-1.3.4.tgz", + "integrity": "sha512-u2UAM7iSM09sQaSG9jtxWFSPgB8boVj50/BoyMvYnhVgGBu+nXIuAcdDUQCsZA44YZgTPPhN/b86JX+jH7SPzQ==", + "license": "0BSD", + "dependencies": { + "unicode-segmenter": "^0.17.0" + } + }, "node_modules/@cloudflare/kv-asset-handler": { "version": "0.5.0", "resolved": "https://registry.npmjs.org/@cloudflare/kv-asset-handler/-/kv-asset-handler-0.5.0.tgz", @@ -1152,6 +1184,342 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@oxlint/binding-android-arm-eabi": { + "version": "1.80.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-android-arm-eabi/-/binding-android-arm-eabi-1.80.0.tgz", + "integrity": "sha512-RM3Plj+biQpxa5d1GOOX6ciDlcUROmm4OZ/pLTpitkQt2mJv4jhtY4cbgaetOm5UKWZe05/TGQ6o1Vl8EOHkrA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-android-arm64": { + "version": "1.80.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-android-arm64/-/binding-android-arm64-1.80.0.tgz", + "integrity": "sha512-YlO5JEf0Yr2bUUlu8O8daVcUxtcGGbcSmyV7E7nSbJbfAdxTE0PFPwgnIlw7wXJaTYjb+qs5hI5q3jxUkI7cAw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-darwin-arm64": { + "version": "1.80.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-darwin-arm64/-/binding-darwin-arm64-1.80.0.tgz", + "integrity": "sha512-BULDOyO3AhsmdWfQeIUCykDt3dd7XZBGLhp1eIh56skRv01O+cNjNPwXMIbeW1x4+pxcln5if72wcRgViVo7PA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-darwin-x64": { + "version": "1.80.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-darwin-x64/-/binding-darwin-x64-1.80.0.tgz", + "integrity": "sha512-YJ4JzLw7N5TDSQFlA0hAQGHvnDZgyypm1yunObVWcWiF9KM7eGCJKYKLgTC2Fi/57OdnBhbj4OkzPGdFQJ6HyA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-freebsd-x64": { + "version": "1.80.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-freebsd-x64/-/binding-freebsd-x64-1.80.0.tgz", + "integrity": "sha512-AYUIk5QnL0s8oWAYsREZwkRYy1SupJTXALo93J1TgzHywxQtdM99FecRMQ87MXEdPQ0j1TmEpeeq3fGNkpvMqg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm-gnueabihf": { + "version": "1.80.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.80.0.tgz", + "integrity": "sha512-9hBZVANupQ89W9dXyE0n8doCyaW5pDyGn3y6XlIMPZ+rIKuyqkr3SNUXmVJIhuvUq0NBU3RBiSXXE69l4XI6KA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm-musleabihf": { + "version": "1.80.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-1.80.0.tgz", + "integrity": "sha512-SvS2uKqzY+pbfuvAHzH4338R6Zwo805GAwrIMVvK1KxoOWCIjZUdfzTCvilD7z6JK91v011+zYMryabhDo2AsQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm64-gnu": { + "version": "1.80.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.80.0.tgz", + "integrity": "sha512-tCLadyqRVL3pQTRPNg7cjXKvcvS4fbyXeQHhKk5BTJ1oftQln5/yIIWbu/Xom/DX41zv2P9QGt6+D/TtQVtY3A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm64-musl": { + "version": "1.80.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.80.0.tgz", + "integrity": "sha512-XfpCNRlOPcLlJl4Bn/FUhjqlR6BVavEykERBf/MV7YA9VZDa5g5znVqYhyviMafcxS9Pe/i/kPvHNO0U6svEHQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-ppc64-gnu": { + "version": "1.80.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.80.0.tgz", + "integrity": "sha512-3I4yMwcFG9NeO8ioY6JBBuKsIm5GL/x7MATt1S4tVWaxPu5HcJ+XnLUbcVBTxG8q2Wu56HSj+NmXQiVYb1lp6A==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-riscv64-gnu": { + "version": "1.80.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-1.80.0.tgz", + "integrity": "sha512-E1wAKymkpe1/E8helzBKdm81OBOF+ezxRyXRMEuik3ZpWDER5CPOKZwF66RsdwW98uwZv8UTFremUQtC1CzdJA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-riscv64-musl": { + "version": "1.80.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-1.80.0.tgz", + "integrity": "sha512-+gLRGD4sIo3+VA++iham5UxD9tKSoJ/VOrROCEXIcknrYtQg6iIQgvjN0cpiRF7N6UYC7pJbvHJlDnMge5LRpQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-s390x-gnu": { + "version": "1.80.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.80.0.tgz", + "integrity": "sha512-aR0PrzHj9leW3NmzBAAP4EzdoBNoJcs9sjnIQPIwyRnBGYrRbXUIpEB5Q39AqK3PLY5JK5uEhDQDiUa1QSAstw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-x64-gnu": { + "version": "1.80.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.80.0.tgz", + "integrity": "sha512-vSVh5cSo3Xxs6ghBCcFJlpbkbENzDog1qXtoXLa/HC3aCrR4XO76GZbXmQoCPHnu99nQpdCeC3H9tdNICfDh7A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-x64-musl": { + "version": "1.80.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-x64-musl/-/binding-linux-x64-musl-1.80.0.tgz", + "integrity": "sha512-FfzBXpNQ8u7/ZI/p8bl73MeZ508Ax3hxWp3SiJpEFiC+BB9XcXy5FAZHTLKDPSzrUpxQZSZJAVdDmuJp/+HDBQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-openharmony-arm64": { + "version": "1.80.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-openharmony-arm64/-/binding-openharmony-arm64-1.80.0.tgz", + "integrity": "sha512-zMzbkumtmprCgRwoYNzcB3iC39fXdJIMLMU33KdCjEGLlJGOEt1+LwQ4LF8ndLzAEKVz4BR0y3V6Xrkk3Nm3yA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-win32-arm64-msvc": { + "version": "1.80.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.80.0.tgz", + "integrity": "sha512-ib6iRcrXsk4t1fm3iKcwksyWh1ZkZXC/2mEzakl0ai2+6HZunf1WWMZ/xP9EJAvw9g9K4UVTC3NF/+G2qLrbTQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-win32-ia32-msvc": { + "version": "1.80.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-1.80.0.tgz", + "integrity": "sha512-xhRWBMpLxZvgKAH6+DJZmpP+W8Y8UdQOSU1JfxSWNXsaBaRGW77j+1hCuNHlzj7OH4SPN8fYd1q0o2qrDtoVyw==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-win32-x64-msvc": { + "version": "1.80.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.80.0.tgz", + "integrity": "sha512-yAnO7lwBYQnz2pcfBPIGQQZWIX5zd5R/1aAKIF3oE+TVj7IhoHcROjOkz3sRDngzqhfPKfFaXqug5j5rE5dn6Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/plugins": { + "version": "1.80.0", + "resolved": "https://registry.npmjs.org/@oxlint/plugins/-/plugins-1.80.0.tgz", + "integrity": "sha512-QRgH1XqQEYNHa4f1vvPQ5fAdNdncHGIUG1ZWLlGIZHky3qwCEeAKYitZNbZMtaXtAQAAFFTOwqUfzESvimqZNA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/Boshen" + } + }, "node_modules/@poppinss/colors": { "version": "4.1.6", "resolved": "https://registry.npmjs.org/@poppinss/colors/-/colors-4.1.6.tgz", @@ -1201,6 +1569,12 @@ "dev": true, "license": "CC0-1.0" }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "license": "MIT" + }, "node_modules/@types/node": { "version": "22.20.1", "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.1.tgz", @@ -1294,6 +1668,12 @@ "@esbuild/win32-x64": "0.28.2" } }, + "node_modules/esm-env": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/esm-env/-/esm-env-1.2.2.tgz", + "integrity": "sha512-Epxrv+Nr/CaL4ZcFGPJIYLWFom+YeV1DqMLHJoEd9SYRxNbaFruBwfEX/kkHUJf55j2+TUbmDcmuilbP1TmXHA==", + "license": "MIT" + }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -1337,6 +1717,55 @@ "node": ">=22.0.0" } }, + "node_modules/oxlint": { + "version": "1.80.0", + "resolved": "https://registry.npmjs.org/oxlint/-/oxlint-1.80.0.tgz", + "integrity": "sha512-5nTiSps4qdbCWLbxzuO00alHkEO2exR9YMN/ig6QXWrLsYSG0KaObOAM+l6oU2LcKPWoSAGYbkZIGEu1ViiWKA==", + "dev": true, + "license": "MIT", + "bin": { + "oxlint": "bin/oxlint" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/sponsors/Boshen" + }, + "optionalDependencies": { + "@oxlint/binding-android-arm-eabi": "1.80.0", + "@oxlint/binding-android-arm64": "1.80.0", + "@oxlint/binding-darwin-arm64": "1.80.0", + "@oxlint/binding-darwin-x64": "1.80.0", + "@oxlint/binding-freebsd-x64": "1.80.0", + "@oxlint/binding-linux-arm-gnueabihf": "1.80.0", + "@oxlint/binding-linux-arm-musleabihf": "1.80.0", + "@oxlint/binding-linux-arm64-gnu": "1.80.0", + "@oxlint/binding-linux-arm64-musl": "1.80.0", + "@oxlint/binding-linux-ppc64-gnu": "1.80.0", + "@oxlint/binding-linux-riscv64-gnu": "1.80.0", + "@oxlint/binding-linux-riscv64-musl": "1.80.0", + "@oxlint/binding-linux-s390x-gnu": "1.80.0", + "@oxlint/binding-linux-x64-gnu": "1.80.0", + "@oxlint/binding-linux-x64-musl": "1.80.0", + "@oxlint/binding-openharmony-arm64": "1.80.0", + "@oxlint/binding-win32-arm64-msvc": "1.80.0", + "@oxlint/binding-win32-ia32-msvc": "1.80.0", + "@oxlint/binding-win32-x64-msvc": "1.80.0" + }, + "peerDependencies": { + "oxlint-tsgolint": ">=7.0.2001", + "vite-plus": "*" + }, + "peerDependenciesMeta": { + "oxlint-tsgolint": { + "optional": true + }, + "vite-plus": { + "optional": true + } + } + }, "node_modules/path-to-regexp": { "version": "6.3.0", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", @@ -1490,6 +1919,12 @@ "pathe": "^2.0.3" } }, + "node_modules/unicode-segmenter": { + "version": "0.17.3", + "resolved": "https://registry.npmjs.org/unicode-segmenter/-/unicode-segmenter-0.17.3.tgz", + "integrity": "sha512-hKZwqBjJDmqNrq1+LjDxck1qLzJFcLLJM2Xq92ORRHOuau6GSHeELmn+uyk6zNH1CK9mogrUJGP9WJCJUQXMAw==", + "license": "MIT" + }, "node_modules/workerd": { "version": "1.20260828.1", "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260828.1.tgz", diff --git a/package.json b/package.json index 2f54ff2..4d037c9 100644 --- a/package.json +++ b/package.json @@ -6,13 +6,20 @@ "typecheck": "tsc --noEmit", "test": "node --test --import tsx tests/*.test.ts", "deploy": "wrangler deploy", - "migrate": "wrangler d1 execute strata --remote --file schema.sql" + "migrate": "wrangler d1 execute strata --remote --file schema.sql", + "lint": "oxlint src tests", + "check": "tsc --noEmit && oxlint src tests && node --test --import tsx tests/*.test.ts" }, "devDependencies": { "@cloudflare/workers-types": "5.20260829.1", + "@oxlint/plugins": "1.80.0", "@types/node": "22.20.1", + "oxlint": "1.80.0", "tsx": "4.23.12", "typescript": "5.9.3", "wrangler": "4.127.1" + }, + "dependencies": { + "@atcute/lexicons": "2.1.0" } } diff --git a/src/index.ts b/src/index.ts index 339c4f9..e20f876 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,8 +4,20 @@ * collection index, and POSTs the result here. this worker only stores and * aggregates; it holds no schedule and never talks to the archive itself. */ -import { IngestParseError, namespaceOf, parseIngestBatch, type SegmentRecord } from "./ingest"; -import page from "./page.html"; +import { safeParse } from "@atcute/lexicons/validations"; + +import { IngestBatchSchema, checkIngestBatch, namespaceOf, type SegmentRecord } from "./ingest"; +import pageSource from "./page.html"; +import ui from "./ui.js"; + +/** the page names its script by content hash so an html/js cache skew on deploy cannot happen. */ +function contentHash(text: string): string { + let h = 2166136261; + for (let i = 0; i < text.length; i++) h = Math.imul(h ^ text.charCodeAt(i), 16777619); + return (h >>> 0).toString(36); +} +const UI_VERSION = contentHash(ui); +const page = pageSource.replace('src="ui.js"', `src="ui.js?v=${UI_VERSION}"`); interface Env { readonly DB: D1Database; @@ -39,7 +51,10 @@ async function putArchive(request: Request, key: string, env: Env): Promise = {}): Response { +/** what this worker answers with: plain JSON built from our own row types (type aliases, so arrays of them fit). */ +type JsonValue = string | number | boolean | null | undefined | readonly JsonValue[] | { readonly [key: string]: JsonValue }; + +function json(body: JsonValue, status = 200, headers: Readonly> = {}): Response { return new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json", "access-control-allow-origin": "*", ...headers }, @@ -97,27 +112,30 @@ async function ingest(request: Request, env: Env): Promise { if (!authorized(request, env)) return json({ error: "unauthorized" }, 401); const length = Number(request.headers.get("content-length") ?? "0"); if (!Number.isFinite(length) || length > MAX_BODY_BYTES) return json({ error: "body too large" }, 413); - let batch; + let body; try { - batch = parseIngestBatch(await request.json()); - } catch (err) { - if (err instanceof IngestParseError) return json({ error: err.message }, 400); + body = await request.json(); + } catch { return json({ error: "invalid json" }, 400); } + const parsed = safeParse(IngestBatchSchema, body); + if (!parsed.ok) return json({ error: parsed.message }, 400); + const problem = checkIngestBatch(parsed.value); + if (problem !== null) return json({ error: problem }, 400); const ingestedAt = Date.now(); - for (const segment of batch.segments) { + for (const segment of parsed.value.segments) { await env.DB.batch(segmentStatements(env.DB, segment, ingestedAt)); } - return json({ ingested: batch.segments.length }); + return json({ ingested: parsed.value.segments.length }); } -interface ProgressRow { +type ProgressRow = { readonly segments: number; readonly max_idx: number | null; readonly max_seq: number | null; readonly events: number | null; readonly updated_at: number | null; -} +}; async function progress(env: Env): Promise { const [row, kinds] = await env.DB.batch([ @@ -126,8 +144,9 @@ async function progress(env: Env): Promise { ), env.DB.prepare("SELECT COUNT(DISTINCT nsid) AS collections, COUNT(DISTINCT ns) AS namespaces FROM segment_collections"), ]); - // SAFETY: the two statements select exactly these columns. + // SAFETY: the first statement selects exactly ProgressRow's columns. const p = row?.results[0] as ProgressRow | undefined; + // SAFETY: the second statement selects exactly these two counts. const k = kinds?.results[0] as { collections: number; namespaces: number } | undefined; return json({ segments: p?.segments ?? 0, @@ -140,13 +159,13 @@ async function progress(env: Env): Promise { }); } -interface HeatRow { +type HeatRow = { readonly bucket: number; readonly key: string; readonly count: number; -} +}; -interface BucketRow { +type BucketRow = { readonly bucket: number; readonly first_idx: number; readonly last_idx: number; @@ -155,7 +174,7 @@ interface BucketRow { readonly min_witnessed_at: number; readonly max_witnessed_at: number; readonly event_count: number; -} +}; const MAX_TOP = 60; @@ -165,20 +184,22 @@ interface HeatQuery { readonly scope: { readonly kind: "namespaces" } | { readonly kind: "namespace"; readonly ns: string } | { readonly kind: "collections" }; } -function parseHeatQuery(url: URL): HeatQuery | string { +type HeatQueryResult = { readonly ok: true; readonly query: HeatQuery } | { readonly ok: false; readonly error: string }; + +function parseHeatQuery(url: URL): HeatQueryResult { const step = Number(url.searchParams.get("step") ?? "10"); const top = Number(url.searchParams.get("top") ?? "12"); - if (!Number.isSafeInteger(step) || step < 1) return "step must be a positive integer"; - if (!Number.isSafeInteger(top) || top < 1 || top > MAX_TOP) return `top must be 1..${MAX_TOP}`; + if (!Number.isSafeInteger(step) || step < 1) return { ok: false, error: "step must be a positive integer" }; + if (!Number.isSafeInteger(top) || top < 1 || top > MAX_TOP) return { ok: false, error: `top must be 1..${MAX_TOP}` }; const ns = url.searchParams.get("ns"); const group = url.searchParams.get("group") ?? "namespaces"; if (ns !== null) { - if (!/^[a-z0-9-]+\.[a-z0-9-]+$/i.test(ns)) return "ns must be an authority like fm.teal"; - return { step, top, scope: { kind: "namespace", ns } }; + if (!/^[a-z0-9-]+\.[a-z0-9-]+$/i.test(ns)) return { ok: false, error: "ns must be an authority like fm.teal" }; + return { ok: true, query: { step, top, scope: { kind: "namespace", ns } } }; } - if (group === "collections") return { step, top, scope: { kind: "collections" } }; - if (group === "namespaces") return { step, top, scope: { kind: "namespaces" } }; - return "group must be namespaces or collections"; + if (group === "collections") return { ok: true, query: { step, top, scope: { kind: "collections" } } }; + if (group === "namespaces") return { ok: true, query: { step, top, scope: { kind: "namespaces" } } }; + return { ok: false, error: "group must be namespaces or collections" }; } /** rows of the heat map: (bucket, key, count) where key is a namespace, or an nsid inside one namespace, or a @@ -239,15 +260,16 @@ function heatKeysStatement(db: D1Database, q: HeatQuery): D1PreparedStatement { } } -interface KeyRow { +type KeyRow = { readonly key: string; readonly count: number; readonly members: number; -} +}; async function heat(url: URL, env: Env): Promise { - const q = parseHeatQuery(url); - if (typeof q === "string") return json({ error: q }, 400); + const parsed = parseHeatQuery(url); + if (!parsed.ok) return json({ error: parsed.error }, 400); + const q = parsed.query; const progressRow = await env.DB.prepare("SELECT MAX(idx) AS max_idx FROM segments").first<{ max_idx: number | null }>(); const maxIdx = progressRow?.max_idx ?? -1; if (Math.floor(maxIdx / q.step) + 1 > MAX_HEAT_BUCKETS) { @@ -267,9 +289,11 @@ async function heat(url: URL, env: Env): Promise { if (buckets === undefined || cells === undefined || keys === undefined) { return json({ error: "batch returned fewer results than statements" }, 500); } - // SAFETY: the statements above select exactly the columns of BucketRow, HeatRow, and KeyRow. + // SAFETY: the first statement selects exactly BucketRow's columns. const bucketRows = buckets.results as BucketRow[]; + // SAFETY: heatRowsStatement selects exactly HeatRow's columns in every scope. const heatRows = cells.results as HeatRow[]; + // SAFETY: heatKeysStatement selects exactly KeyRow's columns in every scope. const keyRows = keys.results as KeyRow[]; return json( { step: q.step, top: q.top, scope: q.scope, buckets: bucketRows, cells: heatRows, keys: keyRows }, @@ -327,7 +351,14 @@ export default { case "/api/partitions": return partitions(url, env); case "/": - return new Response(page, { headers: { "content-type": "text/html; charset=utf-8", "cache-control": "public, max-age=300" } }); + return new Response(page, { headers: { "content-type": "text/html; charset=utf-8", "cache-control": "public, max-age=60" } }); + case "/ui.js": + return new Response(ui, { + headers: { + "content-type": "text/javascript; charset=utf-8", + "cache-control": url.searchParams.get("v") === UI_VERSION ? "public, max-age=31536000, immutable" : "no-cache", + }, + }); case "/api": return json({ name: "strata", diff --git a/src/ingest.ts b/src/ingest.ts index 43ae8e5..b156c48 100644 --- a/src/ingest.ts +++ b/src/ingest.ts @@ -1,103 +1,64 @@ -/** the record contract between the strata flow and this worker. - * - * one row per sealed stream segment, with the per-collection event counts - * decoded from the segment's collection block index. everything here is - * boundary input from an HTTP body and is parsed before it reaches D1. */ +/** the record contract between the strata flow and this worker: one row per sealed stream segment, with the + * per-collection event counts decoded from the segment's collection block index. the schema parses the HTTP + * body at the boundary; `checkIngestBatch` holds the invariants a schema cannot express. */ -export interface CollectionCount { - readonly nsid: string; - readonly count: number; -} - -/** the authority prefix of an nsid: its first two labels (`app.bsky.feed.like` → `app.bsky`). */ -export function namespaceOf(nsid: string): string { - const parts = nsid.split("."); - return parts.length <= 2 ? nsid : `${parts[0]}.${parts[1]}`; -} - -export interface SegmentRecord { - readonly idx: number; - readonly name: string; - readonly sizeBytes: number; - readonly checksum: string; - readonly eventCount: number; - readonly blockCount: number; - readonly minSeq: number; - readonly maxSeq: number; - readonly minWitnessedAt: number; - readonly maxWitnessedAt: number; - readonly collections: readonly CollectionCount[]; -} - -export interface IngestBatch { - readonly segments: readonly SegmentRecord[]; -} - -export class IngestParseError extends Error {} +import { + array, + arrayLength, + constrain, + integer, + integerRange, + object, + string, + stringLength, + type InferOutput, +} from "@atcute/lexicons/validations"; -const MAX_SEGMENTS_PER_BATCH = 200; -const MAX_COLLECTIONS_PER_SEGMENT = 1 << 20; +const count = constrain(integer(), [integerRange(0)]); -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} +export const CollectionCountSchema = object({ + nsid: constrain(string(), [stringLength(1, 255)]), + count, +}); -function integer(source: Record, key: string, where: string): number { - const value = source[key]; - if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) { - throw new IngestParseError(`${where}.${key}: expected a non-negative integer`); - } - return value; -} +export const SegmentRecordSchema = object({ + idx: count, + name: constrain(string(), [stringLength(1, 64)]), + sizeBytes: count, + checksum: constrain(string(), [stringLength(1, 32)]), + eventCount: count, + blockCount: count, + minSeq: count, + maxSeq: count, + minWitnessedAt: count, + maxWitnessedAt: count, + collections: constrain(array(CollectionCountSchema), [arrayLength(0, 1 << 20)]), +}); -function text(source: Record, key: string, where: string, maxLength: number): string { - const value = source[key]; - if (typeof value !== "string" || value.length === 0 || value.length > maxLength) { - throw new IngestParseError(`${where}.${key}: expected a string of 1..${maxLength} chars`); - } - return value; -} +export const IngestBatchSchema = object({ + segments: constrain(array(SegmentRecordSchema), [arrayLength(1, 200)]), +}); -function parseCollection(value: unknown, where: string): CollectionCount { - if (!isRecord(value)) throw new IngestParseError(`${where}: expected an object`); - return { nsid: text(value, "nsid", where, 255), count: integer(value, "count", where) }; -} +export type CollectionCount = InferOutput; +export type SegmentRecord = InferOutput; +export type IngestBatch = InferOutput; -function parseSegment(value: unknown, where: string): SegmentRecord { - if (!isRecord(value)) throw new IngestParseError(`${where}: expected an object`); - const rawCollections = value["collections"]; - if (!Array.isArray(rawCollections) || rawCollections.length > MAX_COLLECTIONS_PER_SEGMENT) { - throw new IngestParseError(`${where}.collections: expected an array`); - } - const collections = rawCollections.map((c, i) => parseCollection(c, `${where}.collections[${i}]`)); - const seen = new Set(); - for (const c of collections) { - if (seen.has(c.nsid)) throw new IngestParseError(`${where}.collections: duplicate nsid ${c.nsid}`); - seen.add(c.nsid); - } - const minSeq = integer(value, "minSeq", where); - const maxSeq = integer(value, "maxSeq", where); - if (maxSeq < minSeq) throw new IngestParseError(`${where}: maxSeq < minSeq`); - return { - idx: integer(value, "idx", where), - name: text(value, "name", where, 64), - sizeBytes: integer(value, "sizeBytes", where), - checksum: text(value, "checksum", where, 32), - eventCount: integer(value, "eventCount", where), - blockCount: integer(value, "blockCount", where), - minSeq, - maxSeq, - minWitnessedAt: integer(value, "minWitnessedAt", where), - maxWitnessedAt: integer(value, "maxWitnessedAt", where), - collections, - }; +/** the authority prefix of an nsid: its first two labels (`app.bsky.feed.like` → `app.bsky`). */ +export function namespaceOf(nsid: string): string { + const parts = nsid.split("."); + return parts.length <= 2 ? nsid : `${parts[0]}.${parts[1]}`; } -export function parseIngestBatch(value: unknown): IngestBatch { - if (!isRecord(value)) throw new IngestParseError("body: expected an object"); - const raw = value["segments"]; - if (!Array.isArray(raw) || raw.length === 0 || raw.length > MAX_SEGMENTS_PER_BATCH) { - throw new IngestParseError(`body.segments: expected 1..${MAX_SEGMENTS_PER_BATCH} segments`); +/** invariants beyond shape: a segment's seq range runs forward and names each collection once. + * returns the first violation as a message, or null when the batch is sound. */ +export function checkIngestBatch(batch: IngestBatch): string | null { + for (const [i, segment] of batch.segments.entries()) { + if (segment.maxSeq < segment.minSeq) return `segments[${i}]: maxSeq < minSeq`; + const seen = new Set(); + for (const c of segment.collections) { + if (seen.has(c.nsid)) return `segments[${i}].collections: duplicate nsid ${c.nsid}`; + seen.add(c.nsid); + } } - return { segments: raw.map((s, i) => parseSegment(s, `segments[${i}]`)) }; + return null; } diff --git a/src/page.html b/src/page.html index 88fb292..1491f7c 100644 --- a/src/page.html +++ b/src/page.html @@ -14,57 +14,94 @@ *{margin:0;padding:0;box-sizing:border-box} html,body{background:var(--bg);color:var(--ink);font:14px/1.65 ui-monospace,"SF Mono",Menlo,Consolas,monospace} a{color:inherit;text-decoration:none} -.wrap{max-width:1180px;margin:0 auto;padding:0 2ch;display:flex;flex-direction:column;min-height:100vh} -header{display:flex;flex-direction:column;align-items:center;gap:.35rem;padding:1.7rem 0 1.4rem} +.wrap{max-width:1100px;margin:0 auto;padding:0 2ch;display:flex;flex-direction:column;min-height:100vh} +header{display:flex;flex-direction:column;align-items:center;gap:.35rem;padding:1.7rem 0 1.4rem;text-align:center} .brand{letter-spacing:.12em;font-size:.9rem;color:var(--hot)} a.brand:hover{color:var(--water)} .livedot{font-size:.74rem;color:var(--dim);font-variant-numeric:tabular-nums} .livedot b{color:var(--water);font-weight:400} -.tag{color:var(--dim);font-size:.8rem;text-align:center;max-width:64ch;margin-top:.3rem} -.tag em{font-style:normal;color:var(--ink)} -main{flex:1;display:flex;flex-direction:column;align-items:center;gap:1.6rem} -.sect{font-size:.72rem;letter-spacing:.35em;text-transform:uppercase;color:var(--dim)} -.grid{display:flex;flex-wrap:wrap;justify-content:center;gap:1.2rem 3ch;width:100%;max-width:72ch} +.tag{color:var(--dim);font-size:.8rem;max-width:64ch;margin-top:.3rem} +em{font-style:normal;color:var(--ink)} +main{flex:1;display:flex;flex-direction:column;gap:1.6rem} +.sect{font-size:.72rem;letter-spacing:.35em;text-transform:uppercase;color:var(--dim);text-align:center} +.grid{display:flex;flex-wrap:wrap;justify-content:center;gap:1.2rem 3ch;width:100%} .stat{flex:0 1 15ch;display:flex;flex-direction:column;align-items:center;gap:.15rem;text-align:center} .stat .v{font-size:1.5rem;font-weight:300;color:var(--hot);letter-spacing:.04em;font-variant-numeric:tabular-nums} .stat .l{font-size:.7rem;color:var(--dim);letter-spacing:.22em;text-transform:uppercase} -.how{color:var(--dim);font-size:.8rem;max-width:76ch;text-align:center;line-height:1.65} -.how em{font-style:normal;color:var(--ink)} +.how{color:var(--dim);font-size:.8rem;max-width:76ch;line-height:1.65;margin:0 auto;text-align:center} .how .glyph{color:var(--water)} -.controls{display:flex;flex-wrap:wrap;justify-content:center;align-items:center;gap:.6rem 2.4ch;color:var(--dim);font-size:.8rem;width:100%} +.controls{display:flex;flex-wrap:wrap;justify-content:center;align-items:center;gap:.6rem 2.4ch;color:var(--dim);font-size:.8rem} .controls label{display:inline-flex;align-items:center;gap:1ch} -select,input,button,textarea{font:inherit;font-size:.8rem;color:var(--ink);background:var(--well);border:1px solid var(--faint);padding:.25rem .8ch;border-radius:0} +select,input,button,textarea{max-width:100%;font:inherit;font-size:.8rem;color:var(--ink);background:var(--well);border:1px solid var(--faint);padding:.25rem .8ch;border-radius:0} select:focus-visible,input:focus-visible,button:focus-visible,textarea:focus-visible{outline:1px solid var(--water);outline-offset:1px} button{cursor:pointer;color:var(--dim)}button:hover{color:var(--hot);border-color:var(--water)}button:disabled{opacity:.5;cursor:default} button[aria-pressed="true"]{color:var(--water);border-color:var(--water)} -input{width:22ch}input::placeholder,textarea::placeholder{color:var(--faint)} -.section{position:relative;width:100%} -canvas#strata{display:block;width:100%;height:auto;cursor:crosshair;touch-action:pan-y} -.reader{width:100%;min-height:3.4rem;padding:.7rem 0 0;border-top:1px dotted var(--faint);font-size:.8rem;color:var(--dim);font-variant-numeric:tabular-nums;line-height:1.6} +input{width:20ch}input::placeholder,textarea::placeholder{color:var(--faint)} + +/* the layers: one grid row per app — name | activity strip | total. the name column is a fixed width so + opening an app never moves anything; children indent inside it. on phones the strip drops under the name. */ +.layers{position:relative;display:flex;flex-direction:column;gap:2px;--name:24ch;--total:7ch} +.lane{display:grid;grid-template-columns:var(--name) minmax(0,1fr) var(--total);column-gap:1.5ch;align-items:center;min-height:14px} +.lane.head{color:var(--dim);font-size:.7rem;letter-spacing:.16em;text-transform:uppercase;margin-bottom:.2rem} +.lane.head .count{text-align:right} +.lane.head .strip{background:none;height:auto} +.name{display:flex;align-items:center;gap:.6ch;background:none;border:0;padding:0;text-align:left;color:var(--ink);font-size:.8rem;min-width:0;cursor:default} +.name .glyph{color:var(--water);width:1.2ch;flex:none} +.name .label{overflow:hidden;text-overflow:ellipsis;white-space:nowrap;direction:rtl;text-align:left} +.name .label bdi{direction:ltr;unicode-bidi:isolate} +.name[aria-expanded]{cursor:pointer} +.name[aria-expanded]:hover .label,.name:focus-visible .label{color:var(--hot)} +.name:focus-visible{outline:1px solid var(--water);outline-offset:2px} +.lane.child .name{padding-left:2.4ch;color:var(--dim);font-size:.76rem} +.lane.child .name .glyph{color:var(--faint)} +.strip{position:relative;height:var(--h,20px);background:var(--well);min-width:0} +.strip svg{display:block;width:100%;height:100%} +.count{font-size:.76rem;color:var(--dim);text-align:right;font-variant-numeric:tabular-nums;white-space:nowrap} +.lane.child .count{color:var(--faint)} +.cursor{position:absolute;top:0;bottom:0;width:0;border-left:1px solid var(--hot);opacity:.7;pointer-events:none;display:none} +.axis .strip{height:32px;background:none} +.axis .era{position:absolute;top:4px;height:2px;background:var(--faint)} +.axis .era.live{background:var(--water)} +.axis .era span{position:absolute;top:6px;left:0;font-size:.7rem;color:var(--dim);white-space:nowrap} +.axis .era.live span{color:var(--water)} +.axis .tick{position:absolute;top:20px;font-size:.7rem;color:var(--dim);white-space:nowrap} +.axis .tick.end{transform:translateX(-100%)} +.reader{min-height:3.4rem;padding:.7rem 0 0;margin-top:.6rem;border-top:1px dotted var(--faint);font-size:.8rem;color:var(--dim);font-variant-numeric:tabular-nums;line-height:1.6} .reader b{color:var(--hot);font-weight:400}.reader i{font-style:normal;color:var(--water)} -.legend{display:flex;flex-wrap:wrap;align-items:center;justify-content:center;gap:.4rem 1.6ch;font-size:.74rem;color:var(--dim);width:100%} +.legend{display:flex;flex-wrap:wrap;align-items:center;justify-content:center;gap:.4rem 1.6ch;font-size:.74rem;color:var(--dim)} .legend .bar{width:22ch;height:8px;border:1px solid var(--faint)} .legend .none{display:inline-block;width:1.2em;height:8px;background:var(--well);border:1px solid var(--faint);vertical-align:-1px;margin-right:.6ch} -.note{color:var(--dim);font-size:.76rem;max-width:74ch;text-align:center;line-height:1.6} -.note em{font-style:normal;color:var(--ink)} +.note{color:var(--dim);font-size:.76rem;max-width:74ch;margin:0 auto;text-align:center;line-height:1.6} .table-wrap{width:100%;overflow-x:auto} table{border-collapse:collapse;font-size:.76rem;font-variant-numeric:tabular-nums;margin:0 auto} th,td{text-align:right;padding:.2rem 1.2ch;border-bottom:1px solid var(--well);white-space:nowrap;color:var(--dim)} td:first-child,th:first-child{text-align:left;color:var(--ink);position:sticky;left:0;background:var(--bg)} th{color:var(--dim);font-weight:400;letter-spacing:.1em;text-transform:uppercase;font-size:.66rem} -.ask{width:100%;max-width:100ch;display:flex;flex-direction:column;gap:.7rem} +.ask{width:100%;max-width:100ch;margin:0 auto;display:flex;flex-direction:column;gap:.7rem} .ask .row{display:flex;flex-wrap:wrap;gap:.5rem 2ch;align-items:center;color:var(--dim);font-size:.8rem} +.ask .row label{display:inline-flex;align-items:center;gap:1ch;max-width:100%;min-width:0} +#ask-ns{width:34ch;max-width:100%;min-width:0} .ask textarea{width:100%;min-height:5.5rem;resize:vertical;line-height:1.5;tab-size:2} .ask .prompts{display:flex;flex-wrap:wrap;gap:.4rem 1ch} .ask .prompts button{font-size:.74rem} .ask .out{font-size:.78rem;color:var(--dim);min-height:1.5rem} .ask .out b{color:var(--hot);font-weight:400} .ask .err{color:#d99a8a;white-space:pre-wrap} +.ask .fine{font-size:.74rem;color:var(--faint)} footer{display:flex;flex-wrap:wrap;justify-content:center;align-items:center;gap:.45rem 3ch;padding:2rem 0 1.5rem;color:var(--dim);font-size:.76rem} footer a{color:var(--dim);border-bottom:1px dotted var(--faint);white-space:nowrap} footer a:hover{color:var(--hot);border-bottom-color:var(--water)} .status{color:var(--faint)} -@media (max-width:640px){.wrap{padding:0 1.5ch}.grid{gap:1rem 2ch}.stat{flex:0 1 11ch}.tag{max-width:38ch}input{width:16ch}} +@media (max-width:640px){ + .wrap{padding:0 1.5ch}.grid{gap:1rem 2ch}.stat{flex:0 1 11ch}.tag{max-width:38ch}input{width:16ch} + .layers{--name:1fr;--total:auto;gap:6px} + .lane{grid-template-columns:1fr auto;row-gap:2px;column-gap:1ch} + .lane .strip{grid-column:1/-1} + .lane.head .strip{display:none} + .lane.child{margin-left:0} + .ask .row label{width:100%} + #ask-ns{width:100%} +} @@ -79,13 +116,13 @@ footer a:hover{color:var(--hot);border-bottom-color:var(--water)}
—records
—apps
—record types
-
—archive chunks
-

each row is an app, named by the identifier its records carry (app.bsky is Bluesky, sh.tangled is Tangled). a taller row has written more records overall. a brighter cell means that app was busier in that slice of the archive. ▸ opens an app to show its record types.

+

each row is an app, named by the identifier its records carry — app.bsky is Bluesky, sh.tangled is Tangled.
+ taller = more records overall · brighter = busier in that slice of the archive · ▸ = open it to see its record types

- +
@@ -135,277 +176,6 @@ footer a:hover{color:var(--hot);border-bottom-color:var(--water)} source - - + diff --git a/src/text-modules.d.ts b/src/text-modules.d.ts index 9b4de38..ea7a05c 100644 --- a/src/text-modules.d.ts +++ b/src/text-modules.d.ts @@ -2,3 +2,7 @@ declare module "*.html" { const text: string; export default text; } +declare module "*/ui.js" { + const text: string; + export default text; +} diff --git a/src/ui.js b/src/ui.js new file mode 100644 index 0000000..0744a43 --- /dev/null +++ b/src/ui.js @@ -0,0 +1,428 @@ +// strata's page: the layers picture (one DOM lane per app), the reader line, and "look inside an app" +// (duckdb-wasm over the app's public parquet files). served by the worker at ui.js; talks to api/* relatively. + +const fmt = new Intl.NumberFormat("en-US"); +const compact = new Intl.NumberFormat("en-US", { notation: "compact", maximumFractionDigits: 1 }); +const dayFmt = new Intl.DateTimeFormat("en-CA", { timeZone: "UTC", year: "numeric", month: "2-digit", day: "2-digit" }); +const timeFmt = new Intl.DateTimeFormat("en-GB", { timeZone: "UTC", hour: "2-digit", minute: "2-digit" }); +const STEPS = [1, 2, 5, 10, 25, 50, 100]; +const FLOOR = 0.14; +const OTHER = "other"; +const OTHER_LABEL = "everything else"; +// the whole-network bootstrap ran 2026-07-28 → 2026-08-03 ~07:00 utc (stream /status: "backfill took 6d 2h") +const BOOTSTRAP_END_US = 1785740400000000; +const ERA_LABEL = { bootstrap: "history, replayed", live: "as it happened" }; + +function byId(id, ctor) { + const node = document.getElementById(id); + if (!(node instanceof ctor)) throw new Error(`missing #${id} (${ctor.name})`); + return node; +} +const el = { + layers: byId("layers", HTMLDivElement), + cursor: byId("cursor", HTMLDivElement), + reader: byId("reader", HTMLDivElement), + step: byId("step", HTMLSelectElement), + scale: byId("scale", HTMLSelectElement), + find: byId("find", HTMLInputElement), + nsList: byId("ns-list", HTMLDataListElement), + more: byId("more", HTMLButtonElement), + tableToggle: byId("table-toggle", HTMLButtonElement), + table: byId("table", HTMLDivElement), + live: byId("live", HTMLSpanElement), + updated: byId("updated", HTMLSpanElement), + legendBar: byId("legend-bar", HTMLSpanElement), + legendNote: byId("legend-note", HTMLSpanElement), + askNs: byId("ask-ns", HTMLSelectElement), + askSize: byId("ask-size", HTMLSpanElement), + askRun: byId("ask-run", HTMLButtonElement), + askEdit: byId("ask-edit", HTMLButtonElement), + askSql: byId("ask-sql", HTMLTextAreaElement), + askFine: byId("ask-fine", HTMLParagraphElement), + askOut: byId("ask-out", HTMLDivElement), + askTable: byId("ask-table", HTMLDivElement), + prompts: byId("prompts", HTMLDivElement), +}; +const stat = { events: byId("s-events", HTMLSpanElement), apps: byId("s-namespaces", HTMLSpanElement), types: byId("s-collections", HTMLSpanElement) }; + +const esc = (s) => String(s).replace(/[&<>"]/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """ }[c])); +const displayName = (key) => (key === OTHER ? OTHER_LABEL : key); +const shortName = (nsid) => nsid.slice(nsid.indexOf(".", nsid.indexOf(".") + 1) + 1); +// one hue, dark → light, with a floor so a cell with anything in it never fades into the empty tone +function shade(t) { + const u = FLOOR + (1 - FLOOR) * Math.max(0, Math.min(1, t)); + return `oklch(${(0.27 + 0.66 * u).toFixed(3)} ${(0.045 + 0.08 * u * (1 - u)).toFixed(3)} 222)`; +} +const rowHeight = (count, maxCount) => Math.round(12 + 34 * (Math.log10(Math.max(1, count)) / Math.log10(Math.max(10, maxCount)))); + +async function api(path) { + const r = await fetch(path); + return r.json(); +} + +const view = { top: 12, expanded: new Map(), pinned: [], namespaces: [], stepChosen: false }; +let picture = null; // { rows, buckets, step } +let hover = null; // { ri, bi } + +for (const s of STEPS) { + const o = document.createElement("option"); + o.value = String(s); + o.textContent = s === 1 ? "1 chunk per column" : `${s} chunks per column`; + el.step.append(o); +} + +async function loadProgress() { + const p = await api("api/progress"); + stat.events.textContent = compact.format(p.events); + stat.apps.textContent = fmt.format(p.namespaces); + stat.types.textContent = fmt.format(p.collections); + if (p.updatedAt) { + const d = new Date(p.updatedAt); + el.live.innerHTML = `· ${fmt.format(p.segments)} chunks read · ${compact.format(p.events)} records`; + el.updated.textContent = `updated ${dayFmt.format(d)} ${timeFmt.format(d)} utc`; + } + if (!view.stepChosen) el.step.value = String(STEPS.find((s) => Math.ceil(p.segments / s) <= 160) ?? STEPS[STEPS.length - 1]); + if (!view.namespaces.length) { + const { namespaces } = await api("api/namespaces"); + view.namespaces = namespaces; + el.nsList.innerHTML = namespaces.map((n) => ``).join(""); + el.askNs.innerHTML = `` + namespaces.filter((n) => n.ns !== "app.bsky").map((n) => ``).join(""); + } +} + +function seriesFor(cells, key, bucketIndex, n) { + const a = new Float64Array(n); + for (const c of cells) { + if (c.key !== key) continue; + const i = bucketIndex.get(c.bucket); + if (i !== undefined) a[i] += c.count; + } + return a; +} + +async function load() { + await loadProgress(); + const step = Number(el.step.value); + const heat = await api(`api/heat?step=${step}&top=${view.top}`); + if (heat.error) { + el.reader.innerHTML = `${esc(heat.error)}`; + return; + } + const buckets = heat.buckets; + const bucketIndex = new Map(buckets.map((b, i) => [b.bucket, i])); + const keys = new Map(heat.keys.map((k) => [k.key, k])); + const order = heat.keys.map((k) => k.key); + const extra = view.pinned.filter((p) => !keys.has(p.ns)); + const extraHeat = await Promise.all(extra.map((p) => api(`api/heat?step=${step}&ns=${p.ns}`))); + extra.forEach((p, i) => { + keys.set(p.ns, { key: p.ns, count: p.count, members: p.members, cells: extraHeat[i].cells }); + order.push(p.ns); + }); + if (heat.cells.some((c) => c.key === OTHER)) { + order.push(OTHER); + keys.set(OTHER, { key: OTHER, count: 0, members: 0 }); + } + const rows = []; + for (const key of order) { + const k = keys.get(key); + const values = k.cells ? seriesFor(k.cells, key, bucketIndex, buckets.length) : seriesFor(heat.cells, key, bucketIndex, buckets.length); + const total = key === OTHER ? values.reduce((a, b) => a + b, 0) : k.count; + rows.push({ key, depth: 0, total, members: k.members, values, expandable: key !== OTHER && k.members > 1 }); + const sub = view.expanded.get(key); + if (sub && sub.step === step) rows.push(...sub.rows); + } + picture = { rows, buckets, step }; + for (const [key, sub] of view.expanded) if (sub.step !== step) expand(key); + render(); +} + +async function expand(key) { + const step = Number(el.step.value); + const sub = await api(`api/heat?step=${step}&ns=${key}`); + const bucketIndex = new Map(sub.buckets.map((b, i) => [b.bucket, i])); + const rows = sub.keys.map((k) => ({ key: k.key, depth: 1, total: k.count, members: 1, values: seriesFor(sub.cells, k.key, bucketIndex, sub.buckets.length), expandable: false })); + view.expanded.set(key, { step, rows }); + if (picture && picture.step === step) { + dropChildren(key); + const at = picture.rows.findIndex((r) => r.key === key && r.depth === 0); + if (at >= 0) picture.rows.splice(at + 1, 0, ...rows); + } + render(); +} +function dropChildren(key) { + const at = picture.rows.findIndex((r) => r.key === key && r.depth === 0); + if (at < 0) return; + let end = at + 1; + while (end < picture.rows.length && picture.rows[end].depth === 1) end++; + picture.rows.splice(at + 1, end - at - 1); +} +function collapse(key) { + view.expanded.delete(key); + if (picture) dropChildren(key); + render(); +} + +const eraOf = (b) => (b.min_witnessed_at < BOOTSTRAP_END_US ? "bootstrap" : "live"); + +function render() { + if (!picture) return; + const { rows, buckets } = picture; + const scale = el.scale.value; + const n = buckets.length; + const maxTotal = Math.max(...rows.filter((r) => r.depth === 0).map((r) => r.total)); + const maxPerRow = rows.map((r) => Math.max(0, ...r.values)); + const logMax = Math.log10(Math.max(10, ...maxPerRow)); + const cellW = Math.max(0.5, 1 - Math.min(0.15, 24 / n)); // a hairline gap when columns are wide enough to show one + + const lanes = rows.map((r, ri) => { + const h = Math.max(r.depth ? 11 : 12, rowHeight(r.total, maxTotal) - (r.depth ? 8 : 0)); + const rects = []; + for (let bi = 0; bi < n; bi++) { + const v = r.values[bi]; + if (v <= 0) continue; + const t = scale === "row" ? v / maxPerRow[ri] : Math.log10(v) / logMax; + rects.push(``); + } + const glyph = r.expandable ? (view.expanded.has(r.key) ? "▾" : "▸") : r.depth ? "·" : ""; + const label = r.depth ? shortName(r.key) : displayName(r.key); + const expandAttr = r.expandable ? ` aria-expanded="${view.expanded.has(r.key)}"` : ""; + return `
` + + `` + + `
` + + `${compact.format(r.total)}
`; + }); + + const eras = []; + let start = 0; + for (let bi = 1; bi <= n; bi++) { + if (bi === n || eraOf(buckets[bi]) !== eraOf(buckets[start])) { + const era = eraOf(buckets[start]); + const wide = bi - start > n / 6; + eras.push(`
${wide ? `${ERA_LABEL[era]}` : ""}
`); + start = bi; + } + } + const ticks = []; + let lastDay = ""; + let lastAt = -1; + for (let bi = 0; bi < n; bi++) { + const day = dayFmt.format(new Date(buckets[bi].min_witnessed_at / 1000)); + if (day !== lastDay && bi - lastAt > n / 8) { + ticks.push(`${day}`); + lastDay = day; + lastAt = bi; + } + } + const axis = `
${eras.join("")}${ticks.join("")}
`; + + el.layers.innerHTML = `
apprecords
${lanes.join("")}${axis}
`; + el.cursor = byId("cursor", HTMLDivElement); + el.legendBar.style.background = `linear-gradient(90deg, ${[0, 0.25, 0.5, 0.75, 1].map(shade).join(",")})`; + el.legendNote.textContent = scale === "row" ? "· brightest = that app's busiest column" : "· same scale for every app (log)"; + renderTable(); +} + +function cellFromPointer(ev) { + if (!picture) return null; + const lane = ev.target instanceof Element ? ev.target.closest(".lane[data-ri]") : null; + if (!lane) return null; + const strip = lane.querySelector(".strip"); + if (!(strip instanceof HTMLElement)) return null; + const rect = strip.getBoundingClientRect(); + const ri = Number(lane.dataset.ri); + const x = (ev.clientX - rect.left) / rect.width; + const bi = x < 0 || x >= 1 ? -1 : Math.floor(x * picture.buckets.length); + return { ri, bi, rect }; +} + +function readCell(c) { + const r = picture.rows[c.ri]; + const what = r.depth ? "record type" : r.key === OTHER ? "the apps not shown" : "app"; + if (c.bi < 0) { + el.reader.innerHTML = `${esc(displayName(r.key))} · ${what} · ${fmt.format(r.total)} records in the archive${r.expandable ? ` across ${r.members} record types · click the name to open` : ""}`; + return; + } + const b = picture.buckets[c.bi]; + const v = r.values[c.bi]; + const from0 = dayFmt.format(new Date(b.min_witnessed_at / 1000)); + if (v <= 0) { + el.reader.innerHTML = `nothing from ${esc(displayName(r.key))} in this slice (received ${from0} utc)`; + return; + } + const share = b.event_count ? (100 * v) / b.event_count : 0; + const from = dayFmt.format(new Date(b.min_witnessed_at / 1000)); + const to = dayFmt.format(new Date(b.max_witnessed_at / 1000)); + const chunk = b.last_idx !== b.first_idx ? `chunks #${b.first_idx}–${b.last_idx}` : `chunk #${b.first_idx}`; + el.reader.innerHTML = + `${esc(displayName(r.key))} wrote ${fmt.format(v)} records in this slice — ${share < 0.01 && v ? "under 0.01" : share.toFixed(2)}% of the ${compact.format(b.event_count)} in it
` + + `archive ${chunk} · received ${from}${to !== from ? ` → ${to}` : ""} utc · record numbers ${fmt.format(b.min_seq)} – ${fmt.format(b.max_seq)}`; +} + +function showCursor(c) { + if (c.bi < 0) { + el.cursor.style.display = "none"; + return; + } + const host = el.layers.getBoundingClientRect(); + const w = c.rect.width / picture.buckets.length; + el.cursor.style.display = "block"; + el.cursor.style.left = `${c.rect.left - host.left + c.bi * w}px`; + el.cursor.style.width = `${Math.max(w, 1)}px`; + el.cursor.style.borderLeftWidth = w >= 3 ? "0" : "1px"; + el.cursor.style.background = w >= 3 ? "rgba(226,235,240,.08)" : "transparent"; +} + +function clearHover() { + hover = null; + el.cursor.style.display = "none"; + el.reader.innerHTML = 'move over the picture to read a cell · open an app to see its record types'; +} + +el.layers.addEventListener("pointermove", (ev) => { + const c = cellFromPointer(ev); + if (!c) { + clearHover(); + return; + } + hover = { ri: c.ri, bi: c.bi }; + showCursor(c); + readCell(c); +}); +el.layers.addEventListener("pointerleave", clearHover); +el.layers.addEventListener("click", (ev) => { + const button = ev.target instanceof Element ? ev.target.closest("button.name[aria-expanded]") : null; + if (!(button instanceof HTMLButtonElement)) return; + const key = button.dataset.key ?? ""; + if (view.expanded.has(key)) collapse(key); + else expand(key); +}); + +function renderTable() { + if (!picture || el.table.hidden) return; + const { rows, buckets } = picture; + el.table.innerHTML = + `${buckets.map((b) => ``).join("")}` + + rows.map((r) => `${Array.from(r.values, (v) => ``).join("")}`).join("") + + `
app / record typerecords#${b.first_idx}
${r.depth ? "  · " : ""}${esc(displayName(r.key))}${fmt.format(r.total)}${v ? fmt.format(v) : ""}
`; +} +el.tableToggle.addEventListener("click", () => { + el.table.hidden = !el.table.hidden; + el.tableToggle.setAttribute("aria-pressed", String(!el.table.hidden)); + renderTable(); +}); +el.more.addEventListener("click", () => { + view.top = Math.min(60, view.top + 12); + el.more.disabled = view.top >= 60; + load(); +}); +el.step.addEventListener("change", () => { + view.stepChosen = true; + load(); +}); +el.scale.addEventListener("change", render); +el.find.addEventListener("change", () => { + const ns = el.find.value.trim().toLowerCase(); + const hit = view.namespaces.find((n) => n.ns === ns); + if (!hit) { + el.reader.innerHTML = `no app called ${esc(ns)} has written to the archive yet`; + return; + } + if (!view.pinned.some((p) => p.ns === ns)) view.pinned.push(hit); + el.find.value = ""; + load(); +}); +window.addEventListener("resize", () => { + if (hover) el.cursor.style.display = "none"; +}); + +// ---- look inside an app ---- +const PROMPTS = [ + ["what it writes", "select collection as record_type, op as action, count(*) as records, count(distinct did) as accounts\nfrom rows group by all order by records desc"], + ["people per day", "select strftime(make_timestamp(time_us), '%Y-%m-%d') as day, count(distinct did) as accounts, count(*) as records\nfrom rows where kind = 'commit' group by 1 order by 1"], + ["when each type first appeared", "select collection as record_type, strftime(make_timestamp(min(time_us)), '%Y-%m-%d %H:%M') as first_seen, count(*) as records\nfrom rows where kind = 'commit' group by 1 order by min(time_us)"], + ["most active accounts", "select did as account, count(*) as records, count(distinct collection) as record_types\nfrom rows where kind = 'commit' group by 1 order by 2 desc limit 20"], + ["creates vs deletes", "select op as action, count(*) as records from rows where kind = 'commit' group by 1 order by 2 desc"], +]; +for (const [label, sql] of PROMPTS) { + const b = document.createElement("button"); + b.type = "button"; + b.textContent = label; + b.addEventListener("click", () => { + el.askSql.value = sql; + if (el.askNs.value) runAsk(); + }); + el.prompts.append(b); +} +el.askSql.value = PROMPTS[0][1]; +el.askEdit.addEventListener("click", () => { + el.askSql.hidden = !el.askSql.hidden; + el.askFine.hidden = el.askSql.hidden; + el.askEdit.setAttribute("aria-pressed", String(!el.askSql.hidden)); + if (!el.askSql.hidden) el.askSql.focus(); +}); + +let duck = null; +let partition = null; +async function duckdb() { + if (duck) return duck; + el.askOut.innerHTML = "loading the query engine…"; + const mod = await import("https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm@1.29.0/+esm"); + const bundle = await mod.selectBundle(mod.getJsDelivrBundles()); + const worker = new Worker(URL.createObjectURL(new Blob([`importScripts("${bundle.mainWorker}");`], { type: "text/javascript" }))); + const db = new mod.AsyncDuckDB(new mod.ConsoleLogger(mod.LogLevel.WARNING), worker); + await db.instantiate(bundle.mainModule, bundle.pthreadWorker); + duck = await db.connect(); + return duck; +} +el.askNs.addEventListener("change", async () => { + partition = null; + el.askRun.disabled = true; + el.askTable.innerHTML = ""; + el.askOut.innerHTML = ""; + if (!el.askNs.value) { + el.askSize.textContent = "pick one"; + return; + } + el.askSize.textContent = "checking…"; + const p = await api(`api/partitions?ns=${el.askNs.value}`); + if (p.error) { + el.askSize.textContent = p.error; + return; + } + partition = p; + const size = p.bytes < 1 << 20 ? `${Math.round(p.bytes / 1024)} KB` : `${(p.bytes / (1 << 20)).toFixed(1)} MB`; + el.askSize.textContent = p.files.length ? `${size} to read from ${p.files.length} chunk${p.files.length === 1 ? "" : "s"}` : "not copied out of the archive yet"; + el.askRun.disabled = !p.files.length; +}); +function formatCell(value, numeric) { + if (value === null || value === undefined) return ""; + return numeric ? fmt.format(value) : esc(value); +} +async function runAsk() { + if (!partition || !partition.files.length) return; + el.askRun.disabled = true; + try { + const conn = await duckdb(); + await conn.query(`create or replace view rows as select * from read_parquet([${partition.files.map((f) => `'${f}'`).join(", ")}])`); + const t = performance.now(); + const res = await conn.query(el.askSql.value); + const fields = res.schema.fields.map((f) => ({ name: f.name, numeric: /Int|Float|Decimal/.test(String(f.type)) })); + const out = res.toArray().map((r) => r.toJSON()); + el.askOut.innerHTML = `${fmt.format(out.length)} row${out.length === 1 ? "" : "s"} in ${(performance.now() - t).toFixed(0)} ms`; + el.askTable.innerHTML = + `${fields.map((f) => ``).join("")}` + + out.slice(0, 500).map((r) => `${fields.map((f) => ``).join("")}`).join("") + + `
${esc(f.name.replaceAll("_", " "))}
${formatCell(r[f.name], f.numeric)}
` + + (out.length > 500 ? `

showing 500 of ${fmt.format(out.length)}

` : ""); + } catch (err) { + el.askOut.innerHTML = `${esc(err instanceof Error ? err.message : err)}`; + } finally { + el.askRun.disabled = false; + } +} +el.askRun.addEventListener("click", runAsk); +el.askSql.addEventListener("keydown", (ev) => { + if ((ev.metaKey || ev.ctrlKey) && ev.key === "Enter") runAsk(); +}); + +load(); diff --git a/tests/ingest.test.ts b/tests/ingest.test.ts index 6090fdb..e8087d6 100644 --- a/tests/ingest.test.ts +++ b/tests/ingest.test.ts @@ -1,7 +1,9 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { IngestParseError, namespaceOf, parseIngestBatch } from "../src/ingest"; +import { safeParse } from "@atcute/lexicons/validations"; + +import { IngestBatchSchema, checkIngestBatch, namespaceOf } from "../src/ingest"; const segment = { idx: 6000, @@ -21,27 +23,33 @@ const segment = { } as const; test("accepts a well-formed batch and preserves values", () => { - const batch = parseIngestBatch({ segments: [segment] }); - assert.equal(batch.segments.length, 1); - assert.deepEqual(batch.segments[0], segment); + const result = safeParse(IngestBatchSchema, { segments: [segment] }); + assert.ok(result.ok); + assert.equal(result.value.segments.length, 1); + assert.deepEqual(result.value.segments[0], segment); + assert.equal(checkIngestBatch(result.value), null); }); test("rejects an empty batch", () => { - assert.throws(() => parseIngestBatch({ segments: [] }), IngestParseError); + assert.equal(safeParse(IngestBatchSchema, { segments: [] }).ok, false); }); -test("rejects duplicate collections within a segment", () => { - const dup = { ...segment, collections: [...segment.collections, { nsid: "app.bsky.feed.like", count: 1 }] }; - assert.throws(() => parseIngestBatch({ segments: [dup] }), /duplicate nsid/); +test("rejects non-integer counts", () => { + const bad = { ...segment, collections: [{ nsid: "app.bsky.feed.like", count: "many" }] }; + assert.equal(safeParse(IngestBatchSchema, { segments: [bad] }).ok, false); }); -test("rejects seq ranges that run backwards", () => { - assert.throws(() => parseIngestBatch({ segments: [{ ...segment, minSeq: 5, maxSeq: 4 }] }), /maxSeq < minSeq/); +test("flags duplicate collections within a segment", () => { + const dup = { ...segment, collections: [...segment.collections, { nsid: "app.bsky.feed.like", count: 1 }] }; + const result = safeParse(IngestBatchSchema, { segments: [dup] }); + assert.ok(result.ok); + assert.match(checkIngestBatch(result.value) ?? "", /duplicate nsid/); }); -test("rejects non-integer counts", () => { - const bad = { ...segment, collections: [{ nsid: "app.bsky.feed.like", count: "many" }] }; - assert.throws(() => parseIngestBatch({ segments: [bad] }), /count: expected a non-negative integer/); +test("flags seq ranges that run backwards", () => { + const result = safeParse(IngestBatchSchema, { segments: [{ ...segment, minSeq: 5, maxSeq: 4 }] }); + assert.ok(result.ok); + assert.match(checkIngestBatch(result.value) ?? "", /maxSeq < minSeq/); }); test("namespaceOf takes the first two labels", () => { diff --git a/tools/oxlint/anti-slop/index.ts b/tools/oxlint/anti-slop/index.ts new file mode 100644 index 0000000..2b4ae22 --- /dev/null +++ b/tools/oxlint/anti-slop/index.ts @@ -0,0 +1,41 @@ +import { eslintCompatPlugin } from "@oxlint/plugins"; + +import { noChainedTypeAssertionsRule } from "./rules/no-chained-type-assertions.ts"; +import { noConditionalEmptyObjectSpreadRule } from "./rules/no-conditional-empty-object-spread.ts"; +import { noKnownValueWideningRule } from "./rules/no-known-value-widening.ts"; +import { noModuleMockingRule } from "./rules/no-module-mocking.ts"; +import { noObjectParametersRule } from "./rules/no-object-parameters.ts"; +import { noReflectApplyRule } from "./rules/no-reflect-apply.ts"; +import { noReflectGetRule } from "./rules/no-reflect-get.ts"; +import { noRuntimeTypeofRule } from "./rules/no-runtime-typeof.ts"; +import { noForbiddenTermInSymbolNamesRule } from "./rules/no-shape-in-symbol-names.ts"; +import { noUnknownParametersRule } from "./rules/no-unknown-parameters.ts"; +import { noUnknownReturnsRule } from "./rules/no-unknown-returns.ts"; +import { noUnknownTypeAliasesRule } from "./rules/no-unknown-type-aliases.ts"; +import { noUnsafeDictionaryTypeRule } from "./rules/no-unsafe-dictionary-type.ts"; +import { noWidenThenAssertRule } from "./rules/no-widen-then-assert.ts"; +import { requireSafetyCommentForTypeAssertionRule } from "./rules/require-safety-comment-for-type-assertion.ts"; + +/** Generic Oxlint rules that reject low-evidence and low-signal implementation patterns. */ +const antiSlopPlugin = eslintCompatPlugin({ + meta: { name: "anti-slop" }, + rules: { + "no-chained-type-assertions": noChainedTypeAssertionsRule, + "no-conditional-empty-object-spread": noConditionalEmptyObjectSpreadRule, + "no-known-value-widening": noKnownValueWideningRule, + "no-module-mocking": noModuleMockingRule, + "no-object-parameters": noObjectParametersRule, + "no-reflect-apply": noReflectApplyRule, + "no-reflect-get": noReflectGetRule, + "no-runtime-typeof": noRuntimeTypeofRule, + "no-unsafe-dictionary-type": noUnsafeDictionaryTypeRule, + "no-shape-in-symbol-names": noForbiddenTermInSymbolNamesRule, + "no-unknown-parameters": noUnknownParametersRule, + "no-unknown-returns": noUnknownReturnsRule, + "no-unknown-type-aliases": noUnknownTypeAliasesRule, + "no-widen-then-assert": noWidenThenAssertRule, + "require-safety-comment-for-type-assertion": requireSafetyCommentForTypeAssertionRule, + }, +}); + +export default antiSlopPlugin; diff --git a/tools/oxlint/anti-slop/rules/no-chained-type-assertions.ts b/tools/oxlint/anti-slop/rules/no-chained-type-assertions.ts new file mode 100644 index 0000000..0d11852 --- /dev/null +++ b/tools/oxlint/anti-slop/rules/no-chained-type-assertions.ts @@ -0,0 +1,77 @@ +import { defineRule } from "@oxlint/plugins"; +import type { ESTree } from "@oxlint/plugins"; + +type TypeAssertionExpression = ESTree.TSAsExpression | ESTree.TSTypeAssertion; + +function isTypeAssertionExpression(node: ESTree.Node): node is TypeAssertionExpression { + return node.type === "TSAsExpression" || node.type === "TSTypeAssertion"; +} + +function unwrapParenthesizedExpression(expression: ESTree.Expression): ESTree.Expression { + let current = expression; + while (current.type === "ParenthesizedExpression") { + current = current.expression; + } + return current; +} + +function isConstAssertion(node: TypeAssertionExpression): boolean { + const { typeAnnotation } = node; + return ( + typeAnnotation.type === "TSTypeReference" && + typeAnnotation.typeName.type === "Identifier" && + typeAnnotation.typeName.name === "const" + ); +} + +function isOutermostAssertionInChain(node: TypeAssertionExpression): boolean { + let current: ESTree.Expression = node; + let parent = node.parent; + + while (parent.type === "ParenthesizedExpression" && parent.expression === current) { + current = parent; + parent = parent.parent; + } + + return !isTypeAssertionExpression(parent) || parent.expression !== current; +} + +function isForbiddenAssertionChain(node: TypeAssertionExpression): boolean { + let assertionCount = 0; + let hasNonConstAssertion = false; + let current: ESTree.Expression = node; + + while (isTypeAssertionExpression(current)) { + assertionCount += 1; + hasNonConstAssertion ||= !isConstAssertion(current); + current = unwrapParenthesizedExpression(current.expression); + } + + return assertionCount > 1 && hasNonConstAssertion; +} + +/** Disallow nested TypeScript type assertions, while permitting chains made only of const assertions. */ +export const noChainedTypeAssertionsRule = defineRule({ + meta: { + type: "problem", + docs: { + description: + "Disallow chained TypeScript as and angle-bracket assertions, including parenthesized chains.", + }, + messages: { + chained: + "This assertion chain discards type evidence. Keep the original precise type, or parse untrusted input at its boundary before narrowing it.", + }, + }, + createOnce(context) { + const checkTypeAssertion = (node: TypeAssertionExpression) => { + if (!isOutermostAssertionInChain(node) || !isForbiddenAssertionChain(node)) return; + context.report({ node, messageId: "chained" }); + }; + + return { + TSAsExpression: checkTypeAssertion, + TSTypeAssertion: checkTypeAssertion, + }; + }, +}); diff --git a/tools/oxlint/anti-slop/rules/no-conditional-empty-object-spread.ts b/tools/oxlint/anti-slop/rules/no-conditional-empty-object-spread.ts new file mode 100644 index 0000000..ae7248d --- /dev/null +++ b/tools/oxlint/anti-slop/rules/no-conditional-empty-object-spread.ts @@ -0,0 +1,49 @@ +import { defineRule } from "@oxlint/plugins"; +import type { ESTree } from "@oxlint/plugins"; + +function unwrapParentheses(node: ESTree.Expression): ESTree.Expression { + let current = node; + while (current.type === "ParenthesizedExpression") { + current = current.expression; + } + return current; +} + +function isEmptyObjectExpression(node: ESTree.Expression): boolean { + return node.type === "ObjectExpression" && node.properties.length === 0; +} + +function isConditionalEmptyObjectSpread(node: ESTree.Expression): boolean { + const conditional = unwrapParentheses(node); + return ( + conditional.type === "ConditionalExpression" && + (isEmptyObjectExpression(conditional.consequent) || + isEmptyObjectExpression(conditional.alternate)) + ); +} + +/** Ban conditional empty-object spreads without changing their omission semantics. */ +export const noConditionalEmptyObjectSpreadRule = defineRule({ + meta: { + type: "suggestion", + docs: { + description: + "Disallow object spreads that conditionally spread an empty object to omit fields.", + }, + messages: { + avoid: + "This conditional spread hides property omission behind an empty object. Build the object in separate statements and add the property only when present.", + }, + }, + createOnce(context) { + return { + SpreadElement(node) { + if (node.parent.type !== "ObjectExpression") return; + + if (isConditionalEmptyObjectSpread(node.argument)) { + context.report({ node, messageId: "avoid" }); + } + }, + }; + }, +}); diff --git a/tools/oxlint/anti-slop/rules/no-known-value-widening.ts b/tools/oxlint/anti-slop/rules/no-known-value-widening.ts new file mode 100644 index 0000000..2a6806c --- /dev/null +++ b/tools/oxlint/anti-slop/rules/no-known-value-widening.ts @@ -0,0 +1,247 @@ +import { defineRule } from "@oxlint/plugins"; + +import { + classifyWideningTarget, + createTypeEnvironment, + isKnownEvidenceExpression, + type TypeEnvironment, + type WideningTarget, +} from "../shared/dictionary-types.ts"; + +import type { ESTree, Scope, SourceCode, Variable } from "@oxlint/plugins"; + +type FunctionExpression = ESTree.ArrowFunctionExpression | ESTree.Function; + +function unwrapExpression(expression: ESTree.Expression): ESTree.Expression { + let current = expression; + while ( + current.type === "ParenthesizedExpression" || + current.type === "TSAsExpression" || + current.type === "TSSatisfiesExpression" || + current.type === "TSTypeAssertion" || + current.type === "TSNonNullExpression" + ) { + current = current.expression; + } + return current; +} + +function resolveVariable( + sourceCode: SourceCode, + identifier: ESTree.IdentifierReference, +): Variable | null { + let scope: Scope | null = sourceCode.getScope(identifier); + while (scope !== null) { + const variable = scope.set.get(identifier.name); + if (variable !== undefined) return variable; + scope = scope.upper; + } + return null; +} + +function variableDeclarator(variable: Variable): ESTree.VariableDeclarator | null { + if (variable.defs.length !== 1) return null; + const [definition] = variable.defs; + return definition?.type === "Variable" && definition.node.type === "VariableDeclarator" + ? definition.node + : null; +} + +function isStableConstVariable(variable: Variable, declarator: ESTree.VariableDeclarator): boolean { + return ( + declarator.parent.type === "VariableDeclaration" && + declarator.parent.kind === "const" && + variable.references.every((reference) => reference.init || !reference.isWrite()) + ); +} + +function hasKnownEvidence( + sourceCode: SourceCode, + expression: ESTree.Expression, + visitedVariables = new Set(), +): boolean { + if (isKnownEvidenceExpression(expression)) return true; + const unwrapped = unwrapExpression(expression); + if (unwrapped.type !== "Identifier") return false; + const variable = resolveVariable(sourceCode, unwrapped); + if (variable === null || visitedVariables.has(variable)) return false; + const declarator = variableDeclarator(variable); + if ( + declarator === null || + declarator.init === null || + !isStableConstVariable(variable, declarator) + ) { + return false; + } + visitedVariables.add(variable); + return hasKnownEvidence(sourceCode, declarator.init, visitedVariables); +} + +function annotationTarget( + annotation: ESTree.TSTypeAnnotation | null | undefined, + environment: TypeEnvironment, +): WideningTarget | null { + return annotation === null || annotation === undefined + ? null + : classifyWideningTarget(annotation.typeAnnotation, environment); +} + +function enclosingFunction(node: ESTree.Node): FunctionExpression | null { + let current: ESTree.Node | null = node.parent; + while (current !== null && current.type !== "Program") { + if ( + current.type === "ArrowFunctionExpression" || + current.type === "FunctionDeclaration" || + current.type === "FunctionExpression" + ) { + return current; + } + current = current.parent; + } + return null; +} + +function sourceKeyName(sourceCode: SourceCode, key: ESTree.PropertyKey): string { + if (key.type === "Identifier" || key.type === "PrivateIdentifier") return key.name; + if (key.type === "Literal") return String(key.value); + return sourceCode.getText(key); +} + +function functionName(sourceCode: SourceCode, owner: FunctionExpression | null): string { + if (owner === null) return "anonymous function"; + if (owner.id !== null) return owner.id.name; + const parent = owner.parent; + if (parent.type === "VariableDeclarator" && parent.id.type === "Identifier") + return parent.id.name; + if (parent.type === "MethodDefinition") return sourceKeyName(sourceCode, parent.key); + return "anonymous function"; +} + +function isEmptyObjectExpression(expression: ESTree.Expression): boolean { + const unwrapped = unwrapExpression(expression); + return unwrapped.type === "ObjectExpression" && unwrapped.properties.length === 0; +} + +function isDictionaryAccumulatorTarget(destination: WideningTarget): boolean { + return destination.kind === "open dictionary" || destination.kind === "generic container"; +} + +function hasParentAssertion(node: ESTree.Node): boolean { + return node.parent?.type === "TSAsExpression" || node.parent?.type === "TSTypeAssertion"; +} + +/** Detect sound syntactic cases where a known value is explicitly widened and loses evidence. */ +export const noKnownValueWideningRule = defineRule({ + meta: { + type: "problem", + docs: { + description: + "Disallow syntactically established values from flowing into explicitly broad or anonymous target types that discard useful evidence.", + }, + messages: { + widening: + "The explicit {{target}} type on {{subject}} discards known type evidence. Keep inference, validate with `satisfies`, or use a named owner contract.", + }, + }, + createOnce(context) { + let environment: TypeEnvironment | null = null; + + const reportFlow = ( + expression: ESTree.Expression, + destination: WideningTarget | null, + subject: string, + ) => { + if (destination === null) return; + if ( + isDictionaryAccumulatorTarget(destination) && + isEmptyObjectExpression(expression) + ) { + return; + } + if (!hasKnownEvidence(context.sourceCode, expression)) return; + context.report({ + node: expression, + messageId: "widening", + data: { subject, target: destination.kind }, + }); + }; + + const targetFromAnnotation = (annotation: ESTree.TSTypeAnnotation | null | undefined) => + environment === null ? null : annotationTarget(annotation, environment); + + return { + Program(node) { + environment = createTypeEnvironment(node); + }, + VariableDeclarator(node) { + if (node.init === null || node.id.type !== "Identifier") return; + reportFlow( + node.init, + targetFromAnnotation(node.id.typeAnnotation), + `binding \`${node.id.name}\``, + ); + }, + PropertyDefinition(node) { + if (node.value === null) return; + reportFlow( + node.value, + targetFromAnnotation(node.typeAnnotation), + `property \`${sourceKeyName(context.sourceCode, node.key)}\``, + ); + }, + AccessorProperty(node) { + if (node.value === null) return; + reportFlow( + node.value, + targetFromAnnotation(node.typeAnnotation), + `property \`${sourceKeyName(context.sourceCode, node.key)}\``, + ); + }, + AssignmentExpression(node) { + if (node.operator !== "=" || node.left.type !== "Identifier") return; + const variable = resolveVariable(context.sourceCode, node.left); + if (variable === null) return; + const declarator = variableDeclarator(variable); + if (declarator === null || declarator.id.type !== "Identifier") return; + reportFlow( + node.right, + targetFromAnnotation(declarator.id.typeAnnotation), + `binding \`${declarator.id.name}\``, + ); + }, + ReturnStatement(node) { + if (node.argument === null) return; + const owner = enclosingFunction(node); + reportFlow( + node.argument, + targetFromAnnotation(owner?.returnType), + `return value of \`${functionName(context.sourceCode, owner)}\``, + ); + }, + ArrowFunctionExpression(node) { + if (node.body.type === "BlockStatement") return; + reportFlow( + node.body, + targetFromAnnotation(node.returnType), + `return value of \`${functionName(context.sourceCode, node)}\``, + ); + }, + TSAsExpression(node) { + if (environment === null || hasParentAssertion(node)) return; + reportFlow( + node.expression, + classifyWideningTarget(node.typeAnnotation, environment), + "assertion", + ); + }, + TSTypeAssertion(node) { + if (environment === null || hasParentAssertion(node)) return; + reportFlow( + node.expression, + classifyWideningTarget(node.typeAnnotation, environment), + "assertion", + ); + }, + }; + }, +}); diff --git a/tools/oxlint/anti-slop/rules/no-module-mocking.ts b/tools/oxlint/anti-slop/rules/no-module-mocking.ts new file mode 100644 index 0000000..d6fb5b4 --- /dev/null +++ b/tools/oxlint/anti-slop/rules/no-module-mocking.ts @@ -0,0 +1,91 @@ +import { defineRule } from "@oxlint/plugins"; + +import type { ESTree, Scope, SourceCode, Variable } from "@oxlint/plugins"; + +const moduleMockMethods = new Set(["doMock", "mock", "unstable_mockModule"]); + +function resolveVariable( + sourceCode: SourceCode, + identifier: ESTree.IdentifierReference, +): Variable | null { + let scope: Scope | null = sourceCode.getScope(identifier); + while (scope !== null) { + const variable = scope.set.get(identifier.name); + if (variable !== undefined) return variable; + scope = scope.upper; + } + return null; +} + +function importedName(node: ESTree.Node): string | null { + if (node.type !== "ImportSpecifier") return null; + return node.imported.type === "Identifier" ? node.imported.name : node.imported.value; +} + +function isTestFrameworkObject( + sourceCode: SourceCode, + expression: ESTree.Expression, +): expression is ESTree.IdentifierReference { + if (expression.type !== "Identifier") return false; + if ( + (expression.name === "vi" || expression.name === "jest") && + sourceCode.isGlobalReference(expression) + ) { + return true; + } + + const variable = resolveVariable(sourceCode, expression); + if (variable === null || variable.defs.length === 0) { + return expression.name === "vi" || expression.name === "jest"; + } + return variable.defs.some((definition) => { + if (definition.type !== "ImportBinding" || definition.parent?.type !== "ImportDeclaration") { + return false; + } + const source = definition.parent.source.value; + const name = importedName(definition.node); + return (source === "vitest" && name === "vi") || (source === "@jest/globals" && name === "jest"); + }); +} + +function moduleMockCall(sourceCode: SourceCode, callee: ESTree.Expression): boolean { + if (!("property" in callee) || !("object" in callee) || !("computed" in callee)) return false; + if (!isTestFrameworkObject(sourceCode, callee.object)) return false; + const property = callee.property; + const method = callee.computed + ? property.type === "Literal" && + (property.value === "doMock" || + property.value === "mock" || + property.value === "unstable_mockModule") + ? property.value + : null + : property.type === "Identifier" + ? property.name + : null; + return method !== null && moduleMockMethods.has(method); +} + +/** Ban test framework module mocking in favor of real dependency seams. */ +export const noModuleMockingRule = defineRule({ + meta: { + type: "problem", + docs: { + description: + "Disallow Vitest and Jest module mocking; tests must replace dependencies through real interfaces.", + }, + messages: { + moduleMock: + "Replace module mocking with dependency injection through a real interface, service layer, or faithful test implementation.", + }, + }, + createOnce(context) { + return { + CallExpression(node) { + if (node.callee.type === "Super" || node.callee.type === "V8IntrinsicExpression") return; + if (moduleMockCall(context.sourceCode, node.callee)) { + context.report({ node, messageId: "moduleMock" }); + } + }, + }; + }, +}); diff --git a/tools/oxlint/anti-slop/rules/no-object-parameters.ts b/tools/oxlint/anti-slop/rules/no-object-parameters.ts new file mode 100644 index 0000000..29b990f --- /dev/null +++ b/tools/oxlint/anti-slop/rules/no-object-parameters.ts @@ -0,0 +1,126 @@ +import { defineRule } from "@oxlint/plugins"; + +import type { ESTree, SourceCode } from "@oxlint/plugins"; + +import { lexicalTypeParameterNames } from "../shared/lexical-type-parameters.ts"; + +type Parameter = ESTree.ParamPattern; +type ParameterOwner = + | ESTree.ArrowFunctionExpression + | ESTree.Function + | ESTree.TSCallSignatureDeclaration + | ESTree.TSConstructSignatureDeclaration + | ESTree.TSConstructorType + | ESTree.TSFunctionType + | ESTree.TSMethodSignature; + +function parameterAnnotation(parameter: Parameter): ESTree.TSTypeAnnotation | null | undefined { + if (parameter.type === "TSParameterProperty") { + return parameterAnnotation(parameter.parameter); + } + if (parameter.type === "RestElement") { + return parameter.typeAnnotation ?? parameterAnnotation(parameter.argument); + } + if (parameter.type === "AssignmentPattern") { + return parameter.typeAnnotation ?? parameter.left.typeAnnotation; + } + return parameter.typeAnnotation; +} + +function parameterName(parameter: Parameter, sourceCode: SourceCode): string { + return parameter.type === "Identifier" + ? parameter.name + : sourceCode.getText(parameter).replace(/\s*:\s*object\s*$/u, ""); +} + +/** Ban the broad object type on function inputs, including local aliases to object. */ +export const noObjectParametersRule = defineRule({ + meta: { + type: "problem", + docs: { + description: + "Disallow object function parameters; inputs must use an owner-provided type and be parsed at their boundary.", + }, + messages: { + objectParameter: + "Parameter `{{parameter}}` uses the broad `object` type. Accept a named owner type; parse external input at its boundary before calling this function.", + }, + }, + createOnce(context) { + const aliases = new Map(); + + const resolvesToObject = ( + type: ESTree.TSType, + shadowedAliases: ReadonlySet, + visited = new Set(), + ): boolean => { + if (type.type === "TSObjectKeyword") return true; + if (type.type === "TSParenthesizedType") + return resolvesToObject(type.typeAnnotation, shadowedAliases, visited); + if (type.type === "TSUnionType") { + return type.types.some((member) => + resolvesToObject(member, shadowedAliases, visited), + ); + } + if ( + type.type !== "TSTypeReference" || + type.typeName.type !== "Identifier" || + (type.typeArguments !== null && + type.typeArguments !== undefined && + type.typeArguments.params.length > 0) || + visited.has(type.typeName.name) || + shadowedAliases.has(type.typeName.name) + ) { + return false; + } + const alias = aliases.get(type.typeName.name); + if (alias === undefined) return false; + const nextVisited = new Set(visited); + nextVisited.add(type.typeName.name); + return resolvesToObject(alias, shadowedAliases, nextVisited); + }; + + const checkParameters = (node: ParameterOwner) => { + const shadowedAliases = lexicalTypeParameterNames( + node, + context.sourceCode.visitorKeys, + ); + for (const parameter of node.params) { + const annotation = parameterAnnotation(parameter); + if (annotation === null || annotation === undefined) continue; + if (!resolvesToObject(annotation.typeAnnotation, shadowedAliases)) continue; + context.report({ + node: annotation.typeAnnotation, + messageId: "objectParameter", + data: { parameter: parameterName(parameter, context.sourceCode) }, + }); + } + }; + + return { + Program(node) { + aliases.clear(); + for (const statement of node.body) { + const declaration = + statement.type === "ExportNamedDeclaration" ? statement.declaration : statement; + if ( + declaration?.type === "TSTypeAliasDeclaration" && + (declaration.typeParameters === null || declaration.typeParameters === undefined) + ) { + aliases.set(declaration.id.name, declaration.typeAnnotation); + } + } + }, + ArrowFunctionExpression: checkParameters, + FunctionDeclaration: checkParameters, + FunctionExpression: checkParameters, + TSCallSignatureDeclaration: checkParameters, + TSConstructSignatureDeclaration: checkParameters, + TSConstructorType: checkParameters, + TSDeclareFunction: checkParameters, + TSEmptyBodyFunctionExpression: checkParameters, + TSFunctionType: checkParameters, + TSMethodSignature: checkParameters, + }; + }, +}); diff --git a/tools/oxlint/anti-slop/rules/no-reflect-apply.ts b/tools/oxlint/anti-slop/rules/no-reflect-apply.ts new file mode 100644 index 0000000..2cc3045 --- /dev/null +++ b/tools/oxlint/anti-slop/rules/no-reflect-apply.ts @@ -0,0 +1,28 @@ +import { defineRule } from "@oxlint/plugins"; + +import { isGlobalReflectMethodCall } from "../shared/reflect-method.ts"; + +/** Ban Reflect.apply, which bypasses ordinary typed function calls. */ +export const noReflectApplyRule = defineRule({ + meta: { + type: "problem", + docs: { + description: + "Disallow Reflect.apply; call typed functions directly or model dynamic dispatch behind an interface.", + }, + messages: { + reflectApply: + "Replace `Reflect.apply` with a typed function call. Model dynamic dispatch behind a named interface.", + }, + }, + createOnce(context) { + return { + CallExpression(node) { + if (node.callee.type === "Super" || node.callee.type === "V8IntrinsicExpression") return; + if (isGlobalReflectMethodCall(context.sourceCode, node.callee, "apply")) { + context.report({ node, messageId: "reflectApply" }); + } + }, + }; + }, +}); diff --git a/tools/oxlint/anti-slop/rules/no-reflect-get.ts b/tools/oxlint/anti-slop/rules/no-reflect-get.ts new file mode 100644 index 0000000..cf630ec --- /dev/null +++ b/tools/oxlint/anti-slop/rules/no-reflect-get.ts @@ -0,0 +1,28 @@ +import { defineRule } from "@oxlint/plugins"; + +import { isGlobalReflectMethodCall } from "../shared/reflect-method.ts"; + +/** Ban Reflect.get, which bypasses ordinary property access and useful type evidence. */ +export const noReflectGetRule = defineRule({ + meta: { + type: "problem", + docs: { + description: + "Disallow Reflect.get; use typed property access or parse dynamic input into a domain type.", + }, + messages: { + reflectGet: + "Replace `Reflect.get` with typed property access. Parse dynamic input into a named domain type before reading it.", + }, + }, + createOnce(context) { + return { + CallExpression(node) { + if (node.callee.type === "Super" || node.callee.type === "V8IntrinsicExpression") return; + if (isGlobalReflectMethodCall(context.sourceCode, node.callee, "get")) { + context.report({ node, messageId: "reflectGet" }); + } + }, + }; + }, +}); diff --git a/tools/oxlint/anti-slop/rules/no-runtime-typeof.ts b/tools/oxlint/anti-slop/rules/no-runtime-typeof.ts new file mode 100644 index 0000000..6a25c24 --- /dev/null +++ b/tools/oxlint/anti-slop/rules/no-runtime-typeof.ts @@ -0,0 +1,67 @@ +import { defineRule } from "@oxlint/plugins"; + +import type { ESTree } from "@oxlint/plugins"; + +type RuntimeFunction = ESTree.ArrowFunctionExpression | ESTree.Function; + +function isRuntimeFunction(node: ESTree.Node): node is RuntimeFunction { + return ( + node.type === "ArrowFunctionExpression" || + node.type === "FunctionDeclaration" || + node.type === "FunctionExpression" + ); +} + +function isInsideTypeGuard(node: ESTree.Node): boolean { + let current: ESTree.Node | null = node.parent; + while (current !== null && current.type !== "Program") { + if (isRuntimeFunction(current)) { + return current.returnType?.typeAnnotation.type === "TSTypePredicate"; + } + current = current.parent; + } + return false; +} + +/** Disallow runtime typeof checks that narrow unparsed values instead of decoding them. */ +export const noRuntimeTypeofRule = defineRule({ + meta: { + type: "problem", + docs: { + description: + "Disallow runtime typeof checks; external values must be decoded into meaningful types at their I/O boundary.", + }, + messages: { + runtimeTypeof: + "A `typeof` check narrows a representation without establishing its contract. Parse input at its I/O boundary, then branch on the domain value.", + }, + schema: [ + { + type: "object", + properties: { + allowInTypeGuards: { type: "boolean" }, + }, + additionalProperties: false, + }, + ], + defaultOptions: [{ allowInTypeGuards: false }], + }, + createOnce(context) { + return { + UnaryExpression(node) { + const option = context.options?.[0]; + const allowInTypeGuards = + typeof option === "object" && + option !== null && + !Array.isArray(option) && + option.allowInTypeGuards === true; + if ( + node.operator === "typeof" && + (!allowInTypeGuards || !isInsideTypeGuard(node)) + ) { + context.report({ node, messageId: "runtimeTypeof" }); + } + }, + }; + }, +}); diff --git a/tools/oxlint/anti-slop/rules/no-shape-in-symbol-names.ts b/tools/oxlint/anti-slop/rules/no-shape-in-symbol-names.ts new file mode 100644 index 0000000..afc00dd --- /dev/null +++ b/tools/oxlint/anti-slop/rules/no-shape-in-symbol-names.ts @@ -0,0 +1,39 @@ +import { defineRule } from "@oxlint/plugins"; +import type { ESTree } from "@oxlint/plugins"; + +const FORBIDDEN_SYMBOL_NAME = "shape"; + +function containsForbiddenSymbolName(name: string): boolean { + return name.toLowerCase().includes(FORBIDDEN_SYMBOL_NAME); +} + +/** Ban the case-insensitive substring "shape" in every JavaScript and TypeScript symbol name. */ +export const noForbiddenTermInSymbolNamesRule = defineRule({ + meta: { + type: "problem", + docs: { + description: + 'Disallow the case-insensitive substring "shape" in JavaScript, TypeScript, private, and JSX symbol names.', + }, + messages: { + forbiddenSymbolName: + 'Rename symbol "{{name}}" for its domain role; "shape" describes structure rather than ownership.', + }, + }, + createOnce(context) { + const reportForbiddenSymbolName = (node: ESTree.Node & { name: string }) => { + if (!containsForbiddenSymbolName(node.name)) return; + context.report({ + node, + messageId: "forbiddenSymbolName", + data: { name: node.name }, + }); + }; + + return { + Identifier: reportForbiddenSymbolName, + PrivateIdentifier: reportForbiddenSymbolName, + JSXIdentifier: reportForbiddenSymbolName, + }; + }, +}); diff --git a/tools/oxlint/anti-slop/rules/no-unknown-parameters.ts b/tools/oxlint/anti-slop/rules/no-unknown-parameters.ts new file mode 100644 index 0000000..cdc6c23 --- /dev/null +++ b/tools/oxlint/anti-slop/rules/no-unknown-parameters.ts @@ -0,0 +1,83 @@ +import { defineRule } from "@oxlint/plugins"; +import type { ESTree } from "@oxlint/plugins"; + +type Parameter = ESTree.ParamPattern; +type ParameterOwner = + | ESTree.ArrowFunctionExpression + | ESTree.Function + | ESTree.TSCallSignatureDeclaration + | ESTree.TSConstructSignatureDeclaration + | ESTree.TSConstructorType + | ESTree.TSFunctionType + | ESTree.TSMethodSignature; + +function parameterAnnotation(parameter: Parameter): ESTree.TSTypeAnnotation | null | undefined { + if (parameter.type === "TSParameterProperty") { + return parameterAnnotation(parameter.parameter); + } + if (parameter.type === "RestElement") { + return parameter.typeAnnotation ?? parameterAnnotation(parameter.argument); + } + if (parameter.type === "AssignmentPattern") { + return parameter.typeAnnotation ?? parameter.left.typeAnnotation; + } + return parameter.typeAnnotation; +} + +function parameterName(parameter: Parameter, sourceText: string): string { + if (parameter.type === "TSParameterProperty") { + return parameterName(parameter.parameter, sourceText); + } + if (parameter.type === "AssignmentPattern") { + return parameterName(parameter.left, sourceText); + } + if (parameter.type === "RestElement") { + return parameterName(parameter.argument, sourceText); + } + return parameter.type === "Identifier" + ? parameter.name + : sourceText.replace(/\s*:\s*unknown\s*$/u, ""); +} + +/** Disallow unknown inputs except explicitly named error-cause enrichment. */ +export const noUnknownParametersRule = defineRule({ + meta: { + type: "problem", + docs: { + description: + "Disallow explicitly unknown function parameters except `cause`; decode unknown input at its I/O boundary instead.", + }, + messages: { + unknownParameter: + "Parameter `{{parameter}}` leaves input unparsed. Accept a named domain type; run the expected schema or parser at the I/O boundary before calling this function.", + }, + }, + createOnce(context) { + const checkParameters = (node: ParameterOwner) => { + for (const parameter of node.params) { + const annotation = parameterAnnotation(parameter); + if (annotation?.typeAnnotation.type !== "TSUnknownKeyword") continue; + const name = parameterName(parameter, context.sourceCode.getText(parameter)); + if (name === "cause") continue; + context.report({ + node: annotation.typeAnnotation, + messageId: "unknownParameter", + data: { parameter: name }, + }); + } + }; + + return { + ArrowFunctionExpression: checkParameters, + FunctionDeclaration: checkParameters, + FunctionExpression: checkParameters, + TSCallSignatureDeclaration: checkParameters, + TSConstructSignatureDeclaration: checkParameters, + TSConstructorType: checkParameters, + TSDeclareFunction: checkParameters, + TSEmptyBodyFunctionExpression: checkParameters, + TSFunctionType: checkParameters, + TSMethodSignature: checkParameters, + }; + }, +}); diff --git a/tools/oxlint/anti-slop/rules/no-unknown-returns.ts b/tools/oxlint/anti-slop/rules/no-unknown-returns.ts new file mode 100644 index 0000000..4b16d6e --- /dev/null +++ b/tools/oxlint/anti-slop/rules/no-unknown-returns.ts @@ -0,0 +1,115 @@ +import { defineRule } from "@oxlint/plugins"; + +import type { ESTree } from "@oxlint/plugins"; + +import { lexicalTypeParameterNames } from "../shared/lexical-type-parameters.ts"; + +type FunctionWithReturnType = + | ESTree.ArrowFunctionExpression + | ESTree.Function + | ESTree.TSCallSignatureDeclaration + | ESTree.TSConstructSignatureDeclaration + | ESTree.TSConstructorType + | ESTree.TSFunctionType + | ESTree.TSMethodSignature; + +function referencedAliasName(type: ESTree.TSType): string | null { + if (type.type === "TSParenthesizedType") return referencedAliasName(type.typeAnnotation); + if (type.type !== "TSTypeReference" || type.typeName.type !== "Identifier") return null; + return type.typeArguments === null || + type.typeArguments === undefined || + type.typeArguments.params.length === 0 + ? type.typeName.name + : null; +} + +/** Ban function contracts that return unknown instead of a parsed domain type. */ +export const noUnknownReturnsRule = defineRule({ + meta: { + type: "problem", + docs: { + description: + "Disallow functions whose explicit return contract is unknown or Promise.", + }, + messages: { + unknownReturn: + "This function exposes `unknown` to its caller. Parse the value at its boundary and return a named domain type.", + }, + }, + createOnce(context) { + const aliases = new Map(); + + const resolvesToUnknown = ( + type: ESTree.TSType, + shadowedAliases: ReadonlySet, + visited = new Set(), + ): boolean => { + if (type.type === "TSUnknownKeyword") return true; + if (type.type === "TSParenthesizedType") { + return resolvesToUnknown(type.typeAnnotation, shadowedAliases, visited); + } + if (type.type === "TSUnionType") { + return type.types.some((member) => + resolvesToUnknown(member, shadowedAliases, visited), + ); + } + if ( + type.type === "TSTypeReference" && + type.typeName.type === "Identifier" && + (type.typeName.name === "Promise" || type.typeName.name === "PromiseLike") + ) { + const value = type.typeArguments?.params[0]; + return value !== undefined && resolvesToUnknown(value, shadowedAliases, visited); + } + const name = referencedAliasName(type); + if (name === null || visited.has(name) || shadowedAliases.has(name)) return false; + const alias = aliases.get(name); + if ( + alias === undefined || + (alias.typeParameters !== null && alias.typeParameters !== undefined) + ) { + return false; + } + const nextVisited = new Set(visited); + nextVisited.add(name); + return resolvesToUnknown(alias.typeAnnotation, shadowedAliases, nextVisited); + }; + + const checkReturnType = (node: FunctionWithReturnType) => { + const annotation = node.returnType; + if (annotation === null || annotation === undefined) return; + if ( + !resolvesToUnknown( + annotation.typeAnnotation, + lexicalTypeParameterNames(node, context.sourceCode.visitorKeys), + ) + ) { + return; + } + context.report({ node: annotation.typeAnnotation, messageId: "unknownReturn" }); + }; + + return { + Program(node) { + aliases.clear(); + for (const statement of node.body) { + const declaration = + statement.type === "ExportNamedDeclaration" ? statement.declaration : statement; + if (declaration?.type === "TSTypeAliasDeclaration") { + aliases.set(declaration.id.name, declaration); + } + } + }, + ArrowFunctionExpression: checkReturnType, + FunctionDeclaration: checkReturnType, + FunctionExpression: checkReturnType, + TSCallSignatureDeclaration: checkReturnType, + TSConstructSignatureDeclaration: checkReturnType, + TSConstructorType: checkReturnType, + TSDeclareFunction: checkReturnType, + TSEmptyBodyFunctionExpression: checkReturnType, + TSFunctionType: checkReturnType, + TSMethodSignature: checkReturnType, + }; + }, +}); diff --git a/tools/oxlint/anti-slop/rules/no-unknown-type-aliases.ts b/tools/oxlint/anti-slop/rules/no-unknown-type-aliases.ts new file mode 100644 index 0000000..3e328fd --- /dev/null +++ b/tools/oxlint/anti-slop/rules/no-unknown-type-aliases.ts @@ -0,0 +1,70 @@ +import { defineRule } from "@oxlint/plugins"; + +import type { ESTree } from "@oxlint/plugins"; + +function referencedAliasName(type: ESTree.TSType): string | null { + if (type.type === "TSParenthesizedType") return referencedAliasName(type.typeAnnotation); + if (type.type !== "TSTypeReference" || type.typeName.type !== "Identifier") return null; + return type.typeArguments === null || + type.typeArguments === undefined || + type.typeArguments.params.length === 0 + ? type.typeName.name + : null; +} + +/** Ban named aliases that merely conceal TypeScript's unknown top type. */ +export const noUnknownTypeAliasesRule = defineRule({ + meta: { + type: "problem", + docs: { + description: + "Disallow type aliases whose resolved type is unknown; unknown must remain visible at an allowed boundary.", + }, + messages: { + unknownAlias: + "Type alias `{{alias}}` hides `unknown`. Keep `unknown` explicit at the parsing boundary or on an allowed `cause` field; otherwise use the parsed owner type.", + }, + }, + createOnce(context) { + const aliases = new Map(); + + const resolvesToUnknown = (type: ESTree.TSType, visited = new Set()): boolean => { + if (type.type === "TSUnknownKeyword") return true; + if (type.type === "TSParenthesizedType") + return resolvesToUnknown(type.typeAnnotation, visited); + const name = referencedAliasName(type); + if (name === null || visited.has(name)) return false; + const alias = aliases.get(name); + if ( + alias === undefined || + (alias.typeParameters !== null && alias.typeParameters !== undefined) + ) { + return false; + } + const nextVisited = new Set(visited); + nextVisited.add(name); + return resolvesToUnknown(alias.typeAnnotation, nextVisited); + }; + + return { + Program(node) { + aliases.clear(); + for (const statement of node.body) { + const declaration = + statement.type === "ExportNamedDeclaration" ? statement.declaration : statement; + if (declaration?.type === "TSTypeAliasDeclaration") { + aliases.set(declaration.id.name, declaration); + } + } + for (const alias of aliases.values()) { + if (!resolvesToUnknown(alias.typeAnnotation, new Set([alias.id.name]))) continue; + context.report({ + node: alias.id, + messageId: "unknownAlias", + data: { alias: alias.id.name }, + }); + } + }, + }; + }, +}); diff --git a/tools/oxlint/anti-slop/rules/no-unsafe-dictionary-type.ts b/tools/oxlint/anti-slop/rules/no-unsafe-dictionary-type.ts new file mode 100644 index 0000000..8c45eed --- /dev/null +++ b/tools/oxlint/anti-slop/rules/no-unsafe-dictionary-type.ts @@ -0,0 +1,134 @@ +import { defineRule } from "@oxlint/plugins"; + +import { + classifyUnsafeDictionary, + classifyUnsafeDictionaryValue, + createTypeEnvironment, + type TypeEnvironment, +} from "../shared/dictionary-types.ts"; + +import type { ESTree } from "@oxlint/plugins"; + +const typeNodeKinds: ReadonlySet = new Set([ + "JSDocNonNullableType", + "JSDocNullableType", + "JSDocUnknownType", + "TSAnyKeyword", + "TSArrayType", + "TSBigIntKeyword", + "TSBooleanKeyword", + "TSConditionalType", + "TSConstructorType", + "TSFunctionType", + "TSImportType", + "TSIndexedAccessType", + "TSInferType", + "TSIntersectionType", + "TSIntrinsicKeyword", + "TSLiteralType", + "TSMappedType", + "TSNamedTupleMember", + "TSNeverKeyword", + "TSNullKeyword", + "TSNumberKeyword", + "TSObjectKeyword", + "TSParenthesizedType", + "TSStringKeyword", + "TSSymbolKeyword", + "TSTemplateLiteralType", + "TSThisType", + "TSTupleType", + "TSTypeLiteral", + "TSTypeOperator", + "TSTypePredicate", + "TSTypeQuery", + "TSTypeReference", + "TSUndefinedKeyword", + "TSUnionType", + "TSUnknownKeyword", + "TSVoidKeyword", +]); + +function isTypeNode(node: ESTree.Node): node is ESTree.TSType { + return typeNodeKinds.has(node.type); +} + +function typeReferenceName(type: ESTree.TSTypeReference): string | null { + return type.typeName.type === "Identifier" ? type.typeName.name : null; +} + +function isInsideTypeAliasDeclaration(node: ESTree.Node): boolean { + let current: ESTree.Node | null = node.parent; + while (current !== null && current.type !== "Program") { + if (current.type === "TSTypeAliasDeclaration") return true; + current = current.parent; + } + return false; +} + +function isPlainAliasConsumerUse(node: ESTree.TSType, environment: TypeEnvironment): boolean { + if (node.type !== "TSTypeReference" || node.typeArguments?.params.length) return false; + const name = typeReferenceName(node); + return name !== null && environment.aliases.has(name) && !isInsideTypeAliasDeclaration(node); +} + +function shouldReportType(node: ESTree.TSType, environment: TypeEnvironment): boolean { + if (isPlainAliasConsumerUse(node, environment)) return false; + if (classifyUnsafeDictionary(node, environment) === null) return false; + let current: ESTree.Node | null = node.parent; + while (current !== null && current.type !== "Program") { + if (isTypeNode(current) && classifyUnsafeDictionary(current, environment) !== null) + return false; + current = current.parent; + } + return true; +} + +/** Disallow object-dictionary contracts whose direct value type is an unsafe escape hatch. */ +export const noUnsafeDictionaryTypeRule = defineRule({ + meta: { + type: "problem", + docs: { + description: + "Disallow object-dictionary contracts whose direct value type is unknown, any, object, {}, or a union/alias containing one of those escape hatches.", + }, + messages: { + unsafeDictionary: + "This dictionary's {{value}} value type gives callers no concrete value contract. Use an owner/schema-derived value type; parse external payloads before insertion.", + }, + }, + createOnce(context) { + let environment: TypeEnvironment | null = null; + const report = (node: ESTree.Node, value: string) => { + context.report({ node, messageId: "unsafeDictionary", data: { value } }); + }; + const reportIfUnsafe = (node: ESTree.TSType) => { + if (environment === null || !shouldReportType(node, environment)) return; + const unsafe = classifyUnsafeDictionary(node, environment); + if (unsafe === null) return; + report(node, unsafe.unsafeValue); + }; + + return { + Program(node) { + environment = createTypeEnvironment(node); + }, + TSTypeReference: reportIfUnsafe, + TSTypeLiteral: reportIfUnsafe, + TSMappedType: reportIfUnsafe, + TSIndexSignature(node) { + if ( + environment === null || + node.typeAnnotation === null || + node.parent.type === "TSTypeLiteral" + ) + return; + const unsafe = classifyUnsafeDictionaryValue( + node.typeAnnotation.typeAnnotation, + environment, + ); + if (unsafe !== null) report(node, unsafe.unsafeValue); + }, + }; + }, +}); diff --git a/tools/oxlint/anti-slop/rules/no-widen-then-assert.ts b/tools/oxlint/anti-slop/rules/no-widen-then-assert.ts new file mode 100644 index 0000000..c5e07f7 --- /dev/null +++ b/tools/oxlint/anti-slop/rules/no-widen-then-assert.ts @@ -0,0 +1,366 @@ +import { defineRule } from "@oxlint/plugins"; +import type { ESTree, Variable } from "@oxlint/plugins"; + +type BroadTypeKind = "top" | "object" | "record"; + +type KnownValueEvidence = { + readonly type: ESTree.TSType | null; +}; + +const functionBoundaryTypes = new Set([ + "ArrowFunctionExpression", + "FunctionDeclaration", + "FunctionExpression", + "TSDeclareFunction", + "TSEmptyBodyFunctionExpression", +]); + +function unwrapExpressionParentheses(expression: ESTree.Expression): ESTree.Expression { + let current = expression; + while (current.type === "ParenthesizedExpression") current = current.expression; + return current; +} + +function unwrapTypeParentheses(type: ESTree.TSType): ESTree.TSType { + let current = type; + while (current.type === "TSParenthesizedType") current = current.typeAnnotation; + return current; +} + +function typeReferenceName(type: ESTree.TSTypeReference): string | null { + return type.typeName.type === "Identifier" ? type.typeName.name : null; +} + +function isUnknownOrAnyType(type: ESTree.TSType): boolean { + const unwrapped = unwrapTypeParentheses(type); + return unwrapped.type === "TSUnknownKeyword" || unwrapped.type === "TSAnyKeyword"; +} + +function isBroadRecordKeyType(type: ESTree.TSType): boolean { + const unwrapped = unwrapTypeParentheses(type); + if ( + unwrapped.type === "TSStringKeyword" || + unwrapped.type === "TSNumberKeyword" || + unwrapped.type === "TSSymbolKeyword" + ) { + return true; + } + if (unwrapped.type === "TSUnionType") return unwrapped.types.every(isBroadRecordKeyType); + return unwrapped.type === "TSTypeReference" && typeReferenceName(unwrapped) === "PropertyKey"; +} + +function isBroadRecordType(type: ESTree.TSType): boolean { + const unwrapped = unwrapTypeParentheses(type); + + if (unwrapped.type === "TSTypeReference") { + if (typeReferenceName(unwrapped) === "Readonly") { + const [inner] = unwrapped.typeArguments?.params ?? []; + return inner !== undefined && isBroadRecordType(inner); + } + + if (typeReferenceName(unwrapped) !== "Record") return false; + const parameters = unwrapped.typeArguments?.params ?? []; + return ( + parameters.length === 2 && + parameters[0] !== undefined && + parameters[1] !== undefined && + isBroadRecordKeyType(parameters[0]) && + isUnknownOrAnyType(parameters[1]) + ); + } + + if (unwrapped.type !== "TSTypeLiteral" || unwrapped.members.length !== 1) return false; + const [member] = unwrapped.members; + const [parameter] = member?.type === "TSIndexSignature" ? member.parameters : []; + return ( + member?.type === "TSIndexSignature" && + member.parameters.length === 1 && + parameter !== undefined && + isBroadRecordKeyType(parameter.typeAnnotation.typeAnnotation) && + isUnknownOrAnyType(member.typeAnnotation.typeAnnotation) + ); +} + +function broadTypeKind(type: ESTree.TSType): BroadTypeKind | null { + const unwrapped = unwrapTypeParentheses(type); + if (unwrapped.type === "TSUnknownKeyword" || unwrapped.type === "TSAnyKeyword") return "top"; + if (unwrapped.type === "TSObjectKeyword") return "object"; + return isBroadRecordType(unwrapped) ? "record" : null; +} + +function assertedExpression( + node: ESTree.TSAsExpression | ESTree.TSTypeAssertion, +): ESTree.Expression { + return unwrapExpressionParentheses(node.expression); +} + +function assertionFromExpression( + expression: ESTree.Expression, +): ESTree.TSAsExpression | ESTree.TSTypeAssertion | null { + const unwrapped = unwrapExpressionParentheses(expression); + return unwrapped.type === "TSAsExpression" || unwrapped.type === "TSTypeAssertion" + ? unwrapped + : null; +} + +function normalizedTypeText(sourceText: string, type: ESTree.TSType): string { + return sourceText.slice(type.start, type.end).replaceAll(/\s+/gu, ""); +} + +function typesHaveSameSyntax( + sourceText: string, + left: ESTree.TSType | null, + right: ESTree.TSType, +): boolean { + return ( + left !== null && + normalizedTypeText(sourceText, unwrapTypeParentheses(left)) === + normalizedTypeText(sourceText, unwrapTypeParentheses(right)) + ); +} + +function isDefinitelyObjectType(type: ESTree.TSType): boolean { + const unwrapped = unwrapTypeParentheses(type); + switch (unwrapped.type) { + case "TSArrayType": + case "TSConstructorType": + case "TSFunctionType": + case "TSMappedType": + case "TSObjectKeyword": + case "TSTupleType": + return true; + case "TSTypeLiteral": + return unwrapped.members.length > 0; + case "TSIntersectionType": + return unwrapped.types.every(isDefinitelyObjectType); + case "TSTypeOperator": + return unwrapped.operator === "readonly" && isDefinitelyObjectType(unwrapped.typeAnnotation); + default: + return false; + } +} + +function isDefinitelyNarrowerRecordType(type: ESTree.TSType): boolean { + const unwrapped = unwrapTypeParentheses(type); + if (unwrapped.type === "TSTypeLiteral") { + return unwrapped.members.some((member) => member.type !== "TSIndexSignature"); + } + + if (unwrapped.type !== "TSTypeReference") return false; + if (typeReferenceName(unwrapped) === "Readonly") { + const [inner] = unwrapped.typeArguments?.params ?? []; + return inner !== undefined && isDefinitelyNarrowerRecordType(inner); + } + if (typeReferenceName(unwrapped) !== "Record") return false; + + const parameters = unwrapped.typeArguments?.params ?? []; + return ( + parameters.length === 2 && parameters[1] !== undefined && !isUnknownOrAnyType(parameters[1]) + ); +} + +function functionBoundary(node: ESTree.Node): ESTree.Node | null { + let current = node.parent; + while (current !== null && current.type !== "Program") { + if (functionBoundaryTypes.has(current.type)) return current; + current = current.parent; + } + return null; +} + +function resolvedVariableForIdentifier( + scopes: readonly { + readonly references: readonly { + readonly identifier: ESTree.Node; + readonly resolved: Variable | null; + }[]; + }[], + identifier: ESTree.IdentifierReference, +): Variable | null { + for (const scope of scopes) { + const reference = scope.references.find( + (candidate) => + candidate.identifier.start === identifier.start && + candidate.identifier.end === identifier.end, + ); + if (reference !== undefined) return reference.resolved; + } + return null; +} + +function variableDeclarator(variable: Variable): ESTree.VariableDeclarator | null { + for (const definition of variable.defs) { + if (definition.type === "Variable" && definition.node.type === "VariableDeclarator") { + return definition.node; + } + } + return null; +} + +function knownValueEvidence( + expression: ESTree.Expression, + scopes: Parameters[0], + boundary: ESTree.Node | null, + visitedVariables: ReadonlySet, +): KnownValueEvidence | null { + const unwrapped = unwrapExpressionParentheses(expression); + + if (unwrapped.type === "TSAsExpression" || unwrapped.type === "TSTypeAssertion") { + if (broadTypeKind(unwrapped.typeAnnotation) !== null) return null; + return { type: unwrapped.typeAnnotation }; + } + + if (unwrapped.type === "Literal" || unwrapped.type === "TemplateLiteral") { + return { type: null }; + } + + if ( + unwrapped.type === "ArrayExpression" || + unwrapped.type === "ArrowFunctionExpression" || + unwrapped.type === "ClassExpression" || + unwrapped.type === "FunctionExpression" || + unwrapped.type === "NewExpression" || + unwrapped.type === "ObjectExpression" + ) { + return { type: null }; + } + + if (unwrapped.type !== "Identifier") return null; + const variable = resolvedVariableForIdentifier(scopes, unwrapped); + if (variable === null || visitedVariables.has(variable)) return null; + + const annotatedIdentifier = variable.identifiers.find( + (identifier) => identifier.typeAnnotation !== null && identifier.typeAnnotation !== undefined, + ); + const annotation = annotatedIdentifier?.typeAnnotation?.typeAnnotation; + if (annotation !== undefined && annotatedIdentifier !== undefined) { + if (functionBoundary(annotatedIdentifier) !== boundary || broadTypeKind(annotation) !== null) { + return null; + } + return { type: annotation }; + } + + const declarator = variableDeclarator(variable); + if ( + declarator === null || + declarator.parent.type !== "VariableDeclaration" || + declarator.parent.kind !== "const" || + declarator.init === null || + variable.references.some((reference) => reference.isWrite() && !reference.init) || + functionBoundary(declarator) !== boundary + ) { + return null; + } + + return knownValueEvidence( + declarator.init, + scopes, + boundary, + new Set([...visitedVariables, variable]), + ); +} + +function widenedBinding( + variable: Variable, + scopes: Parameters[0], +): { + readonly broadKind: BroadTypeKind; + readonly evidence: KnownValueEvidence; + readonly declaredAt: number; + readonly boundary: ESTree.Node | null; +} | null { + const declarator = variableDeclarator(variable); + if ( + declarator === null || + declarator.parent.type !== "VariableDeclaration" || + declarator.parent.kind !== "const" || + declarator.id.type !== "Identifier" || + declarator.init === null || + variable.references.some((reference) => reference.isWrite() && !reference.init) + ) { + return null; + } + + const boundary = functionBoundary(declarator); + const declaredType = declarator.id.typeAnnotation?.typeAnnotation; + const initializerAssertion = assertionFromExpression(declarator.init); + const initializerBroadKind = + initializerAssertion === null ? null : broadTypeKind(initializerAssertion.typeAnnotation); + const declaredBroadKind = declaredType === undefined ? null : broadTypeKind(declaredType); + const broadKind = declaredBroadKind ?? initializerBroadKind; + if (broadKind === null) return null; + + const originalExpression = + initializerAssertion !== null && initializerBroadKind !== null + ? assertedExpression(initializerAssertion) + : declarator.init; + const evidence = knownValueEvidence(originalExpression, scopes, boundary, new Set([variable])); + return evidence === null ? null : { broadKind, evidence, declaredAt: declarator.end, boundary }; +} + +function assertionIsNarrower( + sourceText: string, + broadKind: BroadTypeKind, + evidence: KnownValueEvidence, + assertedType: ESTree.TSType, +): boolean { + if (broadTypeKind(assertedType) !== null) return false; + if (broadKind === "top") return true; + if (typesHaveSameSyntax(sourceText, evidence.type, assertedType)) return true; + if (broadKind === "object") return isDefinitelyObjectType(assertedType); + return isDefinitelyNarrowerRecordType(assertedType); +} + +/** Detect immutable local bindings that erase a known type and are later asserted back to a narrower type. */ +export const noWidenThenAssertRule = defineRule({ + meta: { + type: "problem", + docs: { + description: + "Disallow local const flows that explicitly widen a known value before asserting the widened binding to a narrower type.", + }, + messages: { + widenThenAssert: + 'Binding "{{name}}" discards type evidence and later recreates it with an assertion. Keep the precise type from initialization through use; parse boundary input once.', + }, + }, + createOnce(context) { + let scopes: Parameters[0] = []; + + const checkAssertion = (node: ESTree.TSAsExpression | ESTree.TSTypeAssertion) => { + const expression = assertedExpression(node); + if (expression.type !== "Identifier") return; + + const variable = resolvedVariableForIdentifier(scopes, expression); + if (variable === null) return; + const widened = widenedBinding(variable, scopes); + if ( + widened === null || + node.start <= widened.declaredAt || + functionBoundary(node) !== widened.boundary || + !assertionIsNarrower( + context.sourceCode.text, + widened.broadKind, + widened.evidence, + node.typeAnnotation, + ) + ) { + return; + } + + context.report({ + node, + messageId: "widenThenAssert", + data: { name: expression.name }, + }); + }; + + return { + Program() { + scopes = context.sourceCode.scopeManager.scopes; + }, + TSAsExpression: checkAssertion, + TSTypeAssertion: checkAssertion, + }; + }, +}); diff --git a/tools/oxlint/anti-slop/rules/require-safety-comment-for-type-assertion.ts b/tools/oxlint/anti-slop/rules/require-safety-comment-for-type-assertion.ts new file mode 100644 index 0000000..f1a2ffc --- /dev/null +++ b/tools/oxlint/anti-slop/rules/require-safety-comment-for-type-assertion.ts @@ -0,0 +1,62 @@ +import { defineRule } from "@oxlint/plugins"; + +import type { ESTree, SourceCode } from "@oxlint/plugins"; + +type TypeAssertion = ESTree.TSAsExpression | ESTree.TSTypeAssertion; + +const commentOwnerKinds = new Set([ + "ExpressionStatement", + "PropertyDefinition", + "ReturnStatement", + "ThrowStatement", + "VariableDeclaration", +]); + +function isConstAssertion(node: TypeAssertion): boolean { + return ( + node.typeAnnotation.type === "TSTypeReference" && + node.typeAnnotation.typeName.type === "Identifier" && + node.typeAnnotation.typeName.name === "const" + ); +} + +function hasSafetyComment(sourceCode: SourceCode, node: TypeAssertion): boolean { + let current: ESTree.Node = node; + while (true) { + if ( + sourceCode + .getCommentsBefore(current) + .some((comment) => comment.end <= node.start && /\bSAFETY\s*:/u.test(comment.value)) + ) { + return true; + } + if (commentOwnerKinds.has(current.type) || current.parent.type === "Program") return false; + current = current.parent; + } +} + +/** Require every non-const type assertion to state the invariant TypeScript cannot express. */ +export const requireSafetyCommentForTypeAssertionRule = defineRule({ + meta: { + type: "problem", + docs: { + description: + "Require a nearby SAFETY comment for every TypeScript type assertion except const assertions.", + }, + messages: { + missingSafetyComment: + "This type assertion has no `SAFETY:` justification. State the checked invariant immediately before the assertion or its containing statement.", + }, + }, + createOnce(context) { + const checkAssertion = (node: TypeAssertion) => { + if (isConstAssertion(node) || hasSafetyComment(context.sourceCode, node)) return; + context.report({ node, messageId: "missingSafetyComment" }); + }; + + return { + TSAsExpression: checkAssertion, + TSTypeAssertion: checkAssertion, + }; + }, +}); diff --git a/tools/oxlint/anti-slop/shared/dictionary-types.ts b/tools/oxlint/anti-slop/shared/dictionary-types.ts new file mode 100644 index 0000000..8651700 --- /dev/null +++ b/tools/oxlint/anti-slop/shared/dictionary-types.ts @@ -0,0 +1,502 @@ +import type { ESTree } from "@oxlint/plugins"; + +const BUILT_INS = new Set([ + "Record", + "Readonly", + "Partial", + "Required", + "Pick", + "Omit", + "PropertyKey", + "NonNullable", +]); +const TRANSPARENT_WRAPPERS = new Set(["Readonly", "Partial", "Required", "NonNullable"]); + +type TypeAliasEnvironment = ReadonlyMap; + +type ResolvedType = { + readonly type: ESTree.TSType; + readonly substitutions: TypeAliasEnvironment; +}; + +export type UnsafeDictionary = { + readonly kind: "unsafe-dictionary"; + readonly unsafeValue: "any" | "empty-object" | "object" | "union" | "unknown"; +}; + +export type WideningTargetKind = + | "anonymous object" + | "generic container" + | "object" + | "open dictionary" + | "unknown"; + +export type WideningTarget = { + readonly kind: WideningTargetKind; +}; + +export type TypeEnvironment = { + readonly aliases: ReadonlyMap; + readonly interfaces: ReadonlyMap; + readonly shadowedBuiltIns: ReadonlySet; +}; + +function declaredStatement(statement: ESTree.Statement): ESTree.Node | null { + return statement.type === "ExportNamedDeclaration" || + statement.type === "ExportDefaultDeclaration" + ? (statement.declaration ?? null) + : statement; +} + +export function createTypeEnvironment(program: ESTree.Program): TypeEnvironment { + const aliases = new Map(); + const interfaces = new Map(); + const shadowedBuiltIns = new Set(); + + for (const statement of program.body) { + const declaration = declaredStatement(statement); + if (declaration?.type === "ImportDeclaration") { + for (const specifier of declaration.specifiers) { + if (BUILT_INS.has(specifier.local.name)) shadowedBuiltIns.add(specifier.local.name); + } + continue; + } + + if (declaration?.type === "TSTypeAliasDeclaration") { + const existing = aliases.get(declaration.id.name); + if (existing === undefined) aliases.set(declaration.id.name, declaration); + else shadowedBuiltIns.add(declaration.id.name); + if (BUILT_INS.has(declaration.id.name)) shadowedBuiltIns.add(declaration.id.name); + continue; + } + + if (declaration?.type === "TSInterfaceDeclaration") { + const declarations = interfaces.get(declaration.id.name) ?? []; + declarations.push(declaration); + interfaces.set(declaration.id.name, declarations); + if (BUILT_INS.has(declaration.id.name)) shadowedBuiltIns.add(declaration.id.name); + continue; + } + + if (declaration?.type === "TSEnumDeclaration") { + if (BUILT_INS.has(declaration.id.name)) shadowedBuiltIns.add(declaration.id.name); + continue; + } + + if ( + (declaration?.type === "ClassDeclaration" || + declaration?.type === "FunctionDeclaration") && + declaration.id !== null + ) { + if (BUILT_INS.has(declaration.id.name)) shadowedBuiltIns.add(declaration.id.name); + } + } + + return { aliases, interfaces, shadowedBuiltIns }; +} + +function typeReferenceName(type: ESTree.TSTypeReference): string | null { + return type.typeName.type === "Identifier" ? type.typeName.name : null; +} + +function isBuiltIn(name: string, environment: TypeEnvironment): boolean { + return BUILT_INS.has(name) && !environment.shadowedBuiltIns.has(name); +} + +function isUnappliedReferenceTo(type: ESTree.TSType, name: string): boolean { + const unwrapped = unwrapTransparentType(type); + return ( + unwrapped.type === "TSTypeReference" && + typeReferenceName(unwrapped) === name && + (unwrapped.typeArguments === null || + unwrapped.typeArguments === undefined || + unwrapped.typeArguments.params.length === 0) + ); +} + +function unwrapTransparentType(type: ESTree.TSType): ESTree.TSType { + let current = type; + while ( + current.type === "TSParenthesizedType" || + (current.type === "TSTypeOperator" && current.operator === "readonly") + ) { + current = current.typeAnnotation; + } + return current; +} + +function isNeverType(type: ESTree.TSType): boolean { + return unwrapTransparentType(type).type === "TSNeverKeyword"; +} + +function isEffectivelyEmptyMember(member: ESTree.TSSignature): boolean { + return ( + member.type === "TSPropertySignature" && + member.optional === true && + member.typeAnnotation !== null && + member.typeAnnotation !== undefined && + isNeverType(member.typeAnnotation.typeAnnotation) + ); +} + +function isEffectivelyEmptyTypeLiteral(type: ESTree.TSTypeLiteral): boolean { + return type.members.length === 0 || type.members.every(isEffectivelyEmptyMember); +} + +function isEffectivelyEmptyInterface( + declarations: readonly ESTree.TSInterfaceDeclaration[], +): boolean { + if (declarations.length !== 1) return false; + const [type] = declarations; + return ( + type !== undefined && + type.extends.length === 0 && + (type.body.body.length === 0 || type.body.body.every(isEffectivelyEmptyMember)) + ); +} + +function resolvedSubstitutionArgument( + type: ESTree.TSType, + base: TypeAliasEnvironment, + resolving: ReadonlySet = new Set(), +): ESTree.TSType { + const unwrapped = unwrapTransparentType(type); + if (unwrapped.type !== "TSTypeReference") return type; + const name = typeReferenceName(unwrapped); + if (name === null || resolving.has(name)) return type; + const substitution = base.get(name); + if (substitution === undefined) return type; + const nextResolving = new Set(resolving); + nextResolving.add(name); + return resolvedSubstitutionArgument(substitution, base, nextResolving); +} + +function aliasSubstitution( + alias: ESTree.TSTypeAliasDeclaration, + type: ESTree.TSTypeReference, + base: TypeAliasEnvironment, +): TypeAliasEnvironment | null { + const parameters = alias.typeParameters?.params ?? []; + const arguments_ = type.typeArguments?.params ?? []; + const next = new Map(base); + for (const [index, parameter] of parameters.entries()) { + const argument = arguments_[index] ?? parameter.default; + if (argument === null || argument === undefined) return null; + next.set(parameter.name.name, resolvedSubstitutionArgument(argument, next)); + } + return next; +} + +function unsafeDirectValue( + type: ESTree.TSType, + environment: TypeEnvironment, + substitutions: TypeAliasEnvironment, + resolvingAliases: ReadonlySet, +): UnsafeDictionary["unsafeValue"] | null { + const unwrapped = unwrapTransparentType(type); + if (unwrapped.type === "TSUnknownKeyword") return "unknown"; + if (unwrapped.type === "TSAnyKeyword") return "any"; + if (unwrapped.type === "TSObjectKeyword") return "object"; + if (unwrapped.type === "TSTypeLiteral" && isEffectivelyEmptyTypeLiteral(unwrapped)) + return "empty-object"; + if (unwrapped.type === "TSUnionType") { + return unwrapped.types.some( + (member) => unsafeDirectValue(member, environment, substitutions, resolvingAliases) !== null, + ) + ? "union" + : null; + } + if (unwrapped.type === "TSIntersectionType") { + const unsafeMembers = unwrapped.types.map((member) => + unsafeDirectValue(member, environment, substitutions, resolvingAliases), + ); + if (unsafeMembers.includes("any")) return "any"; + return unsafeMembers.length > 0 && unsafeMembers.every((member) => member !== null) + ? unsafeMembers[0] + : null; + } + if (unwrapped.type !== "TSTypeReference") return null; + const name = typeReferenceName(unwrapped); + if (name === null) return null; + if (TRANSPARENT_WRAPPERS.has(name) && isBuiltIn(name, environment)) { + const wrapped = unwrapped.typeArguments?.params[0]; + return wrapped === undefined + ? null + : unsafeDirectValue(wrapped, environment, substitutions, resolvingAliases); + } + const substitution = substitutions.get(name); + if (substitution !== undefined) { + return isUnappliedReferenceTo(substitution, name) + ? null + : unsafeDirectValue(substitution, environment, substitutions, resolvingAliases); + } + const interfaceDeclarations = environment.interfaces.get(name); + if (interfaceDeclarations !== undefined) { + return isEffectivelyEmptyInterface(interfaceDeclarations) ? "empty-object" : null; + } + const alias = environment.aliases.get(name); + if (alias === undefined || resolvingAliases.has(name)) return null; + const nextSubstitutions = aliasSubstitution(alias, unwrapped, substitutions); + if (nextSubstitutions === null) return null; + const nextResolving = new Set(resolvingAliases); + nextResolving.add(name); + return unsafeDirectValue(alias.typeAnnotation, environment, nextSubstitutions, nextResolving); +} + +function dictionaryValueTypes( + type: ESTree.TSType, + environment: TypeEnvironment, + substitutions: TypeAliasEnvironment, + resolvingAliases: ReadonlySet, +): readonly ResolvedType[] { + const unwrapped = unwrapTransparentType(type); + + if (unwrapped.type === "TSTypeLiteral") { + return unwrapped.members.flatMap((member): readonly ResolvedType[] => + member.type === "TSIndexSignature" && member.typeAnnotation !== null + ? [{ type: member.typeAnnotation.typeAnnotation, substitutions }] + : [], + ); + } + + if (unwrapped.type === "TSMappedType") { + return unwrapped.typeAnnotation === null + ? [] + : [{ type: unwrapped.typeAnnotation, substitutions }]; + } + + if (unwrapped.type !== "TSTypeReference") return []; + const name = typeReferenceName(unwrapped); + if (name === null) return []; + + const substitution = substitutions.get(name); + if (substitution !== undefined) { + return isUnappliedReferenceTo(substitution, name) + ? [] + : dictionaryValueTypes(substitution, environment, substitutions, resolvingAliases); + } + + if (TRANSPARENT_WRAPPERS.has(name) && isBuiltIn(name, environment)) { + const wrapped = unwrapped.typeArguments?.params[0]; + return wrapped === undefined + ? [] + : dictionaryValueTypes(wrapped, environment, substitutions, resolvingAliases); + } + + if (name === "Record" && isBuiltIn(name, environment)) { + const value = unwrapped.typeArguments?.params[1] ?? null; + return value === null ? [] : [{ type: value, substitutions }]; + } + + if ((name === "Pick" || name === "Omit") && isBuiltIn(name, environment)) { + const source = unwrapped.typeArguments?.params[0]; + return source === undefined + ? [] + : dictionaryValueTypes(source, environment, substitutions, resolvingAliases); + } + + const alias = environment.aliases.get(name); + if (alias === undefined || resolvingAliases.has(name)) return []; + const nextSubstitutions = aliasSubstitution(alias, unwrapped, substitutions); + if (nextSubstitutions === null) return []; + const nextResolving = new Set(resolvingAliases); + nextResolving.add(name); + return dictionaryValueTypes(alias.typeAnnotation, environment, nextSubstitutions, nextResolving); +} + +export function classifyUnsafeDictionaryValue( + valueType: ESTree.TSType, + environment: TypeEnvironment, +): UnsafeDictionary | null { + const unsafeValue = unsafeDirectValue(valueType, environment, new Map(), new Set()); + return unsafeValue === null ? null : { kind: "unsafe-dictionary", unsafeValue }; +} + +export function classifyUnsafeDictionary( + type: ESTree.TSType, + environment: TypeEnvironment, +): UnsafeDictionary | null { + for (const valueType of dictionaryValueTypes(type, environment, new Map(), new Set())) { + const unsafeValue = unsafeDirectValue( + valueType.type, + environment, + valueType.substitutions, + new Set(), + ); + if (unsafeValue !== null) return { kind: "unsafe-dictionary", unsafeValue }; + } + return null; +} + +function resolvesToDictionary( + type: ESTree.TSType, + environment: TypeEnvironment, + substitutions: TypeAliasEnvironment, + resolvingAliases: ReadonlySet, +): boolean { + return dictionaryValueTypes(type, environment, substitutions, resolvingAliases).length > 0; +} + +export function classifyWideningTarget( + type: ESTree.TSType, + environment: TypeEnvironment, +): WideningTarget | null { + const unwrapped = unwrapTransparentType(type); + if (unwrapped.type === "TSUnknownKeyword") return { kind: "unknown" }; + if (unwrapped.type === "TSObjectKeyword") return { kind: "object" }; + if (unwrapped.type === "TSTypeLiteral") { + return unwrapped.members.some((member) => member.type === "TSIndexSignature") + ? { kind: "open dictionary" } + : unwrapped.members.length > 0 + ? { kind: "anonymous object" } + : null; + } + if (unwrapped.type === "TSMappedType") return { kind: "open dictionary" }; + if (unwrapped.type !== "TSTypeReference") return null; + const name = typeReferenceName(unwrapped); + if (name === null) return null; + if (TRANSPARENT_WRAPPERS.has(name) && isBuiltIn(name, environment)) { + const wrapped = unwrapped.typeArguments?.params[0]; + return wrapped === undefined ? null : classifyWideningTarget(wrapped, environment); + } + if (name === "Record" && isBuiltIn(name, environment)) return { kind: "open dictionary" }; + const alias = environment.aliases.get(name); + if (alias === undefined) return null; + if ((alias.typeParameters?.params.length ?? 0) > 0) { + const substitutions = aliasSubstitution(alias, unwrapped, new Map()); + return substitutions !== null && + resolvesToDictionary(alias.typeAnnotation, environment, substitutions, new Set([name])) + ? { kind: "generic container" } + : null; + } + const substitutions = aliasSubstitution(alias, unwrapped, new Map()); + if (substitutions === null) return null; + const resolved = classifyAliasBroadTarget( + alias.typeAnnotation, + environment, + substitutions, + new Set([name]), + ); + return resolved; +} + +function isBroadMappedKey( + type: ESTree.TSType, + environment: TypeEnvironment, + substitutions: TypeAliasEnvironment, +): boolean { + const unwrapped = unwrapTransparentType(type); + if ( + unwrapped.type === "TSStringKeyword" || + unwrapped.type === "TSNumberKeyword" || + unwrapped.type === "TSSymbolKeyword" + ) { + return true; + } + if (unwrapped.type === "TSUnionType") { + return unwrapped.types.every((member) => + isBroadMappedKey(member, environment, substitutions), + ); + } + if (unwrapped.type !== "TSTypeReference") return false; + const name = typeReferenceName(unwrapped); + if (name === null) return false; + const substitution = substitutions.get(name); + if (substitution !== undefined && !isUnappliedReferenceTo(substitution, name)) { + return isBroadMappedKey(substitution, environment, substitutions); + } + return name === "PropertyKey" && isBuiltIn(name, environment); +} + +function classifyAliasBroadTarget( + type: ESTree.TSType, + environment: TypeEnvironment, + substitutions: TypeAliasEnvironment, + resolvingAliases: ReadonlySet, +): WideningTarget | null { + const unwrapped = unwrapTransparentType(type); + if (unwrapped.type === "TSUnknownKeyword") return { kind: "unknown" }; + if (unwrapped.type === "TSObjectKeyword") return { kind: "object" }; + if (unwrapped.type === "TSTypeLiteral") { + return unwrapped.members.some((member) => member.type === "TSIndexSignature") + ? { kind: "open dictionary" } + : null; + } + if (unwrapped.type === "TSMappedType") { + return isBroadMappedKey(unwrapped.constraint, environment, substitutions) + ? { kind: "open dictionary" } + : null; + } + if (unwrapped.type !== "TSTypeReference") return null; + const name = typeReferenceName(unwrapped); + if (name === null) return null; + const substitution = substitutions.get(name); + if (substitution !== undefined) { + return isUnappliedReferenceTo(substitution, name) + ? null + : classifyAliasBroadTarget( + substitution, + environment, + substitutions, + resolvingAliases, + ); + } + if (TRANSPARENT_WRAPPERS.has(name) && isBuiltIn(name, environment)) { + const wrapped = unwrapped.typeArguments?.params[0]; + return wrapped === undefined + ? null + : classifyAliasBroadTarget(wrapped, environment, substitutions, resolvingAliases); + } + if (name === "Record" && isBuiltIn(name, environment)) { + return { kind: "open dictionary" }; + } + const alias = environment.aliases.get(name); + if (alias === undefined || resolvingAliases.has(name)) return null; + const nextSubstitutions = aliasSubstitution(alias, unwrapped, substitutions); + if (nextSubstitutions === null) return null; + const nextResolving = new Set(resolvingAliases); + nextResolving.add(name); + return classifyAliasBroadTarget( + alias.typeAnnotation, + environment, + nextSubstitutions, + nextResolving, + ); +} + +export function isPopulatedObjectExpression(expression: ESTree.Expression): boolean { + let current = expression; + while ( + current.type === "ParenthesizedExpression" || + current.type === "TSAsExpression" || + current.type === "TSTypeAssertion" || + current.type === "TSNonNullExpression" + ) { + current = current.expression; + } + return current.type === "ObjectExpression" && current.properties.length > 0; +} + +export function isKnownEvidenceExpression(expression: ESTree.Expression): boolean { + let current = expression; + while ( + current.type === "ParenthesizedExpression" || + current.type === "TSAsExpression" || + current.type === "TSTypeAssertion" || + current.type === "TSNonNullExpression" || + current.type === "TSSatisfiesExpression" + ) { + current = current.expression; + } + if (current.type === "ObjectExpression") return true; + return ( + current.type === "ArrayExpression" || + current.type === "ArrowFunctionExpression" || + current.type === "ClassExpression" || + current.type === "FunctionExpression" || + current.type === "NewExpression" || + current.type === "Literal" || + current.type === "TemplateLiteral" || + current.type === "UnaryExpression" + ); +} diff --git a/tools/oxlint/anti-slop/shared/lexical-type-parameters.ts b/tools/oxlint/anti-slop/shared/lexical-type-parameters.ts new file mode 100644 index 0000000..7cdb18c --- /dev/null +++ b/tools/oxlint/anti-slop/shared/lexical-type-parameters.ts @@ -0,0 +1,61 @@ +import type { ESTree } from "@oxlint/plugins"; + +type VisitorKeys = Readonly>; + +function isNode(value: unknown): value is ESTree.Node { + return ( + typeof value === "object" && + value !== null && + "type" in value && + typeof value.type === "string" + ); +} + +function collectInferTypeParameterNames( + node: ESTree.Node, + visitorKeys: VisitorKeys, + names: Set, +): void { + if (node.type === "TSInferType") names.add(node.typeParameter.name.name); + const record = node as unknown as Readonly>; + for (const key of visitorKeys[node.type] ?? []) { + const value = record[key]; + if (isNode(value)) { + collectInferTypeParameterNames(value, visitorKeys, names); + continue; + } + if (!Array.isArray(value)) continue; + for (const child of value) { + if (isNode(child)) collectInferTypeParameterNames(child, visitorKeys, names); + } + } +} + +/** Collect type binders that are in scope at a node and can shadow module aliases. */ +export function lexicalTypeParameterNames( + node: ESTree.Node, + visitorKeys: VisitorKeys, +): ReadonlySet { + const names = new Set(); + let descendant: ESTree.Node = node; + let current: ESTree.Node | null = node; + while (current !== null && current.type !== "Program") { + if ("typeParameters" in current) { + for (const parameter of current.typeParameters?.params ?? []) { + names.add(parameter.name.name); + } + } + if ( + current.type === "TSMappedType" && + (descendant === current.nameType || descendant === current.typeAnnotation) + ) { + names.add(current.key.name); + } + if (current.type === "TSConditionalType" && descendant === current.trueType) { + collectInferTypeParameterNames(current.extendsType, visitorKeys, names); + } + descendant = current; + current = current.parent; + } + return names; +} diff --git a/tools/oxlint/anti-slop/shared/reflect-method.ts b/tools/oxlint/anti-slop/shared/reflect-method.ts new file mode 100644 index 0000000..39bc218 --- /dev/null +++ b/tools/oxlint/anti-slop/shared/reflect-method.ts @@ -0,0 +1,35 @@ +import type { ESTree, Scope, SourceCode, Variable } from "@oxlint/plugins"; + +function resolveVariable( + sourceCode: SourceCode, + identifier: ESTree.IdentifierReference, +): Variable | null { + let scope: Scope | null = sourceCode.getScope(identifier); + while (scope !== null) { + const variable = scope.set.get(identifier.name); + if (variable !== undefined) return variable; + scope = scope.upper; + } + return null; +} + +function isGlobalReflect(sourceCode: SourceCode, expression: ESTree.Expression): boolean { + if (expression.type !== "Identifier" || expression.name !== "Reflect") return false; + if (sourceCode.isGlobalReference(expression)) return true; + const variable = resolveVariable(sourceCode, expression); + return variable === null || variable.defs.length === 0; +} + +/** Reports whether a call target names one method on the global Reflect object. */ +export function isGlobalReflectMethodCall( + sourceCode: SourceCode, + callee: ESTree.Expression, + methodName: string, +): boolean { + if (!("property" in callee) || !("object" in callee) || !("computed" in callee)) return false; + if (!isGlobalReflect(sourceCode, callee.object)) return false; + const property = callee.property; + return callee.computed + ? property.type === "Literal" && property.value === methodName + : property.type === "Identifier" && property.name === methodName; +} diff --git a/wrangler.jsonc b/wrangler.jsonc index 2ed24c5..e22c599 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -2,7 +2,7 @@ "$schema": "node_modules/wrangler/config-schema.json", "name": "strata", "main": "src/index.ts", - "rules": [{ "type": "Text", "globs": ["**/*.html"], "fallthrough": true }], + "rules": [{ "type": "Text", "globs": ["**/*.html", "**/ui.js"], "fallthrough": true }], "compatibility_date": "2026-08-01", "observability": { "enabled": true }, "workers_dev": false, -- 2.51.2 From 7096df624b25546746b142daf015dc36d9cb64a0 Mon Sep 17 00:00:00 2001 From: zzstoatzz Date: Sat, 29 Aug 2026 01:58:52 -0500 Subject: [PATCH 08/17] page: one left-aligned column, two controls, continuous strips, explanation folded away Stat tiles become a sentence; zoom picks itself; brightness stays relative; the table hides under 'how to read this'; prompts are text links. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01B927dNwNKNYsbQdUoNJwHS --- src/page.html | 180 +++++++++++++++++++++++--------------------------- src/ui.js | 60 +++++++---------- 2 files changed, 105 insertions(+), 135 deletions(-) diff --git a/src/page.html b/src/page.html index 1491f7c..e38750f 100644 --- a/src/page.html +++ b/src/page.html @@ -14,93 +14,89 @@ *{margin:0;padding:0;box-sizing:border-box} html,body{background:var(--bg);color:var(--ink);font:14px/1.65 ui-monospace,"SF Mono",Menlo,Consolas,monospace} a{color:inherit;text-decoration:none} -.wrap{max-width:1100px;margin:0 auto;padding:0 2ch;display:flex;flex-direction:column;min-height:100vh} -header{display:flex;flex-direction:column;align-items:center;gap:.35rem;padding:1.7rem 0 1.4rem;text-align:center} +em{font-style:normal;color:var(--hot)} +.wrap{max-width:1100px;margin:0 auto;padding:0 3ch 2rem;display:flex;flex-direction:column;min-height:100vh;gap:2.4rem} +header{display:flex;flex-direction:column;align-items:center;gap:.35rem;padding:1.7rem 0 0;text-align:center} .brand{letter-spacing:.12em;font-size:.9rem;color:var(--hot)} a.brand:hover{color:var(--water)} .livedot{font-size:.74rem;color:var(--dim);font-variant-numeric:tabular-nums} .livedot b{color:var(--water);font-weight:400} -.tag{color:var(--dim);font-size:.8rem;max-width:64ch;margin-top:.3rem} -em{font-style:normal;color:var(--ink)} -main{flex:1;display:flex;flex-direction:column;gap:1.6rem} -.sect{font-size:.72rem;letter-spacing:.35em;text-transform:uppercase;color:var(--dim);text-align:center} -.grid{display:flex;flex-wrap:wrap;justify-content:center;gap:1.2rem 3ch;width:100%} -.stat{flex:0 1 15ch;display:flex;flex-direction:column;align-items:center;gap:.15rem;text-align:center} -.stat .v{font-size:1.5rem;font-weight:300;color:var(--hot);letter-spacing:.04em;font-variant-numeric:tabular-nums} -.stat .l{font-size:.7rem;color:var(--dim);letter-spacing:.22em;text-transform:uppercase} -.how{color:var(--dim);font-size:.8rem;max-width:76ch;line-height:1.65;margin:0 auto;text-align:center} -.how .glyph{color:var(--water)} -.controls{display:flex;flex-wrap:wrap;justify-content:center;align-items:center;gap:.6rem 2.4ch;color:var(--dim);font-size:.8rem} -.controls label{display:inline-flex;align-items:center;gap:1ch} +main{flex:1;display:flex;flex-direction:column;gap:2.4rem} +section{display:flex;flex-direction:column;gap:.8rem} +.copy{max-width:70ch} +.lede{font-size:1rem;line-height:1.6;color:var(--ink)} +.sect{font-size:.72rem;letter-spacing:.35em;text-transform:uppercase;color:var(--dim)} +.bar{display:flex;flex-wrap:wrap;align-items:center;justify-content:space-between;gap:.5rem 3ch;color:var(--dim);font-size:.8rem} +.bar .tools{display:flex;flex-wrap:wrap;align-items:center;gap:.5rem 2ch} select,input,button,textarea{max-width:100%;font:inherit;font-size:.8rem;color:var(--ink);background:var(--well);border:1px solid var(--faint);padding:.25rem .8ch;border-radius:0} -select:focus-visible,input:focus-visible,button:focus-visible,textarea:focus-visible{outline:1px solid var(--water);outline-offset:1px} +select:focus-visible,input:focus-visible,button:focus-visible,textarea:focus-visible,a:focus-visible,summary:focus-visible{outline:1px solid var(--water);outline-offset:2px} button{cursor:pointer;color:var(--dim)}button:hover{color:var(--hot);border-color:var(--water)}button:disabled{opacity:.5;cursor:default} -button[aria-pressed="true"]{color:var(--water);border-color:var(--water)} -input{width:20ch}input::placeholder,textarea::placeholder{color:var(--faint)} +.linkish{background:none;border:0;padding:0;color:var(--water);cursor:pointer;font-size:inherit} +.linkish:hover{color:var(--hot);border:0} +.linkish:disabled{color:var(--faint)} +input{width:18ch}input::placeholder,textarea::placeholder{color:var(--faint)} -/* the layers: one grid row per app — name | activity strip | total. the name column is a fixed width so - opening an app never moves anything; children indent inside it. on phones the strip drops under the name. */ -.layers{position:relative;display:flex;flex-direction:column;gap:2px;--name:24ch;--total:7ch} -.lane{display:grid;grid-template-columns:var(--name) minmax(0,1fr) var(--total);column-gap:1.5ch;align-items:center;min-height:14px} -.lane.head{color:var(--dim);font-size:.7rem;letter-spacing:.16em;text-transform:uppercase;margin-bottom:.2rem} -.lane.head .count{text-align:right} -.lane.head .strip{background:none;height:auto} +/* the picture: one grid row per app — name | activity strip | total. the name column is fixed so opening an + app never moves the strips; children indent inside it. on phones the strip drops under the name. */ +.layers{position:relative;display:flex;flex-direction:column;gap:3px;--name:22ch;--total:7ch} +.lane{display:grid;grid-template-columns:var(--name) minmax(0,1fr) var(--total);column-gap:2ch;align-items:center} .name{display:flex;align-items:center;gap:.6ch;background:none;border:0;padding:0;text-align:left;color:var(--ink);font-size:.8rem;min-width:0;cursor:default} .name .glyph{color:var(--water);width:1.2ch;flex:none} .name .label{overflow:hidden;text-overflow:ellipsis;white-space:nowrap;direction:rtl;text-align:left} .name .label bdi{direction:ltr;unicode-bidi:isolate} .name[aria-expanded]{cursor:pointer} .name[aria-expanded]:hover .label,.name:focus-visible .label{color:var(--hot)} -.name:focus-visible{outline:1px solid var(--water);outline-offset:2px} .lane.child .name{padding-left:2.4ch;color:var(--dim);font-size:.76rem} .lane.child .name .glyph{color:var(--faint)} .strip{position:relative;height:var(--h,20px);background:var(--well);min-width:0} .strip svg{display:block;width:100%;height:100%} .count{font-size:.76rem;color:var(--dim);text-align:right;font-variant-numeric:tabular-nums;white-space:nowrap} .lane.child .count{color:var(--faint)} -.cursor{position:absolute;top:0;bottom:0;width:0;border-left:1px solid var(--hot);opacity:.7;pointer-events:none;display:none} -.axis .strip{height:32px;background:none} -.axis .era{position:absolute;top:4px;height:2px;background:var(--faint)} +.cursor{position:absolute;top:0;bottom:0;pointer-events:none;display:none;background:rgba(226,235,240,.08);border-left:1px solid var(--hot)} +.axis .strip{height:34px;background:none} +.axis .era{position:absolute;top:2px;height:2px;background:var(--faint)} .axis .era.live{background:var(--water)} .axis .era span{position:absolute;top:6px;left:0;font-size:.7rem;color:var(--dim);white-space:nowrap} .axis .era.live span{color:var(--water)} .axis .tick{position:absolute;top:20px;font-size:.7rem;color:var(--dim);white-space:nowrap} .axis .tick.end{transform:translateX(-100%)} -.reader{min-height:3.4rem;padding:.7rem 0 0;margin-top:.6rem;border-top:1px dotted var(--faint);font-size:.8rem;color:var(--dim);font-variant-numeric:tabular-nums;line-height:1.6} +.reader{min-height:3rem;font-size:.8rem;color:var(--dim);font-variant-numeric:tabular-nums;line-height:1.6} .reader b{color:var(--hot);font-weight:400}.reader i{font-style:normal;color:var(--water)} -.legend{display:flex;flex-wrap:wrap;align-items:center;justify-content:center;gap:.4rem 1.6ch;font-size:.74rem;color:var(--dim)} -.legend .bar{width:22ch;height:8px;border:1px solid var(--faint)} -.legend .none{display:inline-block;width:1.2em;height:8px;background:var(--well);border:1px solid var(--faint);vertical-align:-1px;margin-right:.6ch} -.note{color:var(--dim);font-size:.76rem;max-width:74ch;margin:0 auto;text-align:center;line-height:1.6} +.legend{display:flex;flex-wrap:wrap;align-items:center;gap:.4rem 1.2ch;font-size:.74rem;color:var(--dim)} +.legend .ramp{width:14ch;height:8px;border:1px solid var(--faint)} +.legend .none{display:inline-block;width:1.4em;height:8px;background:var(--well);border:1px solid var(--faint);vertical-align:-1px} +details{max-width:70ch;font-size:.8rem;color:var(--dim)} +summary{cursor:pointer;color:var(--water);list-style:none} +summary::-webkit-details-marker{display:none} +summary:hover{color:var(--hot)} +details[open] summary{margin-bottom:.5rem} +details p{margin:0 0 .5rem;line-height:1.6} +details p:last-child{margin-bottom:0} .table-wrap{width:100%;overflow-x:auto} -table{border-collapse:collapse;font-size:.76rem;font-variant-numeric:tabular-nums;margin:0 auto} -th,td{text-align:right;padding:.2rem 1.2ch;border-bottom:1px solid var(--well);white-space:nowrap;color:var(--dim)} -td:first-child,th:first-child{text-align:left;color:var(--ink);position:sticky;left:0;background:var(--bg)} +table{border-collapse:collapse;font-size:.76rem;font-variant-numeric:tabular-nums} +th,td{text-align:right;padding:.2rem 1.4ch;border-bottom:1px solid var(--well);white-space:nowrap;color:var(--dim)} +td:first-child,th:first-child{text-align:left;color:var(--ink);position:sticky;left:0;background:var(--bg);padding-left:0} th{color:var(--dim);font-weight:400;letter-spacing:.1em;text-transform:uppercase;font-size:.66rem} -.ask{width:100%;max-width:100ch;margin:0 auto;display:flex;flex-direction:column;gap:.7rem} -.ask .row{display:flex;flex-wrap:wrap;gap:.5rem 2ch;align-items:center;color:var(--dim);font-size:.8rem} -.ask .row label{display:inline-flex;align-items:center;gap:1ch;max-width:100%;min-width:0} -#ask-ns{width:34ch;max-width:100%;min-width:0} -.ask textarea{width:100%;min-height:5.5rem;resize:vertical;line-height:1.5;tab-size:2} -.ask .prompts{display:flex;flex-wrap:wrap;gap:.4rem 1ch} -.ask .prompts button{font-size:.74rem} -.ask .out{font-size:.78rem;color:var(--dim);min-height:1.5rem} +.ask .row{display:flex;flex-wrap:wrap;align-items:center;gap:.5rem 1.5ch;color:var(--dim);font-size:.8rem} +.ask select{width:34ch;min-width:0} +.ask .prompts{display:flex;flex-wrap:wrap;gap:.3rem 1ch;font-size:.8rem;color:var(--faint)} +.ask textarea{width:100%;max-width:100ch;min-height:5.5rem;resize:vertical;line-height:1.5;tab-size:2} +.ask .fine{font-size:.74rem;color:var(--faint);max-width:70ch} +.ask .out{font-size:.78rem;color:var(--dim);min-height:1.2rem} .ask .out b{color:var(--hot);font-weight:400} .ask .err{color:#d99a8a;white-space:pre-wrap} -.ask .fine{font-size:.74rem;color:var(--faint)} -footer{display:flex;flex-wrap:wrap;justify-content:center;align-items:center;gap:.45rem 3ch;padding:2rem 0 1.5rem;color:var(--dim);font-size:.76rem} -footer a{color:var(--dim);border-bottom:1px dotted var(--faint);white-space:nowrap} +.ask .blurb{color:var(--dim);font-size:.8rem} +footer{display:flex;flex-wrap:wrap;align-items:center;gap:.4rem 3ch;padding-top:1rem;border-top:1px dotted var(--faint);color:var(--dim);font-size:.76rem} +footer a{border-bottom:1px dotted var(--faint);white-space:nowrap} footer a:hover{color:var(--hot);border-bottom-color:var(--water)} .status{color:var(--faint)} @media (max-width:640px){ - .wrap{padding:0 1.5ch}.grid{gap:1rem 2ch}.stat{flex:0 1 11ch}.tag{max-width:38ch}input{width:16ch} - .layers{--name:1fr;--total:auto;gap:6px} - .lane{grid-template-columns:1fr auto;row-gap:2px;column-gap:1ch} + .wrap{padding:0 2ch 2rem;gap:2rem} + .layers{gap:8px} + .lane{grid-template-columns:1fr auto;row-gap:3px;column-gap:1ch} .lane .strip{grid-column:1/-1} - .lane.head .strip{display:none} - .lane.child{margin-left:0} - .ask .row label{width:100%} - #ask-ns{width:100%} + .ask select{width:100%} + input{width:100%} } @@ -109,64 +105,52 @@ footer a:hover{color:var(--hot);border-bottom-color:var(--water)}
~  s t r e a m  ~ · reading the archive -

strata — what's in the archive. every app on the network writes its own kinds of records; this is how much of each there is, and when it arrived.

-
-
—records
-
—apps
-
—record types
-
+
+

strata — what's in the archive. every app on the network writes its own kinds of records; this is how much of each there is, and when it arrived.

+
-

each row is an app, named by the identifier its records carry — app.bsky is Bluesky, sh.tangled is Tangled.
- taller = more records overall · brighter = busier in that slice of the archive · ▸ = open it to see its record types

- -
- - - - - -
- -
- - -
- nothing from that app in this slice - quieterbusier - -
- -

the archive is stored in chunks of about 3 million records; the columns are those chunks in order, oldest on the left. the dates are when the archive received the records, not when people wrote them: in its first six days (from 2026-07-28) it replayed the network's whole history, so those columns are packed — history, replayed. after that each chunk is a few minutes of live traffic — as it happened. account and identity changes belong to no app and aren't counted. everything else bundles the apps not shown.

- - +
+ taller = more records· + quieterbusier, within that app· + nothing in that slice +
+
+ how to read this +

each row is an app, named by the identifier its records carry — app.bsky is Bluesky, sh.tangled is Tangled. ▸ opens an app to show its record types.

+

columns are the archive in order, oldest on the left, in chunks of about 3 million records. the dates are when the archive received the records, not when people wrote them: in its first six days it replayed the network's whole history (history, replayed); after that each chunk is a few minutes of live traffic (as it happened).

+

account and identity changes belong to no app and aren't counted. everything else bundles the apps not shown.

+
+ + -
look inside an app
-
-

pick an app and every record it has ever written is searched right here, in your browser, from the public archive. Bluesky itself is too big for that and is shown from summaries only.

+
+ look inside an app +

every record an app has ever written, searched in your browser from the public archive. Bluesky is too big for this and is shown from summaries only.

- - pick one + +
- +
-
+