From 20f2efabe72245d8a768786a0dba4eb0d5afd935 Mon Sep 17 00:00:00 2001 From: Orual Date: Wed, 5 Aug 2026 16:38:19 -0400 Subject: [PATCH] PM-86: reconcile cleanup rebase semantics Epic: PM-86 --- .current-epic/dependencies.dot | 8 + .current-epic/manifest.txt | 2 + .current-epic/merge-gate.json | 4 +- .current-epic/plan-F.md | 394 +++--- .current-epic/plan-G.md | 2 +- .current-epic/task-L.md | 53 +- .current-epic/task-state.json | 46 +- .gitignore | 4 +- AGENTS.md | 2 + crates/polymodel-ldraw-core/src/cache.rs | 5 +- crates/polymodel-ldraw-core/src/lib.rs | 29 +- crates/polymodel-ldraw-core/src/model.rs | 602 +++++++- crates/polymodel-ldraw-core/src/parser.rs | 488 +++++-- crates/polymodel-ldraw-core/src/types.rs | 156 ++- crates/polymodel-ldraw-testkit/src/gates.rs | 8 +- .../polymodel-renderer-worker/src/worker.rs | 190 ++- flake.nix | 1 + justfile | 274 +++- src/appview/download_tests.rs | 160 ++- src/appview/read_tests.rs | 385 ++++++ src/appview/test_support.rs | 196 ++- src/thing_detail.rs | 24 +- tools/check-devshell-tools-only.py | 25 + tools/in-dev-shell | 27 + tools/ldraw-compat-harness/src/main.rs | 1209 ++++++++++++++--- tools/test-just-nix.sh | 136 ++ 26 files changed, 3533 insertions(+), 897 deletions(-) create mode 100644 tools/check-devshell-tools-only.py create mode 100755 tools/in-dev-shell create mode 100755 tools/test-just-nix.sh diff --git a/.current-epic/dependencies.dot b/.current-epic/dependencies.dot index d40ddc9..0dc5c03 100644 --- a/.current-epic/dependencies.dot +++ b/.current-epic/dependencies.dot @@ -13,6 +13,8 @@ digraph epic_dependencies { I [label="I: PM-82\nTEXMAP and embedded\ntexture pipeline"]; J [label="J: PM-83\nFormat registry and\npublishing/viewer integration"]; K [label="K: PM-84\nInteroperability, fuzz,\nperformance, visual baselines"]; + L [label="L: PM-90\nCorrect LDraw face lighting\nand top/bottom brightness"]; + M [label="M: PM-91\nClose-range depth corruption\nand camera clipping precision"]; // A (PM-87) blocks: F, G, J A -> F; @@ -62,5 +64,11 @@ digraph epic_dependencies { // J (PM-83) blocks: K J -> K; + // PM-90 and PM-91 are operator-approved dynamic tasks discovered during PM-79 visual validation. + F -> L; + F -> M; + // K (PM-84) blocks: none + // L (PM-90) blocks: none + // M (PM-91) blocks: none } diff --git a/.current-epic/manifest.txt b/.current-epic/manifest.txt index 922105b..cf7979e 100644 --- a/.current-epic/manifest.txt +++ b/.current-epic/manifest.txt @@ -17,3 +17,5 @@ H PM-81 https://radiant-industries.atlassian.net/browse/PM-81 LDraw points, surf I PM-82 https://radiant-industries.atlassian.net/browse/PM-82 LDraw TEXMAP and embedded texture pipeline J PM-83 https://radiant-industries.atlassian.net/browse/PM-83 Format registry and LDraw publishing/viewer integration K PM-84 https://radiant-industries.atlassian.net/browse/PM-84 Interoperability, fuzzing, performance, and visual baselines +L PM-90 https://radiant-industries.atlassian.net/browse/PM-90 Correct LDraw face lighting and top/bottom brightness +M PM-91 https://radiant-industries.atlassian.net/browse/PM-91 Fix close-range renderer depth corruption and camera clipping precision diff --git a/.current-epic/merge-gate.json b/.current-epic/merge-gate.json index ea2d438..c78cc67 100644 --- a/.current-epic/merge-gate.json +++ b/.current-epic/merge-gate.json @@ -1,5 +1,5 @@ { - "holder": "task-C", - "last_merged": "task-D", + "holder": "task-F", + "last_merged": "task-C", "queue": [] } diff --git a/.current-epic/plan-F.md b/.current-epic/plan-F.md index 227540f..4d4bdc0 100644 --- a/.current-epic/plan-F.md +++ b/.current-epic/plan-F.md @@ -1,305 +1,243 @@ -# PM-79 / Task F — Minimal LDraw vertical slice and renderer scene IR +# PM-79 / Task F — Minimal LDraw vertical slice and generic renderer scene IR -## Contract +## Contract and cold-start context -- **Epic:** PM-86 — [https://radiant-industries.atlassian.net/browse/PM-86](https://radiant-industries.atlassian.net/browse/PM-86) -- **Task:** PM-79 — [https://radiant-industries.atlassian.net/browse/PM-79](https://radiant-industries.atlassian.net/browse/PM-79) -- **Task prefix:** F -- **Task cache:** `/home/orual/Projects/epic-86/.current-epic/task-F.md` -- **Epic workspace/bookmark:** `/home/orual/Projects/epic-86`, `epic-86` -- **Manifest/dependencies/state:** `/home/orual/Projects/epic-86/.current-epic/manifest.txt`, `dependencies.dot`, `task-state.json` -- **Base bookmark:** `main` -- **Expected task workspace/bookmark:** `/home/orual/Projects/epic-86-task-F`, `epic-86-task-F` -- **Integrated baseline:** `/home/orual/Projects/epic-86-task-G`, `epic-86@99436d09` +- **Epic:** PM-86 — https://radiant-industries.atlassian.net/browse/PM-86 +- **Task:** PM-79 — https://radiant-industries.atlassian.net/browse/PM-79 +- **Prefix:** F +- **Task cache:** `.current-epic/task-F.md` +- **Manifest/dependencies/findings:** `.current-epic/manifest.txt`, `.current-epic/dependencies.dot`, `.current-epic/findings-F.jsonl` +- **Epic workspace/bookmark:** `/home/orual/Projects/epic-86`, `epic-86`; **base:** `main` +- **Task workspace/bookmark:** `/home/orual/Projects/epic-86-task-F`, `epic-86-task-F` +- **Parent truth:** consume the then-current `epic-86@` tip and reachable PM-86 reconciliation ancestry. Do not pin or revive `99436d09`; `.current-epic/task-state.json` is stale coordination metadata, not ancestry truth. +- **Oracle:** `/home/orual/Projects/three-d-reference`, detached clean tag `0.19.0`, commit `9b66a7459cb72bb317b58e791747871337bebb33`. -## Context and outcome +F owns the dependency-light renderer scene contract and the sole LDraw-to-world boundary for one ordinary Thing → Model → Part → File compound flow. Keep one three-d context, camera/orbit/lights/target/readback/OffscreenCanvas lifecycle and current ordinary `Model` path. F adds the generic scene IR, E-owned semantic export consumption, custom world-space primitives, and shared Interactive/Preview output; it does not add a route, demo model, direct LDraw URL, upload flow, parser duplicate, production second renderer, or H’s complete classifier. -F owns the sole LDraw-to-renderer/world boundary for the normal Thing detail/viewer product flow plus ordinary-format parity. The acceptance composition is an ordinary Thing→Model→Part→File record: the normal publish UI/record contract uploads an LDraw multipart/MPD root and all companion sources, projection and G’s compound resolver discover verified descriptors, E parses them, and F renders them in that Thing’s existing viewer endpoint. There is no special viewer route, demo-model entry, separate upload flow, or direct-URL LDraw path. E owns parsing, semantic records, BFC, inclusion traversal, colour/material state, canonical projection, and semantic reservations. G is completed and integrated at `99436d09` and owns verified content-bound compound resource acquisition plus worker protocol/session/epoch/cancellation plumbing. +## Safety, scope, and replace-vs-edit decisions -The operator-visible composition uses authoritative LDraw primitives exported by E: type 1 occurrences, type 2 explicit lines, type 3 triangles, type 4 quads, and type 5 conditional lines. Generic IR point rendering is separately proved with a renderer-owned lower-level fixture; F does not invent a type-0 LDraw point syntax. H/I/J/K remain out of scope. +Before source work, from the task workspace run `pwd`, `jj workspace root`, `jj workspace list`, `jj bookmark list`, `jj status`, and the one-line `jj log` checks for `epic-86@` and `main@`. Stop without editing if the root is not `/home/orual/Projects/epic-86-task-F`, the bookmark is not `epic-86-task-F`, this is the default workspace, the working copy is `main`, or the task change is not based on the then-current `epic-86@` tip. Do not edit manifests, dependency graphs, state, merge gates, event logs, branches, or workspaces. -## Safety check — stop on `main` +| Area | Decision | Frozen implementation shape | +|---|---|---| +| `crates/polymodel-renderer-protocol/src/lib.rs` and new `src/scene.rs` | Focused edit plus leaf module | Add branded IDs, units, provenance/diagnostics, deterministic scene tables/occurrences/primitives/bounds/stats, and appended `LengthUnit::LdrawUnit`; preserve ordinary wire fixtures and protocol independence from E/mesh/three-d. | +| `crates/polymodel-ldraw-core/src/{geom,model,types,resolve,lib}.rs` plus owned export module | Focused semantic extension | Retain scanner/parser/BFC/resolver/canonical projection. Add owned render export, explicit per-model LDMesh-compatible payload, and complete ordered primitive provenance; do not replace the parser or reparse bytes. | +| `crates/polymodel-renderer-worker/src/worker.rs` | Replace compound completion | Replace the complete `load_compound_source` → `finish_compound_source` chain and its byte-posting flow with worker-internal verified materialization → one E parse → one semantic export → one adaptation → pending resources → namespace output → metadata/readback observation → cleanup. `load_compound_source` must return a worker-internal verified materialization/source handle, never the removed protocol `CompoundSceneSource`. Do not add branches around the obsolete boundary. | +| `crates/polymodel-renderer-worker/src/main.rs` and target layout | Add reusable library, retain WASM bin, add native bin | Add `src/lib.rs` as the reusable worker/scene/render library; retain `src/main.rs` as the `renderer_worker` WASM bin; add exact native `src/bin/ldraw_preview.rs` binary named `ldraw_preview`; add integration test `crates/polymodel-renderer-worker/tests/ldraw_preview.rs`. | +| worker Cargo/Just surface | Focused build-system edit | In `crates/polymodel-renderer-worker/Cargo.toml`, add `polymodel-ldraw-core = { path = "../polymodel-ldraw-core" }` as a worker-only dependency alongside the existing mesh/protocol dependencies; app code remains protocol-only and `just verify-renderer-split` proves no `polymodel-ldraw-core`, mesh, three-d, or parser dependency enters the app WASM tree. Keep production WASM dependencies and add native-only `three-d` window support for the named preview target/test. Public Just recipes dispatch through `tools/in-dev-shell`; add private implementations behind them. | +| `src/viewer.rs`, `src/thing_detail.rs`, appview fixtures | Focused integration edit | Consume metadata-only events and the current-`LoadIdentity`-gated test readback hook through the existing normal Thing endpoint and Preview request path. | +| corpus/e2e fixtures | Focused fixture extension | Reuse current `/did:plc:aaaaaaaaaaaaaaaaaaaaaaaa/thing/parametric-enclosure` anchor and existing resolver keys; add only named real byte files/helper/test assertions below. | -Before any workspace or source operation, run: +A module/function is replaced when it cannot express both namespaces without violating the worker-internal verified-byte boundary; `finish_compound_source` is therefore an explicit replacement. A three-d fork is permitted only on compile evidence from the pinned `0.19.0` source that a required generic public hook is inaccessible; isolate only that generic hook in a pinned sibling path and keep all Polymodel scene/classifier logic outside it. -```sh -pwd -jj workspace root -jj workspace list -jj bookmark list -jj log -r 'epic-86@' -n 1 --no-graph -jj log -r 'main@' -n 1 --no-graph -jj status -``` +## Frozen contracts -Stop if the workspace root is not `/home/orual/Projects/epic-86-task-F`, the active bookmark is `main`, the workspace is the default workspace, or the task is not based on the parent’s integrated Epic tip. Create/use the expected sibling workspace and start a fresh change: - -```sh -jj workspace add /home/orual/Projects/epic-86-task-F -r epic-86 -cd /home/orual/Projects/epic-86-task-F -jj new -m 'PM-79: add minimal LDraw renderer scene boundary' -``` +### Units and world boundary -Do not reset unrelated changes. Do not edit `.current-epic/manifest.txt`, `.current-epic/dependencies.dot`, `.current-epic/task-state.json`, `.current-epic/merge-gate.json`, `events.log`, branches, or workspaces. +`1 LDU = 1/64 inch = 0.396875 mm = 0.000396875 m` exactly, represented as `Units::declared(LengthUnit::LdrawUnit)`. E parsing, inclusion, semantic export, instancing, BFC, bounds, and F adaptation remain finite `f64` LDU; no scaling occurs before the renderer/world boundary. Apply exactly once: -## Grounded replace-vs-edit decisions +```text +(x, y, z) -> (x, z, -y) +B = [[1, 0, 0], [0, 0, 1], [0, -1, 0]] +det(B) = +1 +``` -| Area | Decision | Plan | -|---|---|---| -| `crates/polymodel-renderer-protocol/src/lib.rs`, new `src/scene.rs` | **Edit/add** | Keep integrated `CompoundLdraw`, `StartCompoundLoad`, `CompoundSceneSource`, `CompoundResourcesReady`, `InstallSucceeded`, and `LdrawUnit`; add a dependency-light serde-safe IR. Protocol remains an app leaf independent of E, mesh, and `three-d`. Preserve wire bytes; explicitly version unavoidable changes. | -| `crates/polymodel-ldraw-core/src/geom.rs`, `model.rs`, `lib.rs`, new export module | **Focused edit/add; do not replace parser** | Extend the existing `GeometryRecord`/`InstanceRecord`/`OwnedParseResult` seam with an owned render export. Keep scanner, parser, BFC, colours, traversal, limits, canonical projection, and reservation ownership. Never reconstruct geometry from counts or parse source in F. | -| New `crates/polymodel-renderer-worker/src/scene.rs` | **Add** | Define a worker-private installed-scene abstraction with two close-parallel payload modes: the existing rich ordinary `ModelMesh`/`three_d::Model` path, preserving transformed parts, PBR colours/materials/textures/hierarchy and triangle/point geometry; and a richer structured-primitive path carrying surfaces, explicit edges, conditional edges, points, occurrences, and materials. Adapt E's LDraw export into the structured mode, but name and shape the mode by renderer capability rather than LDraw so another structured/CAD source can use it later without parser knowledge. | -| `crates/polymodel-renderer-worker/src/worker.rs` | **Focused renderer generalization** | Generalize installed/pending payload, admission, draw dispatch, and Preview to operate on the two scene modes under one lifecycle contract. Preserve the existing ordinary conversion and PBR payload intact. Both modes share reducer transitions, ownership/swap, camera fitting, controls, frame scheduling, resize, render target/readback, Preview encoding, disposal, and context retention; only resource construction and per-primitive draw dispatch differ. | -| `crates/polymodel-renderer-worker/Cargo.toml` | **Minimal edit** | Add E core for production. Add `three-d`'s `window` feature only to the native Preview render test target/dev dependency; production worker features remain unchanged. Keep protocol leaf-safe and keep `three-d`, `three-d-asset`, and mesh parsing worker-only. | -| Native Preview runner and render test | **Add** | Add one native `three-d/window` Preview runner backed by a shared render function. Interactive invocation opens a visible window for operator inspection and writes the rendered PNG; the automated test invokes the same function with a hidden window/offscreen `RenderTarget`, reads pixels, and compares the deterministic image. Run the automated mode under Xvfb when no display server is available; do not add EGL/OSMesa or a software renderer. | -| `src/thing_detail.rs`, `src/viewer.rs`, appview/indexing publish/projection seams | **Focused edit/add** | Exercise the normal Thing detail viewer for an ordinary Thing→Model→Part→File LDraw composition; do not add a viewer route, DEMO_MODELS entry, special source, upload flow, or product surface. Preserve ordinary format behavior. | -| `e2e/tests/viewer.spec.ts` | **Focused edit** | Exercise normal Thing detail wiring, installed stats, source identity, and no-fallback behaviour; keep existing context-retention tests. Pixel correctness belongs to the native Preview render test, not browser e2e. | -| `justfile` | **Minimal edit** | Add `preview-ldraw` to launch the visible native runner and a focused automated native Preview test recipe if the existing test command cannot address it directly. Do not alter unrelated recipes. | -| `.current-epic/*`, branches, manifests | **Do not edit** | Preserve orchestration state and repository rails. | +Preserve winding; compute source normals before B, use inverse-transpose for non-rigid occurrence transforms, apply B after that, and compute bounds after primitive points, occurrence transforms, and B. Correct stale protocol sites `crates/polymodel-renderer-protocol/src/lib.rs:326` and `:1055-1056` from `1/64 inch`/`0.396875 mm` to the task value. Test exact stud/module vectors separately. -## Scene IR contract to freeze +### Generic scene IR -- Branded deterministic IDs: `SceneId`, `OccurrenceId`, `GeometryId`, `MaterialId`, `TextureId`, provenance and diagnostic IDs. IDs are opaque, never public vector indices. -- `Scene` owns `Units`, provenance, diagnostics, root, ordered geometries/materials and flat occurrences, bounds, and full `SceneStats`; LDraw uses `Units::declared(LengthUnit::LdrawUnit)`. -- `Occurrence` is flat for this slice: stable `OccurrenceId`, source/target geometry identity, source span/material/BFC as needed, and one composed `world_from_local` transform to root. Repeated subfiles share geometry but retain distinct stable occurrence IDs/transforms. Nested parent/children and E parent IDs are not required; hierarchy generalization is not implemented here. -- `Geometry` owns authoritative type-2/3/4/5 payloads: generic `Point`, `Triangle`, `Quad`, `Line`, and `ConditionalLine { endpoints, controls }`. Generic points are renderer-fixture-only; no LDraw type-0 syntax is added. LDraw topology and occurrence material/BFC state remain available through the adapter. -- Material carries E-resolved colour/edge/finish/material state with deterministic fallback. TEXMAP remains provenance/diagnostics only; F does not sample it. -- Bounds and counts come from actual emitted geometry after hierarchy transforms, in native LDU; stats include vertices, triangles, quads, lines, conditional lines, and points. -- Sole world basis: `(x,y,z) -> (x,z,-y)`, matrix `[[1,0,0],[0,0,1],[0,-1,0]]`, determinant `+1`; preserve winding. Metric is declaration/provenance only: `1 LDU = 1/2500 m = 0.4 mm`. -- Ordinary `ModelMesh` inputs retain current STL/OBJ/glTF/3MF units, hierarchy, materials, textures, camera behavior, and output. +Add dependency-light serde/postcard-safe `SceneId`, `NodeId`/`OccurrenceId`, `GeometryId`, `MaterialId`, `TextureId`, `ProvenanceId`, and `DiagnosticId` (opaque deterministic IDs, not vector indices). `Scene` owns declared units, source/provenance, diagnostics, root, ordered geometry/material/texture tables, flat occurrences, bounds, and checked stats. Geometry retains semantic points, indexed triangles, quads, explicit lines, and conditional endpoints/control points; GPU upload alone may triangulate quads. Occurrences retain source/target identity, source span, material/BFC state, composed transform, and separate identity for repeated child instances. Materials retain colour/edge/finish/local mapping; F does not sample TEXMAP. Counts derive from emitted transformed semantic geometry. Ordinary wire-compatible `MeshStats { vertices, triangles }` remains unchanged. -## Integrated baseline and exact G handoff +### E export: two distinct structures -This plan is grounded in `/home/orual/Projects/epic-86-task-G` at `epic-86@99436d09`, where G is completed and integrated. Consume these exact symbols, not a parallel URL path: +The implementor must add and name a new per-model `LdMeshSemantic` structure; it does **not** already exist and must not be implied to be `GeometryRecord`. For every model, construct it during E semantic processing and compare it to the pinned C++ LDParse/LDMesh oracle: -- `MeshFormat::CompoundLdraw`, `LengthUnit::LdrawUnit`. -- `RendererCommand::StartCompoundLoad`, `CompoundLoadPlan`, `WorkerFetchDescriptor`, `validate`, `encoded_size`. -- `CompoundSceneSource { primary, descriptors: Vec<(WorkerFetchDescriptor, Vec)> }` and `handoff_compound_scene`. -- `RendererEvent::CompoundResourcesReady { namespace, identity, source }` (verified-resource boundary, not readiness). -- Worker `handle_compound_load`, `load_compound_source`, `compound_is_current`, `fetch_descriptor_bytes`, `finish_compound_source`. -- `RendererEvent::InstallSucceeded`, `LoadEvent::InstallSucceeded`, `LoadIdentity { load_epoch, scene_generation }`, `LoadReducer`, `ResourceToken`, `TransitionOutcome`. -- Viewer `ViewerMeshSource`, `CompoundLogicalIntent`, `PendingCompoundLoad`, `/app/ldraw/compound-resources`, and acquisition/restart epoch checks. +- `positions: Vec<[f64; 3]>` in native source LDU order; +- `normals: Option>`, populated only when the oracle exposes authoritative normals for that model, otherwise `None`; +- `indices: Vec` forward triangle indices; +- `bf_indices: Vec` reverse/back-face index stream exactly as oracle-observable; +- `step_offsets: Vec` direct-mesh index offsets in LDParse index space. -Old G review claims that G lacked compound handoff, parser boundary, or integration are **rebutted-stale** by this baseline. F consumes G’s verified bytes/resource set; `CompoundResourcesReady` must never be treated as renderer readiness. +Keep a separate ordered `Vec` with one source primitive per record, never grouped batches. Enumerate its fields explicitly: `line_type`, `colour`, ordered `vertices`, source model identity, exact source span, effective BFC state, winding/reversal/inversion state, TEXMAP association, and stable primitive/provenance identity. Type 2/3/4/5 roles remain exact; quads and source provenance remain semantic until GPU upload. Add every required field to all `GeometryRecord` constructors and serializers, update every `crates/polymodel-ldraw-core` test and `crates/polymodel-ldraw-testkit` fixture/snapshot that constructs or decodes it, and reject missing fields: no `serde(default)`, permissive defaults, compatibility shims, or hidden backcompat path. Also retain model identity/path/default colour/local colour mappings, child/include identity and spans, occurrence transforms/colour inheritance, sharing, and typed materialization provenance. Compare positions, authoritative normals, forward/reverse indices, step offsets, primitive order/counts/identity, BFC/material and source spans against `crates/polymodel-ldraw-testkit/corpus/expected/*.json` oracle fields; metadata counts alone are insufficient. -## Authoritative LDU and transform rules +The semantic export remains `f64` and must first reject non-finite coordinates/normals. For each oracle-comparable value, perform a checked finite `f64 -> f32` conversion, compare the exact f32-rounded representation (`to_bits`) with the decoded C++ oracle f32 encoding/fixture value, and separately retain/assert the original finite f64 semantic value; do not compare f64 directly to f32 or silently hide precision loss. If an oracle fixture is decimal-formatted rather than bit-encoded, use the named oracle float-format tolerance derived from that formatter and assert the rounded f32 bits separately. -PM-75/PM-79 is authoritative: **`1 LDU = 1/2500 m = 0.4 mm`**. Integrated protocol prose saying `1/64 inch = 0.396875 mm` is stale documentation; correct it when touching that comment, but do not convert coordinates. `Units::declared(LengthUnit::LdrawUnit)` records declaration only. +### Compound boundary and stale protocol cleanup -Retain native finite `f64` LDU coordinates. Sole basis: +Verified descriptor bytes stay inside the worker. Remove or replace `CompoundResourcesReady` at `crates/polymodel-renderer-protocol/src/lib.rs:684-690` with namespace success events and, if retained, a test/metadata-only identity/count/bounds event containing no bytes, vectors, raw text, or descriptors. Explicitly remove/relocate the obsolete byte transport at protocol `:611-632` (`CompoundSceneSource`, `handoff_compound_scene`) and update/remove its golden tests at `:997-1012`; delete worker uses at `worker.rs:1361`, `:1407-1412`, `:1421`, `:1448` and main-thread use at `src/viewer.rs:1806-1815`. Any genuinely worker-internal source container belongs in the worker library, not protocol. `InstallSucceeded` is Interactive readiness only; `PreviewImageRendered { request_id, png }` is Preview success only. No source bytes cross the worker bridge. -```text -(x,y,z) -> (x,z,-y) -B = [[1,0,0], [0,0,1], [0,-1,0]], det(B) = +1 -``` +### Operator-owned browser and ledger boundaries -Compute normals from LDU triangles before basis; use inverse-transpose for non-rigid/non-uniform occurrence linear transforms, then apply `B` once. Preserve winding—no second flip. Compute bounds after primitive points, composed occurrence transforms, then basis. Test stud vectors independently from the metric. +Do not add or restore `renderer-test-readback`, `RendererReadback`, `TestReadback`, `window.__POLYMODEL_TEST_READBACKS__`, `e2e-viewer`, or `serve-headless`. Do not add browser e2e to `test-all`. The existing compound browser test remains a transport test for the public plan and resource routes; do not turn it into renderer/readback acceptance or remove its transport assertions. -## Dependency graph and E-owned export contract +Do not add reservation classes, charges, ownership, tests, or other behaviour to `polymodel-renderer-ledger`. PM-88 is frozen. F uses the existing ledger API where the existing implementation already does so, but this task does not expand or redesign ledger admission. Rendering evidence belongs to the shared native renderer/preview path; official-library evidence belongs to focused resolver/native integration coverage. -```text -protocol::scene (leaf IR; app-safe) - ^ ^ - | | -E LDraw core (export) mesh (ordinary input) - \ / - renderer-worker (adapter + three-d/GPU) - ^ - app/viewer -``` +## Numbered implementation steps -The protocol remains independent of E, mesh, and `three-d`; E remains independent of renderer protocol. Put the narrow serde/postcard-safe IR in `polymodel-renderer-protocol/src/scene.rs`. The worker depends on E and owns adaptation/GPU resources; E does not depend on the renderer protocol. No speculative scene crate or format registry. - -G descriptor roots are the exact appview strings `mpd`, `manifest`, `models`, `parts`, `p`, `external`, `official-parts`, and `official-p` (`src/appview/ldraw.rs:341-352`). Map them deterministically before E parsing: `mpd -> RootId::CurrentMpd` and current-MPD `Materialization`; `manifest -> RootId::UploadedManifest` and manifest/project `Materialization`; `models|parts|p|external -> RootId::UploadedLdraw` and uploaded-LDraw `Materialization`; `official-parts|official-p -> RootId::OfficialLibrary` and official-library `Materialization`. Unknown roots fail closed. For each verified descriptor construct `Materialization::new(NormalizedPath::new(descriptor.path), mapped_root, bytes, true, Some(descriptor.key/root provenance))`; reject invalid path, length/digest mismatch, duplicate `(mapped_root,path)`, missing/duplicate primary, or route root/path mismatch. The unique descriptor whose key equals `source.primary` supplies `ParseOptions.root_name` and `resolved_root`; use the worker owner and an explicit empty `target_selections: &[]` because the integrated plan has no selection table. Never guess by vector order or synthesize E parent IDs. - -Add a public E export module (for example `polymodel-ldraw-core/src/render_export.rs`) with exact owned types: - -```rust -pub struct RenderExport { - pub source: RenderSourceIdentity, - pub units: RenderUnits, // native LDU - pub geometries: Vec, - pub occurrences: Vec, - pub materials: Vec, - pub diagnostics: Vec, - pub provenance: RenderProvenance, - pub bounds: Option, - pub stats: RenderStats, -} -pub struct RenderGeometry { - pub id: GeometryIdentity, - pub model_id: String, - pub primitives: Vec, - pub bfc: BfcFrame, - pub material: MaterialIdentity, -} -pub enum RenderPrimitive { - Triangle { vertices: [Point3; 3], winding: Winding }, - Line { endpoints: [Point3; 2] }, - ConditionalLine { endpoints: [Point3; 2], controls: [Point3; 2] }, -} -pub struct RenderOccurrence { - pub id: OccurrenceIdentity, - pub source_geometry: GeometryIdentity, - pub target_geometry: GeometryIdentity, - pub world_from_local: Transform, - pub bfc: BfcFrame, - pub colour: ColourCode, - pub source_model: String, - pub target_model: String, -} -``` +### 1. Ground the live parent and load implementation skills -Also own `RenderSourceIdentity` (canonical path/root/content hash), `RenderUnits`, deterministic branded `GeometryIdentity`/`OccurrenceIdentity`/`MaterialIdentity`, `RenderMaterialState` (resolved `ColourData`, edge/finish/material state and fallback), `RenderProvenance`, `RenderDiagnostic` (severity/code/message/span), and checked `RenderStats` (vertices, triangles, quads, lines, conditional lines, points). `OwnedParseResult::into_render_export(self) -> Result` is the sole ownership transfer; `OwnedRenderExport` retains E `SemanticArenas`/reservation until drop or adapter consumption. Preserve `OwnedParseResult::project()`, `OwnedProjection`, and `CanonicalRecord` unchanged as compatibility projections. +Verify workspace/ancestry safety, inspect the then-current A/B/D/E/G symbols, and inspect the pinned three-d 0.19 public `Geometry`, `Material`, `Program`, `Gm`, buffer/context, window, and readback seams. Load the renderer/frontend skills applicable to worker protocol, native rendering, and browser tests. -The existing `GeometryRecord` retains type-2/3/4/5 arrays in `ModelData::geometry`/`ModelSummary::geometry`; `InstanceRecord` retains type-1 transform, inversion, BFC, colour, and target identity. Extend this seam to copy owned payloads and flat occurrences with composed transforms and source/target geometry identity. Split type 4 deterministically as triangles `(0,1,2)` and `(0,2,3)` with E-resolved BFC winding exactly once. Do not parse source lines again in F. +**Done when:** live parent tip and pinned oracle/reference are recorded in the task change; no work starts on `main` or stale `task-state`; required public three-d seams are identified from the exact pinned source. -**Point contract:** do not invent an LDraw type-0 point syntax. The protocol IR includes generic `Primitive::Point`, tested only with a renderer-owned lower-level fixture. The normal Thing MPD/companion composition uses authoritative types 1–5; if point acceptance is required, prove generic point drawing separately and state the distinction. H owns broader LDraw point semantics. +### 2. Add protocol scene IR, units, and byte-boundary cleanup -## Renderer IR, adapter, and GPU ownership +Implement the scene module and appended LDU unit. Remove the stale `CompoundSceneSource`/`handoff_compound_scene`/byte-bearing event sites listed above and update main handling to metadata-only/current-identity events. Preserve ordinary wire golden values and protocol dependency split; do not add browser test-readback protocol. -`scene.rs` owns branded `SceneId`, `OccurrenceId`, `GeometryId`, `MaterialId`, `TextureId`, provenance/diagnostic IDs; `Scene`, flat `Occurrence`, `Geometry`, `Material`, `TextureRef`, `Bounds`, `SceneStats`, `Units`, and `Primitive`. `Scene` owns ordered geometry/material tables, flat occurrences, units, provenance, diagnostics, bounds, and full stats. Each occurrence owns source/target geometry identity and a composed `world_from_local` transform; no nested parent/children hierarchy is required in F. `Geometry` owns indexed triangles/indices/normals, explicit lines, conditional endpoints/controls, and point positions. IDs/order are deterministic from content identity and traversal order. +**Done when:** protocol native/WASM checks via Just pass; scene IDs/order/serde/postcard and exact LDU tests pass; no protocol event carries verified bytes; exact stale sites are corrected or removed; no browser readback feature or event exists. -The existing ordinary parser continues producing rich `ModelMesh` values: transformed parts, PBR colours/materials, textures, hierarchy, triangle geometry, and point clouds. STEP continues reaching this path through its existing glTF conversion. Preserve that payload and conversion intact. +### 3. Extend E with oracle-matched semantic export -Generalize the worker's installed scene into two close-parallel capability modes: `Ordinary` retains the existing `three_d::Model` representation; `Structured` owns surface batches, explicit edge batches, conditional-edge controls, points, occurrence transforms, and resolved materials. E's LDraw export adapts into `Structured`, but the mode itself contains no LDraw parser concepts and is reusable by a future structured/CAD source. Both modes implement one worker-private behaviour contract for bounds/stats, admission, camera fit, draw submission, Preview, readback, controls, resize, lifecycle swap, and disposal. The WASM viewer, native visible runner, and native automated Preview test invoke this same behaviour; only mode-specific resource construction and primitive draw dispatch differ. TEXMAP remains metadata only; F does not sample it. +Add `LdMeshSemantic` and the distinct ordered `GeometryRecord` fields, construct both once during E semantic processing, preserve type 1–5 provenance, BFC/material/TEXMAP state, models/children/occurrences, and exact step offsets. Add typed root mapping and materialization validation: app descriptor roots `mpd`, `manifest`, `models`/`parts`/`p`/`external`, `official-parts`/`official-p` map respectively to `CurrentMpd`, `UploadedManifest`, `UploadedLdraw`, `OfficialLibrary`; unknown roots, invalid paths, missing/duplicate primary, duplicate `(RootId, normalized path)`, route mismatch, length/digest mismatch, and descriptor identity mismatch fail before parse. -Worker-only `PendingScene`/`InstalledScene` own `GpuSceneResources`: triangle batches as `three_d::CpuMesh` uploaded to `three_d::Mesh` and drawn via `three_d::Gm`; explicit lines via `three_d::Line`; points via `three_d::Points`; conditional endpoints/controls in a worker-owned conditional batch, reduced to visible endpoint segments submitted through `three_d::Line`. The deterministic conditional visibility rule projects endpoints and controls and draws the endpoint iff control points are on opposite sides of the projected endpoint line (cross-product signs product `<= 0`, zero visible). A scene is valid with triangles **or** lines/conditional lines/points; empty bounds are an install error. Camera framing uses transformed bounds and existing worker camera/control setup. Reserve before allocation; build pending completely; swap only after reducer-accepted `LoadEvent::InstallSucceeded`; drop pending/old resources and reservations exactly once on failure, stale, cancel, resize, or dispose. No TEXMAP sampling. Keep wire `MeshStats { vertices, triangles }` unchanged; full primitive counts remain internal/browser-visible stats. No implicit wire field additions. +**Done when:** `just test-ldraw-core`, `just ldraw-differential`, and `just ldraw-corpus` compare actual export fields against the pinned C++ oracle; positions, authoritative normals, indices, `bf_indices`, step offsets, ordered records/identity, BFC/material, and source spans are asserted, with checked finite f64 values and exact f32-rounded/oracle encoding comparisons, and semantic drops separately observed from worker ledger snapshots. In `src/appview/read_tests.rs`, named tests `ldraw_descriptor_root_mapping_accepts_supported_roots` and `ldraw_descriptor_rejects_malformed_plans_before_parse` use `e2e/fixtures/ldraw/descriptor-matrix.json` to cover `mpd`, `manifest`, `models`/`parts`/`p`/`external`, `official-parts`/`official-p`, unknown roots, invalid paths, missing/duplicate primary, duplicate `(RootId, normalized path)`, route mismatch, length/digest mismatch, and descriptor identity mismatch. -Expose per-kind counts and bounds only through one narrow test/debug-gated payload for the normal Thing browser proof and worker tests; never claim a general browser API: +### 4. Add reusable worker library and custom three-d objects -```text -DebugFrameReadback { - identity: LoadIdentity, - viewport_width: u32, - viewport_height: u32, - dpr: u32, - clear_rgba: [u8; 4], - non_clear_pixels: u32, - triangles: u32, - lines: u32, - conditional_lines: u32, - points: u32, - bounds_min: [f32; 3], - bounds_max: [f32; 3], -} -``` +Move reusable scene construction, adaptation, camera/draw/readback/PNG functions into `crates/polymodel-renderer-worker/src/lib.rs`. Retain `src/main.rs` as WASM `renderer_worker`; add `src/bin/ldraw_preview.rs` as native `ldraw_preview`; add `crates/polymodel-renderer-worker/tests/ldraw_preview.rs` as the automated named-target integration test. Use public three-d 0.19 `Geometry`, `Material`, `Program`, `Gm`, and low-level buffers/context for world-space surfaces, lines, portable points, and conditional endpoints/controls. Do not use built-in 2D `three_d::Line` or invent `three_d::Points`; keep ordinary `Model` and glTF hierarchy/PBR/textures unchanged. F owns only the minimal conditional visibility rule/seam; H owns the complete classifier. -Emit it only after an actual worker frame draw/readback on the fixed viewport/DPR/clear colour. `non_clear_pixels > 0` is the viability sentinel; DOM readiness, canvas existence, or counts alone are insufficient. +**Done when:** pure worker tests cover basis/determinant/winding/normals/bounds, finite/nonempty geometry, primitive objects, conditional controls/rule, pending/installed ownership, and ordinary format parity; the library is shared by WASM worker and native preview with one scene/render path. No ledger API or admission expansion is introduced. -## Numbered implementation steps +### 5. Replace compound completion and unify Interactive/Preview -### 1. Freeze protocol IR and workspace safety +Replace the complete `load_compound_source` → `finish_compound_source` chain with verify → worker-internal typed materialization → parse once → export once → adapt once → build pending candidate → namespace output; remove the protocol `CompoundSceneSource` return type from that chain. The adapter maps E model identity/path/default and local colours to scene provenance/material tables; ordered `GeometryRecord` vertices/topology/BFC/winding/TEXMAP/provenance to scene geometry and material references; and instance source/target identity, composed transforms, inherited colour, inversion, and spans to distinct flat occurrences. It derives scene bounds/stats only from emitted transformed semantic geometry. Check session/namespace/load epoch/scene generation/acquisition/restart identity, reducer identity, abort, and disposal before/after each boundary. Interactive posts/accepts `InstallSucceeded` through the reducer before swapping candidate for old scene and then performs actual draw/readback. Preview renders the same candidate path and posts `PreviewImageRendered`; it never posts Interactive readiness. On error/cancel/stale/resize/dispose release pending semantic/GPU values and preserve the existing cancellation and cleanup behaviour without adding ledger classes or admission semantics. -From `/home/orual/Projects/epic-86-task-F`, verify workspace/bookmark safety and `epic-86@99436d09`. Add the leaf IR, generic point primitive, and unit declaration while preserving all integrated G symbols and existing wire fixtures. In `crates/polymodel-renderer-protocol/src/lib.rs`, explicitly replace the stale LDU comment at lines 326-327 and the stale numeric assertion at lines 1054-1056 that encode `0.396875`; preserve the `LdrawUnit` discriminant and assert metadata `1/2500 m = 0.4 mm` without scaling coordinates. +**Done when:** lifecycle tests match the finite table below; no stale/cancelled completion publishes success or replaces the old scene; parse/export/adapt counters are exactly once on successful Interactive and Preview; abort slots are empty after terminal cleanup. -**Done when:** native/WASM protocol builds, mixed-scene serde/order/ID tests pass, existing wire golden is unchanged, and protocol has no E/mesh/three-d dependency. +### 6. Ship optional official-library fallback and seed the pinned OMR product path -### 2. Add E render export, not a second parser +Restore the historical pinned Nix `ldraw-parts` derivation (`2026-06`, `complete.zip`, fixed hash) as `packages.ldraw-parts`. Add optional server runtime configuration `POLYMODEL_LDRAW_LIBRARY_DIR` at the process environment boundary and carry it internally as a validated/branded immutable library root. A configured server serves only `official-parts` and `official-p` resources from that tree; MPD/manifest/models/parts/p/external project resources remain PDS/projection-backed. Preserve resolver precedence so MPD/project-provided resources override official fallback. Canonicalize paths beneath `parts/` and `p/`, reject traversal and symlink escapes, and include pinned library version/source identity, byte length, and digest in descriptors. When the package is not configured or a required official part is absent, resolution fails explicitly before rendering; never require publishers to upload official-library bytes to their PDS and never partially render. -Extend `GeometryRecord`/`InstanceRecord`/`OwnedParseResult` with the exact export contract. Retain type-2/3/4/5 arrays, type-1 hierarchy/transforms, identities, BFC, colour/material state, TEXMAP metadata, diagnostics, bounds/stats, and reservation lease. Split quads deterministically in E; do not add type-0 parsing. +Keep the existing normal route and anchor `/did:plc:aaaaaaaaaaaaaaaaaaaaaaaa/thing/parametric-enclosure` and request body `at://did:plc:aaaaaaaaaaaaaaaaaaaaaaaa/space.polymodel.library.thing/parametric-enclosure`. Use the existing accepted compatibility fixture `crates/polymodel-ldraw-testkit/corpus/omr-10030-1.mpd` (manifest SHA-256 `a7f56cfbd71445d6fa218a102161608d52f63cd0b3b55062276c7e0f54b03ebb`) and its authoritative `corpus/expected/omr-10030-1.expected.json`; do not create a smaller preview/product-only fixture. Dev/native Just recipes inject the restored Nix derivation root; server integration tests configure the same package through `POLYMODEL_LDRAW_LIBRARY_DIR`. -**Done when:** exact primitive-array export and repeated transformed-child identity tests pass; canonical projection remains stable; dropping `OwnedRenderExport` releases E reservation. +Add `seed_ldraw_compound_fixture` beside the existing `seed_if_empty`/`seed` helpers in `src/indexing/sample_data.rs`, and make the appview fixture routes serve the exact OMR plus official-library fallback bytes with real CID, SHA-256, length, MIME/type, root/path, version, and source identity. Tests compare fetched bytes directly to corpus/library bytes and assert local official fallback without PDS uploads, project override precedence, missing-library and missing-part failure, traversal/symlink rejection, and the OMR's actual model graph, STEP offsets, repeated transformed occurrences, BFC/material state, ordered primitive identity/source spans, and nonempty bounds against its expected record. -### 3. Add the structured scene mode in close parallel +**Done when:** the existing `/app/ldraw/compound-resources` and resource routes deliver the pinned OMR primary plus required local official-library resources, the normal Thing detail viewer consumes them without an alternate route or PDS copies, project resources override official fallback, absent/malformed official resources fail closed, and root/integrity/identity tests reject malformed plans before E parse. -Generalize pending/installed scene behaviour around `Ordinary` and `Structured` payloads. Keep ordinary `ModelMesh` conversion and `three_d::Model` construction unchanged. Implement the E-export adapter into the structured mode; validate finite/checked/non-empty payloads, map E materials/fallbacks, and construct surface, explicit-edge, conditional-edge, and generic-point resources. Compute LDU normals first, inverse-transpose where needed, then apply the determinant-`+1` basis once with no winding flip and compute post-transform bounds. Reuse common camera fitting, controls, render targets, Preview encoding, lifecycle, and disposal while dispatching each mode's actual drawable resources. +### 7. Freeze native preview recipes and automated target -**Done when:** structured adapter tests prove topology/endpoints/controls/point resources, point/line-only acceptance, basis/stud vectors, normals, transformed bounds, materials, and typed errors; ordinary rich PBR/texture/hierarchy payloads remain unchanged; both modes pass the same behaviour contract for install, frame, resize, Preview, and disposal. +Edit the Justfile public dispatch surface and private implementations with these exact recipes: -### 4. Consume G handoff through E to install +- `just test-ldraw-core` — focused E/export/oracle unit and fixture tests; +- `just ldraw-wasm-check` — host-WASM compile/test gate for `polymodel-ldraw-core` and `polymodel-ldraw-testkit`; +- `just test-renderer-native` — under the recipe-owned headless GL/Xvfb environment, invokes `crates/polymodel-renderer-worker/tests/ldraw_preview.rs` against `crates/polymodel-ldraw-testkit/corpus/omr-10030-1.mpd` plus the pinned official-library resolver inputs, at fixed `1280x720`, and asserts readback/PNG/stats/bounds and external-part resolution; +- `just preview-ldraw *ARGS` — invokes the exact native binary `ldraw_preview` and forwards `ARGS` unchanged; with no arguments it uses the same pinned OMR fixture/resolver, fixed `1280x720` viewport, and deterministic output `target/ldraw-preview/omr-10030-1.png`; `--help`, input, resolver/library root, output PNG, width, and height are documented; -Retain G’s `load_compound_source` path: validate plan; reserve network/retained/decoded/WASM classes before allocation; fetch each descriptor route; verify exact length/SHA-256 via `verify_part_file`; check `compound_is_current`; construct `CompoundSceneSource`; call `handoff_compound_scene`; release fetch reservation. Never derive a direct LDraw URL. +The private implementations remain behind `tools/in-dev-shell`; public recipe names are the only repository command surface. The recipes own a deterministic headless GL environment (Xvfb with fixed screen/depth or the repository's equivalent proven headless three-d context) and fail when that environment, the OMR bytes, or any pinned official-library dependency is unavailable. `just preview-ldraw` and `just test-renderer-native` call the same library scene construction, resolver, camera, draw, real readback, and PNG encoder; add no second renderer backend. -Then construct E `Materialization` entries from descriptor path and verified bytes, select `source.primary`, use the current `ReservationOwner`, invoke `LdrawParser::parse_bytes` exactly once, and call `into_render_export`. The worker keeps the verified source/bytes; the main thread never receives bytes or sends them back. Check negotiated session, namespace, load epoch, scene generation, acquisition/restart epoch, source identity, descriptor integrity, current identity after parse/adapt, and `Installing(identity)` before output. Check abort between descriptors and before parse/adapt/output. Clean every E/fetch/scene/GPU reservation and candidate on error, cancel, supersession, stale, resize, and dispose. +**Done when:** `just preview-ldraw --help`, zero-argument `just preview-ldraw`, and `just test-renderer-native` execute the named target without ambient display configuration; `target/ldraw-preview/omr-10030-1.png` has a valid PNG signature, exact `1280x720` dimensions, non-clear pixels, expected oracle-derived primitive counts/bounds, and evidence that official-library parts were resolved and rendered; missing official dependencies fail the test; `just build-renderer-worker` still builds the retained `renderer_worker` WASM bin; `just verify-renderer-split` proves the app remains protocol-only. -Both `RendererNamespace::Interactive` and `RendererNamespace::Preview` use the same verified-source → E parse/export → LDraw scene-adapter → worker-private `PendingScene` pipeline. In `finish_compound_source`, no `CompoundResourcesReady` event may be posted before this shared parse/export/adapt stage. Interactive reserves/allocates GPU resources, reducer-commits `LoadEvent::InstallSucceeded`, swaps the installed scene, and publishes `RendererEvent::InstallSucceeded` with wire-stable `MeshStats`; after an actual draw it may emit the narrow gated debug readback. Preview uses the same candidate scene resources and camera to render the fixed requested output to PNG, then publishes the existing `RendererEvent::PreviewImageRendered { request_id, png }` protocol shape; it does not publish Interactive `InstallSucceeded`. Ordinary Preview PNG behavior remains unchanged for non-compound formats, and no parallel parser/raw URL path is added. +### 8. Preserve browser transport coverage and add native rendering parity -`CompoundResourcesReady` remains an informational verified-source event only and must be posted after the shared parse/export/adapt stage and after the namespace-specific output has been accepted: for Interactive, after reducer-committed `InstallSucceeded`, installed swap, and the actual draw/readback; for Preview, after the shared scene has rendered and `PreviewImageRendered { request_id, png }` has been accepted for posting. It is never readiness and must not precede parse/export/adapt. Exactly three lifecycle coupling cases cover both namespaces without a Cartesian matrix: failure/cancel cleanup, superseding stale completion, and resize/dispose ownership cleanup. +Keep the existing compound case in `e2e/tests/viewer.spec.ts` as a public plan/resource transport test with its current descriptor, HTTP, byte-stability, and compound-boundary assertions. Do not add browser readback, screenshot, Preview PNG, or renderer acceptance to that test. Add focused resolver/native integration coverage for a compact model that exercises the built-in official library, proving typed official roots, precedence, transitive resolution, exact bytes, and successful native rendering. Retain and strengthen ordinary STL/OBJ/glTF/3MF parity in the shared native scene/render path using the existing ordinary fixtures. -**Done when:** Interactive reaches `Ready` only through reducer-accepted `InstallSucceeded`; Preview renders the same E export through the ordinary PNG path and publishes `PreviewImageRendered` with a nonempty valid PNG; `CompoundResourcesReady` is posted only afterward and is never readiness; stale/cancel/error paths leak nothing; ordinary formats/textures remain green. +**Done when:** `just e2e` preserves the compound transport test unchanged in purpose, and `just test-renderer` plus `just test-renderer-native` prove official-library resolution and actual rendering without browser readback infrastructure. -### 5. Prove Preview output natively and exercise the normal Thing flow +### 9. Validate, review, and hand off -Use the normal publish UI/record contract to compose an ordinary Thing→Model→Part→File record containing an LDraw multipart/MPD root at `models/main.ldr` plus every companion source. Seed those ordinary records in the test database, including actual type-2/3/4/5 geometry, BFC/material state, a repeated transformed child, and the real CID/SHA-256/byte-length values in file metadata and manifest edges. Assert projection and G’s `/app/ldraw/compound-resources` resolver discover the primary and companions; the worker must verify source identity/boundaries before passing bytes to E. Navigate to that Thing’s normal detail viewer endpoint. Do not add a fixture/product route, DEMO_MODELS entry, separate upload flow, direct URL, or fallback source. +Run the exact Just surface below, inspect `jj diff`/`jj status`, and commit only intended implementation, test, fixture, and recipe files with trailers: -Add a native Preview runner compiled with test/tool-only `three-d/window`. Its normal invocation opens a visible window, loads the seeded verified compound inputs through E export and the same F scene adapter/Preview camera path, renders continuously for operator inspection, and writes the current Preview PNG on startup or an explicit capture action. Provide a documented `just preview-ldraw` command that launches this runner directly, without the app server, browser, database service, or Playwright. +```text +PM-79: add minimal LDraw renderer scene boundary -Make the deterministic renderer oracle browser-free by reusing the runner's render function in an automated mode: create a hidden `WindowedContext`, render into an offscreen `RenderTarget`, read pixels, and compare the produced image at a fixed viewport against the committed expected PNG (exactly, or with one explicitly justified tight pixel tolerance if the native GL implementation requires it). Run automated mode under Xvfb in display-less environments. The test must assert valid PNG dimensions/encoding, non-clear pixels, expected primitive stats/bounds, and image comparison; scene snapshots alone are insufficient. The visible runner and automated test must not maintain separate scene construction, camera, or draw implementations. +Epic: PM-86 +Task: PM-79 +``` -Browser e2e proves product wiring and straightforward live-renderer behaviour rather than duplicating the native pixel baseline: the normal Thing detail endpoint exposes the expected Thing/source/current-load identities, reaches reducer-accepted `InstallSucceeded` with nonzero installed stats, and draws a non-clear frame without falling back to STL or a special route. Apply one deterministic orbit input and one resize; assert the same retained worker/context and load identity produce another non-clear frame with updated viewport dimensions, without reacquisition, reinstall, or context recreation. Compound Preview uses the same E export/scene adapter, renderer core, camera/frame path, and returns a valid nonempty PNG through the existing Preview event. +Resolve critical/high implementation-review findings before parent handoff. Report the consumed parent tip, commit, file groups, exact recipe results, oracle evidence, fixture byte/CID/digest evidence, event-security evidence, lifecycle evidence, and clean task status. Parent merges only after all gates and review clearance. -**Done when:** `just preview-ldraw` opens a visible native window showing the seeded multipart model and writes its Preview PNG; the shared hidden-window automated mode produces the expected image without a browser; normal Thing detail navigation reaches compound LDraw with no fallback and correct identities/readiness; compound Preview returns the same-scene PNG; no special route or product surface exists. +## Finite lifecycle acceptance table -## Minimal functional test set +`P` means Preview. Successful rows require the listed success event exactly once; interrupted rows require zero success events. `parse/export/adapt` are exact invocation counts. `abort` is the namespace abort slot. `candidate/old` states are after cleanup. This table validates F lifecycle behaviour only; it does not add or alter PM-88 ledger requirements. -Do not add fuzzing, performance suites, publishing registry work, TEXMAP sampling, broad visual baselines, or H/I/J/K scope. Add only these focused tests: +| Case | Namespace/checkpoint | Expected event(s) | parse/export/adapt | abort | candidate / old scene | semantic drop | +|---|---|---|---:|---|---|---| +| `I-fetch-fail` | Interactive, fetch | one error; no readiness | `0/0/0` | empty | none / old retained | none | +| `P-fetch-cancel` | Preview, fetch | one cancellation; no PNG | `0/0/0` | empty | none / unchanged | fetched values dropped | +| `I-preparse-cancel` | Interactive, pre-parse | one cancellation; no readiness | `0/0/0` | empty | none / old retained | materializations dropped | +| `P-preparse-cancel` | Preview, pre-parse | one cancellation; no PNG | `0/0/0` | empty | none / unchanged | materializations dropped | +| `I-postparse-pre-adapt-stale` | Interactive, after parse before adapt | stale ignored; no readiness | `1/1/0` | empty | candidate absent / old retained | export dropped | +| `P-postparse-pre-adapt-stale` | Preview, after parse before adapt | stale ignored; no PNG | `1/1/0` | empty | candidate absent / unchanged | export dropped | +| `I-postadapt-pre-output-error` | Interactive, after adapt before install | one error; no readiness | `1/1/1` | empty | candidate dropped / old retained | candidate/export dropped | +| `P-postadapt-pre-output-error` | Preview, after adapt before PNG | one error; no PNG | `1/1/1` | empty | candidate dropped / unchanged | candidate/export dropped | +| `I-stale-completion` | Interactive, completion after newer identity | stale ignored; no readiness/swap | `1/1/1` | empty | stale candidate dropped / newer scene retained | candidate/export dropped | +| `P-stale-completion` | Preview, pre-publication after newer identity | stale ignored; no PNG | `1/1/1` | empty | stale candidate dropped / unchanged | candidate/export dropped | +| `I-success` | Interactive, reducer-accepted install then draw | one `InstallSucceeded` | `1/1/1` | empty | candidate installed / old released once | old scene released once | +| `P-success` | Preview, output publication | one `PreviewImageRendered` | `1/1/1` | empty | temporary candidate dropped / unchanged | candidate/export dropped after PNG | +| `I-resize` | Interactive, installed scene | no reload/readiness | unchanged `1/1/1` | empty | installed retained / same identity | none | +| `I-dispose` | Interactive, during fetch/parse/adapt/install | one disposal; no late readiness; late completion ignored | at-most `1/1/1`, never repeated | empty after disposal | pending candidate dropped / old scene retained | all temporary semantic/export/adapt values dropped | +| `P-resize` | Preview, before publication | no reload/PNG publication; resize is rejected as non-Interactive and late output is ignored | unchanged `1/1/1` | empty | temporary candidate dropped / unchanged | all temporary semantic/export/adapt values dropped | +| `P-dispose` | Preview, during/after output | one disposal; no late PNG | at-most `1/1/1`, never repeated | empty | candidate dropped / unchanged | all temporary values dropped | -1. **E export (Tier 1):** exact type-2/3/4/5 primitive arrays, deterministic quad split, repeated transformed child occurrence IDs/shared geometry identity, BFC/material metadata, provenance, bounds/stats, and reservation drop. -2. **IR/protocol (Tier 1):** serde/order/ID stability for triangle, explicit line, conditional line, and generic point; one pre-existing wire golden and unit compatibility check. -3. **Math (Tier 1):** exact declaration `1/2500 m = 0.4 mm`, no scaling, basis/stud vectors, determinant `+1`, winding preservation, normal inverse-transpose timing, and bounds order. -4. **Adapter (Tier 1):** E export to triangle/line/conditional draw resources; renderer-owned generic point fixture to a point draw resource; point/line-only acceptance; malformed/non-finite/empty errors; pending/installed ownership and full internal stats. -5. **Ordinary parity (Tier 1/worker):** compact STL/OBJ/glTF/3MF adapter smoke, retained hierarchy/material behavior, and existing glTF texture assertion. -6. **Exactly three real-ledger lifecycle coupling cases (Tier 1 worker/reducer, no Cartesian matrix):** (a) parse/adapter/install failure or cancel returns reservations and disposes pending resources; (b) superseding identity makes late completion stale and unable to replace the scene; (c) resize/dispose swaps or clears ownership without leaking the old scene. -7. **Native Preview runner/render test:** `just preview-ldraw` opens a visible native `three-d/window` runner for operator inspection and PNG capture from deterministic multipart inputs. The automated mode reuses the exact scene/camera/draw function with a hidden `WindowedContext` (Xvfb in display-less CI), renders through the real Preview camera and offscreen `RenderTarget`, reads pixels, encodes PNG, and compares with the committed expected image at a fixed viewport. Assert dimensions, non-clear pixels, primitive stats/bounds, and expected image; this is the primary renderer proof and uses no browser. -8. **Live renderer browser behaviour:** navigate to the seeded Thing’s normal detail endpoint and assert projection/G compound discovery, source identity/no STL fallback, reducer-accepted `InstallSucceeded`, nonzero installed stats, and one non-clear frame. Apply one deterministic orbit and one resize, then assert another non-clear frame with updated viewport while worker/context/load identity remain unchanged and no reacquisition or reinstall occurs. Request ordinary compound Preview and assert valid nonempty PNG delivery. This is a focused behaviour test, not a visual baseline; do not add a special route or DEMO_MODELS entry. +## Test strategy and exact validation order -## Commands and validation order +Tests are Tier 1 pure protocol/E/adapter/reducer/oracle tests, Tier 3 real appview/worker/browser integration, and Tier 4 native GL/readback tests; no provider-backed tests apply. Every behavior covers success, failure, cancellation/supersession, malformed input, and cleanup through the named recipe rather than raw build/test commands. -Run from `/home/orual/Projects/epic-86-task-F` on `epic-86-task-F`: +Run in this order from `/home/orual/Projects/epic-86-task-F` after safety checks: -```sh -pwd -jj workspace root -jj log -r 'epic-86@' -n 1 --no-graph -jj status +```text just check just fix -cargo test -p polymodel-renderer-protocol -cargo test -p polymodel-renderer-ledger -just preview-ldraw --help -cargo test -p polymodel-ldraw-core -cargo test -p polymodel-renderer-worker --lib -just verify-renderer-split +just test-ldraw-core +just ldraw-wasm-check just build-renderer-worker +just verify-renderer-split +just test-renderer-native +just preview-ldraw --help +just preview-ldraw just test-renderer -cd e2e && npm install && npm test -- viewer.spec.ts -cd .. +just e2e just test-all +jj diff --summary jj diff jj status ``` -Use the repository’s documented generated-file prerequisite if a fresh workspace lacks `src/env.rs`; never hand-edit generated output. Resolve command failures; do not weaken acceptance. - -## F1 finding dispositions - -1. **Stale premise that G is incomplete/no integrated handoff — rebutted stale:** baseline is `epic-86@99436d09`; use `CompoundResourcesReady`, `CompoundSceneSource`, `handoff_compound_scene`, `load_compound_source`, and the reducer seam. -2. **LDU ambiguity — fixed:** PM-79 freezes `1 LDU = 1/2500 m = 0.4 mm`; integrated `1/64 inch` prose is stale documentation and cannot control geometry. -3. **E export underspecified — fixed:** exact `RenderExport`, `OwnedRenderExport`, primitive/occurrence/material/provenance/diagnostic/units/stats contracts and reservation lifetime are specified; canonical projection remains. -4. **Parser payload loss/worker duplication — fixed:** export from existing `GeometryRecord`/`InstanceRecord`/`OwnedParseResult`; retain type 2–5 arrays in E, split quads in E, parse once, no F parser. -5. **Invented LDraw point syntax — fixed:** generic IR point plus renderer-owned fixture only; operator LDraw fixture uses authoritative types 1–5; H owns broader point semantics. -6. **IR dependency inversion — fixed:** IR is a narrow protocol leaf; E does not depend on protocol/mesh/three-d; worker owns adaptation. -7. **GPU/stats/line-only behavior — fixed:** pending/installed resources own primitive buffers, reservations, bounds, full internal stats, compatibility `MeshStats`, and swap/dispose rules. -8. **Normals/basis/bounds — fixed:** LDU normals first, inverse-transpose where needed, determinant-`+1` basis once, no winding flip, explicit bounds order. -9. **G-to-install lifecycle — fixed:** verified resource -> content-bound handoff -> E materialization/parser/export -> adapter -> GPU install -> reducer `InstallSucceeded`, with checks and cleanup at every boundary. -10. **Special-route/product-flow ambiguity — corrected by operator:** the earlier `/viewer/ldraw-vertical-slice` and demo/STL-switch choices are removed. Acceptance uses an ordinary Thing→Model→Part→File MPD composition, normal publish/projection/G compound discovery, and that Thing’s normal detail viewer endpoint; no new route, DEMO_MODELS entry, upload flow, or direct URL. -11. **Interactive-only/Preview ambiguity — corrected by operator:** the earlier Interactive-only disposition is removed. Interactive and compound Preview share verified source→E parse/export→F adapter; they diverge only at output (`InstallSucceeded` versus ordinary preview PNG-ready event). -12. **Test sprawl/weak viability/H-I-J-K leakage — fixed:** focused E/IR/math/worker/ordinary-parity tests, exactly three lifecycle cases without a namespace Cartesian matrix, normal Thing browser flow with actual Interactive readback and same-scene Preview PNG, and explicit exclusions. +Do not add `e2e`, an `e2e-viewer` recipe, or new renderer/ledger gates to the `test-all` dependency list. Run `just e2e` explicitly to preserve the existing browser transport coverage. The public `just ldraw-wasm-check` recipe dispatches `_ldraw-wasm-check`; all other required F validation uses the existing Just surface without expanding PM-88 ledger gates. Do not write raw Cargo/npm/dx/nix/wasm-bindgen commands in this plan; all repository builds, tests, generation, browser, renderer, and oracle work is invoked through plain `just ` public dispatch. -## Review expectations +## Complete finding disposition — all 34 supplied IDs -Block protocol/app dependency on E/mesh/three-d, parser duplication, direct LDraw URL derivation, separate upload/route/demo product surfaces, bypass of G bytes, an LDraw-named or parser-aware installed-scene abstraction, separate renderer lifecycle/frame/Preview implementations by mode, replacement or flattening of ordinary `ModelMesh` transformed-part/PBR/colour/texture/hierarchy payloads, speculative format registry work, or duplicated camera/render-target/readback logic between native Preview and the WASM viewer; missing type-2/3/4/5 payloads, fabricated type-0 syntax, lost flat occurrence identity/transforms, nondeterministic IDs/order, or canonical projection breakage; hidden scale conversion, wrong LDU value, negative determinant, second winding flip, wrong normal timing, or pre-transform bounds; allocation before reservation, leaks, pending/live swap before reducer acceptance, readiness on `CompoundResourcesReady`, stale replacement, or broken cancel/resize/dispose cleanup; ordinary STL/OBJ/glTF/3MF or texture regression; Preview not sharing E export/adapter or not publishing a valid PNG; second WebGL context; DOM-only e2e; or H/I/J/K/fuzz/performance/publishing/TEXMAP/visual-baseline scope creep. +Round-two architecture review marked the architecture CLEAN, but medium findings are fixed by default. Parent-accepted rebuttals remain parent-owned and are preserved in substance. -## Commit and ready-for-parent-merge handoff - -Commit on `/home/orual/Projects/epic-86-task-F` at bookmark `epic-86-task-F`: - -```text -PM-79: add minimal LDraw renderer scene boundary - -Epic: PM-86 -Task: PM-79 -``` - -Before parent merge, report task workspace/bookmark/commit; grouped file list; exact command results including `just test-all`; clean `jj diff`/`jj status`; confirmation that no runtime state/manifest/dependency files changed; integrated baseline and exact G symbols consumed; exact normal Thing→Model→Part→File MPD/companion composition and real CIDs/digests/lengths; root mapping/materialization decisions; authoritative LDU value and both corrected stale sites; proof G’s verified `CompoundSceneSource` bytes were parsed once by E inside the worker and never round-tripped through main; proof Interactive `InstallSucceeded` is readiness while `CompoundResourcesReady` is informational; proof compound Preview used the same E export/adapter and produced a valid PNG; normal Thing detail endpoint/no fallback; fixed-viewport pixel/readback and nonzero gated stats; zero reservation snapshots for the exactly three lifecycle cases; and all reviewer dispositions. Parent merges only after trailers, focused/full gates, measured real-flow pixels and Preview PNG, and review clearance. \ No newline at end of file +| Finding ID | Disposition | Required plan response | +|---|---|---| +| `F-plan-2-A1` | **Fixed** | Use current `epic-86@` ancestry; prohibit hidden `99436d09`. | +| `F-plan-2-A2` | **Fixed** | F supplies only the custom conditional primitive/minimal rule/seam; H owns the complete classifier. | +| `F-plan-2-A3` | **Corrected after operator clarification** | PM-79 task contract is authoritative: `1 LDU = 1/64 inch = 0.396875 mm = 0.000396875 m` exactly. Preserve the correct protocol comments and exact conversion tests. | +| `F-plan-2-A4` | **Fixed** | Ground completion on current `epic-86@` and live symbols, not stale baseline evidence. | +| `F-plan-2-A5` | **Fixed** | Keep app descriptor root strings distinct from typed E `RootId`; freeze the eight-to-four mapping and provenance. | +| `F-plan-2-C1` | **Fixed** | Use public three-d 0.19 custom `Geometry`/`Material`/`Program`/`Gm` and low-level seams; prohibit built-in 2D `Line` and nonexistent `Points`. | +| `F-plan-2-C2` | **Fixed** | Replace `finish_compound_source` with the complete worker-internal pipeline and cleanup checkpoints. | +| `F-plan-2-C3` | **Rebutted-accepted by parent; preserved** | Reachable PM-86/PM-78 trailers and parent validation are the completion evidence; stale portable state is not truth. | +| `F-plan-2-C4` | **Fixed** | Typed materialization/root/path/hash/length/span/source identity is required; no guessed free-form provenance. | +| `F-plan-2-C5` | **Fixed** | Add explicit per-model LDMesh payload, ordered flat `GeometryRecord`, forward/reverse topology, BFC/winding, colours, spans, steps, sharing, transforms, and TEXMAP identity. | +| `F-plan-2-C6` | **Fixed** | Freeze `src/lib.rs`, retained `src/main.rs`/`renderer_worker`, `src/bin/ldraw_preview.rs`/`ldraw_preview`, `tests/ldraw_preview.rs`, and exact Just recipes. | +| `F-plan-2-C7` | **Superseded in part by explicit operator direction** | E owns semantic data. Preserve existing PM-88 consumption, but do not add ledger classes, charges, reservation ownership, admission behaviour, or ledger tests in F. | +| `F-plan-2-B1` | **Fixed** | Remove byte-bearing `CompoundResourcesReady` and `CompoundSceneSource` bridge; bytes remain worker-internal. | +| `F-plan-2-B2` | **Fixed** | Exact-once Interactive/Preview event-order and stale/cancel tests are in the finite table. | +| `F-plan-2-B3` | **Fixed** | Named native binary/library/integration test share one scene/render/readback/PNG path and Just dispatch; zero-argument preview uses pinned `omr-10030-1.mpd`, official-library resolution, fixed headless GL/viewport/output, and fail-closed assertions. | +| `F-plan-2-B4` | **Fixed** | The normal Thing helper serves pinned `omr-10030-1.mpd` and required official-library bytes with real CIDs/digests/lengths; tests assert resolved external parts, model graph/STEP/repeated-transform/BFC/material semantics, primitive provenance, and bounds against the expected record. | +| `F-plan-2-B5` | **Fixed** | Preserve ordinary STL/OBJ/glTF/3MF path and test glTF hierarchy/bounds/PBR/textures plus common frame/Preview behavior. | +| `F-plan-2-B6` | **Superseded in part by explicit operator direction** | The lifecycle table covers F namespaces, checkpoints, stale/cancel/error/resize/dispose, abort, candidate/old, and semantic drops without adding ledger requirements. | +| `F-plan-2-B7` | **Fixed** | Every supported root plus unknown/path/primary/duplicate/integrity/route mismatch fails closed before parse. | +| `F-plan-2-B8` | **Superseded by explicit operator direction** | Do not restore browser test-readback features, events, globals, hooks, or dedicated recipes. Native rendering tests provide pixel evidence; browser compound coverage remains transport-only. | +| `F-plan-2-B9` | **Fixed** | Preserve semantic quads/provenance; triangulate only during GPU upload. | +| `F-plan-3-B1` | **Fixed** | Concrete `lib.rs`, retained WASM `main.rs`, named `ldraw_preview` bin, named integration test, exact public recipes, and argument forwarding are specified. | +| `F-plan-3-B2` | **Fixed** | Existing Thing/e2e resolver anchors reuse pinned `omr-10030-1.mpd`, its expected record, and required official-library bytes through `seed_ldraw_compound_fixture`; actual bytes, external resolution, and parsed export assertions are required without a duplicate product-only fixture. | +| `F-plan-3-B3` | **Superseded in part by explicit operator direction** | Lifecycle coverage retains namespace/checkpoint/outcome/event counts/exact-once counters/abort/scenes/drops without adding ledger requirements. | +| `F-plan-3-B4` | **Superseded by explicit operator direction** | Browser readback features, events, hooks, globals, and e2e assertions remain removed. | +| `F-plan-3-C1` | **Fixed** | `LdMeshSemantic` fields are explicit and separate from enumerated ordered `GeometryRecord`; construction is during E semantic processing and oracle comparison is required. | +| `F-plan-3-C2` | **Fixed** | Exact stale LDU comment/assertion sites `protocol/src/lib.rs:326` and `:1055-1056` are named. | +| `F-plan-3-C3` | **Fixed** | Exact `CompoundSceneSource`/`handoff_compound_scene` protocol, worker, viewer, and golden-test cleanup/relocation sites are named. | +| `F-plan-4-B1` | **Superseded in part by explicit operator direction** | The lifecycle table retains `I-dispose` and `P-resize` event/state/drop assertions without adding ledger requirements. | +| `F-plan-4-B2` | **Superseded by explicit operator direction** | Do not restore `renderer-test-readback`, `e2e-viewer`, `serve-headless`, browser readback wiring, or browser e2e in `test-all`. | +| `F-plan-4-B3` | **Superseded in part by explicit operator direction** | Retain appview mapping/malformed tests, official-library resolver/native coverage, ordinary format parity, and existing Just validation without expanding `test-all`. | +| `F-plan-4-C1` | **Fixed** | Semantic coordinates stay finite `f64`; checked conversion compares exact f32-rounded bits/oracle encoding (or the named oracle-format tolerance with separate rounded-bit assertion), preserving precision visibility and prohibiting naive f64 equality. | +| `F-plan-4-C2` | **Fixed** | `crates/polymodel-renderer-worker/Cargo.toml` explicitly adds worker-only `polymodel-ldraw-core`; the app remains protocol-only and `just verify-renderer-split` proves the dependency boundary. | +| `F-plan-4-C3` | **Fixed** | Every required `GeometryRecord` field is updated across constructors, serializers, tests, fixtures, and snapshots; missing fields are rejected without permissive serde defaults or compatibility shims. | + +## Review expectations and handoff + +Reviewers block hidden baseline pins, main/default-workspace edits, stale state as truth, a second production renderer/context/camera/readback path, wrong LDU/basis/winding/normal/bounds timing, parser duplication, lossy/grouped geometry, missing LDMesh fields or oracle comparison, byte-bearing protocol events, readiness before reducer acceptance, stale scene replacement, non-exact parse/export/adapt, weak resolver/native fixture tests, raw repository commands, a broad H classifier, a Polymodel-specific three-d fork, missing worker-only LDraw dependency, direct f64-to-f32 equality, or permissive GeometryRecord schema evolution. Reviewers must respect the operator-owned prohibitions on browser readback restoration, `e2e-viewer`, `test-all` expansion, compound browser transport-test mutation, and ledger changes. The implementation reviewer must verify `jj diff` contains only intended source/test/fixture/Just files and that all 34 finding dispositions remain satisfied. + +Ready-for-parent-merge handoff includes workspace/bookmark/commit and consumed parent tip; grouped files; exact `just` results including `just test-all`; clean `jj diff`/`jj status`; pinned-oracle semantic evidence for `LdMeshSemantic` plus ordered records; exact root mapping and real byte fixture CID/digest/length/type/BFC/material/repeated-child evidence; LDU/basis/no-scaling proof; proof bytes never crossed the worker bridge; reducer-accepted Interactive readiness; shared Preview/native/browser readback/PNG evidence; finite lifecycle ledger-zero/drop evidence; and confirmation that F’s conditional-line seam is ready for H without renderer surgery. diff --git a/.current-epic/plan-G.md b/.current-epic/plan-G.md index 2fd8a23..cdd6b89 100644 --- a/.current-epic/plan-G.md +++ b/.current-epic/plan-G.md @@ -22,7 +22,7 @@ The cached task contract in `.current-epic/task-G.md` is authoritative. Jira is G consumes PM-76’s typed verified resolver, PM-87’s identity/reducer state, and PM-88’s reservation ledger. It must provide a URL-free, principal-bound compound LDraw plan and replace the current bare postcard stream with a negotiated, versioned app↔worker protocol. PM-76 currently exposes `ResolvedResource::ByteSource { resource_uri, blob_cids }`; G must add its adjacent descriptor/route seam without duplicating PM-76’s schema, route table, or verification state machine. -LDraw scale is exact: `1 LDU = 1/64 inch = 0.396875 mm = 0.000396875 m`; `0.4 mm` is display rounding only. Convert exactly once at the worker scene/IR boundary. +LDraw scale is exact: `1 LDU = 1/64 inch = 0.396875 mm = 0.000396875 m`. Convert exactly once at the worker scene/IR boundary. ## Isolation and safety gate diff --git a/.current-epic/task-L.md b/.current-epic/task-L.md index 698ee66..b9ac70b 100644 --- a/.current-epic/task-L.md +++ b/.current-epic/task-L.md @@ -1,34 +1,37 @@ -# Task L — Dynamic: Ledger-backed WASM memory enforcement +# Task L — PM-90: Correct LDraw face lighting and top/bottom brightness -**Created by:** Orchestrator (dynamic task) -**Depends on:** B (PM-88, merged) -**Epic:** PM-86 +**Jira:** [PM-90](https://radiant-industries.atlassian.net/browse/PM-90) +**Status:** To Do | **Issue type:** Bug +**Execution prefix:** L +**Epic:** [PM-86](https://radiant-industries.atlassian.net/browse/PM-86) -## Goal +## Origin and approval -Replace the talc allocator's `WasmGrowAndClaim` growth source with a `LedgerBackedSource` that checks the reservation ledger before every `memory.grow`. This makes the ledger a real enforcement boundary instead of a cooperating counter. +Operator-approved dynamic Epic task created during PM-79 visual validation on 2026-08-06. PM-79 fixed missing/invisible LDraw geometry and delivered an operator-verified native picker; the operator accepted separating the remaining face-lighting discrepancy into this task. -## Context +## Dependency -The current ledger (`polymodel-renderer-ledger`) is `#![forbid(unsafe_code)]`, std-only, and has no connection to the actual allocator. It checks caller-supplied estimates at representative call sites but cannot prevent unanticipated allocations from growing memory. +Depends on F (PM-79) completion and integration into the PM-86 Epic branch. -Talc 5.0.4 is the global allocator: `TalcLock`. Talc exposes a `Source` trait where `Source::acquire` is called on every memory growth request. By implementing a custom `Source` that wraps `WasmGrowAndExtend` and checks the ledger before delegating, we get real enforcement: if the budget is exceeded, `memory.grow` is never called and the allocation fails. +## Problem + +Known-colour LDraw parts can show top and bottom faces with opposite brightness/colour from the expected result. Existing numeric topology checks found no material colour swap, but visual production-path evidence remains unresolved. ## Scope -1. Create a `LedgerBackedSource` implementing talc's `Source` trait -2. On `acquire`: calculate the page delta, check `ledger.reserve(owner, ResourceClass::WasmPages, bytes)`, delegate to inner source on success, return `Err(())` on failure -3. Track reservations for release on `resize`/drop -4. Replace `WasmGrowAndClaim` with `LedgerBackedSource` in the worker's `#[global_allocator]` -5. Update the ledger README to remove the "cannot enforce" justification and document the real enforcement boundary -6. Add tests proving that allocation fails when the ledger budget is exhausted -7. Keep the ledger crate itself `#![forbid(unsafe_code)]`; the `LedgerBackedSource` lives in the worker crate (allocator code, `unsafe` is appropriate) - -## Acceptance - -- When the ledger's `WasmPages` budget is exhausted, new allocations fail (panic-on-OOM or alloc_error, not silent growth) -- When the budget is not exhausted, allocations succeed normally -- The ledger's accounting matches actual memory pages grown (not estimates) -- The README documents the real enforcement boundary honestly -- `just check` and `just fix` pass -- Worker wasm check passes +Diagnose matched identity/reflected and front/back production renders across authored BFC winding, generated triangle order, local/world normals, reflection/inversion composition, culling, effective LDraw colour, PhysicalMaterial properties, light direction conventions, and native/static/web parity. + +Fix shared production rendering policy. Do not disable depth testing, culling, or lighting, and do not replace affected materials with unlit materials. + +## Acceptance criteria + +- Matched front/back and identity/reflected renders identify the first incorrect boundary. +- Numeric tests cover indices, normals, light-facing sign, and material identity. +- Native persistent, static readback, and web worker use equivalent lighting/material semantics. +- Known-colour top and bottom faces render with expected relative appearance under controlled lights. +- Existing LDraw geometry counts, picker behaviour, ordinary format parity, and renderer split remain green. +- Run the complete jira-epic-run dynamic-task pipeline. + +## Model routing override + +While ZAI is rate-limited, use `neuralwatt/glm-5.2-flex(medium)` for architecture and long-context GLM review lanes. Preserve standard Codex correctness and test-quality lanes. diff --git a/.current-epic/task-state.json b/.current-epic/task-state.json index 413119c..d37f0dc 100644 --- a/.current-epic/task-state.json +++ b/.current-epic/task-state.json @@ -1,6 +1,6 @@ { "schema_version": 1, - "updated_at": "2026-08-01T18:00:00Z", + "updated_at": "2026-08-06T18:35:00Z", "updated_by": "polytoken", "epic": { "jira_key": "PM-86", @@ -42,9 +42,7 @@ "last_error": null }, "C": { - "phase": "merging", - "job_id": null, - "change_id": "985003c9", + "phase": "completed", "jira_key": "PM-76", "jira_url": "https://radiant-industries.atlassian.net/browse/PM-76", "title": "LDraw library projection and ATProto dependency manifest", @@ -54,7 +52,7 @@ "plan_path": ".current-epic/plan-C.md", "task_context_dir": "../epic-86-task-C/.current-task", "job_id": null, - "change_id": null, + "change_id": "65f00d02", "last_error": null }, "D": { @@ -74,7 +72,7 @@ "last_error": null }, "E": { - "phase": "in-progress", + "phase": "completed", "jira_key": "PM-78", "jira_url": "https://radiant-industries.atlassian.net/browse/PM-78", "title": "Thomas-compatible Rust LDraw semantic core", @@ -83,12 +81,12 @@ "task_cache": ".current-epic/task-E.md", "plan_path": ".current-epic/plan-E.md", "task_context_dir": "../epic-86-task-E/.current-task", - "job_id": "general-purpose:implementor-e5", - "change_id": null, + "job_id": null, + "change_id": "ukyukrormltsqqtuoussstzyzqvwtnnv", "last_error": null }, "F": { - "phase": "pending", + "phase": "merging", "jira_key": "PM-79", "jira_url": "https://radiant-industries.atlassian.net/browse/PM-79", "title": "Minimal LDraw vertical slice and generic scene IR contract", @@ -98,7 +96,7 @@ "plan_path": ".current-epic/plan-F.md", "task_context_dir": "../epic-86-task-F/.current-task", "job_id": null, - "change_id": null, + "change_id": "svzwtnklsrpkqqnnotrolqxrnlzlrrwu", "last_error": null }, "G": { @@ -170,6 +168,34 @@ "job_id": null, "change_id": null, "last_error": null + }, + "L": { + "phase": "pending", + "jira_key": "PM-90", + "jira_url": "https://radiant-industries.atlassian.net/browse/PM-90", + "title": "Correct LDraw face lighting and top/bottom brightness", + "workspace": "../epic-86-task-L", + "bookmark": "epic-86-task-L", + "task_cache": ".current-epic/task-L.md", + "plan_path": ".current-epic/plan-L.md", + "task_context_dir": "../epic-86-task-L/.current-task", + "job_id": null, + "change_id": null, + "last_error": null + }, + "M": { + "phase": "pending", + "jira_key": "PM-91", + "jira_url": "https://radiant-industries.atlassian.net/browse/PM-91", + "title": "Fix close-range renderer depth corruption and camera clipping precision", + "workspace": "../epic-86-task-M", + "bookmark": "epic-86-task-M", + "task_cache": ".current-epic/task-M.md", + "plan_path": ".current-epic/plan-M.md", + "task_context_dir": "../epic-86-task-M/.current-task", + "job_id": null, + "change_id": null, + "last_error": null } } } diff --git a/.gitignore b/.gitignore index 63e069f..4db5ea5 100644 --- a/.gitignore +++ b/.gitignore @@ -40,5 +40,5 @@ rustc-ice-*.txt !/public/models/box_textured.png !.cargo/ !.cargo/config.toml -target/ unanchored -tools/ldraw-compat-harness/target/ +# Cargo build outputs, including standalone tools with their own target directories. +target/ diff --git a/AGENTS.md b/AGENTS.md index 1ba834f..b29d2da 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -50,6 +50,8 @@ just test-all just serve ``` +Use Just recipes exclusively for all repository builds, checks, tests, generation, migrations, E2E, renderer, and oracle work. Invoke the command surface with plain `just [args]`; do not bypass it with raw `cargo`, `dx`, `npm`, `wasm-bindgen`, or equivalent commands. When a required operation lacks a recipe, add the focused recipe first. + Before review or handoff, run at least `just fix` and all relevant tests. Use `just test-all` for substantial changes. In a fresh checkout or jj workspace, run `just check` once before `just fix` so the workspace dependencies and generated build outputs are available. The frontend no longer generates or imports a `src/env.rs` module; browser configuration must not receive server-only `POLYMODEL_*` secrets. Server runtime configuration is loaded by the server modules from the process environment. diff --git a/crates/polymodel-ldraw-core/src/cache.rs b/crates/polymodel-ldraw-core/src/cache.rs index b969fe3..53cbad1 100644 --- a/crates/polymodel-ldraw-core/src/cache.rs +++ b/crates/polymodel-ldraw-core/src/cache.rs @@ -10,9 +10,8 @@ impl NormalizedPath { pub fn new(path: &str) -> Result { let normalized_separators = path.replace('\\', "/"); let bytes = normalized_separators.as_bytes(); - let drive_qualified = bytes.len() >= 2 - && bytes[0].is_ascii_alphabetic() - && bytes[1] == b':'; + let drive_qualified = + bytes.len() >= 2 && bytes[0].is_ascii_alphabetic() && bytes[1] == b':'; if normalized_separators.starts_with('/') || normalized_separators.starts_with("//") || drive_qualified diff --git a/crates/polymodel-ldraw-core/src/lib.rs b/crates/polymodel-ldraw-core/src/lib.rs index b6898bc..12e76a9 100644 --- a/crates/polymodel-ldraw-core/src/lib.rs +++ b/crates/polymodel-ldraw-core/src/lib.rs @@ -35,16 +35,27 @@ mod tests { } #[test] fn normalized_paths_reject_rooted_include_forms() { - for path in ["/parts/a.dat", "\\\\server\\share\\a.dat", "C:/parts/a.dat", "z:\\parts\\a.dat"] { - assert!(matches!( - NormalizedPath::new(path), - Err(LdrawError::Diagnostic(Diagnostic { - code: DiagnosticCode::PathForbiddenSyntax, - .. - })) - ), "path should be rejected: {path}"); + for path in [ + "/parts/a.dat", + "\\\\server\\share\\a.dat", + "C:/parts/a.dat", + "z:\\parts\\a.dat", + ] { + assert!( + matches!( + NormalizedPath::new(path), + Err(LdrawError::Diagnostic(Diagnostic { + code: DiagnosticCode::PathForbiddenSyntax, + .. + })) + ), + "path should be rejected: {path}" + ); } - assert_eq!(NormalizedPath::new("parts/./a.dat").unwrap().as_str(), "parts/a.dat"); + assert_eq!( + NormalizedPath::new("parts/./a.dat").unwrap().as_str(), + "parts/a.dat" + ); } #[test] diff --git a/crates/polymodel-ldraw-core/src/model.rs b/crates/polymodel-ldraw-core/src/model.rs index e5c2607..7e477d8 100644 --- a/crates/polymodel-ldraw-core/src/model.rs +++ b/crates/polymodel-ldraw-core/src/model.rs @@ -3,11 +3,12 @@ use crate::cache::{CacheKey, NormalizedPath}; use crate::geom::{Bounds3, ColourCode, GeometryRecord, Transform, deserialize_optional_bounds3}; use crate::mpd::OwnedVirtualFile; use crate::scanner::{LineType, Token, TokenKind}; -use crate::texmap::{TexmapEvent, TexmapState, TextureDescriptor}; +use crate::texmap::{TexmapAssociation, TexmapEvent, TexmapState, TextureDescriptor}; use crate::types::{Diagnostic, LimitCounters, SCHEMA_VERSION, SemanticArenas, Span}; use serde::{Deserialize, Serialize}; use std::borrow::Cow; use std::collections::BTreeMap; +use std::fmt; #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] pub struct ColourData { @@ -24,19 +25,74 @@ pub enum ColourEdge { Data(ColourData), } -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ColourFinish { + Chrome, + Pearlescent, + Rubber, + MatteMetallic, + Metal, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ColourFabricKind { + Velvet, + Canvas, + String, + Fur, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ColourMaterialKind { + Glitter, + Speckle, + Fabric { variant: Option }, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ColourMaterial<'src> { + /// The source spelling of the material kind, retained for compatibility. + #[serde(borrow)] + pub kind: Cow<'src, str>, + pub typed_kind: ColourMaterialKind, + pub value: Option, + pub fraction: Option, + pub vfraction: Option, + pub size: Option, + pub min_size: Option, + pub max_size: Option, + #[serde(borrow)] + pub extra: Vec>, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct Colour<'src> { pub code: ColourCode, #[serde(borrow)] pub name: Option>, pub data: ColourData, pub edge: ColourEdge, + #[serde(default)] + pub finishes: Vec, + #[serde(default)] + pub material: Option>, } -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +const FIRST_LOCAL_COLOUR_SLOT: u32 = 512; +const LAST_LOCAL_COLOUR_SLOT: u32 = 65_534; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ColourSlotError; + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct ColourTable<'src> { #[serde(borrow)] entries: BTreeMap>, + local_slots: BTreeMap, + next_local_slot: u32, } impl<'src> Default for ColourTable<'src> { @@ -49,6 +105,8 @@ impl<'src> ColourTable<'src> { pub fn new() -> Self { Self { entries: BTreeMap::new(), + local_slots: BTreeMap::new(), + next_local_slot: FIRST_LOCAL_COLOUR_SLOT, } } @@ -72,6 +130,10 @@ impl<'src> ColourTable<'src> { } } + pub fn local_slot(&self, code: impl Into) -> Option { + self.local_slots.get(&code.into()).copied() + } + pub fn len(&self) -> usize { self.entries.len() } @@ -80,6 +142,16 @@ impl<'src> ColourTable<'src> { self.entries.is_empty() } + pub(crate) fn merge_scope(&mut self, scope: ColourScope<'src>) { + self.entries.extend(scope.committed.into_iter().flatten()); + self.local_slots.extend(scope.local_slots); + self.next_local_slot = scope.next_local_slot; + } + + pub(crate) fn next_local_slot(&self) -> u32 { + self.next_local_slot + } + pub(crate) fn into_owned(self) -> ColourTable<'static> { ColourTable { entries: self @@ -93,85 +165,382 @@ impl<'src> ColourTable<'src> { name: colour.name.map(|name| Cow::Owned(name.into_owned())), data: colour.data, edge: colour.edge, + finishes: colour.finishes, + material: colour.material.map(|material| ColourMaterial { + kind: Cow::Owned(material.kind.into_owned()), + typed_kind: material.typed_kind, + value: material.value, + fraction: material.fraction, + vfraction: material.vfraction, + size: material.size, + min_size: material.min_size, + max_size: material.max_size, + extra: material + .extra + .into_iter() + .map(|value| Cow::Owned(value.into_owned())) + .collect(), + }), }, ) }) .collect(), + local_slots: self.local_slots, + next_local_slot: self.next_local_slot, } } } +/// File-local transactional colour definitions. +/// +/// Records are invisible until `commit`. Dropping or rolling back a scope +/// discards both definitions and provisional local slots, so failed parses do +/// not affect later virtual files or parses. +#[derive(Clone, Debug, PartialEq)] +pub struct ColourScope<'src> { + committed: Vec>>, + active: BTreeMap>, + local_slots: BTreeMap, + active_slot_start: u32, + next_local_slot: u32, +} + +impl<'src> ColourScope<'src> { + pub fn new() -> Self { + Self::with_first_local_slot(FIRST_LOCAL_COLOUR_SLOT) + } + + pub(crate) fn with_first_local_slot(first_local_slot: u32) -> Self { + Self { + committed: Vec::new(), + active: BTreeMap::new(), + local_slots: BTreeMap::new(), + active_slot_start: first_local_slot, + next_local_slot: first_local_slot, + } + } + + pub fn record(&mut self, colour: Colour<'src>) -> Result { + let code = colour.code; + if let Some(slot) = self.local_slots.get(&code).copied() { + self.active.insert(code, colour); + return Ok(slot); + } + let slot = self.allocate_local_slot()?; + self.local_slots.insert(code, slot); + self.active.insert(code, colour); + Ok(slot) + } + + pub fn commit(&mut self) { + if !self.active.is_empty() { + self.committed.push(std::mem::take(&mut self.active)); + } + self.active_slot_start = self.next_local_slot; + } + + pub fn rollback(&mut self) { + self.active.clear(); + self.local_slots + .retain(|_, slot| u32::from(*slot) < self.active_slot_start); + self.next_local_slot = self.active_slot_start; + } + + pub fn has_uncommitted(&self) -> bool { + !self.active.is_empty() + } + + pub fn get(&self, code: impl Into) -> Option<&Colour<'src>> { + if !self.active.is_empty() { + return None; + } + let code = code.into(); + self.committed.iter().rev().find_map(|map| map.get(&code)) + } + + pub fn find(&self, code: impl Into, find_edge: bool) -> Option<&ColourData> { + let mut code = code.into(); + for (map_index, map) in self.committed.iter().enumerate().rev() { + let Some(colour) = map.get(&code) else { + continue; + }; + if !find_edge { + return Some(&colour.data); + } + match &colour.edge { + ColourEdge::Data(data) => return Some(data), + ColourEdge::Code(edge_code) => { + if let Some(edge) = map.get(edge_code) { + return Some(&edge.data); + } + code = *edge_code; + for older_map in self.committed[..map_index].iter().rev() { + if let Some(edge) = older_map.get(&code) { + return Some(&edge.data); + } + } + return None; + } + } + } + None + } + + pub fn local_slot(&self, code: impl Into) -> Option { + self.local_slots.get(&code.into()).copied() + } + + fn allocate_local_slot(&mut self) -> Result { + if self.next_local_slot > LAST_LOCAL_COLOUR_SLOT { + return Err(ColourSlotError); + } + let slot = u16::try_from(self.next_local_slot).map_err(|_| ColourSlotError)?; + self.next_local_slot += 1; + Ok(slot) + } +} + +impl Default for ColourScope<'_> { + fn default() -> Self { + Self::new() + } +} + pub(crate) fn parse_colour<'src>(tokens: &[Token<'src>]) -> Option> { - if tokens.len() < 5 || tokens.get(2)?.kind != TokenKind::Identifier { + if tokens.len() < 9 + || tokens.first()?.kind != TokenKind::Zero + || tokens.get(1)?.kind != TokenKind::Colour + || tokens.get(2)?.kind != TokenKind::Identifier + { + return None; + } + let mut index = 3; + if !matches_keyword(tokens.get(index), TokenKind::Code) { return None; } - if tokens[2].text.ends_with(';') { + let code = parse_colour_code(tokens.get(index + 1)?.text)?; + index += 2; + if !matches_keyword(tokens.get(index), TokenKind::Value) { return None; } - let fields = &tokens[3..]; - if fields.len() % 2 != 0 { + let (r, g, b) = parse_rgb(tokens.get(index + 1)?.text)?; + index += 2; + if !matches_keyword(tokens.get(index), TokenKind::Edge) { return None; } + let edge_token = tokens.get(index + 1)?.text; + let edge = parse_rgb(edge_token) + .map(|(r, g, b)| { + ColourEdge::Data(ColourData { + r, + g, + b, + alpha: None, + luminance: None, + }) + }) + .or_else(|| parse_colour_code(edge_token).map(ColourEdge::Code))?; + index += 2; - let mut code = None; - let mut value = None; - let mut edge = None; - let mut alpha = None; - let mut luminance = None; - for pair in fields.chunks_exact(2) { - let key = pair[0].kind; - let value_token = &pair[1]; - match key { - TokenKind::Code if code.is_none() => code = parse_colour_code(value_token.text), - TokenKind::Value if value.is_none() => value = parse_rgb(value_token.text), - TokenKind::Edge if edge.is_none() => { - edge = parse_rgb(value_token.text) - .map(|(r, g, b)| { - ColourEdge::Data(ColourData { - r, - g, - b, - alpha: None, - luminance: None, - }) - }) - .or_else(|| parse_colour_code(value_token.text).map(ColourEdge::Code)); + let mut data = ColourData { + r, + g, + b, + alpha: None, + luminance: None, + }; + let mut finishes = Vec::new(); + let mut material = None; + while index < tokens.len() { + match tokens[index].kind { + TokenKind::Alpha => { + if data.alpha.is_some() { + return None; + } + data.alpha = Some(parse_byte(tokens.get(index + 1)?.text)?); + index += 2; } - TokenKind::Alpha if alpha.is_none() => alpha = value_token.text.parse().ok(), - TokenKind::Luminance if luminance.is_none() => { - luminance = value_token.text.parse().ok() + TokenKind::Luminance => { + if data.luminance.is_some() { + return None; + } + data.luminance = Some(parse_byte(tokens.get(index + 1)?.text)?); + index += 2; } TokenKind::Chrome | TokenKind::Pearlescent | TokenKind::Rubber | TokenKind::MatteMetallic - | TokenKind::Metal - | TokenKind::Material => return None, + | TokenKind::Metal => { + if !finishes.is_empty() || material.is_some() { + return None; + } + finishes.push(match tokens[index].kind { + TokenKind::Chrome => ColourFinish::Chrome, + TokenKind::Pearlescent => ColourFinish::Pearlescent, + TokenKind::Rubber => ColourFinish::Rubber, + TokenKind::MatteMetallic => ColourFinish::MatteMetallic, + TokenKind::Metal => ColourFinish::Metal, + _ => unreachable!(), + }); + index += 1; + } + TokenKind::Material => { + if !finishes.is_empty() || material.is_some() { + return None; + } + material = Some(parse_material(&tokens[index + 1..])?); + index = tokens.len(); + } _ => return None, } } - Some(Colour { - code: code?, + code, name: Some(Cow::Borrowed(tokens[2].text)), - data: ColourData { - r: value?.0, - g: value?.1, - b: value?.2, - alpha, - luminance, - }, - edge: edge?, + data, + edge, + finishes, + material, + }) +} + +fn matches_keyword(token: Option<&Token<'_>>, expected: TokenKind) -> bool { + token.is_some_and(|token| token.kind == expected) +} + +fn parse_material<'src>(tokens: &[Token<'src>]) -> Option> { + let kind_token = tokens.first()?; + let typed_kind = match kind_token.text { + "GLITTER" => ColourMaterialKind::Glitter, + "SPECKLE" => ColourMaterialKind::Speckle, + "FABRIC" => ColourMaterialKind::Fabric { variant: None }, + _ => return None, + }; + if matches!(typed_kind, ColourMaterialKind::Fabric { .. }) { + let variant = tokens.get(1).map(|token| match token.text { + "VELVET" => Some(ColourFabricKind::Velvet), + "CANVAS" => Some(ColourFabricKind::Canvas), + "STRING" => Some(ColourFabricKind::String), + "FUR" => Some(ColourFabricKind::Fur), + _ => None, + }); + if tokens.len() > 2 || variant.is_some_and(|variant| variant.is_none()) { + return None; + } + return Some(ColourMaterial { + kind: Cow::Borrowed(kind_token.text), + typed_kind: ColourMaterialKind::Fabric { + variant: variant.flatten(), + }, + value: None, + fraction: None, + vfraction: None, + size: None, + min_size: None, + max_size: None, + extra: Vec::new(), + }); + } + let mut value = None; + let mut fraction = None; + let mut vfraction = None; + let mut size = None; + let mut min_size = None; + let mut max_size = None; + let mut index = 1; + while index < tokens.len() { + let key = tokens[index].text; + if index + 1 >= tokens.len() { + return None; + } + let value_token = &tokens[index + 1]; + match key { + "VALUE" if value.is_none() => { + let (r, g, b) = parse_rgb(value_token.text)?; + value = Some(ColourData { + r, + g, + b, + alpha: None, + luminance: None, + }); + } + "ALPHA" if value.is_some() && value.as_ref()?.alpha.is_none() => { + value.as_mut()?.alpha = Some(parse_byte(value_token.text)?); + } + "LUMINANCE" if value.is_some() && value.as_ref()?.luminance.is_none() => { + value.as_mut()?.luminance = Some(parse_byte(value_token.text)?); + } + "FRACTION" if fraction.is_none() => fraction = Some(parse_fraction(value_token.text)?), + "VFRACTION" if vfraction.is_none() => { + vfraction = Some(parse_fraction(value_token.text)?) + } + "SIZE" if size.is_none() => size = Some(parse_positive(value_token.text)?), + "MINSIZE" if min_size.is_none() => min_size = Some(parse_positive(value_token.text)?), + "MAXSIZE" if max_size.is_none() => max_size = Some(parse_positive(value_token.text)?), + _ => return None, + } + index += 2; + } + let is_glitter = matches!(typed_kind, ColourMaterialKind::Glitter); + let is_speckle = matches!(typed_kind, ColourMaterialKind::Speckle); + if value.is_none() || fraction.is_none() { + return None; + } + if is_glitter { + if vfraction.is_none() || size.is_none() || min_size.is_some() || max_size.is_some() { + return None; + } + } else if is_speckle + && (vfraction.is_some() || size.is_some() || min_size.is_none() || max_size.is_none()) + { + return None; + } + if size.is_some() == (min_size.is_some() || max_size.is_some()) + || min_size.is_some() != max_size.is_some() + { + return None; + } + Some(ColourMaterial { + kind: Cow::Borrowed(kind_token.text), + typed_kind, + value, + fraction, + vfraction, + size, + min_size, + max_size, + extra: Vec::new(), }) } fn parse_colour_code(text: &str) -> Option { - let value = text.parse::().ok()?; - Some(ColourCode(value)) + (!text.is_empty() && text.bytes().all(|byte| byte.is_ascii_digit())) + .then(|| text.parse::().ok()) + .flatten() + .map(ColourCode) +} + +fn parse_byte(text: &str) -> Option { + (!text.is_empty() && text.bytes().all(|byte| byte.is_ascii_digit())) + .then(|| text.parse::().ok()) + .flatten() + .and_then(|value| u8::try_from(value).ok()) +} + +fn parse_fraction(text: &str) -> Option { + let value = text.parse::().ok()?; + (value > 0.0 && value < 1.0 && value.is_finite()).then_some(value) +} + +fn parse_positive(text: &str) -> Option { + let value = text.parse::().ok()?; + (value > 0.0 && value.is_finite()).then_some(value) } fn parse_rgb(text: &str) -> Option<(u8, u8, u8)> { - let hex = text.strip_prefix('#')?; + let hex = text.strip_prefix('#').or_else(|| text.strip_prefix("0x"))?; if hex.len() != 6 { return None; } @@ -231,6 +600,8 @@ pub struct SemanticRecord<'src> { pub limits: Vec>, pub texmap_events: Vec>, #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub texmap_geometry: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] pub data_payloads: Vec, } impl<'src> SemanticRecord<'src> { @@ -261,6 +632,7 @@ impl<'src> SemanticRecord<'src> { .into_iter() .map(TexmapEvent::into_owned) .collect(), + texmap_geometry: self.texmap_geometry, data_payloads: self.data_payloads, } } @@ -270,8 +642,20 @@ pub struct InstanceRecord { pub instance_id: String, pub source_model: String, pub target_model: String, + #[serde(default)] + pub target_key: Option, pub transform: Transform, pub inverted: bool, + #[serde(default)] + pub bfc: Option, + #[serde(default)] + pub colour: Option, + #[serde(default)] + pub texmap_state: Option, + #[serde(default)] + pub texmap_reference: Option, + #[serde(default)] + pub texmap: Option, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -299,6 +683,23 @@ pub struct CanonicalRecord<'src> { pub provenance_id: Cow<'src, str>, } +impl<'src> CanonicalRecord<'src> { + pub fn into_owned(self) -> CanonicalRecord<'static> { + CanonicalRecord { + schema_version: Cow::Owned(self.schema_version.into_owned()), + syntax: self + .syntax + .into_iter() + .map(SyntaxRecord::into_owned) + .collect(), + semantic: self.semantic.into_owned(), + scene: self.scene, + diagnostics: self.diagnostics, + provenance_id: Cow::Owned(self.provenance_id.into_owned()), + } + } +} + #[derive(Clone, Debug)] pub(crate) struct DataPayloadBuilder { pub(crate) filename: String, @@ -312,7 +713,20 @@ pub(crate) struct Include<'src> { pub transform: Transform, pub(crate) span: Span, pub(crate) inverted: bool, + pub(crate) bfc: BfcFrame, + pub(crate) colour: ColourCode, + pub(crate) texmap_state: TexmapState, + pub(crate) texmap: Option>, + pub(crate) texmap_association: Option, +} + +#[derive(Clone, Debug)] +pub(crate) enum ModelEvent<'src> { + Step, + Indices(u64), + Include(Cow<'src, str>), } + #[derive(Clone, Debug)] pub(crate) struct ModelData<'src> { pub(crate) file_id: u32, @@ -321,12 +735,18 @@ pub(crate) struct ModelData<'src> { pub(crate) bfc: BfcFrame, pub(crate) colour: ColourCode, pub(crate) steps: Vec>, + /// LDParse-compatible direct-mesh index offsets recorded at each STEP. + /// This is distinct from the root semantic step labels used by policy. + pub(crate) step_ends: Vec, + pub(crate) mesh_index_count: u64, + pub(crate) events: Vec>, pub(crate) texmap_state: TexmapState, pub(crate) texmap_descriptor: Option>, pub(crate) texmap_stack: Vec<(TexmapState, Option>, bool)>, pub(crate) texmap_next_span: Option, pub(crate) texmap_fallback_seen: bool, pub(crate) texmap_events: Vec>, + pub(crate) texmap_geometry: Vec, pub(crate) data_payload: Option, pub(crate) data_payloads: Vec, pub(crate) includes: Vec>, @@ -350,6 +770,66 @@ pub struct ParseResult<'src> { pub arenas: SemanticArenas, pub provenance_id: Cow<'src, str>, } + +/// An owned parse result. The retained semantic result and its reservation are +/// one object: projection borrows or clones data while this owner remains live. +pub struct OwnedParseResult { + pub result: ParseResult<'static>, +} + +impl fmt::Debug for OwnedParseResult { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("OwnedParseResult") + .field("schema_version", &SCHEMA_VERSION) + .field("reservation_id", &self.result.arenas.reservation_id()) + .finish() + } +} + +impl std::ops::Deref for OwnedParseResult { + type Target = ParseResult<'static>; + + fn deref(&self) -> &Self::Target { + &self.result + } +} + +impl OwnedParseResult { + pub(crate) fn from_parse_result(result: ParseResult<'_>) -> Self { + Self { + result: result.into_owned(), + } + } + + pub fn project(&self) -> CanonicalRecord<'static> { + self.result.project() + } + + pub fn ledger_reservation_id(&self) -> u64 { + self.result.arenas.reservation_id() + } + + /// Move both the canonical projection and its retained arena owner together. + pub fn into_projection(self) -> OwnedProjection { + OwnedProjection { + canonical: self.result.project(), + arenas: self.result.arenas, + } + } +} + +pub struct OwnedProjection { + pub canonical: CanonicalRecord<'static>, + pub arenas: SemanticArenas, +} + +impl OwnedProjection { + pub fn project(&self) -> &CanonicalRecord<'static> { + &self.canonical + } +} + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct ModelSummary { pub id: String, @@ -362,6 +842,8 @@ pub struct ModelSummary { pub quads: u64, pub lines: u64, pub conditional_lines: u64, + /// LDParse-authoritative direct-mesh index offsets for this model's steps. + pub step_ends: Vec, pub geometry: Vec, pub bfc: BfcState, } @@ -377,31 +859,25 @@ impl<'src> ParseResult<'src> { } } - pub fn into_project(self) -> CanonicalRecord<'src> { - CanonicalRecord { - schema_version: Cow::Borrowed(SCHEMA_VERSION), - syntax: self.syntax, - semantic: self.semantic, - scene: self.scene, - diagnostics: self.diagnostics, - provenance_id: self.provenance_id, - } - } - - pub fn into_owned_project(self) -> CanonicalRecord<'static> { - CanonicalRecord { - schema_version: Cow::Borrowed(SCHEMA_VERSION), + pub(crate) fn into_owned(self) -> ParseResult<'static> { + ParseResult { syntax: self .syntax .into_iter() - .map(|record| record.into_owned()) + .map(SyntaxRecord::into_owned) .collect(), semantic: self.semantic.into_owned(), scene: self.scene, diagnostics: self.diagnostics, + counters: self.counters, + files: self.files, + models: self.models, + colours: self.colours.into_owned(), + arenas: self.arenas, provenance_id: Cow::Owned(self.provenance_id.into_owned()), } } + pub fn ledger_reservation_id(&self) -> u64 { self.arenas.reservation_id() } diff --git a/crates/polymodel-ldraw-core/src/parser.rs b/crates/polymodel-ldraw-core/src/parser.rs index e4f02cc..194431a 100644 --- a/crates/polymodel-ldraw-core/src/parser.rs +++ b/crates/polymodel-ldraw-core/src/parser.rs @@ -1,23 +1,24 @@ +#![allow(clippy::too_many_arguments)] + use crate::BfcFrame; use crate::bfc::{BfcState, bfc}; use crate::cache::{CacheKey, NormalizedPath}; use crate::geom::{parse_primitive, process_geometry}; use crate::model::{ - CanonicalRecord, ColourTable, DataPayload, DataPayloadBuilder, ModelData, ParseResult, + ColourScope, ColourTable, DataPayload, DataPayloadBuilder, ModelData, ModelEvent, ParseResult, SemanticRecord, SyntaxRecord, parse_colour, }; use crate::mpd::{OwnedVirtualFile, VirtualFile, split_mpd}; -use crate::resolve::ResolveRequest; use crate::scanner::{LineType, ScannedLine, TokenKind, scan_lines}; use crate::texmap::{TexmapState, TextureDescriptor, texmap}; use crate::traversal::traverse; use crate::types::{ - DEFAULT_COLOUR, DEFAULT_PROVENANCE, BorrowedMaterialization, Diagnostic, DiagnosticCode, - LdrawLimits, LimitCounters, LimitKind, ParseError, ParseOptions, ParserProfile, - SemanticArenas, Span, add_diag, limit_error, + DEFAULT_COLOUR, DEFAULT_PROVENANCE, Diagnostic, DiagnosticCode, LdrawLimits, LimitCounters, + LimitKind, Materialization, ParseError, ParseOptions, ParserProfile, SemanticArenas, Span, + add_diag, limit_error, }; use std::borrow::Cow; -use std::collections::{BTreeSet, HashSet}; +use std::collections::BTreeSet; pub struct LdrawParser; impl Default for LdrawParser { @@ -26,79 +27,17 @@ impl Default for LdrawParser { } } impl LdrawParser { - pub async fn parse( - &self, - bytes: Vec, - mut options: ParseOptions<'_>, - resolver: &R, - ) -> Result, ParseError> { - let root_bytes = bytes; - let mut resolved = Vec::::new(); - let mut requested = HashSet::<(NormalizedPath, crate::types::RootId)>::new(); - loop { - let borrowed = resolved - .iter() - .map(|materialization| BorrowedMaterialization { - path: materialization.path.clone(), - root: materialization.root, - content_hash: materialization.content_hash.clone(), - bytes: &materialization.bytes, - available: materialization.available, - }) - .collect::>(); - let parsed = self.parse_bytes(&root_bytes, options.clone(), Some(&borrowed))?; - let mut next = None; - for model in &parsed.models { - for (include_index, include) in model.includes.iter().enumerate() { - let path = match NormalizedPath::new(include) { - Ok(path) => path, - Err(_) => continue, - }; - let root = options.resolved_root; - if !requested.insert((path.clone(), root)) { - continue; - } - next = Some(ResolveRequest { - source: model.cache_key.clone(), - path, - root, - depth: model.depth.saturating_add(1), - span: model.include_spans.get(include_index).copied(), - }); - break; - } - if next.is_some() { - break; - } - } - let Some(request) = next else { - return Ok(parsed.into_owned_project()); - }; - if options.fetch_count >= options.limits.fetches { - return Err(limit_error(LimitKind::Fetches, request.span)); - } - options.fetch_count = options - .fetch_count - .checked_add(1) - .ok_or(ParseError::Overflow("fetch count"))?; - let materialization = resolver.resolve(request).await?; - if !materialization.has_valid_hash() { - return Err(crate::resolve::ResolveError::Rejected { - path: materialization.path, - message: "resolver returned bytes with a mismatched content hash".into(), - } - .into()); - } - resolved.push(materialization); - } - } - + /// Parse immutable root bytes plus caller-admitted materializations. + /// + /// Discovery, fetching, precedence, and host policy are intentionally not + /// part of this core API. The adapter must provide every materialization it + /// wants the deterministic parser to consider. pub fn parse_bytes<'src, 'a>( &self, bytes: &'src [u8], options: ParseOptions<'a>, - materializations: Option<&[BorrowedMaterialization<'src>]>, - ) -> Result, ParseError> { + materializations: &[Materialization], + ) -> Result { let mut counters = LimitCounters::default(); let input_len = u64::try_from(bytes.len()).map_err(|_| ParseError::Overflow("resource bytes"))?; @@ -108,14 +47,14 @@ impl LdrawParser { if options.fetch_count > options.limits.fetches { return Err(limit_error(LimitKind::Fetches, None)); } - if options.cancellation.cancelled { + if options.cancellation.checkpoint() { return Err(ParseError::Cancelled); } let budget = options.budget(bytes.len())?; let mut arenas = SemanticArenas::new(&options, budget)?; let scanned = scan_lines(bytes, &options.limits)?; for line in &scanned { - if options.cancellation.cancelled { + if options.cancellation.checkpoint() { return Err(ParseError::Cancelled); } let line_bytes = @@ -130,10 +69,23 @@ impl LdrawParser { && line.tokens.get(1).map(|token| token.kind) == Some(TokenKind::File) && line.tokens.len() >= 3 }); - if !has_mpd_file { + if has_mpd_file { + for line in &scanned { + let is_file = line.line_type == Some(LineType::Zero) + && line.tokens.get(1).map(|token| token.kind) == Some(TokenKind::File) + && line.tokens.len() >= 3; + let is_nofile = line.line_type == Some(LineType::Zero) + && line.tokens.get(1).map(|token| token.kind) == Some(TokenKind::NoFile); + if is_file || is_nofile { + counters + .add(LimitKind::Commands, 1, &options.limits) + .map_err(|name| limit_error(name, Some(line.span)))?; + } + } + } else { files[0].name = Cow::Owned(options.root_name.clone()); } - if options.cancellation.cancelled { + if options.cancellation.checkpoint() { return Err(ParseError::Cancelled); } let file_count = u64::try_from(files.len()) @@ -162,19 +114,39 @@ impl LdrawParser { let mut colours = ColourTable::new(); let mut texture_ids = BTreeSet::new(); for file in &files { - models.push(parse_model( + let mut scope = ColourScope::with_first_local_slot(colours.next_local_slot()); + let model_result = parse_model( file, &options, &mut counters, &mut diagnostics, &mut arenas, - &mut colours, + &mut scope, &mut texture_ids, - )?); + ); + let model = match model_result { + Ok(model) => model, + Err(error) => { + scope.rollback(); + return Err(error); + } + }; + scope.commit(); + colours.merge_scope(scope); + models.push(model); } + let source_model_count = models.len(); let mut materialized_keys = BTreeSet::new(); - for materialization in materializations.into_iter().flatten() { + for materialization in materializations { + if !materialization.has_valid_hash() { + return Err(ParseError::Diagnostic(Diagnostic { + code: DiagnosticCode::MaterializationHashMismatch, + severity: crate::types::Severity::Error, + message: format!("materialization hash mismatch for {}", materialization.path), + span: None, + })); + } if !materialization.available || materialization.bytes.is_empty() { continue; } @@ -196,7 +168,7 @@ impl LdrawParser { &options.limits, ) .map_err(|name| limit_error(name, None))?; - let resolved_scanned = scan_lines(materialization.bytes, &options.limits)?; + let resolved_scanned = scan_lines(&materialization.bytes, &options.limits)?; let file = VirtualFile { id: u32::try_from(files.len()) .map_err(|_| ParseError::Overflow("virtual file id"))?, @@ -214,15 +186,24 @@ impl LdrawParser { .add(LimitKind::Files, 1, &options.limits) .map_err(|name| limit_error(name, None))?; files.push(file.clone()); - let mut model = parse_model( + let mut scope = ColourScope::with_first_local_slot(colours.next_local_slot()); + let mut model = match parse_model( &file, &options, &mut counters, &mut diagnostics, &mut arenas, - &mut colours, + &mut scope, &mut texture_ids, - )?; + ) { + Ok(model) => model, + Err(error) => { + scope.rollback(); + return Err(error); + } + }; + scope.commit(); + colours.merge_scope(scope); model.key = CacheKey { canonical_path: materialization.path.clone(), resolved_root: materialization.root, @@ -230,7 +211,14 @@ impl LdrawParser { }; models.push(model); } - let (scene, summaries) = traverse(&models, &options, &mut counters, &mut diagnostics)?; + project_ldparse_step_ends(&mut models, source_model_count); + let (scene, summaries) = traverse( + &models, + &options, + &mut counters, + &mut diagnostics, + &mut arenas, + )?; let root = models.first(); let semantic = SemanticRecord { canonical_path: root.map(|m| Cow::Owned(m.path.to_string())), @@ -257,9 +245,10 @@ impl LdrawParser { .collect() }) .unwrap_or_default(), + texmap_geometry: root.map(|m| m.texmap_geometry.clone()).unwrap_or_default(), data_payloads: root.map(|m| m.data_payloads.clone()).unwrap_or_default(), }; - Ok(ParseResult { + Ok(crate::OwnedParseResult::from_parse_result(ParseResult { syntax, semantic, scene, @@ -270,7 +259,7 @@ impl LdrawParser { colours: colours.into_owned(), arenas, provenance_id: Cow::Borrowed(DEFAULT_PROVENANCE), - }) + })) } } @@ -280,10 +269,10 @@ fn parse_model<'src>( counters: &mut LimitCounters, diagnostics: &mut Vec, arenas: &mut SemanticArenas, - colours: &mut ColourTable<'src>, + colours: &mut ColourScope<'src>, texture_ids: &mut BTreeSet>, ) -> Result, ParseError> { - if options.cancellation.cancelled { + if options.cancellation.checkpoint() { return Err(ParseError::Cancelled); } let path = NormalizedPath::new(&file.name)?; @@ -306,12 +295,16 @@ fn parse_model<'src>( bfc: BfcFrame::default(), colour: DEFAULT_COLOUR, steps: Vec::new(), + step_ends: Vec::new(), + mesh_index_count: 0, + events: Vec::new(), texmap_state: TexmapState::Inactive, texmap_descriptor: None, texmap_stack: Vec::new(), texmap_next_span: None, texmap_fallback_seen: false, texmap_events: Vec::new(), + texmap_geometry: Vec::new(), data_payload: None, data_payloads: Vec::new(), includes: Vec::new(), @@ -326,7 +319,7 @@ fn parse_model<'src>( if line.blank { continue; } - if options.cancellation.cancelled { + if options.cancellation.checkpoint() { return Err(ParseError::Cancelled); } if counters @@ -391,6 +384,112 @@ fn parse_model<'src>( Ok(model) } +fn project_ldparse_step_ends(models: &mut [ModelData<'_>], source_model_count: usize) { + #[derive(Clone, Copy)] + struct ScanFrame { + source: usize, + next_event: usize, + } + + fn find_model(models: &[ModelData<'_>], limit: usize, name: &str) -> Option { + let normalized = NormalizedPath::new(name).ok()?; + models + .iter() + .take(limit) + .position(|model| model.path == normalized) + } + + let source_limit = source_model_count.min(models.len()); + for model in models.iter_mut() { + model.step_ends.clear(); + } + + // LDParse's callbacks are stateful pointers, not a tree-fold over model + // contents. Keep the parser's source cursor stack separate from the + // callback target: EOF always retargets callbacks to the root, while the + // saved include cursor is replayed against the now-cached child. + let mut callback_indices = vec![0_u64; models.len()]; + let mut cached = BTreeSet::new(); + let mut completed = vec![false; source_limit]; + let mut scan_stack = Vec::new(); + + for outer in 0..source_limit { + if completed[outer] { + continue; + } + cached.insert(outer); + let mut frame = ScanFrame { + source: outer, + next_event: 0, + }; + let mut callback_target = outer; + + loop { + if frame.next_event >= models[frame.source].events.len() { + completed[frame.source] = true; + // ModelBuilder::handleEOF calls recordTo(&target), where the + // EOF callback was retained on the root model. + callback_target = 0; + let Some(parent) = scan_stack.pop() else { + break; + }; + frame = parent; + continue; + } + + let event = models[frame.source].events[frame.next_event].clone(); + match event { + ModelEvent::Step => { + let offset = callback_indices[callback_target]; + models[callback_target].step_ends.push(offset); + frame.next_event += 1; + } + ModelEvent::Indices(count) => { + callback_indices[callback_target] = + callback_indices[callback_target].saturating_add(count); + frame.next_event += 1; + } + ModelEvent::Include(name) => { + let child = find_model(models, source_limit, name.as_ref()); + if let Some(child) = child + && !cached.contains(&child) + { + // Do not advance the include cursor. The real + // parser saves this line and reprocesses it after + // the child reaches EOF. + scan_stack.push(frame); + cached.insert(child); + frame = ScanFrame { + source: child, + next_event: 0, + }; + callback_target = child; + continue; + } + // A cached include is an ordinary callback and consumes + // its source line without changing the callback target. + frame.next_event += 1; + } + } + } + } + + // Caller-admitted materializations are parsed independently of the MPD + // stream and therefore retain their file-local callback target. + for model in models.iter_mut().skip(source_limit) { + let mut indices = 0_u64; + let mut step_ends = Vec::new(); + for event in &model.events { + match event { + ModelEvent::Step => step_ends.push(indices), + ModelEvent::Indices(count) => indices = indices.saturating_add(*count), + ModelEvent::Include(_) => {} + } + } + model.step_ends.extend(step_ends); + } +} + fn finalize_data<'src>( model: &mut ModelData<'src>, arenas: &mut SemanticArenas, @@ -434,9 +533,14 @@ fn finalize_data<'src>( return Ok(()); } }; - if options.cancellation.cancelled { + if options.cancellation.checkpoint() { return Err(ParseError::Cancelled); } + let decoded_len = + u64::try_from(decoded.len()).map_err(|_| ParseError::Overflow("DATA resource bytes"))?; + counters + .add(LimitKind::ResourceBytes, decoded_len, &options.limits) + .map_err(|kind| limit_error(kind, Some(payload.span)))?; arenas.charge_data_bytes(options, decoded.len())?; model.data_payloads.push(DataPayload { filename: payload.filename, @@ -468,15 +572,16 @@ fn process_line<'src>( diagnostics: &mut Vec, counters: &mut LimitCounters, arenas: &mut SemanticArenas, - colours: &mut ColourTable<'src>, + colours: &mut ColourScope<'src>, texture_ids: &mut BTreeSet>, ) -> Result<(), ParseError> { let profile = options.profile; let limits = &options.limits; let primitive = parse_primitive(line); + let bang_colon = line.line_type == Some(LineType::Zero) + && line.tokens.get(1).map(|token| token.text) == Some("!:"); if let Some(payload) = model.data_payload.as_mut() { - let continuation = line.line_type == Some(LineType::Zero) - && line.tokens.get(1).map(|token| token.text) == Some("!:"); + let continuation = bang_colon && model.texmap_state == TexmapState::Inactive; if continuation { let encoded = line .tokens @@ -508,30 +613,49 @@ fn process_line<'src>( } else if matches!( line.line_type, Some(LineType::One | LineType::Two | LineType::Three | LineType::Four | LineType::Five) - ) { - if primitive.is_some() { - let descriptor = model.texmap_descriptor.take(); - model.texmap_state = TexmapState::Active; - process_geometry( - model, - line, - primitive, - profile, - diagnostics, - counters, - limits, - arenas, - options, - )?; - model.texmap_state = TexmapState::Inactive; - if let Some(descriptor) = descriptor { - let _ = texture_ids.insert(descriptor); - } - model.texmap_next_span = None; + ) && primitive.is_some() + { + let descriptor = model.texmap_descriptor.clone(); + model.texmap_state = TexmapState::Active; + model.texmap_descriptor = descriptor.clone(); + let diagnostic_count = diagnostics.len(); + process_geometry( + model, + line, + primitive, + profile, + diagnostics, + counters, + limits, + arenas, + options, + )?; + if diagnostics.len() != diagnostic_count { + model.texmap_state = TexmapState::AwaitingNext; + model.texmap_descriptor = descriptor; return Ok(()); } + model.texmap_state = TexmapState::Inactive; + model.texmap_descriptor = None; + if let Some(descriptor) = descriptor { + let _ = texture_ids.insert(descriptor); + } + model.texmap_next_span = None; + return Ok(()); } } + if bang_colon { + return process_bang_colon( + model, + line, + profile, + diagnostics, + counters, + limits, + arenas, + options, + ); + } if line.line_type == Some(LineType::Zero) { let preserves_invert = line.tokens.get(1).map(|token| token.kind) == Some(TokenKind::Bfc) && line.tokens.get(2).map(|token| token.kind) == Some(TokenKind::InvertNext) @@ -563,6 +687,112 @@ fn process_line<'src>( ) } +fn process_bang_colon<'src>( + model: &mut ModelData<'src>, + line: &ScannedLine<'src>, + profile: ParserProfile, + diagnostics: &mut Vec, + counters: &mut LimitCounters, + limits: &LdrawLimits, + arenas: &mut SemanticArenas, + options: &ParseOptions<'_>, +) -> Result<(), ParseError> { + if !matches!( + model.texmap_state, + TexmapState::Active | TexmapState::Fallback + ) { + return add_diag( + profile, + diagnostics, + counters, + limits, + DiagnosticCode::TexmapBangColonOutOfScope, + line.span, + "bang-colon geometry requires an active TEXMAP frame", + true, + ); + } + let Some(embedded_type) = line.tokens.get(2).and_then(|token| match token.kind { + TokenKind::One => Some(LineType::One), + TokenKind::Two => Some(LineType::Two), + TokenKind::Three => Some(LineType::Three), + TokenKind::Four => Some(LineType::Four), + TokenKind::Five => Some(LineType::Five), + _ => None, + }) else { + return add_diag( + profile, + diagnostics, + counters, + limits, + DiagnosticCode::TexmapInvalidBangColonGeometry, + line.span, + "bang-colon requires an embedded line type 1 through 5", + true, + ); + }; + if matches!( + model.texmap_state, + TexmapState::Active | TexmapState::Fallback + ) && line.tokens.iter().skip(2).any(|token| token.text == "!:") + { + return add_diag( + profile, + diagnostics, + counters, + limits, + DiagnosticCode::TexmapNestedBangColon, + line.span, + "bang-colon geometry cannot be nested", + true, + ); + } + let embedded_line = ScannedLine { + line_type: Some(embedded_type), + raw: line.raw, + raw_tail: line.raw_tail, + tokens: line.tokens[2..].to_vec(), + span: line.span, + ending: line.ending.clone(), + blank: false, + }; + let embedded_primitive = crate::geom::parse_primitive(&embedded_line); + if embedded_primitive.is_none() + || embedded_primitive.as_ref().is_some_and(|primitive| { + matches!( + (embedded_type, primitive), + (LineType::One, crate::geom::ParsedPrimitive::Points(_)) + | ( + LineType::Two | LineType::Three | LineType::Four | LineType::Five, + crate::geom::ParsedPrimitive::Include(_) + ) + ) + }) + { + return add_diag( + profile, + diagnostics, + counters, + limits, + DiagnosticCode::TexmapInvalidBangColonGeometry, + line.span, + "malformed bang-colon geometry", + true, + ); + } + process_geometry( + model, + &embedded_line, + embedded_primitive, + profile, + diagnostics, + counters, + limits, + arenas, + options, + ) +} + fn process_meta<'src>( model: &mut ModelData<'src>, line: &ScannedLine<'src>, @@ -570,7 +800,7 @@ fn process_meta<'src>( diagnostics: &mut Vec, counters: &mut LimitCounters, arenas: &mut SemanticArenas, - colours: &mut ColourTable<'src>, + colours: &mut ColourScope<'src>, texture_ids: &mut BTreeSet>, ) -> Result<(), ParseError> { let profile = options.profile; @@ -642,13 +872,15 @@ fn process_meta<'src>( return Ok(()); } match tokens[1].kind { - TokenKind::Step => { + TokenKind::Step | TokenKind::Rotstep => { let step = model .steps .len() .checked_add(1) .ok_or(ParseError::Overflow("step number"))?; model.steps.push(format!("step-{step:08}").into()); + model.step_ends.push(model.mesh_index_count); + model.events.push(ModelEvent::Step); model.bfc.invert_next = false; model.texmap_state = TexmapState::Inactive; model.texmap_descriptor = None; @@ -663,11 +895,23 @@ fn process_meta<'src>( } if let Some(colour) = parse_colour(tokens) { let code = colour.code; - if colours.get(code).is_none() { + if colours.local_slot(code).is_none() { arenas.charge_local_colour(options)?; } - colours.record(colour); - model.colour = code; + if colours.record(colour).is_err() { + add_diag( + profile, + diagnostics, + counters, + limits, + DiagnosticCode::ColourSlotsLimit, + line.span, + "local colour slot table is exhausted", + true, + )?; + } else { + model.colour = code; + } } else { add_diag( profile, diff --git a/crates/polymodel-ldraw-core/src/types.rs b/crates/polymodel-ldraw-core/src/types.rs index 17f902d..5c85103 100644 --- a/crates/polymodel-ldraw-core/src/types.rs +++ b/crates/polymodel-ldraw-core/src/types.rs @@ -1,4 +1,5 @@ use crate::cache::NormalizedPath; +use crate::cache::{CacheKey, Sha256Hash}; use crate::geom::ColourCode; use miette::SourceSpan; use polymodel_renderer_ledger::{ @@ -7,6 +8,10 @@ use polymodel_renderer_ledger::{ }; use serde::{Deserialize, Serialize}; use std::fmt; +use std::sync::{ + Arc, + atomic::{AtomicBool, AtomicU64, Ordering}, +}; use thiserror::Error; pub const SCHEMA_VERSION: &str = "ldraw-canonical-v1"; @@ -25,18 +30,14 @@ pub const DEFAULT_TEXTURES: u64 = 1_024; pub const DEFAULT_FETCHES: u64 = 8; pub const DEFAULT_DIAGNOSTICS: u64 = 10_000; -#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Hash, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum ParserProfile { + #[default] Strict, Compatibility, Lossless, } -impl Default for ParserProfile { - fn default() -> Self { - Self::Strict - } -} #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] pub enum Severity { @@ -96,10 +97,15 @@ pub enum DiagnosticCode { TexmapFallbackOutOfScope, TexmapDuplicateFallback, TexmapInvalidEnd, + TexmapInvalidBangColonGeometry, + TexmapBangColonOutOfScope, + TexmapNestedBangColon, DataInvalid, DataTruncated, IncludeDepthLimit, GraphCycle, + AmbiguousMaterialization, + MaterializationHashMismatch, ResourceBytesLimit, CommandsLimit, InstancesLimit, @@ -145,10 +151,15 @@ impl fmt::Display for DiagnosticCode { Self::TexmapFallbackOutOfScope => "TEXMAP_FALLBACK_OUT_OF_SCOPE", Self::TexmapDuplicateFallback => "TEXMAP_DUPLICATE_FALLBACK", Self::TexmapInvalidEnd => "TEXMAP_INVALID_END", + Self::TexmapInvalidBangColonGeometry => "TEXMAP_INVALID_BANG_COLON_GEOMETRY", + Self::TexmapBangColonOutOfScope => "TEXMAP_BANG_COLON_OUT_OF_SCOPE", + Self::TexmapNestedBangColon => "TEXMAP_NESTED_BANG_COLON", Self::DataInvalid => "DATA_INVALID", Self::DataTruncated => "DATA_TRUNCATED", Self::IncludeDepthLimit => "INCLUDE_DEPTH_LIMIT", Self::GraphCycle => "GRAPH_CYCLE", + Self::AmbiguousMaterialization => "AMBIGUOUS_MATERIALIZATION", + Self::MaterializationHashMismatch => "MATERIALIZATION_HASH_MISMATCH", Self::ResourceBytesLimit => "RESOURCE_BYTES_LIMIT", Self::CommandsLimit => "COMMANDS_LIMIT", Self::InstancesLimit => "INSTANCES_LIMIT", @@ -205,8 +216,6 @@ pub enum LdrawError { Overflow(&'static str), #[error("parse cancelled")] Cancelled, - #[error(transparent)] - Resolver(#[from] crate::resolve::ResolveError), } pub type ParseError = LdrawError; @@ -323,16 +332,117 @@ impl LdrawLimits { } } -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +/// A target-safe live cancellation signal shared by a host and the parser. +/// +/// The parser checks this signal at every phase boundary; it never snapshots a +/// boolean before scanning. `AtomicBool` is available on native and WASM targets +/// and does not require a host runtime or an async executor. +#[derive(Clone, Debug)] pub struct CancellationPolicy { - pub cancelled: bool, + cancelled: Arc, + checkpoints: Arc, + /// `u64::MAX` is the disabled sentinel; every other value is a 1-based + /// checkpoint threshold, including zero for the next checkpoint. + cancel_after: Arc, } + impl Default for CancellationPolicy { fn default() -> Self { - Self { cancelled: false } + Self { + cancelled: Arc::new(AtomicBool::new(false)), + checkpoints: Arc::new(AtomicU64::new(0)), + cancel_after: Arc::new(AtomicU64::new(u64::MAX)), + } } } +impl CancellationPolicy { + pub fn new() -> Self { + Self::default() + } + + pub fn cancel(&self) { + self.cancelled.store(true, Ordering::Release); + } + + /// Request cancellation after the given number of live checkpoints. + /// + /// This is intentionally a host-test and deterministic orchestration seam: + /// callers can cancel a deep traversal without another thread or a timing + /// dependency. A value of zero cancels at the next checkpoint; the default + /// policy has no checkpoint threshold. + pub fn cancel_after_checkpoints(&self, checkpoints: u64) { + let threshold = checkpoints.max(1); + self.cancel_after.store(threshold, Ordering::Release); + } + + pub(crate) fn checkpoint(&self) -> bool { + let count = self + .checkpoints + .fetch_add(1, Ordering::AcqRel) + .saturating_add(1); + let threshold = self.cancel_after.load(Ordering::Acquire); + self.is_cancelled() || (threshold != u64::MAX && count >= threshold) + } + + pub fn is_cancelled(&self) -> bool { + self.cancelled.load(Ordering::Acquire) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Materialization { + pub path: NormalizedPath, + pub root: RootId, + pub content_hash: Sha256Hash, + pub bytes: Vec, + pub available: bool, + pub provenance: Option, +} + +impl Materialization { + pub fn new( + path: NormalizedPath, + root: RootId, + bytes: Vec, + available: bool, + provenance: Option, + ) -> Self { + let content_hash = CacheKey::new(path.clone(), root, &bytes).content_hash; + Self { + path, + root, + content_hash, + bytes, + available, + provenance, + } + } + + pub fn unavailable(path: NormalizedPath, root: RootId, provenance: Option) -> Self { + Self::new(path, root, Vec::new(), false, provenance) + } + + pub fn key(&self) -> CacheKey { + CacheKey { + canonical_path: self.path.clone(), + resolved_root: self.root, + content_hash: self.content_hash.clone(), + } + } + + pub fn has_valid_hash(&self) -> bool { + CacheKey::new(self.path.clone(), self.root, &self.bytes).content_hash == self.content_hash + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct TargetSelection { + pub source: CacheKey, + pub span: Span, + pub target: CacheKey, +} + #[derive(Clone)] pub struct ParseOptions<'a> { pub profile: ParserProfile, @@ -344,17 +454,10 @@ pub struct ParseOptions<'a> { pub resolved_root: RootId, pub fetch_count: u64, pub cancellation: CancellationPolicy, -} - -/// A borrowed target table used by the compatibility `parse_bytes` entry point. -/// New callers should use [`crate::Materialization`] and [`crate::Resolver`]. -#[derive(Clone, Debug)] -pub struct BorrowedMaterialization<'a> { - pub path: NormalizedPath, - pub root: RootId, - pub content_hash: crate::cache::Sha256Hash, - pub bytes: &'a [u8], - pub available: bool, + /// Exact resolver decisions, keyed by source model identity and include span. + /// An empty table requests the deterministic PM-75 precedence fallback, which + /// still rejects duplicate candidates at the same path/root. + pub target_selections: &'a [TargetSelection], } /// Caller-owned storage for source buffers used during a parse. @@ -398,6 +501,7 @@ impl<'a> ParseOptions<'a> { resolved_root: RootId::UploadedManifest, fetch_count: 0, cancellation: CancellationPolicy::default(), + target_selections: &[], } } pub fn default_budget(input_len: usize) -> Result { @@ -477,6 +581,7 @@ pub(crate) fn limit_error(kind: LimitKind, span: Option) -> ParseError { }) } +#[allow(clippy::too_many_arguments)] pub(crate) fn add_diag( profile: ParserProfile, diagnostics: &mut Vec, @@ -565,6 +670,13 @@ impl SemanticArenas { self.child(options, 128) } + pub(crate) fn charge_geometry_record( + &mut self, + options: &ParseOptions<'_>, + ) -> Result<(), ParseError> { + self.child(options, 256) + } + pub(crate) fn charge_cache_entry( &mut self, options: &ParseOptions<'_>, diff --git a/crates/polymodel-ldraw-testkit/src/gates.rs b/crates/polymodel-ldraw-testkit/src/gates.rs index aff7457..b11acc2 100644 --- a/crates/polymodel-ldraw-testkit/src/gates.rs +++ b/crates/polymodel-ldraw-testkit/src/gates.rs @@ -439,10 +439,10 @@ impl CorpusGate for NamedGate { Ok(()) } "colours_and_ldconfig" => { - if let Some(model) = expected.oracle_expected.root_model.as_ref() { - if u64::from(actual.semantic.colour_state.0) != model.default_color { - return fail("candidate.semantic.colour_state"); - } + if let Some(model) = expected.oracle_expected.root_model.as_ref() + && u64::from(actual.semantic.colour_state.0) != model.default_color + { + return fail("candidate.semantic.colour_state"); } Ok(()) } diff --git a/crates/polymodel-renderer-worker/src/worker.rs b/crates/polymodel-renderer-worker/src/worker.rs index 9224aa2..66ddcb7 100644 --- a/crates/polymodel-renderer-worker/src/worker.rs +++ b/crates/polymodel-renderer-worker/src/worker.rs @@ -224,20 +224,20 @@ impl WorkerState { return; } }; - let canvas_reservation = match self.ledger.reserve( - owner, - ResourceClass::CanvasBackbuffers, - canvas_bytes, - ) { - Ok(reservation) => reservation, - Err(error) => { - self.post_event(&RendererEvent::Error { - fatal: true, - message: format!("canvas admission: {error}"), - }); - return; - } - }; + let canvas_reservation = + match self + .ledger + .reserve(owner, ResourceClass::CanvasBackbuffers, canvas_bytes) + { + Ok(reservation) => reservation, + Err(error) => { + self.post_event(&RendererEvent::Error { + fatal: true, + message: format!("canvas admission: {error}"), + }); + return; + } + }; let opts = js_sys::Object::new(); let _ = js_sys::Reflect::set(&opts, &"desynchronized".into(), &JsValue::from_bool(true)); @@ -395,15 +395,33 @@ fn scene_cpu_bytes(model: &ModelMesh) -> Result { match &primitive.geometry { Geometry::Triangles(mesh) => { add_bytes(&mut bytes, (mesh.positions.len() as u64).saturating_mul(12))?; - add_bytes(&mut bytes, mesh.normals.as_ref().map_or(0, |v| (v.len() as u64) * 12))?; - add_bytes(&mut bytes, mesh.tangents.as_ref().map_or(0, |v| (v.len() as u64) * 16))?; - add_bytes(&mut bytes, mesh.uvs.as_ref().map_or(0, |v| (v.len() as u64) * 8))?; - add_bytes(&mut bytes, mesh.colors.as_ref().map_or(0, |v| (v.len() as u64) * 16))?; + add_bytes( + &mut bytes, + mesh.normals.as_ref().map_or(0, |v| (v.len() as u64) * 12), + )?; + add_bytes( + &mut bytes, + mesh.tangents.as_ref().map_or(0, |v| (v.len() as u64) * 16), + )?; + add_bytes( + &mut bytes, + mesh.uvs.as_ref().map_or(0, |v| (v.len() as u64) * 8), + )?; + add_bytes( + &mut bytes, + mesh.colors.as_ref().map_or(0, |v| (v.len() as u64) * 16), + )?; add_bytes(&mut bytes, mesh.indices.len().unwrap_or(0) as u64 * 4)?; } Geometry::Points(points) => { - add_bytes(&mut bytes, (points.positions.len() as u64).saturating_mul(12))?; - add_bytes(&mut bytes, points.colors.as_ref().map_or(0, |v| (v.len() as u64) * 16))?; + add_bytes( + &mut bytes, + (points.positions.len() as u64).saturating_mul(12), + )?; + add_bytes( + &mut bytes, + points.colors.as_ref().map_or(0, |v| (v.len() as u64) * 16), + )?; } } } @@ -458,8 +476,8 @@ fn admit_scene( let mut gpu_texture_bytes = 0; let mut textures = Vec::with_capacity(texture_inputs.len()); for input in texture_inputs { - let decoded = texture_base_bytes(input) - .map_err(|error| format!("texture base charge: {error}"))?; + let decoded = + texture_base_bytes(input).map_err(|error| format!("texture base charge: {error}"))?; let gpu = texture_charge(input) .map_err(|error| format!("texture charge: {error}"))? .checked_add(GPU_TEXTURE_OVERHEAD_BYTES) @@ -619,7 +637,9 @@ mod tests { .outcome, polymodel_renderer_protocol::TransitionOutcome::Committed ); - assert!(matches!(reducer.state(), polymodel_renderer_protocol::LoadState::Loading(id) if *id == identity)); + assert!( + matches!(reducer.state(), polymodel_renderer_protocol::LoadState::Loading(id) if *id == identity) + ); assert!(matches!( reducer .reduce(LoadEvent::LoadSucceeded { @@ -630,7 +650,10 @@ mod tests { .state, polymodel_renderer_protocol::LoadState::Installing(id) if id == identity )); - assert!(!matches!(reducer.state(), polymodel_renderer_protocol::LoadState::Ready(_))); + assert!(!matches!( + reducer.state(), + polymodel_renderer_protocol::LoadState::Ready(_) + )); assert!(matches!( reducer .reduce(LoadEvent::InstallSucceeded { @@ -702,7 +725,10 @@ mod tests { }) .unwrap_or_else(|error| panic!("{format:?} fixture should parse: {error:?}")); assert_eq!(mesh.format, format); - assert!(mesh.triangle_count() > 0, "{format:?} must contain triangles"); + assert!( + mesh.triangle_count() > 0, + "{format:?} must contain triangles" + ); } } @@ -964,6 +990,7 @@ fn handle_load_mesh( ) { return; } + st.load_epoch = identity.load_epoch; if let Some(renderer) = st.renderer.as_mut() { renderer.model = None; } @@ -980,10 +1007,12 @@ fn handle_load_mesh( } let state_clone = state.clone(); + let load_epoch = identity.load_epoch; + let scene_generation = state.borrow().scene_generation; spawn_local(async move { let load_reservations = { let st = state_clone.borrow(); - let owner = owner(identity.load_epoch, st.scene_generation); + let owner = owner(load_epoch, scene_generation); st.ledger.composite( owner, &[ @@ -1005,13 +1034,28 @@ fn handle_load_mesh( ) }; let Ok(_load_reservations) = load_reservations else { - if let Ok(st) = state_clone.try_borrow() { - st.post_event(&RendererEvent::LoadFailed { + if let Ok(mut st) = state_clone.try_borrow_mut() { + let error = LoadError { + code: LoadErrorCode::Renderer, + detail: "renderer admission rejected mesh load".into(), + }; + let result = st.load_reducer.reduce(LoadEvent::LoadFailed { namespace, identity, - code: LoadErrorCode::Fetch, - detail: "renderer admission rejected mesh load".into(), + error: error.clone(), + candidate: None, }); + if matches!( + result.outcome, + polymodel_renderer_protocol::TransitionOutcome::Committed + ) { + st.post_event(&RendererEvent::LoadFailed { + namespace, + identity, + code: error.code, + detail: error.detail, + }); + } } return; }; @@ -1046,50 +1090,58 @@ fn handle_load_mesh( identity, }); let (pw, ph) = (st.pixel_width, st.pixel_height); - let Some(renderer) = st.renderer.as_ref() else { - let error = LoadError { - code: LoadErrorCode::Renderer, - detail: "renderer context unavailable".into(), - }; - st.load_reducer.reduce(LoadEvent::InstallFailed { - namespace, - identity, - error: error.clone(), - candidate: None, - }); - st.post_event(&RendererEvent::LoadFailed { - namespace, - identity, - code: error.code, - detail: error.detail, - }); - return; - }; - st.scene_generation = st.scene_generation.saturating_add(1); - let scene_owner = owner(st.load_epoch, st.scene_generation); - let (scene_reservations, texture_reservations) = match admit_scene( - &st.ledger, - scene_owner, - &model, - ) { - Ok(reservations) => reservations, - Err(message) => { + let context = match st + .renderer + .as_ref() + .map(|renderer| renderer.context.clone()) + { + Some(context) => context, + None => { + let error = LoadError { + code: LoadErrorCode::Renderer, + detail: "renderer context unavailable".into(), + }; st.load_reducer.reduce(LoadEvent::InstallFailed { namespace, identity, - error: LoadError { code: LoadErrorCode::Install, detail: message.clone() }, + error: error.clone(), candidate: None, }); st.post_event(&RendererEvent::LoadFailed { namespace, identity, - code: LoadErrorCode::Install, - detail: message, + code: error.code, + detail: error.detail, }); return; } }; - let candidate = match build_scene(&renderer.context, &model, pw, ph) { + st.scene_generation = st.scene_generation.saturating_add(1); + let scene_owner = owner(st.load_epoch, st.scene_generation); + let (scene_reservations, texture_reservations) = + match admit_scene(&st.ledger, scene_owner, &model) { + Ok(reservations) => reservations, + Err(detail) => { + let error = LoadError { + code: LoadErrorCode::Install, + detail, + }; + st.load_reducer.reduce(LoadEvent::InstallFailed { + namespace, + identity, + error: error.clone(), + candidate: None, + }); + st.post_event(&RendererEvent::LoadFailed { + namespace, + identity, + code: error.code, + detail: error.detail, + }); + return; + } + }; + let candidate = match build_scene(&context, &model, pw, ph) { Ok(candidate) => candidate, Err(detail) => { drop(scene_reservations); @@ -1502,7 +1554,10 @@ fn handle_render_preview_image( return; }; if st.preview_reducer.current_identity() != Some(identity) - || st.preview_abort.as_ref().is_none_or(|slot| slot.identity != identity) + || st + .preview_abort + .as_ref() + .is_none_or(|slot| slot.identity != identity) { return; } @@ -1962,7 +2017,10 @@ fn handle_command(state: &Rc>, cmd: RendererCommand) { identity, } => { if namespace == RendererNamespace::Preview - && st.preview_abort.as_ref().is_some_and(|slot| slot.identity == identity) + && st + .preview_abort + .as_ref() + .is_some_and(|slot| slot.identity == identity) && let Some(slot) = st.preview_abort.take() { slot.controller.abort(); @@ -2089,9 +2147,7 @@ fn handle_command(state: &Rc>, cmd: RendererCommand) { st.interactive_compound_abort .take() .map(|slot| slot.controller.abort()); - st.preview_abort - .take() - .map(|slot| slot.controller.abort()); + st.preview_abort.take().map(|slot| slot.controller.abort()); st.pending_scene = None; st.disposed = true; st.ledger.retire(owner(st.load_epoch, st.scene_generation)); diff --git a/flake.nix b/flake.nix index f1f17bd..1f0912f 100644 --- a/flake.nix +++ b/flake.nix @@ -191,6 +191,7 @@ PLAYWRIGHT_HOST_PLATFORM_OVERRIDE = "ubuntu-24.04"; shellHook = '' + export POLYMODEL_NIX_SHELL=1 export PATH="${pkgs.dioxus-cli}/bin:${pkgs.wasm-bindgen-cli}/bin:$PATH"; export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$NIX_LD_LIBRARY_PATH" export CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER="wasm-bindgen-test-runner" export CHROME="${pkgs.chromium}/bin/chromium" diff --git a/justfile b/justfile index e9237d6..89f4f2a 100644 --- a/justfile +++ b/justfile @@ -7,122 +7,259 @@ set dotenv-load := false # hermetic — see `set dotenv-load := false` above. db-url := 'sqlite:./data/polymodel.db' +# Public recipes are dispatch boundaries; implementations are underscore-prefixed +# so Just hides them from --list while keeping the dependency graph in-shell. default: @just --list # Format Rust sources. Run clippy/check separately; clippy --fix can corrupt RSX. fix: - cargo fmt --all + @tools/in-dev-shell _fix # Run clippy without --fix; automatic clippy rewrites can corrupt Dioxus RSX. lint: - cargo clippy --workspace --all-targets --features server -- -D warnings -A clippy::useless_format - cargo clippy -p polymodel --all-targets --features server -- -D warnings -A clippy::useless_format - cargo clippy -p polymodel --target wasm32-unknown-unknown --features web -- -D warnings -A clippy::useless_format + @tools/in-dev-shell _lint # Compile checks across the whole workspace plus the app's server/wasm targets. # The server check resolves compile-time `query!` macros against the committed # `.sqlx` offline cache; regenerate it with `just sqlx-prepare` after changing # SQL or migrations. check: - cargo check --workspace - cargo check -p polymodel --features server - cargo check -p polymodel --target wasm32-unknown-unknown --features web - RUSTFLAGS='--cfg getrandom_backend="wasm_js"' cargo check -p polymodel-renderer-worker --target wasm32-unknown-unknown + @tools/in-dev-shell _check # Create and migrate the SQLite projection database. # Required when regenerating the `.sqlx` cache, and re-run after editing migrations. migrate: - mkdir -p ./data - cargo sqlx migrate run --source migrations --database-url '{{ db-url }}' + @tools/in-dev-shell _migrate # Regenerate the committed `.sqlx` offline query cache after changing SQL. # Self-contained: migrates the local database first, then points `cargo sqlx # prepare` at it via DATABASE_URL so no manual env setup is required. -sqlx-prepare: migrate - DATABASE_URL='{{ db-url }}' cargo sqlx prepare -- -p polymodel --features server +sqlx-prepare: + @tools/in-dev-shell _sqlx-prepare # Run unit tests across the workspace with the app's server-only code enabled. test: - cargo nextest run --workspace --features server + @tools/in-dev-shell _test + +# Test the Just/Nix-shell dispatch boundary without evaluating the flake. +test-just-nix: + bash tools/test-just-nix.sh + +# Guard the repair shell from depending on repository build outputs. This must +# remain runnable even when Nix evaluation or repository builds are broken. +check-devshell-tools-only: + python3 tools/check-devshell-tools-only.py # Run server-feature tests (app only; polymodel-api has no server feature). test-server: - cargo nextest run -p polymodel --features server + @tools/in-dev-shell _test-server # Run focused std-only ledger tests, wasm compilation, and its dependency-tree proof. test-renderer-ledger: - cargo test -p polymodel-renderer-ledger - cargo check -p polymodel-renderer-ledger --target wasm32-unknown-unknown - @cargo tree -p polymodel-renderer-ledger --target wasm32-unknown-unknown --edges normal - @if cargo tree -p polymodel-renderer-ledger --target wasm32-unknown-unknown --edges normal | grep -qE 'three-d|three-d-asset|stl_io|polymodel-mesh|polymodel-renderer-protocol|web-sys|wasm-bindgen'; then echo "FAIL: forbidden ledger dependency found"; exit 1; else echo "PASS: std-only ledger dependency tree"; fi + @tools/in-dev-shell _test-renderer-ledger # Run browser-backed wasm tests for the renderer worker. test-renderer: - RUSTFLAGS='--cfg getrandom_backend="wasm_js"' cargo test -p polymodel-renderer-worker --target wasm32-unknown-unknown + @tools/in-dev-shell _test-renderer # Validate the authored PM-77 corpus, provenance, matrix, gates, and snapshots. -ldraw-corpus: ldraw-oracle-integrity ldraw-oracle-check - cargo test -p polymodel-ldraw-testkit - cargo run --manifest-path tools/ldraw-compat-harness/Cargo.toml -- validate - cargo run --manifest-path tools/ldraw-compat-harness/Cargo.toml -- inventory - cargo run --manifest-path tools/ldraw-compat-harness/Cargo.toml -- corpus +ldraw-corpus: + @tools/in-dev-shell _ldraw-corpus # Compile the semantic core and reusable corpus/gate code for the E-facing WASM target. ldraw-wasm-check: - cargo check --tests -p polymodel-ldraw-core --target wasm32-unknown-unknown - cargo check --tests -p polymodel-ldraw-testkit --target wasm32-unknown-unknown + @tools/in-dev-shell _ldraw-wasm-check # Ensure expected/gate/harness/fuzz paths use authored data and the real core adapter. ldraw-oracle-integrity: - @if grep -RInE '(^|[^[:alnum:]_])canonical[[:space:]]*\(|(^|[^[:alnum:]_])parse_syntax[[:space:]]*\(' crates/polymodel-ldraw-testkit tools/ldraw-compat-harness tools/ldraw-oracle-generator --include='*.rs' --include='*.json' | grep -vE 'serialize_canonical|serialize[[:space:]]*\('; then echo 'FAIL: heuristic oracle helper call found'; exit 1; else echo 'PASS: no canonical/parse_syntax helper calls'; fi - @if grep -RInE 'polymodel_ldraw_core|InProcessRustAdapter|polymodel-ldraw-core' crates/polymodel-ldraw-testkit/corpus/expected tools/ldraw-oracle-generator --include='*.json' --include='*.rs'; then echo 'FAIL: oracle generator/records depend on Rust semantic core'; exit 1; else echo 'PASS: oracle generator/records are core-independent'; fi + @tools/in-dev-shell _ldraw-oracle-integrity # Check committed expected records against a fresh oracle-only generation. -ldraw-oracle-check: require-nix - cargo run --manifest-path tools/ldraw-oracle-generator/Cargo.toml -- check crates/polymodel-ldraw-testkit/corpus crates/polymodel-ldraw-testkit/corpus/expected +ldraw-oracle-check: + @tools/in-dev-shell _ldraw-oracle-check # Run bounded, deterministic native adversarial/property smoke. ldraw-fuzz-smoke: - cargo run --manifest-path tools/ldraw-compat-harness/Cargo.toml -- fuzz + @tools/in-dev-shell _ldraw-fuzz-smoke # Generate oracle-only v3 projections into a temporary directory by default. -ldraw-oracle-generate: require-nix - cargo run --manifest-path tools/ldraw-oracle-generator/Cargo.toml -- generate +ldraw-oracle-generate: + @tools/in-dev-shell _ldraw-oracle-generate # Require the native LDParse oracle toolchain before running oracle-backed gates. require-nix: - @command -v nix >/dev/null 2>&1 || { echo "FAIL: nix is required for PM-77 oracle gates"; exit 1; } + @tools/in-dev-shell _require-nix # Run the offline native LDParse compatibility boundary. -ldraw-differential: require-nix - cargo run --manifest-path tools/ldraw-compat-harness/Cargo.toml -- differential +ldraw-differential: + @tools/in-dev-shell _ldraw-differential # Run all local validation expected before review. Keep the PM-77 recipes explicit # so corpus, WASM, fuzz, and Nix-backed differential gates remain visible. -test-all: require-nix fix check lint test test-server test-renderer-ledger test-renderer ldraw-corpus ldraw-wasm-check ldraw-fuzz-smoke ldraw-differential +test-all: + @tools/in-dev-shell _test-all # Run browser end-to-end tests. e2e: - cd e2e && npm test + @tools/in-dev-shell _e2e # Start the Dioxus dev server. Builds the renderer worker first if missing. serve *ARGS: - @if [ ! -f public/renderer_worker_bg.wasm ] || [ ! -f public/renderer_worker_loader.js ]; then echo "renderer worker not built — running just build-renderer-worker"; just build-renderer-worker; fi - dx serve {{ ARGS }} + @tools/in-dev-shell _serve {{ ARGS }} # Build the renderer worker WASM bundle (separate from the app WASM). build-renderer-worker: + @tools/in-dev-shell _build-renderer-worker + +# Build and stage the single-threaded OpenCascade.js STEP conversion artifacts. +build-opencascade-step: + @tools/in-dev-shell _build-opencascade-step + +# Build the Dioxus app for web (includes renderer worker build). +build-web: + @tools/in-dev-shell _build-web + +# Build the Dioxus app for web (release, optimized). Strips debug symbols and runs wasm-opt. +build-web-release: + @tools/in-dev-shell _build-web-release + +# Verify the renderer split: no heavy deps in the app, worker wasm exists, bundle size under ceiling. +verify-renderer-split: + @tools/in-dev-shell _verify-renderer-split + +# Regenerate crates/polymodel-api from authored lexicons via the local Jacquard checkout. +generate-api: + @tools/in-dev-shell _generate-api +# Validate portable event-sourced GC coordination state. +validate-gc-state *ARGS: + @tools/in-dev-shell _validate-gc-state {{ ARGS }} + +# Validate GC jj workspace/bookmark rails before or after forgetting workspaces. +validate-gc-jj-state *ARGS: + @tools/in-dev-shell _validate-gc-jj-state {{ ARGS }} + +# Check that build-time environment handling does not expose server secrets to the frontend. +check-public-env *ARGS: + @tools/in-dev-shell _check-public-env {{ ARGS }} + +# Guard compile-time SQL usage in handwritten production Rust. +check-production-sql *ARGS: + @tools/in-dev-shell _check-production-sql {{ ARGS }} + +# Validate current documentation references and architecture maps. +check-doc-references *ARGS: + @tools/in-dev-shell _check-doc-references {{ ARGS }} + +# Guard the single production appview content-type normalizer and its callers. +check-content-type *ARGS: + @tools/in-dev-shell _check-content-type {{ ARGS }} + +# Validate the borrowed proxy response-reference parser and raw forwarding boundary. +check-proxy-response *ARGS: + @tools/in-dev-shell _check-proxy-response {{ ARGS }} +# Create a sibling jj workspace for a Jira ticket or feature slug, then start a fresh change. +workspace-create name: + @tools/in-dev-shell _workspace-create {{ name }} + +# List jj workspaces. +workspace-list: + @tools/in-dev-shell _workspace-list + +# Forget a jj workspace after the work is safely landed or intentionally abandoned. +workspace-forget name: + @tools/in-dev-shell _workspace-forget {{ name }} + +# Show current jj state and diff. +status: + @tools/in-dev-shell _status + +# Clean build artifacts. +clean: + @tools/in-dev-shell _clean + +_fix: + cargo fmt --all + +_lint: + cargo clippy --workspace --all-targets --features server -- -D warnings -A clippy::useless_format + cargo clippy -p polymodel --all-targets --features server -- -D warnings -A clippy::useless_format + cargo clippy -p polymodel --target wasm32-unknown-unknown --features web -- -D warnings -A clippy::useless_format + +_check: + cargo check --workspace + cargo check -p polymodel --features server + cargo check -p polymodel --target wasm32-unknown-unknown --features web + RUSTFLAGS='--cfg getrandom_backend="wasm_js"' cargo check -p polymodel-renderer-worker --target wasm32-unknown-unknown + +_migrate: + mkdir -p ./data + cargo sqlx migrate run --source migrations --database-url '{{ db-url }}' + +_sqlx-prepare: _migrate + DATABASE_URL='{{ db-url }}' cargo sqlx prepare -- -p polymodel --features server + +_test: + cargo nextest run --workspace --features server + +_test-server: + cargo nextest run -p polymodel --features server + +_test-renderer-ledger: + cargo test -p polymodel-renderer-ledger + cargo check -p polymodel-renderer-ledger --target wasm32-unknown-unknown + @cargo tree -p polymodel-renderer-ledger --target wasm32-unknown-unknown --edges normal + @if cargo tree -p polymodel-renderer-ledger --target wasm32-unknown-unknown --edges normal | grep -qE 'three-d|three-d-asset|stl_io|polymodel-mesh|polymodel-renderer-protocol|web-sys|wasm-bindgen'; then echo "FAIL: forbidden ledger dependency found"; exit 1; else echo "PASS: std-only ledger dependency tree"; fi + +_test-renderer: + RUSTFLAGS='--cfg getrandom_backend="wasm_js"' cargo test -p polymodel-renderer-worker --target wasm32-unknown-unknown + +_ldraw-corpus: _ldraw-oracle-integrity _ldraw-oracle-check + cargo test -p polymodel-ldraw-testkit + cargo run --manifest-path tools/ldraw-compat-harness/Cargo.toml -- validate + cargo run --manifest-path tools/ldraw-compat-harness/Cargo.toml -- inventory + cargo run --manifest-path tools/ldraw-compat-harness/Cargo.toml -- corpus + +_ldraw-wasm-check: + cargo check --tests -p polymodel-ldraw-core --target wasm32-unknown-unknown + cargo check --tests -p polymodel-ldraw-testkit --target wasm32-unknown-unknown + +_ldraw-oracle-integrity: + @if grep -RInE '(^|[^[:alnum:]_])canonical[[:space:]]*\(|(^|[^[:alnum:]_])parse_syntax[[:space:]]*\(' crates/polymodel-ldraw-testkit tools/ldraw-compat-harness tools/ldraw-oracle-generator --include='*.rs' --include='*.json' | grep -vE 'serialize_canonical|serialize[[:space:]]*\('; then echo 'FAIL: heuristic oracle helper call found'; exit 1; else echo 'PASS: no canonical/parse_syntax helper calls'; fi + @if grep -RInE 'polymodel_ldraw_core|InProcessRustAdapter|polymodel-ldraw-core' crates/polymodel-ldraw-testkit/corpus/expected tools/ldraw-oracle-generator --include='*.json' --include='*.rs'; then echo 'FAIL: oracle generator/records depend on Rust semantic core'; exit 1; else echo 'PASS: oracle generator/records are core-independent'; fi + +_ldraw-oracle-check: _require-nix + cargo run --manifest-path tools/ldraw-oracle-generator/Cargo.toml -- check crates/polymodel-ldraw-testkit/corpus crates/polymodel-ldraw-testkit/corpus/expected + +_ldraw-fuzz-smoke: + cargo run --manifest-path tools/ldraw-compat-harness/Cargo.toml -- fuzz + +_ldraw-oracle-generate: _require-nix + cargo run --manifest-path tools/ldraw-oracle-generator/Cargo.toml -- generate + +_require-nix: + @command -v nix >/dev/null 2>&1 || { echo "FAIL: nix is required for PM-77 oracle gates"; exit 1; } + +_ldraw-differential: _require-nix + cargo run --manifest-path tools/ldraw-compat-harness/Cargo.toml -- differential + +_test-all: _require-nix _fix _check _lint _test _test-server _test-renderer-ledger _test-renderer _ldraw-corpus _ldraw-wasm-check _ldraw-fuzz-smoke _ldraw-differential + +_e2e: + cd e2e && npm test + +_serve *ARGS: + @if [ ! -f public/renderer_worker_bg.wasm ] || [ ! -f public/renderer_worker_loader.js ]; then echo "renderer worker not built — running just _build-renderer-worker"; just _build-renderer-worker; fi + dx serve {{ ARGS }} + +_build-renderer-worker: RUSTFLAGS='--cfg getrandom_backend="wasm_js"' cargo build -p polymodel-renderer-worker --bin renderer_worker --target wasm32-unknown-unknown --profile worker-release wasm-bindgen target/wasm32-unknown-unknown/worker-release/renderer_worker.wasm --target web --out-dir public --no-typescript @if command -v wasm-opt &>/dev/null; then wasm-opt public/renderer_worker_bg.wasm -O4 --enable-bulk-memory --enable-simd --enable-nontrapping-float-to-int --enable-sign-ext -o public/renderer_worker_bg.wasm; else echo "wasm-opt not found, skipping optimization"; fi -# Build and stage the single-threaded OpenCascade.js STEP conversion artifacts. -build-opencascade-step: +_build-opencascade-step: node tools/renderer-step/build-and-stage-opencascade.mjs -# Build the Dioxus app for web (includes renderer worker build). -build-web: build-renderer-worker +_build-web: _build-renderer-worker dx build --platform web -# Build the Dioxus app for web (release, optimized). Strips debug symbols and runs wasm-opt. -build-web-release: build-renderer-worker build-opencascade-step +_build-web-release: _build-renderer-worker _build-opencascade-step #!/usr/bin/env bash set -e dx build --platform web --release --debug-symbols=false @@ -139,8 +276,7 @@ build-web-release: build-renderer-worker build-opencascade-step echo "==> wasm-opt not found, skipping optimization" fi -# Verify the renderer split: no heavy deps in the app, worker wasm exists, bundle size under ceiling. -verify-renderer-split: build-web +_verify-renderer-split: _build-web @echo "==> AC.1: checking app dependency tree for forbidden crates..." @if cargo tree -p polymodel --target wasm32-unknown-unknown --features web --edges normal 2>/dev/null | grep -qE '(^| )three-d v|three-d-asset v|stl_io v|polymodel-mesh v'; then echo "FAIL: forbidden crate found in app dependency tree"; exit 1; else echo "PASS: no forbidden crates"; fi @echo "==> AC.1b: checking app Cargo.toml for forbidden deps..." @@ -151,55 +287,45 @@ verify-renderer-split: build-web @test -f public/renderer_worker_bg.wasm && echo "PASS: worker wasm exists" || { echo "FAIL: worker wasm not found"; exit 1; } @echo "==> All renderer-split checks passed." -# Regenerate crates/polymodel-api from authored lexicons via the local Jacquard checkout. -generate-api: +_generate-api: nix run ../jacquard -# Validate portable event-sourced GC coordination state. -validate-gc-state *ARGS: + +_validate-gc-state *ARGS: python3 tools/validate-gc-state.py {{ ARGS }} -# Validate GC jj workspace/bookmark rails before or after forgetting workspaces. -validate-gc-jj-state *ARGS: +_validate-gc-jj-state *ARGS: python3 tools/validate-gc-jj-state.py {{ ARGS }} -# Check that build-time environment handling does not expose server secrets to the frontend. -check-public-env *ARGS: +_check-public-env *ARGS: python3 tools/check-public-env.py {{ ARGS }} -# Guard compile-time SQL usage in handwritten production Rust. -check-production-sql *ARGS: +_check-production-sql *ARGS: python3 tools/check-production-sql.py {{ ARGS }} -# Validate current documentation references and architecture maps. -check-doc-references *ARGS: +_check-doc-references *ARGS: python3 tools/check-doc-references.py {{ ARGS }} -# Guard the single production appview content-type normalizer and its callers. -check-content-type *ARGS: +_check-content-type *ARGS: python3 tools/check-content-type.py {{ ARGS }} -# Validate the borrowed proxy response-reference parser and raw forwarding boundary. -check-proxy-response *ARGS: +_check-proxy-response *ARGS: python3 tools/check-proxy-response.py {{ ARGS }} -# Create a sibling jj workspace for a Jira ticket or feature slug, then start a fresh change. -workspace-create name: + +_workspace-create name: root="$(jj workspace root)" && workspace_path="$(dirname "$root")/{{ name }}" && jj workspace add "$workspace_path" -r @ root="$(jj workspace root)" && workspace_path="$(dirname "$root")/{{ name }}" && cd "$workspace_path" && jj desc -m "{{ name }}" + root="$(jj workspace root)" && workspace_path="$(dirname "$root")/{{ name }}" && direnv allow "$workspace_path" && direnv exec "$workspace_path" true -# List jj workspaces. -workspace-list: +_workspace-list: jj workspace list -# Forget a jj workspace after the work is safely landed or intentionally abandoned. -workspace-forget name: +_workspace-forget name: jj workspace forget {{ name }} -# Show current jj state and diff. -status: +_status: jj status jj diff --summary -# Clean build artifacts. -clean: +_clean: cargo clean rm -rf target/dx diff --git a/src/appview/download_tests.rs b/src/appview/download_tests.rs index c53a719..cf5764e 100644 --- a/src/appview/download_tests.rs +++ b/src/appview/download_tests.rs @@ -38,66 +38,146 @@ async fn get_part_file_malformed_uri_is_bad_request() { } #[tokio::test] -async fn get_part_file_ldraw_resource_is_hard_blocked() { +async fn get_part_file_wrong_collection_is_bad_request() { let state = state().await; let app = crate::appview::router().with_state(state); let resp = app .oneshot( Request::builder() - .uri("/xrpc/space.polymodel.library.getPartFile?uri=at%3A%2F%2Fdid%3Aplc%3Ax%2Fspace.polymodel.library.ldrawResource%2Fr1") + .uri("/xrpc/space.polymodel.library.getPartFile?uri=at%3A%2F%2Fdid%3Aplc%3Ax%2Fspace.polymodel.library.thing%2Ft1") .body(Body::empty()) .unwrap(), ) .await .unwrap(); - assert_eq!(resp.status(), StatusCode::NOT_FOUND); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + let body = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + let json: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(json["error"], "InvalidRequest"); } #[tokio::test] -async fn get_part_file_wrong_collection_is_bad_request() { +async fn get_part_file_missing_part_is_not_found() { let state = state().await; let app = crate::appview::router().with_state(state); let resp = app .oneshot( Request::builder() - .uri("/xrpc/space.polymodel.library.getPartFile?uri=at%3A%2F%2Fdid%3Aplc%3Ax%2Fspace.polymodel.library.thing%2Ft1") + .uri("/xrpc/space.polymodel.library.getPartFile?uri=at%3A%2F%2Fdid%3Aplc%3Ax%2Fspace.polymodel.library.part%2Fnone") .body(Body::empty()) .unwrap(), ) .await .unwrap(); - assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + assert_eq!(resp.status(), StatusCode::NOT_FOUND); let body = axum::body::to_bytes(resp.into_body(), usize::MAX) .await .unwrap(); let json: serde_json::Value = serde_json::from_slice(&body).unwrap(); - assert_eq!(json["error"], "InvalidRequest"); + assert_eq!(json["error"], "RecordNotFound"); } #[tokio::test] -async fn get_part_file_missing_part_is_not_found() { - let state = state().await; +async fn get_part_file_streams_projected_manifest_in_offset_order() { + let (pds, server) = loopback_pds(false, None).await; + let pool = pool().await; + let part_uri = format!("at://{DID_A}/space.polymodel.library.part/download"); + let file = json!({ + "mimeType": "model/stl", + "size": 4, + "chunks": [ + {"blob": {"$type": "blob", "ref": {"$link": "bafkreibb44q4gwsyep63iux2f6pquyjmot5zklqgsj2itrvspjb3qf562q"}, "mimeType": "model/stl", "size": 2}, "offset": 2, "size": 2}, + {"blob": {"$type": "blob", "ref": {"$link": "bafkreih3ryqpylsmh4siyygdtplff46bgrzjro4xpofu2widxbifkyqgam"}, "mimeType": "model/stl", "size": 2}, "offset": 0, "size": 2} + ] + }); + sqlx::query("INSERT INTO parts (did, rkey, uri, cid, name, format, file_json, created_at, indexed_at, record_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") + .bind(DID_A) + .bind("download") + .bind(&part_uri) + .bind("part-cid") + .bind("Bracket.stl") + .bind("stl") + .bind(file.to_string()) + .bind(1_i64) + .bind(1_i64) + .bind(file!()) + .execute(&pool) + .await + .unwrap(); + let bootstrap = crate::oauth::bootstrap_oauth(pool.clone(), Some("http://localhost")) + .expect("ephemeral OAuth bootstrap for tests"); + let state = AppState::new_with_resolver(pool, bootstrap, loopback_resolver(&pds)); let app = crate::appview::router().with_state(state); let resp = app .oneshot( Request::builder() - .uri("/xrpc/space.polymodel.library.getPartFile?uri=at%3A%2F%2Fdid%3Aplc%3Ax%2Fspace.polymodel.library.part%2Fnone") + .uri(format!( + "/xrpc/space.polymodel.library.getPartFile?uri={}", + urlencoding::encode(&part_uri) + )) .body(Body::empty()) .unwrap(), ) .await .unwrap(); - assert_eq!(resp.status(), StatusCode::NOT_FOUND); - let body = axum::body::to_bytes(resp.into_body(), usize::MAX) + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!(resp.headers()["content-type"], "model/stl"); + assert_eq!(resp.headers()["content-length"], "4"); + assert_eq!( + resp.headers()["content-disposition"], + "attachment; filename=\"Bracket.stl\"" + ); + assert_eq!(body_text(resp).await, "abcd"); + server.abort(); +} + +#[tokio::test] +async fn get_part_file_returns_500_when_first_blob_fetch_fails() { + let (pds, server) = loopback_pds(true, None).await; + let pool = pool().await; + let part_uri = format!("at://{DID_A}/space.polymodel.library.part/download"); + let file = json!({ + "mimeType": "model/stl", + "size": 2, + "chunks": [{"blob": {"$type": "blob", "ref": {"$link": "bafkreih3ryqpylsmh4siyygdtplff46bgrzjro4xpofu2widxbifkyqgam"}, "mimeType": "model/stl", "size": 2}, "offset": 0, "size": 2}] + }); + sqlx::query("INSERT INTO parts (did, rkey, uri, cid, name, format, file_json, created_at, indexed_at, record_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") + .bind(DID_A).bind("download").bind(&part_uri).bind("part-cid").bind("Bracket.stl") + .bind("stl").bind(file.to_string()).bind(1_i64).bind(1_i64).bind(file!()) + .execute(&pool).await.unwrap(); + let bootstrap = crate::oauth::bootstrap_oauth(pool.clone(), Some("http://localhost")) + .expect("ephemeral OAuth bootstrap for tests"); + let state = AppState::new_with_resolver(pool, bootstrap, loopback_resolver(&pds)); + let app = crate::appview::router().with_state(state); + let resp = app + .oneshot( + Request::builder() + .uri(format!( + "/xrpc/space.polymodel.library.getPartFile?uri={}", + urlencoding::encode(&part_uri) + )) + .body(Body::empty()) + .unwrap(), + ) .await .unwrap(); - let json: serde_json::Value = serde_json::from_slice(&body).unwrap(); - assert_eq!(json["error"], "RecordNotFound"); + assert_eq!(resp.status(), StatusCode::INTERNAL_SERVER_ERROR); + let body = body_text(resp).await; + let error: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(error["error"], "InternalServerError"); + assert!( + error["message"] + .as_str() + .is_some_and(|message| message.contains("blob fetch")) + ); + server.abort(); } #[tokio::test] async fn get_part_file_later_blob_failure_truncates_committed_stream() { - let failed_cid = "bafkreiape5vm4aofohlvq72uc7nw6golsxa44ser6photbjne6gkhboeii"; + let failed_cid = "bafkreibb44q4gwsyep63iux2f6pquyjmot5zklqgsj2itrvspjb3qf562q"; let (pds, server) = loopback_pds(false, Some(failed_cid.to_string())).await; let pool = pool().await; let part_uri = format!("at://{DID_A}/space.polymodel.library.part/download"); @@ -106,7 +186,7 @@ async fn get_part_file_later_blob_failure_truncates_committed_stream() { "size": 4, "chunks": [ {"blob": {"$type": "blob", "ref": {"$link": failed_cid}, "mimeType": "model/stl", "size": 2}, "offset": 2, "size": 2}, - {"blob": {"$type": "blob", "ref": {"$link": "bafkreif2nycdsf3vvit7hhpxegmlmoknx6m7yuvdt4tkzpk7hv4jqo3ulq"}, "mimeType": "model/stl", "size": 2}, "offset": 0, "size": 2} + {"blob": {"$type": "blob", "ref": {"$link": "bafkreih3ryqpylsmh4siyygdtplff46bgrzjro4xpofu2widxbifkyqgam"}, "mimeType": "model/stl", "size": 2}, "offset": 0, "size": 2} ] }); sqlx::query("INSERT INTO parts (did, rkey, uri, cid, name, format, file_json, created_at, indexed_at, record_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") @@ -129,7 +209,11 @@ async fn get_part_file_later_blob_failure_truncates_committed_stream() { ) .await .unwrap(); - assert_eq!(resp.status(), StatusCode::OK); + if resp.status() != StatusCode::OK { + let status = resp.status(); + let body = body_text(resp).await; + panic!("expected successful streaming response, got {status}: {body}"); + } assert_eq!(resp.headers()["content-length"], "4"); let mut body = resp.into_body(); let mut bytes = Vec::new(); @@ -147,7 +231,10 @@ async fn get_part_file_later_blob_failure_truncates_committed_stream() { } } } - assert!(stream_error, "later PDS failure must surface as a body error"); + assert!( + stream_error, + "later PDS failure must surface as a body error" + ); assert_eq!(&bytes[..], b"ab"); server.abort(); } @@ -175,11 +262,27 @@ async fn anonymous_compound_surface_is_present_without_legacy_routes_or_route_ro assert_eq!(response.status(), StatusCode::NOT_FOUND); for (method, path, body) in [ - (Method::GET, "/app/parts/did:plc:x/abc/download", Body::empty()), + ( + Method::GET, + "/app/parts/did:plc:x/abc/download", + Body::empty(), + ), (Method::GET, "/app/ldraw/did:plc:x/abc", Body::empty()), - (Method::GET, "/xrpc/space.polymodel.library.ldrawResource?uri=at%3A%2F%2Fdid%3Aplc%3Ax%2Fresource%2Fabc", Body::empty()), - (Method::GET, "/xrpc/space.polymodel.library.getLdrawResource?uri=at%3A%2F%2Fdid%3Aplc%3Ax%2Fresource%2Fabc", Body::empty()), - (Method::POST, "/xrpc/space.polymodel.library.mintLdrawResource", Body::from("{}")), + ( + Method::GET, + "/xrpc/space.polymodel.library.ldrawResource?uri=at%3A%2F%2Fdid%3Aplc%3Ax%2Fresource%2Fabc", + Body::empty(), + ), + ( + Method::GET, + "/xrpc/space.polymodel.library.getLdrawResource?uri=at%3A%2F%2Fdid%3Aplc%3Ax%2Fresource%2Fabc", + Body::empty(), + ), + ( + Method::POST, + "/xrpc/space.polymodel.library.mintLdrawResource", + Body::from("{}"), + ), ] { let response = app .clone() @@ -206,13 +309,12 @@ async fn anonymous_compound_surface_is_present_without_legacy_routes_or_route_ro "ldraw_official_snapshots", "ldraw_legacy_resources", ] { - let exists: Option = sqlx::query_scalar( - "SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?", - ) - .bind(table) - .fetch_optional(&pool) - .await - .unwrap(); + let exists: Option = + sqlx::query_scalar("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?") + .bind(table) + .fetch_optional(&pool) + .await + .unwrap(); assert!( exists.is_some(), "PM-76 projection table must remain: {table}" diff --git a/src/appview/read_tests.rs b/src/appview/read_tests.rs index 6e6bfbc..9e758c1 100644 --- a/src/appview/read_tests.rs +++ b/src/appview/read_tests.rs @@ -1,5 +1,10 @@ use super::test_support::*; use super::views; +use crate::indexing::projection::{ProjectionEvent, ProjectionInput, project_event}; +use http_body_util::BodyExt; +use jacquard_common::types::value::{Data, to_data}; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; #[tokio::test] async fn get_thing_hydrates_thing_with_ordered_models_and_record() { @@ -488,3 +493,383 @@ async fn union_serializes_type_tag() { Some("space.polymodel.actor.defs#profileView") ); } + +#[tokio::test] +async fn ordinary_projection_feeds_compound_resolver_and_route() { + use crate::ldraw::resolve::SqliteResolver; + use crate::ldraw::{CanonicalPath, LdrawResourceResolver}; + use base64::Engine as _; + + let bytes = b"0 FILE main.ldr\\n0 NOFILE\\n"; + let (cid, _) = test_blob(bytes); + let digest = Sha256::digest(bytes); + let project = format!("at://{DID_A}/space.polymodel.library.thing/ordinary"); + let model = format!("at://{DID_A}/space.polymodel.library.model/main"); + let part = format!("at://{DID_A}/space.polymodel.library.part/main"); + let pool = pool().await; + let event = |seq: u64, collection: &str, rkey: &str, record: Option| ProjectionEvent { + seq, + did: DID_A.to_owned(), + collection: collection.to_owned(), + rkey: rkey.to_owned(), + action: "create".to_owned(), + record, + cid: Some(format!("record-{seq}")), + }; + + project_event( + &pool, + &ProjectionInput::Record(event( + 1, + "space.polymodel.library.thing", + "ordinary", + Some( + to_data(&json!({ + "name": "Ordinary compound", + "license": "MIT", + "createdAt": "2024-01-01T00:00:00.000Z", + "models": [{ "uri": model, "cid": "bafyreimodel" }] + })) + .unwrap(), + ), + )), + ) + .await + .unwrap(); + project_event( + &pool, + &ProjectionInput::Record(event( + 2, + "space.polymodel.library.model", + "main", + Some( + to_data(&json!({ + "name": "Main", + "createdAt": "2024-01-01T00:00:00.000Z", + "parts": [{ "uri": part, "cid": "bafyreipart" }] + })) + .unwrap(), + ), + )), + ) + .await + .unwrap(); + project_event( + &pool, + &ProjectionInput::Record(event( + 3, + "space.polymodel.library.part", + "main", + Some(to_data(&json!({ + "name": "main.ldr", + "format": "LDraw", + "createdAt": "2024-01-01T00:00:00.000Z", + "file": { + "mimeType": "application/x-ldraw", + "size": bytes.len(), + "digest": { "$bytes": base64::engine::general_purpose::STANDARD.encode(digest) }, + "chunks": [{ + "blob": { "$type": "blob", "ref": { "$link": cid }, "mimeType": "application/x-ldraw", "size": bytes.len() }, + "offset": 0, + "size": bytes.len() + }] + } + })).unwrap()), + )), + ) + .await + .unwrap(); + + let project_uri = jacquard_common::types::string::AtUri::new_owned(&project).unwrap(); + let resource = SqliteResolver::new(pool.clone()) + .resolve( + &project_uri, + &CanonicalPath::parse("models/main.ldr").unwrap(), + ) + .await + .unwrap(); + assert_eq!(resource.source.resource_uri.as_ref(), part); + assert_eq!(resource.target.blob_cids, vec![cid.clone()]); + + let mut blobs = HashMap::new(); + blobs.insert(cid, bytes.to_vec()); + let (pds, server) = loopback_pds_with_blobs(blobs).await; + let bootstrap = crate::oauth::bootstrap_oauth(pool.clone(), Some("http://localhost")) + .expect("ephemeral OAuth bootstrap for tests"); + let projection = pool.clone(); + let state = AppState::new_with_resolver(pool, bootstrap, loopback_resolver(&pds)); + let app = crate::appview::router().with_state(state); + let response = app + .clone() + .oneshot( + Request::builder() + .method(Method::POST) + .uri("/app/ldraw/compound-resources") + .header("content-type", "application/json") + .body(Body::from( + json!({ "project": project, "path": "models/main.ldr" }).to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let plan: serde_json::Value = serde_json::from_slice( + &axum::body::to_bytes(response.into_body(), 1 << 20) + .await + .unwrap(), + ) + .unwrap(); + assert_eq!(plan["primary"], "models/main.ldr"); + assert_eq!(plan["descriptors"].as_array().unwrap().len(), 1); + let descriptor = &plan["descriptors"][0]; + assert_eq!(descriptor["key"], "models/main.ldr"); + assert_eq!(descriptor["target_cids"][0], resource.target.blob_cids[0]); + + let fetched = app + .clone() + .oneshot( + Request::builder() + .uri(descriptor["route"].as_str().unwrap()) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(fetched.status(), StatusCode::OK); + assert_eq!( + fetched + .into_body() + .collect() + .await + .unwrap() + .to_bytes() + .as_ref(), + bytes.as_slice() + ); + + let route = descriptor["route"].as_str().unwrap().to_owned(); + let route_variant = |mutate: &dyn Fn(&mut serde_json::Value)| { + let encoded = route.strip_prefix("/app/ldraw/resources/").unwrap(); + let (payload, _) = encoded.rsplit_once('.').unwrap(); + let bytes = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(payload) + .unwrap(); + let mut payload: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + mutate(&mut payload); + let bytes = serde_json::to_vec(&payload).unwrap(); + let digest = Sha256::digest(&bytes); + format!( + "/app/ldraw/resources/{}.{}", + base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(bytes), + digest + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::() + ) + }; + let fetch_status = |route: String| { + let app = app.clone(); + async move { + app.oneshot(Request::builder().uri(route).body(Body::empty()).unwrap()) + .await + .unwrap() + .status() + } + }; + + // A route whose MAC is changed must fail before any source lookup. + let mut tampered = route.clone().into_bytes(); + let last = tampered.len() - 1; + tampered[last] = if tampered[last] == b'0' { b'1' } else { b'0' }; + assert_eq!( + fetch_status(String::from_utf8(tampered).unwrap()).await, + StatusCode::NOT_FOUND + ); + + // Re-signing a descriptor does not grant it another project's membership. + let project_b = format!("at://{DID_B}/space.polymodel.library.thing/other"); + assert_eq!( + fetch_status(route_variant( + &|payload| payload["project"] = json!(project_b) + )) + .await, + StatusCode::NOT_FOUND + ); + assert_eq!( + fetch_status(route_variant(&|payload| payload["source"] = json!( + "at://did:plc:aaaaaaaaaaaaaaaaaaaaaaaa/space.polymodel.library.part/other" + ))) + .await, + StatusCode::NOT_FOUND + ); + assert_eq!( + fetch_status(route_variant( + &|payload| payload["path"] = json!("models/other.ldr") + )) + .await, + StatusCode::NOT_FOUND + ); + assert_eq!( + fetch_status(route_variant( + &|payload| payload["len"] = json!(bytes.len() + 1) + )) + .await, + StatusCode::NOT_FOUND + ); + assert_eq!( + fetch_status(route_variant( + &|payload| payload["cids"] = json!(["bafkreiinvalid"]) + )) + .await, + StatusCode::NOT_FOUND + ); + assert_eq!( + fetch_status(route_variant( + &|payload| payload["sha256"] = json!(vec![7_u8; 32]) + )) + .await, + StatusCode::NOT_FOUND + ); + + // A source metadata update invalidates an otherwise valid old binding. + sqlx::query("UPDATE ldraw_resources SET sha256 = ? WHERE resource_uri = ?") + .bind(vec![9_u8; 32]) + .bind(&part) + .execute(&projection) + .await + .unwrap(); + assert_eq!(fetch_status(route).await, StatusCode::NOT_FOUND); + server.abort(); +} + +#[tokio::test] +async fn anonymous_compound_route_traverses_mpd_cycles_and_manifest_only_companions() { + let project = format!("at://{DID_A}/space.polymodel.library.thing/compound"); + let root_uri = format!("at://{DID_A}/space.polymodel.library.part/root"); + let child_uri = format!("at://{DID_A}/space.polymodel.library.part/child"); + let companion_uri = format!("at://{DID_A}/space.polymodel.library.part/companion"); + let root_bytes = b"0 FILE root.ldr\n1 16 0 0 0 1 0 0 0 1 0 0 0 0 child.dat\n0 FILE child.dat\n1 16 0 0 0 1 0 0 0 1 0 0 0 0 root.ldr\n0 NOFILE\n"; + let child_bytes = b"1 16 0 0 0 1 0 0 0 1 0 0 0 0 models/root.ldr\n"; + let companion_bytes = b"0 FILE companion.dat\n0 NOFILE\n"; + let (root_cid, _) = test_blob(root_bytes); + let (child_cid, _) = test_blob(child_bytes); + let (companion_cid, _) = test_blob(companion_bytes); + let mut blobs = HashMap::new(); + blobs.insert(root_cid.clone(), root_bytes.to_vec()); + blobs.insert(child_cid.clone(), child_bytes.to_vec()); + blobs.insert(companion_cid.clone(), companion_bytes.to_vec()); + let (pds, server) = loopback_pds_with_blobs(blobs).await; + let pool = pool().await; + seed_compound_resource( + &pool, + &project, + &root_uri, + "models", + "models/root.ldr", + root_bytes, + &root_cid, + ) + .await; + seed_compound_resource( + &pool, + &project, + &child_uri, + "mpd", + "mpd/child.dat", + child_bytes, + &child_cid, + ) + .await; + seed_compound_resource( + &pool, + &project, + &companion_uri, + "mpd", + "mpd/companion.dat", + companion_bytes, + &companion_cid, + ) + .await; + seed_manifest_edge( + &pool, + ManifestEdge { + source_uri: &root_uri, + root: "mpd", + path: "mpd/child.dat", + target_uri: &child_uri, + target_cid: &child_cid, + bytes: child_bytes, + ordinal: 0, + }, + ) + .await; + seed_manifest_edge( + &pool, + ManifestEdge { + source_uri: &root_uri, + root: "mpd", + path: "mpd/companion.dat", + target_uri: &companion_uri, + target_cid: &companion_cid, + bytes: companion_bytes, + ordinal: 1, + }, + ) + .await; + seed_manifest_edge( + &pool, + ManifestEdge { + source_uri: &child_uri, + root: "models", + path: "models/root.ldr", + target_uri: &root_uri, + target_cid: &root_cid, + bytes: root_bytes, + ordinal: 0, + }, + ) + .await; + let bootstrap = crate::oauth::bootstrap_oauth(pool.clone(), Some("http://localhost")) + .expect("ephemeral OAuth bootstrap for tests"); + let state = AppState::new_with_resolver(pool, bootstrap, loopback_resolver(&pds)); + let app = crate::appview::router().with_state(state); + let request = || { + Request::builder() + .method(Method::POST) + .uri("/app/ldraw/compound-resources") + .header("content-type", "application/json") + .body(Body::from( + json!({"project": project, "path": "models/root.ldr"}).to_string(), + )) + .unwrap() + }; + let first = app.clone().oneshot(request()).await.unwrap(); + assert_eq!(first.status(), StatusCode::OK); + let first: serde_json::Value = serde_json::from_slice( + &axum::body::to_bytes(first.into_body(), 1 << 20) + .await + .unwrap(), + ) + .unwrap(); + let second = app.oneshot(request()).await.unwrap(); + assert_eq!(second.status(), StatusCode::OK); + let second: serde_json::Value = serde_json::from_slice( + &axum::body::to_bytes(second.into_body(), 1 << 20) + .await + .unwrap(), + ) + .unwrap(); + assert_eq!(first, second); + let keys = first["descriptors"] + .as_array() + .unwrap() + .iter() + .map(|descriptor| descriptor["key"].as_str().unwrap()) + .collect::>(); + assert_eq!( + keys, + ["models/root.ldr", "mpd/child.dat", "mpd/companion.dat"] + ); + server.abort(); +} diff --git a/src/appview/test_support.rs b/src/appview/test_support.rs index 30df566..4587c50 100644 --- a/src/appview/test_support.rs +++ b/src/appview/test_support.rs @@ -11,6 +11,7 @@ pub(crate) use axum::body::Body; pub(crate) use axum::http::{Method, Request, StatusCode}; pub(crate) use dioxus::prelude::dioxus_fullstack::FullstackContext; +pub(crate) use http_body_util::BodyExt; use jacquard::identity::PublicResolver; use jacquard::identity::resolver::{DidStep, PlcSource, ResolverOptions}; use jacquard::oauth::authstore::ClientAuthStore; @@ -164,10 +165,12 @@ pub(crate) async fn loopback_pds( .into_response(); } match cid { - "bafkreif2nycdsf3vvit7hhpxegmlmoknx6m7yuvdt4tkzpk7hv4jqo3ulq" => { + "bafkreif2nycdsf3vvit7hhpxegmlmoknx6m7yuvdt4tkzpk7hv4jqo3ulq" + | "bafkreih3ryqpylsmh4siyygdtplff46bgrzjro4xpofu2widxbifkyqgam" => { (StatusCode::OK, Body::from("ab")).into_response() } - "bafkreiape5vm4aofohlvq72uc7nw6golsxa44ser6photbjne6gkhboeii" => { + "bafkreiape5vm4aofohlvq72uc7nw6golsxa44ser6photbjne6gkhboeii" + | "bafkreibb44q4gwsyep63iux2f6pquyjmot5zklqgsj2itrvspjb3qf562q" => { (StatusCode::OK, Body::from("cd")).into_response() } _ => (StatusCode::NOT_FOUND, Body::from("unknown blob")).into_response(), @@ -181,13 +184,185 @@ pub(crate) async fn loopback_pds( (endpoint, task) } +pub(crate) async fn loopback_pds_with_blobs( + blobs: std::collections::HashMap>, +) -> (String, tokio::task::JoinHandle<()>) { + use axum::extract::Query; + use axum::response::IntoResponse; + use axum::routing::any; + let listener = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)) + .await + .unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let did_doc_endpoint = endpoint.clone(); + let app = axum::Router::new() + .route( + "/xrpc/com.atproto.identity.resolveDid", + any(move || { + let service_endpoint = did_doc_endpoint.clone(); + async move { + axum::Json(json!({ + "didDoc": { + "@context": ["https://www.w3.org/ns/did/v1"], + "id": DID_A, + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": service_endpoint + }] + } + })) + } + }), + ) + .route( + "/xrpc/com.atproto.sync.getBlob", + any( + move |Query(params): Query>| { + let blobs = blobs.clone(); + async move { + let cid = params.get("cid").map(String::as_str).unwrap_or_default(); + match blobs.get(cid) { + Some(bytes) => { + (StatusCode::OK, Body::from(bytes.clone())).into_response() + } + None => { + (StatusCode::NOT_FOUND, Body::from("unknown blob")).into_response() + } + } + } + }, + ), + ); + let task = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + (endpoint, task) +} + +pub(crate) fn test_blob(bytes: &[u8]) -> (String, Vec) { + use sha2::Digest; + let digest = sha2::Sha256::digest(bytes); + let mut cid_bytes = vec![0x01, 0x55, 0x12, 0x20]; + cid_bytes.extend_from_slice(&digest); + let alphabet = b"abcdefghijklmnopqrstuvwxyz234567"; + let mut cid = String::from("b"); + let mut buffer = 0_u16; + let mut bits = 0_u8; + for byte in cid_bytes { + buffer = (buffer << 8) | u16::from(byte); + bits += 8; + while bits >= 5 { + bits -= 5; + cid.push(alphabet[((buffer >> bits) & 0x1f) as usize] as char); + buffer &= (1_u16 << bits).wrapping_sub(1); + } + } + if bits != 0 { + cid.push(alphabet[((buffer << (5 - bits)) & 0x1f) as usize] as char); + } + (cid, digest.to_vec()) +} + +pub(crate) async fn seed_compound_resource( + pool: &SqlitePool, + project: &str, + uri: &str, + root: &str, + path: &str, + bytes: &[u8], + cid: &str, +) { + use sha2::Digest; + let digest = sha2::Sha256::digest(bytes).to_vec(); + let source_identity = crate::ldraw::source_identity_for( + &jacquard_common::types::string::AtUri::new_owned(uri).unwrap(), + &[cid.to_owned()], + &digest, + bytes.len() as i64, + ); + sqlx::query( + "INSERT INTO ldraw_resources (resource_uri, owner_did, project_uri, record_cid, rkey, root, canonical_path, mime_type, byte_length, sha256, ordered_blob_cids, provenance, licence, notices_json, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ) + .bind(uri) + .bind(DID_A) + .bind(project) + .bind("record-cid") + .bind(path.rsplit('/').next().unwrap()) + .bind(root) + .bind(path) + .bind("application/x-ldraw") + .bind(bytes.len() as i64) + .bind(digest) + .bind(serde_json::to_string(&vec![cid]).unwrap()) + .bind("test") + .bind("CC-BY-4.0") + .bind("[]") + .bind(1_i64) + .bind(1_i64) + .execute(pool) + .await + .unwrap(); + sqlx::query( + "INSERT INTO ldraw_verification (resource_uri, source_identity, state, diagnostic, updated_at) VALUES (?, ?, 'verified', NULL, ?)", + ) + .bind(uri) + .bind(source_identity) + .bind(1_i64) + .execute(pool) + .await + .unwrap(); + sqlx::query( + "INSERT INTO ldraw_resource_memberships (resource_uri, project_uri, root, canonical_path, provenance) VALUES (?, ?, ?, ?, ?)", + ) + .bind(uri) + .bind(project) + .bind(root) + .bind(path) + .bind("test") + .execute(pool) + .await + .unwrap(); +} + +pub(crate) struct ManifestEdge<'a> { + pub(crate) source_uri: &'a str, + pub(crate) root: &'a str, + pub(crate) path: &'a str, + pub(crate) target_uri: &'a str, + pub(crate) target_cid: &'a str, + pub(crate) bytes: &'a [u8], + pub(crate) ordinal: i64, +} + +pub(crate) async fn seed_manifest_edge(pool: &SqlitePool, edge: ManifestEdge<'_>) { + use sha2::Digest; + sqlx::query( + "INSERT INTO ldraw_manifest_files (resource_uri, root, canonical_path, target_resource_uri, target_cid, byte_length, sha256, mime_type, ordinal) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ) + .bind(edge.source_uri) + .bind(edge.root) + .bind(edge.path) + .bind(edge.target_uri) + .bind(edge.target_cid) + .bind(edge.bytes.len() as i64) + .bind(sha2::Sha256::digest(edge.bytes).to_vec()) + .bind("application/x-ldraw") + .bind(edge.ordinal) + .execute(pool) + .await + .unwrap(); +} + pub(crate) fn loopback_resolver(endpoint: &str) -> PublicResolver { - let mut options = ResolverOptions::default(); - options.plc_source = PlcSource::PlcDirectory { - base: Uri::parse(format!("{endpoint}/")).unwrap().to_owned(), + let options = ResolverOptions { + plc_source: PlcSource::PlcDirectory { + base: Uri::parse(format!("{endpoint}/")).unwrap().to_owned(), + }, + pds_fallback: Some(Uri::parse(endpoint.to_owned()).unwrap().to_owned()), + did_order: vec![DidStep::PdsResolveDid], + ..Default::default() }; - options.pds_fallback = Some(Uri::parse(endpoint.to_owned()).unwrap().to_owned()); - options.did_order = vec![DidStep::PdsResolveDid]; PublicResolver::new(reqwest::Client::new(), options) } @@ -430,10 +605,3 @@ pub(crate) async fn seed_list(pool: &SqlitePool, did: &str, rkey: &str, name: &s .unwrap(); uri } - -pub(crate) async fn body_text(resp: axum::response::Response) -> String { - let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20) - .await - .unwrap(); - String::from_utf8(bytes.to_vec()).unwrap() -} diff --git a/src/thing_detail.rs b/src/thing_detail.rs index bc978a7..39c0062 100644 --- a/src/thing_detail.rs +++ b/src/thing_detail.rs @@ -19,6 +19,8 @@ use crate::client::PolymodelClient; use crate::session::SessionIdentity; use crate::viewer::{AssetViewer, ViewerFallbackImage, ViewerMeshSource, ViewerSourceFormat}; +const COMPOUND_PRIMARY_PATH: &str = "models/main.ldr"; + #[derive(Clone, Debug, PartialEq, Eq)] enum DetailState { Loading, @@ -82,16 +84,26 @@ fn part_download_href(uri: &AtUri) -> Option { )) } -fn viewer_source_for_part(part: &PartDetail) -> Option { +fn viewer_source_for_part(part: &PartDetail, compound_project: &AtUri) -> Option { let url = part_download_href(&part.view.uri)?; let name = clean_text(Some(part.view.name.as_ref())) .unwrap_or("Untitled part") .to_string(); let format = part_viewer_format(part, &url, &name)?; + let compound_path = if matches!( + format, + ViewerSourceFormat::Renderer(MeshFormat::CompoundLdraw) + ) { + COMPOUND_PRIMARY_PATH.to_string() + } else { + String::new() + }; Some(ViewerMeshSource { key: part.view.uri.as_str().to_string(), name, url, + compound_project: compound_project.as_str().to_string(), + compound_path, format, expected_size: Some(part.view.file.size), expected_digest: part.view.file.digest.as_deref().map(|b| b.to_vec()), @@ -121,6 +133,14 @@ fn viewer_format_candidate(candidate: &str) -> Option { { return Some(ViewerSourceFormat::Step); } + if matches!( + value.as_str(), + "ldraw" | "ldr" | "mpd" | "application/x-ldraw" | "model/ldraw" + ) || extension_is(&value, "ldr") + || extension_is(&value, "mpd") + { + return Some(ViewerSourceFormat::Renderer(MeshFormat::CompoundLdraw)); + } if matches!(value.as_str(), "stl" | "model/stl" | "application/sla") { return Some(ViewerSourceFormat::Renderer(MeshFormat::Stl)); } @@ -480,7 +500,7 @@ fn DetailPopulated( let viewer_source = selected_model .as_ref() .and_then(|selected| selected.parts.first()) - .and_then(viewer_source_for_part); + .and_then(|part| viewer_source_for_part(part, &thing.uri)); let instructions = thing_record.instructions; rsx! { diff --git a/tools/check-devshell-tools-only.py b/tools/check-devshell-tools-only.py new file mode 100644 index 0000000..cbb3bf6 --- /dev/null +++ b/tools/check-devshell-tools-only.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 + +from pathlib import Path + +flake = Path("flake.nix").read_text() +start = flake.index("devShells.default = pkgs.mkShell {") +depth = 0 +end = None +for index, character in enumerate(flake[start:], start): + if character == "{": + depth += 1 + elif character == "}": + depth -= 1 + if depth == 0: + end = index + 1 + break +if end is None: + raise SystemExit("FAIL: devShells.default block is not balanced") +block = flake[start:end] + +for marker in ("inputsFrom", "self.packages", "ldrawOracle", "packages.default"): + if marker in block: + raise SystemExit(f"FAIL: devShells.default depends on repository output via {marker}") + +print("PASS: devShells.default contains tools and external libraries only") diff --git a/tools/in-dev-shell b/tools/in-dev-shell new file mode 100755 index 0000000..52c366f --- /dev/null +++ b/tools/in-dev-shell @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail + +if (($# == 0)); then + echo "usage: tools/in-dev-shell _private-recipe [args ...]" >&2 + exit 2 +fi + +private_recipe=$1 +shift + +if ! just_bin=$(command -v just); then + echo "error: just is required to dispatch Polymodel recipes" >&2 + exit 127 +fi + +if [[ "${POLYMODEL_NIX_SHELL:-}" == 1 ]]; then + exec "$just_bin" "$private_recipe" "$@" +fi + +if ! direnv_bin=$(command -v direnv); then + echo "error: direnv is required to enter the Polymodel dev shell" >&2 + exit 127 +fi + +printf '==> entering Polymodel dev shell for: %s\n' "${private_recipe#_}" >&2 +exec "$direnv_bin" exec . just "$private_recipe" "$@" diff --git a/tools/ldraw-compat-harness/src/main.rs b/tools/ldraw-compat-harness/src/main.rs index 3a3b187..ea35e35 100644 --- a/tools/ldraw-compat-harness/src/main.rs +++ b/tools/ldraw-compat-harness/src/main.rs @@ -1,19 +1,20 @@ //! Native PM-77 corpus validator and D-phase LDParse oracle harness. use polymodel_ldraw_core::{ - DiagnosticCode, InProcessRustAdapter, LdrawError, ParserProfile as CoreProfile, + AdapterRequest, CancellationPolicy, InProcessRustAdapter, ParserProfile as CoreProfile, RootId, }; use polymodel_ldraw_testkit::{ - coverage_report, inventory, named_gates, serialize_canonical, validate_inventory, - CanonicalRecord, ExpectedOutcome, FixtureCase, CorpusGateSet, OracleDiagnostic, OracleError, - OracleModel, OracleProjection, Profile, SCHEMA_VERSION, ORACLE_SCHEMA_VERSION, GATE_NAMES, - LDRAW_LIMITS, + CorpusGateSet, GATE_NAMES, LDRAW_LIMITS, ORACLE_PATCH_IDENTITY, ORACLE_PROJECTION_RULE_VERSION, + ORACLE_SCHEMA_VERSION, ORACLE_WRAPPER_SCHEMA_VERSION, OracleColor, OracleDiagnostic, OracleError, + OracleProjection, + CANDIDATE_OBSERVATION_SCHEMA_VERSION, Profile, SCHEMA_VERSION, coverage_report, inventory, + named_gates, validate_inventory, }; use polymodel_renderer_ledger::{ReservationLedger, ReservationOwner}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::{ - env, - fs, + env, fs, + io::Read, path::{Path, PathBuf}, process::{Command, ExitCode, Output, Stdio}, thread, @@ -25,7 +26,7 @@ const LD_PARSE_REVISION: &str = "0fe78dcee22982f26cceced1b483340faa4d04e9"; const FUZZ_SEED: u64 = 0x504d3737_4435; const FUZZ_ITERATIONS: usize = 128; const MAX_MUTATED_BYTES: usize = 1024 * 1024; -const ORACLE_TIMEOUT: Duration = Duration::from_secs(2); +const ORACLE_TIMEOUT: Duration = Duration::from_secs(10); const FUZZ_BUDGET: Duration = Duration::from_secs(30); #[derive(Debug, Serialize)] @@ -43,13 +44,115 @@ struct Report<'a> { mutations: usize, } +#[derive(Debug, Serialize)] +struct Snapshot<'a> { + id: &'a str, + oracle_expected: Option, + oracle_error: Option, +} + +#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)] +#[serde(deny_unknown_fields)] +struct OraclePosition(Option, Option, Option); + +#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)] +#[serde(deny_unknown_fields)] +struct OracleMesh { + vertex_count: usize, + positions: Vec, + normal_count: usize, + normals: Vec, + indices: Vec, + bf_indices: Vec, + triangle_count: usize, + bf_triangle_count: usize, +} + +#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)] +#[serde(deny_unknown_fields)] +struct OracleTransform { + translation: OraclePosition, + matrix: Vec>, +} + +#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)] +#[serde(deny_unknown_fields)] +struct OracleChild { + index: usize, + bfc: String, + transform: OracleTransform, + child_name: Option, + child_path: Option, +} + +#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)] +#[serde(deny_unknown_fields)] +struct OracleModelDocument { + name: String, + model_index: u32, + path: String, + certify: String, + winding: String, + default_color: u64, + step_ends: Vec, + local_colors: Vec, + mesh: OracleMesh, + children: Vec, +} + +#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)] +#[serde(deny_unknown_fields)] +struct OracleToken { + kind: String, + kind_code: i32, + value: serde_json::Value, + line: i32, + column: i32, +} + +#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)] +#[serde(deny_unknown_fields)] +struct OracleTokenLine { + index: usize, + text: String, + tokens: Vec, +} + +#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)] +#[serde(deny_unknown_fields)] +struct OracleTokenizedModel { + model_index: usize, + name: String, + lines: Vec, +} + +#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)] +#[serde(deny_unknown_fields)] +struct OracleGeometry { + triangles: usize, + quads: usize, + lines: usize, + conditional_lines: usize, + includes: usize, + triangle_positions: Vec>, +} + #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] struct OracleDocument { schema_version: String, + oracle_revision: String, + wrapper_schema_version: String, + projection_rule_version: String, + patch_identity: String, fixture_path: String, status: String, - models: Vec, + model_names: Vec, + root_model: Option, + models: Vec, + tokenized_models: Vec, + tokenized_geometry: Option, + resolved_geometry: Option, diagnostics: Vec, error: Option, } @@ -59,11 +162,12 @@ fn main() -> ExitCode { let result = match command.as_str() { "validate" => validate(), "inventory" => report("inventory"), - "corpus" => report("corpus"), + "corpus" => corpus(), + "snapshot" => snapshot(), "fuzz" => fuzz(), "differential" => differential(), other => Err(format!( - "unknown command {other}; use validate, inventory, corpus, fuzz, or differential" + "unknown command {other}; use validate, inventory, corpus, snapshot, fuzz, or differential" )), }; match result { @@ -93,16 +197,6 @@ fn report(command: &'static str) -> Result { gates .validate_inventory(&fixtures) .map_err(|error| error.to_string())?; - for fixture in &fixtures { - if fixture.expected.outcome == ExpectedOutcome::Rejected { - validate_rejected(fixture)?; - continue; - } - let actual = actual_record(fixture)?; - for outcome in gates.evaluate(fixture, &actual) { - outcome.map_err(|error| error.to_string())?; - } - } let coverage = coverage_report(&fixtures); serde_json::to_string_pretty(&Report { schema_version: SCHEMA_VERSION, @@ -120,6 +214,117 @@ fn report(command: &'static str) -> Result { .map_err(|error| error.to_string()) } +fn corpus() -> Result { + let fixtures = inventory(); + validate_inventory(&fixtures)?; + let gates = CorpusGateSet::new(named_gates()).map_err(|error| error.to_string())?; + gates + .validate_inventory(&fixtures) + .map_err(|error| error.to_string())?; + let mut evaluated = 0usize; + for (index, fixture) in fixtures.iter().enumerate() { + let ledger = ReservationLedger::new(); + let parsed = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + if fixture.outcome == polymodel_ldraw_testkit::ExpectedOutcome::Cancelled { + let cancellation = CancellationPolicy::new(); + cancellation.cancel_after_checkpoints(1); + InProcessRustAdapter::parse(polymodel_ldraw_core::AdapterRequest { + fixture_id: fixture.id, + bytes: fixture.bytes, + profile: core_profile(fixture.profile), + provenance_id: fixture.id, + owner: ReservationOwner::preview(78, index as u64 + 1, 1), + ledger: &ledger, + semantic_budget: Some(16 * 1024 * 1024), + root_name: fixture.id, + root: RootId::UploadedManifest, + materializations: &[], + target_selections: &[], + cancellation, + }) + } else { + parse_fixture_with_root( + fixture, + ReservationOwner::preview(78, index as u64 + 1, 1), + &ledger, + ) + } + })) + .map_err(|_| format!("{} core parser panicked", fixture.id))?; + match (fixture.outcome, parsed) { + (polymodel_ldraw_testkit::ExpectedOutcome::Rejected, Err(error)) + | (polymodel_ldraw_testkit::ExpectedOutcome::Cancelled, Err(error)) => { + let results = gates.evaluate_rejection(fixture, &error); + if results.is_empty() { + return Err(format!("{} has no applicable rejection gates", fixture.id)); + } + for result in results { + result.map_err(|error| error.to_string())?; + evaluated += 1; + } + } + (polymodel_ldraw_testkit::ExpectedOutcome::Rejected, Ok(_)) + | (polymodel_ldraw_testkit::ExpectedOutcome::Cancelled, Ok(_)) => { + return Err(format!("{} unexpectedly accepted", fixture.id)); + } + (_, Ok(parsed)) => { + let actual = parsed.project(); + let results = gates.evaluate_with_files(fixture, &actual, &parsed.files); + if results.is_empty() { + return Err(format!("{} has no applicable acceptance gates", fixture.id)); + } + for result in results { + result.map_err(|error| error.to_string())?; + evaluated += 1; + } + drop(parsed); + } + (_, Err(error)) => return Err(format!("{} unexpectedly rejected: {error}", fixture.id)), + } + if ledger.snapshot().reservations != 0 { + return Err(format!("{} leaked parser reservations", fixture.id)); + } + } + Ok(serde_json::json!({ + "schema_version": SCHEMA_VERSION, + "command": "corpus", + "status": "pass", + "fixtures": fixtures.len(), + "gates": evaluated, + "profiles": 3, + "oracle_revision": LD_PARSE_REVISION, + "oracle_schema": ORACLE_SCHEMA_VERSION + }).to_string()) +} + +fn snapshot() -> Result { + let fixtures = inventory(); + let root = root_dir(); + let binary = oracle_binary(&root)?; + let mut snapshots = Vec::with_capacity(fixtures.len()); + for fixture in &fixtures { + let path = root + .join("crates/polymodel-ldraw-testkit") + .join(fixture.relative_path); + let (oracle_expected, oracle_error) = + match run_oracle( + &binary, + &path, + &root.join("crates/polymodel-ldraw-testkit/corpus"), + Duration::from_secs(10), + ) { + Ok(document) => (Some(projection(document, &path, fixture.bytes)?), None), + Err(error) => (None, Some(error)), + }; + snapshots.push(Snapshot { + id: fixture.id, + oracle_expected, + oracle_error, + }); + } + serde_json::to_string_pretty(&snapshots).map_err(|error| error.to_string()) +} + fn validate() -> Result { let fixtures = inventory(); validate_inventory(&fixtures)?; @@ -127,19 +332,26 @@ fn validate() -> Result { .map_err(|error| error.to_string())? .validate_inventory(&fixtures) .map_err(|error| error.to_string())?; - if LDRAW_LIMITS.iter().any(|(name, limit)| name.is_empty() || *limit == 0) { + if LDRAW_LIMITS + .iter() + .any(|(name, limit)| name.is_empty() || *limit == 0) + { return Err("invalid LDraw semantic limit".to_owned()); } for fixture in &fixtures { let encoded = serde_json::to_vec(&fixture.expected).map_err(|error| error.to_string())?; let second = serde_json::to_vec(&fixture.expected).map_err(|error| error.to_string())?; if encoded != second { - return Err(format!("non-deterministic expected serialization: {}", fixture.id)); + return Err(format!( + "non-deterministic expected serialization: {}", + fixture.id + )); } - if let Some(projection) = &fixture.expected.oracle_projection { - if projection.status.is_empty() { - return Err(format!("empty authored oracle projection status: {}", fixture.id)); - } + if fixture.expected.oracle_expected.status.is_empty() { + return Err(format!( + "empty generated oracle projection status: {}", + fixture.id + )); } } Ok(serde_json::json!({ @@ -173,14 +385,20 @@ fn run_checked(program: &str, args: &[&str], cwd: &Path) -> Result Result { - if !Command::new("nix").arg("--version").status().map_err(|error| format!("could not start nix: {error}"))?.success() { + let nix_status = Command::new("nix") + .arg("--version") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .map_err(|error| format!("could not start nix: {error}"))?; + if !nix_status.success() { return Err("nix is required for the LDParse differential oracle".to_owned()); } let output = run_checked( "nix", &[ "build", - "path:/home/orual/Projects/epic-86#ldraw-oracle", + &format!("path:{}#ldraw-oracle", root.display()), "--no-link", "--print-out-paths", ], @@ -195,77 +413,344 @@ fn oracle_binary(root: &Path) -> Result { .to_owned(); let binary = Path::new(&store_path).join("bin/ldraw-oracle"); if !binary.is_file() { - return Err(format!("oracle binary is missing from successful Nix output: {}", binary.display())); + return Err(format!( + "oracle binary is missing from successful Nix output: {}", + binary.display() + )); } Ok(binary) } -fn run_oracle(binary: &Path, fixture: &Path, timeout: Duration) -> Result { +fn run_oracle(binary: &Path, fixture: &Path, corpus: &Path, timeout: Duration) -> Result { + let fixture_name = fixture + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| format!("fixture path is not a UTF-8 filename: {}", fixture.display()))?; let mut child = Command::new(binary) - .arg(fixture) + .arg(fixture_name) + .current_dir(corpus) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() .map_err(|error| format!("could not start oracle: {error}"))?; + let mut stdout_pipe = child + .stdout + .take() + .ok_or_else(|| "oracle stdout pipe unavailable".to_owned())?; + let mut stderr_pipe = child + .stderr + .take() + .ok_or_else(|| "oracle stderr pipe unavailable".to_owned())?; + let stdout_reader = thread::spawn(move || { + let mut bytes = Vec::new(); + let _ = stdout_pipe.read_to_end(&mut bytes); + bytes + }); + let stderr_reader = thread::spawn(move || { + let mut bytes = Vec::new(); + let _ = stderr_pipe.read_to_end(&mut bytes); + bytes + }); let deadline = Instant::now() + timeout; let status = loop { - match child.try_wait().map_err(|error| format!("oracle wait failed: {error}"))? { + match child + .try_wait() + .map_err(|error| format!("oracle wait failed: {error}"))? + { Some(status) => break status, None if Instant::now() >= deadline => { let _ = child.kill(); let _ = child.wait(); - return Err(format!("oracle exceeded {}ms for {}", timeout.as_millis(), fixture.display())); + return Err(format!( + "oracle exceeded {}ms for {}", + timeout.as_millis(), + fixture.display() + )); } None => thread::sleep(Duration::from_millis(5)), } }; - let output = child - .wait_with_output() - .map_err(|error| format!("could not collect oracle output: {error}"))?; - if !status.success() && !output.stdout.is_empty() { - return Err(format!("oracle exited {status}: {}", String::from_utf8_lossy(&output.stderr))); + let stdout = stdout_reader + .join() + .map_err(|_| "oracle stdout reader panicked".to_owned())?; + let stderr = stderr_reader + .join() + .map_err(|_| "oracle stderr reader panicked".to_owned())?; + if !status.success() { + return Err(format!( + "oracle exited {status}: {}", + String::from_utf8_lossy(&stderr) + )); } - let document: OracleDocument = serde_json::from_slice(&output.stdout) - .map_err(|error| format!("invalid oracle JSON for {}: {error}; stderr: {}", fixture.display(), String::from_utf8_lossy(&output.stderr)))?; + let stdout = String::from_utf8(stdout).map_err(|error| { + format!( + "oracle emitted non-UTF-8 stdout for {}: {error}", + fixture.display() + ) + })?; + let lines = stdout + .lines() + .filter(|line| !line.trim().is_empty()) + .collect::>(); + if lines.len() != 1 { + return Err(format!( + "oracle must emit exactly one JSON line for {}; got {}; stderr: {}", + fixture.display(), + lines.len(), + String::from_utf8_lossy(&stderr) + )); + } + let mut raw = serde_json::from_str::(lines[0]).map_err(|error| { + format!( + "invalid oracle JSON for {}: {error}; stderr: {}", + fixture.display(), + String::from_utf8_lossy(&stderr) + ) + })?; + let object = raw + .as_object_mut() + .ok_or_else(|| format!("oracle JSON for {} is not an object", fixture.display()))?; + match object.get("status").and_then(serde_json::Value::as_str) { + Some("ok") => { + object.entry("error").or_insert(serde_json::Value::Null); + if let Some(resolved) = object + .get_mut("resolved_geometry") + .and_then(serde_json::Value::as_object_mut) + { + resolved + .entry("triangle_positions") + .or_insert_with(|| serde_json::Value::Array(Vec::new())); + } + } + Some("error") => { + for field in ["model_names", "models", "tokenized_models"] { + object + .entry(field.to_owned()) + .or_insert_with(|| serde_json::Value::Array(Vec::new())); + } + if let Some(error) = object + .get_mut("error") + .and_then(serde_json::Value::as_object_mut) + { + error + .entry("diagnostics") + .or_insert_with(|| serde_json::Value::Array(Vec::new())); + } + } + _ => {} + } + let document = serde_json::from_value::(raw).map_err(|error| { + format!( + "invalid oracle JSON for {}: {error}; stderr: {}", + fixture.display(), + String::from_utf8_lossy(&stderr) + ) + })?; validate_oracle_document(&document, fixture)?; Ok(document) } fn validate_oracle_document(document: &OracleDocument, fixture: &Path) -> Result<(), String> { if document.schema_version != ORACLE_SCHEMA_VERSION { - return Err(format!("{} has schema {}", fixture.display(), document.schema_version)); + return Err(format!( + "{} has schema {}", + fixture.display(), + document.schema_version + )); + } + if document.oracle_revision != LD_PARSE_REVISION + || document.wrapper_schema_version != ORACLE_WRAPPER_SCHEMA_VERSION + || document.projection_rule_version != ORACLE_PROJECTION_RULE_VERSION + || document.patch_identity != ORACLE_PATCH_IDENTITY + { + return Err(format!( + "{} runtime oracle attestation does not match the checked-in contract", + fixture.display() + )); } if document.fixture_path.is_empty() { return Err(format!("{} has empty fixture_path", fixture.display())); } match document.status.as_str() { - "ok" if document.error.is_none() => {} - "error" if document.error.as_ref().is_some_and(|error| error.kind == "construct_null") => {} - status => return Err(format!("{} has invalid status/error shape: {status}", fixture.display())), - } - let mut previous = None; - for (index, model) in document.models.iter().enumerate() { - if previous == Some(model.name.as_str()) { - return Err(format!("{} contains duplicate model {}", fixture.display(), model.name)); + "ok" if document.error.is_none() && document.root_model.is_some() => {} + "error" if document.error.is_some() && document.root_model.is_none() => {} + status => { + return Err(format!( + "{} has invalid status/error shape: {status}", + fixture.display() + )); } - if model.model_index != index as u32 { - return Err(format!("{} has non-sequential model indices", fixture.display())); + } + if !polymodel_ldraw_testkit::unique_strings(document.model_names.clone()) { + return Err(format!( + "{} contains duplicate model names", + fixture.display() + )); + } + if let Some(root) = &document.root_model { + if document.model_names.first() != Some(&root.name) { + return Err(format!( + "{} root model is not first in model_names", + fixture.display() + )); } - previous = Some(model.name.as_str()); - if !matches!(model.certify.as_str(), "unknown" | "certified" | "uncertified") { - return Err(format!("{} has invalid certification {}", fixture.display(), model.certify)); + if !matches!( + root.certify.as_str(), + "unknown" | "certified" | "uncertified" + ) { + return Err(format!( + "{} has invalid certification {}", + fixture.display(), + root.certify + )); } } Ok(()) } -fn projection(document: OracleDocument) -> OracleProjection { - OracleProjection { +fn project_model(model: OracleModelDocument) -> polymodel_ldraw_testkit::OracleModel { + polymodel_ldraw_testkit::OracleModel { + name: model.name, + model_index: model.model_index, + path: model.path, + certify: model.certify, + winding: model.winding, + default_color: model.default_color, + step_ends: model.step_ends.clone(), + local_colors: model.local_colors, + mesh: polymodel_ldraw_testkit::OracleMesh { + vertex_count: model.mesh.vertex_count, + positions: model + .mesh + .positions + .into_iter() + .map(|p| polymodel_ldraw_testkit::OraclePosition(p.0, p.1, p.2)) + .collect(), + normal_count: model.mesh.normal_count, + normals: model + .mesh + .normals + .into_iter() + .map(|p| polymodel_ldraw_testkit::OraclePosition(p.0, p.1, p.2)) + .collect(), + indices: model.mesh.indices, + bf_indices: model.mesh.bf_indices, + triangle_count: model.mesh.triangle_count, + bf_triangle_count: model.mesh.bf_triangle_count, + }, + children: model + .children + .into_iter() + .map(|child| polymodel_ldraw_testkit::OracleChild { + index: child.index, + bfc: child.bfc, + transform: polymodel_ldraw_testkit::OracleTransform { + translation: polymodel_ldraw_testkit::OraclePosition( + child.transform.translation.0, + child.transform.translation.1, + child.transform.translation.2, + ), + matrix: child.transform.matrix, + }, + child_name: child.child_name, + child_path: child.child_path, + }) + .collect(), + } +} + +fn projection( + document: OracleDocument, + fixture: &Path, + fixture_bytes: &[u8], +) -> Result { + let models = document + .models + .into_iter() + .map(project_model) + .collect::>(); + let root_model = document.root_model.map(project_model); + if document.status == "ok" && models.is_empty() { + return Err(format!( + "{} successful oracle document has no models", + fixture.display() + )); + } + let tokenized_models = document + .tokenized_models + .into_iter() + .map(|model| polymodel_ldraw_testkit::OracleTokenizedModel { + model_index: model.model_index, + name: model.name, + lines: model + .lines + .into_iter() + .map(|line| polymodel_ldraw_testkit::OracleTokenLine { + index: line.index, + text: line.text, + tokens: line + .tokens + .into_iter() + .map(|token| polymodel_ldraw_testkit::OracleToken { + kind: token.kind, + kind_code: token.kind_code, + value: token.value, + line: token.line, + column: token.column, + }) + .collect(), + }) + .collect(), + }) + .collect(); + let geometry = |geometry: Option| { + geometry.map(|value| polymodel_ldraw_testkit::OracleGeometry { + triangles: value.triangles, + quads: value.quads, + lines: value.lines, + conditional_lines: value.conditional_lines, + includes: value.includes, + triangle_positions: value + .triangle_positions + .into_iter() + .map(|points| { + points + .into_iter() + .map(|p| polymodel_ldraw_testkit::OraclePosition(p.0, p.1, p.2)) + .collect() + }) + .collect(), + }) + }; + let fixture_path = Path::new(&document.fixture_path) + .strip_prefix(root_dir().join("crates/polymodel-ldraw-testkit")) + .map(|path| format!("crates/polymodel-ldraw-testkit/{}", path.to_string_lossy().replace('\\', "/"))) + .unwrap_or_else(|_| { + format!( + "crates/polymodel-ldraw-testkit/corpus/{}", + document.fixture_path.replace('\\', "/") + ) + }); + Ok(OracleProjection { + schema_version: document.schema_version, + oracle_revision: document.oracle_revision, + wrapper_schema_version: document.wrapper_schema_version, + projection_rule_version: document.projection_rule_version, + patch_identity: document.patch_identity, + fixture_path, + fixture_sha256: polymodel_ldraw_testkit::hex_digest(fixture_bytes), status: document.status, - models: document.models, + model_names: document.model_names, + root_model, + models, + tokenized_models, + tokenized_geometry: geometry(document.tokenized_geometry), + resolved_geometry: geometry(document.resolved_geometry), diagnostics: document.diagnostics, error: document.error, - } + authority_rules: serde_json::from_str(include_str!("../../ldraw-oracle-generator/rules.json")) + .map_err(|error| format!("invalid checked-in authority rules: {error}"))?, + }) } fn core_profile(profile: Profile) -> CoreProfile { @@ -276,51 +761,284 @@ fn core_profile(profile: Profile) -> CoreProfile { } } -fn actual_record(fixture: &FixtureCase) -> Result, String> { - let ledger = ReservationLedger::new(); - let parsed = InProcessRustAdapter::parse_fixture( - fixture.id, - fixture.bytes, - core_profile(fixture.expected.profile), - ReservationOwner::preview(1, 1, 1), - &ledger, - ) - .map_err(|error| format!("core parse failed for {}: {error}", fixture.id))?; - let record = parsed.project(); - if ledger.snapshot().reservations == 0 { - return Err(format!("core reservation was released before projection: {}", fixture.id)); - } - let _ = record; - Ok(parsed.into_owned_project()) +fn fixture_root_name(fixture: &polymodel_ldraw_testkit::FixtureCase) -> &str { + Path::new(fixture.relative_path) + .file_name() + .and_then(|name| name.to_str()) + .expect("fixture relative path must have a UTF-8 filename") +} + +fn parse_fixture_with_root<'a>( + fixture: &'a polymodel_ldraw_testkit::FixtureCase, + owner: ReservationOwner, + ledger: &'a ReservationLedger, +) -> Result { + InProcessRustAdapter::parse(AdapterRequest { + fixture_id: fixture.id, + bytes: fixture.bytes, + profile: core_profile(fixture.profile), + provenance_id: fixture.id, + owner, + ledger, + semantic_budget: Some(16 * 1024 * 1024), + root_name: fixture_root_name(fixture), + root: RootId::UploadedManifest, + materializations: &[], + target_selections: &[], + cancellation: CancellationPolicy::default(), + }) } -fn validate_rejected(fixture: &FixtureCase) -> Result<(), String> { +fn candidate_observation( + parsed: &polymodel_ldraw_core::OwnedParseResult, +) -> Result { + let result = &parsed.result; + Ok(serde_json::json!({ + "schema_version": CANDIDATE_OBSERVATION_SCHEMA_VERSION, + "status": "ok", + "syntax": result.syntax, + "semantic": { + "canonical_path": result.semantic.canonical_path, + "root_identity": result.semantic.root_identity, + "target_identity": result.semantic.target_identity, + "cache_identity": result.semantic.cache_identity, + "bfc_state": result.semantic.bfc_state, + "colour_state": result.semantic.colour_state, + "steps": result.semantic.steps, + "limits": result.semantic.limits, + "texmap_events": result.semantic.texmap_events, + "texmap_geometry": result.semantic.texmap_geometry, + "data_payloads": result.semantic.data_payloads, + }, + "scene": result.scene, + "diagnostics": result.diagnostics, + "models": result.models, + "files": result.files, + "counters": result.counters, + "non_comparable": [ + "oracle_expected.tokenized_models", + "oracle_expected.tokenized_geometry", + "oracle_expected.resolved_geometry.triangle_positions", + "oracle_expected.models[*].mesh.positions", + "oracle_expected.models[*].mesh.normals", + "oracle_expected.models[*].mesh.indices", + "oracle_expected.models[*].mesh.bf_indices", + ], + })) +} + +fn compare_candidate_to_oracle( + fixture: &polymodel_ldraw_testkit::FixtureCase, + oracle: &OracleProjection, +) -> Result<(), String> { let ledger = ReservationLedger::new(); - let error = InProcessRustAdapter::parse_fixture( - fixture.id, - fixture.bytes, - core_profile(fixture.expected.profile), - ReservationOwner::preview(1, 1, 1), + let parsed = parse_fixture_with_root( + fixture, + ReservationOwner::preview(78, 0, 1), &ledger, ) - .err() - .ok_or_else(|| format!("{} expected a rejected core parse", fixture.id))?; - let actual = match error { - LdrawError::Diagnostic(diagnostic) => diagnostic, - other => return Err(format!("{} rejected with unexpected error: {other}", fixture.id)), - }; - let expected = fixture - .expected - .canonical - .diagnostics - .first() - .ok_or_else(|| format!("{} has no authored rejection diagnostic", fixture.id))?; - if actual != *expected { + .map_err(|error| format!("{} candidate rejected while oracle status is {}: {error}", fixture.id, oracle.status))?; + let candidate = parsed.project(); + if oracle.status != "ok" { + if oracle.status != "error" + || oracle.error.is_none() + || oracle.root_model.is_some() + || oracle.resolved_geometry.is_some() + { + return Err(format!( + "{} oracle unavailable-surface shape is invalid: status={} error={} root_model={} resolved_geometry={}", + fixture.id, + oracle.status, + oracle.error.is_some(), + oracle.root_model.is_some(), + oracle.resolved_geometry.is_some() + )); + } + drop(parsed); + if ledger.snapshot().reservations != 0 { + return Err(format!("{} differential candidate leaked reservation", fixture.id)); + } + return Ok(()); + } + let resolved = oracle + .resolved_geometry + .as_ref() + .ok_or_else(|| format!("{} oracle success has no resolved geometry", fixture.id))?; + let oracle_triangles = resolved.triangles as u64; + if candidate.scene.triangles != oracle_triangles { + return Err(format!("{} triangle mismatch: oracle={} candidate={}", fixture.id, oracle_triangles, candidate.scene.triangles)); + } + if parsed.result.models.len() != oracle.models.len() { return Err(format!( - "{} rejection diagnostic mismatch: actual {actual:?}, expected {expected:?}", - fixture.id + "{} model count mismatch: oracle={} candidate={}", + fixture.id, + oracle.models.len(), + parsed.result.models.len() )); } + let mut oracle_step_count = 0usize; + let mut consumed_candidate_indices = std::collections::BTreeSet::new(); + for (model_index, oracle_model) in oracle.models.iter().enumerate() { + if oracle_model.model_index as usize != model_index { + return Err(format!("{} oracle model order/index mismatch at {model_index}", fixture.id)); + } + let matching_candidate_indices = parsed + .result + .models + .iter() + .enumerate() + .filter_map(|(candidate_index, candidate)| { + if consumed_candidate_indices.contains(&candidate_index) { + return None; + } + let candidate_name = Path::new(&candidate.path) + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or_default(); + let candidate_stem = Path::new(candidate_name) + .file_stem() + .and_then(|name| name.to_str()) + .unwrap_or_default(); + (candidate_name.eq_ignore_ascii_case(&oracle_model.name) + || (model_index == 0 + && oracle_model.name == format!("pm77-{candidate_stem}"))) + .then_some(candidate_index) + }) + .collect::>(); + let candidate_index = match matching_candidate_indices.as_slice() { + [] => { + return Err(format!( + "{} missing candidate model[{}] {}", + fixture.id, model_index, oracle_model.name + )); + } + [candidate_index] => *candidate_index, + indices => { + return Err(format!( + "{} ambiguous candidate model[{}] {} matched indices {:?}", + fixture.id, model_index, oracle_model.name, indices + )); + } + }; + consumed_candidate_indices.insert(candidate_index); + let candidate_model = &parsed.result.models[candidate_index]; + let candidate_name = Path::new(&candidate_model.path) + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or_default(); + let candidate_stem = Path::new(candidate_name) + .file_stem() + .and_then(|name| name.to_str()) + .unwrap_or_default(); + let root_alias = model_index == 0 + && oracle_model.name == format!("pm77-{candidate_stem}"); + let oracle_path_name = Path::new(&oracle_model.path) + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or_default(); + let mpd_path_alias = model_index == 0 + && oracle_path_name.eq_ignore_ascii_case(candidate_name); + if model_index == 0 && !mpd_path_alias && candidate_model.path != oracle_model.path { + return Err(format!( + "{} model[{}] path mismatch: oracle={} candidate={}", + fixture.id, model_index, oracle_model.path, candidate_model.path + )); + } + if candidate_name != oracle_model.name + && !candidate_name.eq_ignore_ascii_case(&oracle_model.name) + && !root_alias + { + return Err(format!( + "{} model[{}] name mismatch: oracle={} candidate={}", + fixture.id, model_index, oracle_model.name, candidate_name + )); + } + if candidate_model.step_ends != oracle_model.step_ends { + return Err(format!( + "{} step_ends mismatch for model[{}] {}: oracle={:?} candidate={:?}", + fixture.id, model_index, oracle_model.name, oracle_model.step_ends, candidate_model.step_ends + )); + } + let expected_certification = polymodel_ldraw_testkit::normalize_certification(&oracle_model.certify) + .ok_or_else(|| format!("{} invalid oracle certification at model[{}]", fixture.id, model_index))?; + let candidate_certification = match candidate_model.bfc { + polymodel_ldraw_core::BfcState::Unknown => "unknown", + polymodel_ldraw_core::BfcState::Certified => "certified", + polymodel_ldraw_core::BfcState::Uncertified => "uncertified", + }; + if candidate_certification != expected_certification { + return Err(format!( + "{} certification mismatch for model[{}]: oracle={} candidate={}", + fixture.id, model_index, expected_certification, candidate_certification + )); + } + let tokenized_model = oracle + .tokenized_models + .get(model_index) + .ok_or_else(|| format!("{} missing tokenized model[{}]", fixture.id, model_index))?; + let source_counts = tokenized_model.lines.iter().filter_map(|line| { + line.tokens.first().and_then(|token| match token.kind_code { + 1 => Some((1usize, 0, 0, 0, 0)), + 2 => Some((0, 1, 0, 0, 0)), + 3 => Some((0, 0, 1, 0, 0)), + 4 => Some((0, 0, 0, 1, 0)), + 5 => Some((0, 0, 0, 0, 1)), + _ => None, + }) + }).fold((0usize, 0, 0, 0, 0), |counts, line| { + ( + counts.0 + line.0, + counts.1 + line.1, + counts.2 + line.2, + counts.3 + line.3, + counts.4 + line.4, + ) + }); + if candidate_model.triangles as usize != source_counts.2 + || candidate_model.quads as usize != source_counts.3 + || candidate_model.lines as usize != source_counts.1 + || candidate_model.conditional_lines as usize != source_counts.4 + || candidate_model.includes.len() != source_counts.0 + { + return Err(format!( + "{} source geometry mismatch for model[{}]: oracle={:?} candidate=({},{},{},{},{})", + fixture.id, + model_index, + source_counts, + candidate_model.includes.len(), + candidate_model.lines, + candidate_model.triangles, + candidate_model.quads, + candidate_model.conditional_lines, + )); + } + let expected_children = source_counts.0; + if candidate_model.includes.len() != expected_children { + return Err(format!("{} child count mismatch for model[{}]: oracle={} candidate={}", fixture.id, model_index, expected_children, candidate_model.includes.len())); + } + oracle_step_count = oracle_step_count + .checked_add(oracle_model.step_ends.len()) + .ok_or_else(|| format!("{} oracle step count overflow", fixture.id))?; + } + if fixture.id == "external-ldr-tools" { + let root_step_count = oracle + .root_model + .as_ref() + .map(|model| model.step_ends.len()) + .unwrap_or_default(); + if root_step_count != 52 || oracle_step_count != 109 { + return Err(format!( + "{} expected root/submodel step_ends distribution 52/57, got root={} models_total={}", + fixture.id, + root_step_count, + oracle_step_count + )); + } + } + drop(parsed); + if ledger.snapshot().reservations != 0 { + return Err(format!("{} differential candidate leaked reservation", fixture.id)); + } Ok(()) } @@ -329,21 +1047,36 @@ fn differential() -> Result { validate_inventory(&fixtures)?; let root = root_dir(); let binary = oracle_binary(&root)?; - for fixture in &fixtures { - let path = root.join("crates/polymodel-ldraw-testkit").join(fixture.relative_path); - let actual = run_oracle(&binary, &path, ORACLE_TIMEOUT)?; - let actual_projection = projection(actual); - let expected_projection = fixture - .expected - .oracle_projection - .as_ref() - .ok_or_else(|| format!("missing authored oracle projection: {}", fixture.id))?; - polymodel_ldraw_testkit::compare_oracle_projection( - expected_projection, + for (index, fixture) in fixtures.iter().enumerate() { + let path = root + .join("crates/polymodel-ldraw-testkit") + .join(fixture.relative_path); + let actual = run_oracle( + &binary, + &path, + &root.join("crates/polymodel-ldraw-testkit/corpus"), + ORACLE_TIMEOUT, + )?; + let actual_projection = projection(actual, &path, fixture.bytes)?; + let expected_oracle = &fixture.expected.oracle_expected; + polymodel_ldraw_testkit::compare_oracle_expected( + expected_oracle, &actual_projection, fixture.id, ) - .map_err(|error| format!("{} (source: {}; oracle revision: {}; schema: {})", error, fixture.provenance.id, LD_PARSE_REVISION, ORACLE_SCHEMA_VERSION))?; + .map_err(|error| { + format!( + "{} (source: {}; oracle revision: {}; schema: {})", + error, fixture.provenance.id, LD_PARSE_REVISION, ORACLE_SCHEMA_VERSION + ) + })?; + if fixture.outcome == polymodel_ldraw_testkit::ExpectedOutcome::Accepted + || fixture.outcome == polymodel_ldraw_testkit::ExpectedOutcome::Preserved + || fixture.outcome == polymodel_ldraw_testkit::ExpectedOutcome::Limited + { + compare_candidate_to_oracle(fixture, &actual_projection) + .map_err(|error| format!("fixture {index}: {error}"))?; + } } serde_json::to_string_pretty(&Report { schema_version: SCHEMA_VERSION, @@ -361,6 +1094,30 @@ fn differential() -> Result { .map_err(|error| error.to_string()) } +fn mutation_operator(seed: u64, index: usize) -> &'static str { + match (seed + .wrapping_add(index as u64) + .wrapping_mul(6364136223846793005) + .wrapping_add(1) + % 4) as usize + { + 0 => "insert_meta", + 1 => "delete_bytes", + 2 => "replace_byte", + _ => "normalize_crlf", + } +} + +fn mutation_fixture_index(index: usize, fixture_count: usize) -> usize { + // Four complete rounds cover every fixture; the remaining mutations repeat + // the first four fixture slots with distinct indices and operators. + index % fixture_count +} + +fn mutation_index_is_authorized(index: usize, fixture_count: usize) -> bool { + index < FUZZ_ITERATIONS && fixture_count > 0 +} + fn mutate(seed: u64, index: usize, source: &[u8]) -> Vec { let mut state = seed.wrapping_add(index as u64); state = state.wrapping_mul(6364136223846793005).wrapping_add(1); @@ -394,61 +1151,11 @@ fn mutate(seed: u64, index: usize, source: &[u8]) -> Vec { output } -fn bounded_mutation_rejection(error: &LdrawError) -> bool { - match error { - LdrawError::Cancelled => true, - LdrawError::Diagnostic(diagnostic) => matches!( - diagnostic.code, - DiagnosticCode::InvalidUtf8 - | DiagnosticCode::PathOutOfRoot - | DiagnosticCode::PathForbiddenSyntax - | DiagnosticCode::PathEmpty - | DiagnosticCode::InvalidType1 - | DiagnosticCode::InvalidType2 - | DiagnosticCode::InvalidType3 - | DiagnosticCode::InvalidType4 - | DiagnosticCode::InvalidType5 - | DiagnosticCode::InvalidLineType - | DiagnosticCode::InvalidColour - | DiagnosticCode::MetaEmpty - | DiagnosticCode::TexmapNextType0 - | DiagnosticCode::ColourSlotsLimit - | DiagnosticCode::TexmapUnknownMetaStop - | DiagnosticCode::BfcInvalidDirective - | DiagnosticCode::BfcRecertifyUncertified - | DiagnosticCode::BfcInvalidCertifyArgument - | DiagnosticCode::BfcInvertNextOutsideCertified - | DiagnosticCode::BfcInvalidContext - | DiagnosticCode::TexmapInvalidArity - | DiagnosticCode::TexmapInvalidMethod - | DiagnosticCode::TexmapNextNested - | DiagnosticCode::TexmapInvalidValue - | DiagnosticCode::TexmapFallbackOutOfScope - | DiagnosticCode::TexmapDuplicateFallback - | DiagnosticCode::TexmapInvalidEnd - | DiagnosticCode::DataInvalid - | DiagnosticCode::DataTruncated - | DiagnosticCode::IncludeDepthLimit - | DiagnosticCode::GraphCycle - | DiagnosticCode::ResourceBytesLimit - | DiagnosticCode::CommandsLimit - | DiagnosticCode::InstancesLimit - | DiagnosticCode::TrianglesLimit - | DiagnosticCode::LinesLimit - | DiagnosticCode::TexturesLimit - | DiagnosticCode::FetchesLimit - | DiagnosticCode::DiagnosticsLimit - ), - LdrawError::LdrawDiagnostic(_) - | LdrawError::Reservation(_) - | LdrawError::Overflow(_) => false, - } -} - fn fuzz() -> Result { let fixtures = inventory(); validate_inventory(&fixtures)?; - let binary = oracle_binary(&root_dir())?; + let root = root_dir(); + let binary = oracle_binary(&root)?; let temporary = env::temp_dir().join(format!("polymodel-pm77-fuzz-{}", std::process::id())); fs::create_dir_all(&temporary).map_err(|error| error.to_string())?; let started = Instant::now(); @@ -457,7 +1164,13 @@ fn fuzz() -> Result { if started.elapsed() >= FUZZ_BUDGET { return Err(format!("fuzz budget exceeded before mutation {index}")); } - let fixture = &fixtures[index % fixtures.len()]; + if !mutation_index_is_authorized(index, fixtures.len()) { + return Err(format!( + "mutation {index} has no authorized fixture/profile/operator contract" + )); + } + let fixture_index = mutation_fixture_index(index, fixtures.len()); + let fixture = &fixtures[fixture_index]; let bytes = mutate(FUZZ_SEED, index, fixture.bytes); if bytes.len() > MAX_MUTATED_BYTES { return Err(format!("mutation {index} exceeded 1 MiB")); @@ -469,72 +1182,125 @@ fn fuzz() -> Result { let digest = Sha256::digest(&bytes); let path = temporary.join(format!("{index}-{}.ldr", hex::encode(digest))); fs::write(&path, &bytes).map_err(|error| error.to_string())?; + let operator = mutation_operator(FUZZ_SEED, index); let ledger = ReservationLedger::new(); - let parsed = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let first = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { InProcessRustAdapter::parse_fixture( fixture.id, &bytes, - core_profile(fixture.expected.profile), + core_profile(fixture.profile), ReservationOwner::preview(1, index as u64 + 1, 1), &ledger, ) })) - .map_err(|_| format!("mutation {index} ({}) core parser panicked", fixture.id))?; - match parsed { - Ok(parsed) => { - let first = serialize_canonical(&parsed.project()) - .map_err(|error| format!("mutation {index} ({}) projection failed: {error}", fixture.id))?; - drop(parsed); - if ledger.snapshot().reservations != 0 { - return Err(format!("mutation {index} ({}) leaked ledger reservations", fixture.id)); - } - let repeat_ledger = ReservationLedger::new(); - let repeat = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - InProcessRustAdapter::parse_fixture( - fixture.id, - &bytes, - core_profile(fixture.expected.profile), - ReservationOwner::preview(1, index as u64 + 1, 1), - &repeat_ledger, - ) - })) - .map_err(|_| format!("mutation {index} ({}) repeated core parser panicked", fixture.id))? - .map_err(|error| format!("mutation {index} ({}) was nondeterministically rejected: {error}", fixture.id))?; - let second = serialize_canonical(&repeat.project()) - .map_err(|error| format!("mutation {index} ({}) repeated projection failed: {error}", fixture.id))?; - drop(repeat); - if repeat_ledger.snapshot().reservations != 0 { - return Err(format!("mutation {index} ({}) repeated parse leaked ledger reservations", fixture.id)); - } - if first != second { - return Err(format!("mutation {index} ({}) accepted parse is not deterministic", fixture.id)); - } - let oracle = run_oracle(&binary, &path, ORACLE_TIMEOUT) - .map_err(|error| format!("mutation {index} ({}) failed: {error}", fixture.id))?; - if oracle.status != "ok" { - return Err(format!("mutation {index} ({}) accepted core parse has oracle status {}", fixture.id, oracle.status)); - } - } - Err(error) if bounded_mutation_rejection(&error) => { - if index == 0 { - match error { - LdrawError::Diagnostic(diagnostic) - if diagnostic.code == DiagnosticCode::InvalidUtf8 => {} - other => { - return Err(format!( - "mutation 0 ({}) expected INVALID_UTF8, got {other}", - fixture.id - )); - } - } - } - if ledger.snapshot().reservations != 0 { - return Err(format!("mutation {index} ({}) rejected parse leaked ledger reservations", fixture.id)); - } - } - Err(error) => { - return Err(format!("mutation {index} ({}) core parse failed: {error}", fixture.id)); + .map_err(|_| { + format!( + "mutation {index} ({operator}) ({}) core parser panicked", + fixture.id + ) + })?; + let first_observation = match first { + Ok(parsed) => candidate_observation(&parsed).and_then(|observation| { + serde_json::to_vec(&observation) + }).map_err(|error| { + format!( + "mutation {index} ({operator}) ({}) candidate observation failed: {error}", + fixture.id + ) + }), + Err(error) => Err(format!("error:{error:?}")), + }; + if ledger.snapshot().reservations != 0 { + return Err(format!( + "mutation {index} ({operator}) ({}) leaked ledger reservations", + fixture.id + )); + } + let repeat_ledger = ReservationLedger::new(); + let repeat = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + InProcessRustAdapter::parse_fixture( + fixture.id, + &bytes, + core_profile(fixture.profile), + ReservationOwner::preview(1, index as u64 + 1, 1), + &repeat_ledger, + ) + })) + .map_err(|_| { + format!( + "mutation {index} ({operator}) ({}) repeated core parser panicked", + fixture.id + ) + })?; + let second_observation = match repeat { + Ok(parsed) => candidate_observation(&parsed).and_then(|observation| { + serde_json::to_vec(&observation) + }).map_err(|error| { + format!( + "mutation {index} ({operator}) ({}) repeated candidate observation failed: {error}", + fixture.id + ) + }), + Err(error) => Err(format!("error:{error:?}")), + }; + if repeat_ledger.snapshot().reservations != 0 { + return Err(format!( + "mutation {index} ({operator}) ({}) repeated parse leaked ledger reservations", + fixture.id + )); + } + if first_observation != second_observation { + return Err(format!( + "mutation {index} ({operator}) ({}) core outcome is not deterministic", + fixture.id + )); + } + let oracle_observation = |oracle: Result, repeated: bool| { + match oracle { + Ok(document) => serde_json::to_vec(&projection(document, &path, &bytes)?), + Err(error) => Ok(format!("error:{error}").into_bytes()), } + .map_err(|error| { + format!( + "mutation {index} ({operator}) ({}) {} oracle observation failed: {error}", + fixture.id, + if repeated { "repeated" } else { "initial" } + ) + }) + }; + let oracle_bytes = oracle_observation( + run_oracle( + &binary, + &path, + &root.join("crates/polymodel-ldraw-testkit/corpus"), + ORACLE_TIMEOUT, + ), + false, + )?; + let oracle_repeat_bytes = oracle_observation( + run_oracle( + &binary, + &path, + &root.join("crates/polymodel-ldraw-testkit/corpus"), + ORACLE_TIMEOUT, + ), + true, + )?; + if oracle_bytes != oracle_repeat_bytes { + let first_difference = oracle_bytes + .iter() + .zip(&oracle_repeat_bytes) + .position(|(left, right)| left != right) + .unwrap_or(oracle_bytes.len().min(oracle_repeat_bytes.len())); + return Err(format!( + "mutation {index} ({operator}) ({}) oracle outcome is not deterministic: first_difference={}, initial_len={}, repeat_len={}, initial_sha256={}, repeat_sha256={}", + fixture.id, + first_difference, + oracle_bytes.len(), + oracle_repeat_bytes.len(), + hex::encode(Sha256::digest(&oracle_bytes)), + hex::encode(Sha256::digest(&oracle_repeat_bytes)), + )); } } Ok(()) @@ -559,7 +1325,10 @@ fn fuzz() -> Result { mod hex { pub fn encode(bytes: impl AsRef<[u8]>) -> String { - bytes.as_ref().iter().map(|byte| format!("{byte:02x}")).collect() + bytes + .as_ref() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() } } - diff --git a/tools/test-just-nix.sh b/tools/test-just-nix.sh new file mode 100755 index 0000000..46263fd --- /dev/null +++ b/tools/test-just-nix.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +HELPER="$ROOT/tools/in-dev-shell" +REAL_JUST=$(command -v just) +TMP=$(mktemp -d) +trap 'rm -rf "$TMP"' EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +assert_args() { + local actual=$1 + shift + printf '%s\0' "$@" > "$TMP/expected.args" + cmp "$TMP/expected.args" "$actual" || fail "unexpected argv in $actual" +} + +mkdir -p "$TMP/bin" "$TMP/no-direnv" +ln -s "$(command -v bash)" "$TMP/bin/bash" +ln -s "$(command -v bash)" "$TMP/no-direnv/bash" + +cat > "$TMP/bin/just" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +if [[ "${1:-}" == _child && -n "${FAKE_JUST_NESTED_ARGS:-}" ]]; then + printf '%s\0' "$@" > "$FAKE_JUST_NESTED_ARGS" +elif (($#)); then + printf '%s\0' "$@" > "$FAKE_JUST_ARGS" +else + : > "$FAKE_JUST_ARGS" +fi +if [[ "${1:-}" == _nested ]]; then + exec "$HELPER" _child 'nested value; $HOME' +fi +exit "${FAKE_JUST_EXIT:-0}" +EOF +chmod +x "$TMP/bin/just" +ln -s "$TMP/bin/just" "$TMP/no-direnv/just" + +cat > "$TMP/bin/direnv" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\0' "$@" > "$FAKE_DIRENV_ARGS" +[[ ${1:-} == exec && ${2:-} == . && ${3:-} == just ]] || exit 64 +shift 3 +export POLYMODEL_NIX_SHELL=1 +export FAKE_JUST_NESTED_ARGS="${FAKE_JUST_NESTED_ARGS:-}" +exec "$FAKE_JUST" "$@" +EOF +chmod +x "$TMP/bin/direnv" + +run_outside() { + : > "$TMP/direnv.args" + : > "$TMP/just.args" + : > "$TMP/nested.args" + env -u POLYMODEL_NIX_SHELL \ + PATH="$TMP/bin" \ + FAKE_DIRENV_ARGS="$TMP/direnv.args" \ + FAKE_JUST_ARGS="$TMP/just.args" \ + FAKE_JUST_NESTED_ARGS="$TMP/nested.args" \ + FAKE_JUST="$TMP/bin/just" \ + HELPER="$HELPER" \ + "$HELPER" "$@" +} + +run_outside _demo 'path with spaces; $(printf injected)' '--flag=value' +printf '%s\0' exec . just _demo 'path with spaces; $(printf injected)' '--flag=value' > "$TMP/expected.args" +cmp "$TMP/expected.args" "$TMP/direnv.args" || fail 'outside direnv argv changed' +printf '%s\0' _demo 'path with spaces; $(printf injected)' '--flag=value' > "$TMP/expected.args" +cmp "$TMP/expected.args" "$TMP/just.args" || fail 'outside Just argv changed' + +set +e +env -u POLYMODEL_NIX_SHELL \ + PATH="$TMP/bin" \ + FAKE_DIRENV_ARGS="$TMP/direnv.args" \ + FAKE_JUST_ARGS="$TMP/just.args" \ + FAKE_JUST="$TMP/bin/just" \ + HELPER="$HELPER" \ + FAKE_JUST_EXIT=37 \ + "$HELPER" _exit-test +status=$? +set -e +[[ $status -eq 37 ]] || fail "exit status was $status, expected 37" + +: > "$TMP/just.args" +env POLYMODEL_NIX_SHELL=1 \ + PATH="$TMP/bin" \ + FAKE_JUST_ARGS="$TMP/just.args" \ + "$HELPER" _direct 'inside value' +assert_args "$TMP/just.args" _direct 'inside value' + +: > "$TMP/direnv.args" +: > "$TMP/just.args" +env -u POLYMODEL_NIX_SHELL \ + PATH="$TMP/bin" \ + FAKE_DIRENV_ARGS="$TMP/direnv.args" \ + FAKE_JUST_ARGS="$TMP/just.args" \ + FAKE_JUST_NESTED_ARGS="$TMP/nested.args" \ + FAKE_JUST="$TMP/bin/just" \ + HELPER="$HELPER" \ + "$HELPER" _nested +assert_args "$TMP/just.args" _nested +assert_args "$TMP/nested.args" _child 'nested value; $HOME' +[[ $(wc -c < "$TMP/direnv.args") -gt 0 ]] || fail 'nested public/private dispatch skipped outer shell entry' + +set +e +env -u POLYMODEL_NIX_SHELL PATH="$TMP/no-direnv" "$HELPER" _missing-direnv 2>"$TMP/missing.stderr" +status=$? +set -e +[[ $status -eq 127 ]] || fail "missing direnv status was $status, expected 127" +grep -F 'direnv is required' "$TMP/missing.stderr" >/dev/null || fail 'missing direnv diagnostic missing' + +TEST_JUSTFILE="$TMP/justfile" +cat > "$TEST_JUSTFILE" </dev/null || fail 'workspace-create does not allow the new workspace envrc' +grep -F 'direnv exec "$workspace_path" true' "$ROOT/justfile" >/dev/null || fail 'workspace-create does not verify the new workspace envrc' + +printf 'PASS: in-dev-shell dispatch boundary\n' -- 2.51.2