From 1b93c17118976f6157a77e70c2d52a003ecd3d91 Mon Sep 17 00:00:00 2001 From: Corbin Crutchley Date: Fri, 24 Jul 2026 23:54:35 -0700 Subject: [PATCH] feat: manage persistent browser build cache --- PLAN.md | 4 +- README.md | 6 +++ crates/browser-runtime/src/lib.rs | 58 +++++++++++++++++++++++++++- crates/memory-fs/src/lib.rs | 64 +++++++++++++++++++++++++++++++ docs/architecture.md | 4 +- web/index.html | 1 + web/src/main.ts | 22 ++++++++++- web/src/probe.ts | 7 +++- web/src/probe.worker.ts | 21 ++++++++-- web/tests/probe.spec.ts | 54 ++++++++++++++++++++++++++ 10 files changed, 231 insertions(+), 10 deletions(-) diff --git a/PLAN.md b/PLAN.md index 0205075..3c72f30 100644 --- a/PLAN.md +++ b/PLAN.md @@ -1891,7 +1891,9 @@ The first pipe checkpoint adds AArch64 `pipe2` with process-owned descriptors an The first direct child-process checkpoint implements a deliberately constrained spawn/exec model rather than claiming `fork`. AArch64 `clone` accepts exactly `CLONE_VM | CLONE_VFORK | SIGCHLD`, one suspended parent, and an optional child stack; it rejects nested or unreaped children, TID/TLS modes, and other flags. PID 2 can run directly or replace itself from the guest filesystem, after which exit restores the parent CPU, memory, interpreter, signals, identity, and configuration while retaining aggregate resource counters, output, trace history, and filesystem mutations. `getpid`, `getppid`, `gettid`, and `wait4` expose the child identity and encoded exit status. Native and Chromium fixtures cover both direct child exit and child `execve` followed by parent resumption. Blocking pipe wakeups, general clone modes, build cache policy, and interactive-performance work remain. -The constrained child now inherits a cloned descriptor table backed by shared host open-file descriptions and process-owned pipe queues. Shared file offsets and child writes survive parent restoration, while a child close removes only its table entry and closes the underlying host handle only when no suspended-parent reference remains. Pipe reader/writer lifetime likewise accounts for both tables, so child `O_CLOEXEC` cleanup cannot discard the parent's endpoints or queued data. The spawn/exec fixture now passes bytes through an inherited close-on-exec pipe before replacement, then validates the data and EOF after `wait4`; native runtime and Chromium-facing coverage use that same ELF and filesystem image. Blocking pipe suspension/wakeup, general clone modes, build cache policy, and interactive-performance work remain. +The constrained child now inherits a cloned descriptor table backed by shared host open-file descriptions and process-owned pipe queues. Shared file offsets and child writes survive parent restoration, while a child close removes only its table entry and closes the underlying host handle only when no suspended-parent reference remains. Pipe reader/writer lifetime likewise accounts for both tables, so child `O_CLOEXEC` cleanup cannot discard the parent's endpoints or queued data. The spawn/exec fixture now passes bytes through an inherited close-on-exec pipe before replacement, then validates the data and EOF after `wait4`; native runtime and Chromium-facing coverage use that same ELF and filesystem image. Blocking pipe suspension/wakeup, general clone modes, and interactive-performance work remain. + +The Phase 6 build-cache policy now designates `/project/.cache` as the bounded persistent namespace for guest compiler artifacts. It naturally shares the project snapshot's filesystem quota and OPFS durability rather than introducing a second storage format. A new atomic snapshot transformation removes that file-or-directory tree while retaining all unrelated source and installed toolchain paths; the browser exposes it as **Clear build cache** and persists the canonical result. Rust regressions verify nested removal, idempotence, path confinement, and source retention, while Chromium imports a snapshot containing both cache data and source and verifies the exported result. Blocking pipe suspension/wakeup, general clone modes, and interactive-performance work remain. Do not begin the full web IDE before item 30 passes. diff --git a/README.md b/README.md index 436d644..6f7bc22 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,12 @@ child's exact status. Native and Chromium hosts share the same regressions. General fork/clone modes, concurrent children, and blocking pipe scheduling remain future milestones. +Browser-local build tools should place reusable artifacts under +`/project/.cache`. That namespace is included in the bounded OPFS-backed +project snapshot and therefore survives runs and reloads. The **Clear build +cache** action removes that tree atomically while preserving source, installed +toolchains, and other project files. + The browser build generates its Memory64, JSPI, and P-code `.wasm` probes before starting Vite. Generated artifacts are not committed. Select **Uploaded AArch64 ELF** to run an external static executable with a chosen `argv[0]` and one argument per line; the executable is transferred directly to the runtime Worker. Run the opt-in Phase 4 interpreter/translator benchmark in Chromium with all diff --git a/crates/browser-runtime/src/lib.rs b/crates/browser-runtime/src/lib.rs index 6c3900f..79ea9f9 100644 --- a/crates/browser-runtime/src/lib.rs +++ b/crates/browser-runtime/src/lib.rs @@ -113,6 +113,7 @@ const TERMINAL_INPUT_ELF: &[u8] = include_bytes!("../../../guest-tests/terminal-input/terminal-input.aarch64.elf"); const EXECVE_LAUNCHER_ELF: &[u8] = include_bytes!("../../../guest-tests/execve-launcher/execve-launcher.aarch64.elf"); +const BUILD_CACHE_PATH: &[u8] = b"/project/.cache"; #[cfg(test)] const EXECVE_TOOLCHAIN_IMAGE: &[u8] = include_bytes!("../../../guest-tests/execve-launcher/toolchain.bnfs"); @@ -631,6 +632,31 @@ pub fn write_filesystem_file( .map_err(|error| JsError::new(&error.to_string())) } +/// Remove the persistent build-cache namespace from a project snapshot. +/// +/// # Errors +/// +/// Returns a JavaScript error when the current snapshot is malformed or the +/// resulting canonical snapshot cannot be encoded. +#[wasm_bindgen] +pub fn clear_filesystem_build_cache( + max_filesystem_bytes: u64, + current_snapshot: &[u8], +) -> Result, JsError> { + let mut filesystem = if current_snapshot.is_empty() { + MemoryFileSystem::new(max_filesystem_bytes) + } else { + MemoryFileSystem::from_snapshot(max_filesystem_bytes, current_snapshot) + .map_err(|error| JsError::new(&error.to_string()))? + }; + filesystem + .clear_project_tree(BUILD_CACHE_PATH) + .map_err(|error| JsError::new(&format!("could not clear build cache: {error:?}")))?; + filesystem + .export_snapshot() + .map_err(|error| JsError::new(&error.to_string())) +} + /// Stateful browser guest that can suspend and resume blocking terminal input. #[wasm_bindgen] pub struct BrowserGuestSession { @@ -1286,12 +1312,15 @@ impl HostTerminal for CapturedTerminal { #[cfg(test)] mod tests { use binarrow_execution_ir::{BasicBlock, LiftedInstruction, Operation, Value, ValueSource}; + use binarrow_host_api::HostFileSystem; + use binarrow_memory_fs::MemoryFileSystem; use binarrow_runtime_core::GuestAddress; use super::{ BrowserBlockExecutor, COMPILER_HELLO_ELF, Credentials, EXECVE_TOOLCHAIN_IMAGE, Interpreter, - ProcessConfig, ProcessParameters, SPAWN_EXEC_IMAGE, execute_fixture, fixture, load_process, - program_environment, start_fixture, start_program, translate_fixture_entry_inner, + ProcessConfig, ProcessParameters, SPAWN_EXEC_IMAGE, clear_filesystem_build_cache, + execute_fixture, fixture, load_process, program_environment, start_fixture, start_program, + translate_fixture_entry_inner, }; const DEFAULT_INSTRUCTIONS: u64 = 10_000_000; @@ -1521,6 +1550,31 @@ mod tests { assert_eq!(read.stdout, "persistent project data\n"); } + #[test] + fn clears_only_the_persistent_build_cache_namespace() { + let mut filesystem = MemoryFileSystem::new(DEFAULT_FILESYSTEM); + filesystem + .replace_project_file_contents(b"/project/main.c", b"int main(void) {}") + .unwrap(); + filesystem.create_directory(b"/project/.cache").unwrap(); + filesystem + .create_directory(b"/project/.cache/clang") + .unwrap(); + filesystem + .replace_project_file_contents(b"/project/.cache/clang/module", b"cache") + .unwrap(); + let snapshot = filesystem.export_snapshot().unwrap(); + + let cleared = clear_filesystem_build_cache(DEFAULT_FILESYSTEM, &snapshot).unwrap(); + let restored = MemoryFileSystem::from_snapshot(DEFAULT_FILESYSTEM, &cleared).unwrap(); + + assert_eq!( + restored.read_file(b"/project/main.c"), + Some(&b"int main(void) {}"[..]) + ); + assert_eq!(restored.read_file(b"/project/.cache/clang/module"), None); + } + #[test] fn executes_the_vfs_lifecycle_fixture() { let result = execute_fixture( diff --git a/crates/memory-fs/src/lib.rs b/crates/memory-fs/src/lib.rs index 5df509f..17a279c 100644 --- a/crates/memory-fs/src/lib.rs +++ b/crates/memory-fs/src/lib.rs @@ -137,6 +137,34 @@ impl MemoryFileSystem { Ok(()) } + /// Remove one project file or directory tree while preserving unrelated + /// project contents. A missing target is already clear and succeeds. + /// + /// # Errors + /// + /// Returns a stable filesystem error when the path is not a normalized + /// descendant of `/project`. + pub fn clear_project_tree(&mut self, path: &[u8]) -> Result<(), FileSystemError> { + let normalized = normalize_path(path)?; + if normalized != path || !path.starts_with(b"/project/") { + return Err(FileSystemError::PermissionDenied); + } + let removed_files = self + .files + .keys() + .filter(|candidate| **candidate == normalized || is_descendant(candidate, &normalized)) + .cloned() + .collect::>(); + for removed in removed_files { + if let Some(bytes) = self.files.remove(&removed) { + self.stored_bytes -= bytes.len() as u64; + } + } + self.directories + .retain(|candidate| *candidate != normalized && !is_descendant(candidate, &normalized)); + Ok(()) + } + /// Restore persistent regular files from a deterministic snapshot. /// Ephemeral `/tmp` files are never accepted from snapshots. /// @@ -1060,6 +1088,42 @@ mod tests { ); } + #[test] + fn clears_one_project_tree_without_touching_source_files() { + let mut filesystem = MemoryFileSystem::new(64); + filesystem + .replace_project_file_contents(b"/project/main.c", b"source") + .unwrap(); + filesystem.create_directory(b"/project/.cache").unwrap(); + filesystem + .create_directory(b"/project/.cache/clang") + .unwrap(); + let cached = filesystem + .open(b"/project/.cache/clang/module", CREATE_READ_WRITE) + .unwrap(); + filesystem.write(cached, b"cached bytes").unwrap(); + filesystem.close(cached).unwrap(); + + filesystem.clear_project_tree(b"/project/.cache").unwrap(); + + assert_eq!( + filesystem.read_file(b"/project/main.c"), + Some(&b"source"[..]) + ); + assert_eq!(filesystem.read_file(b"/project/.cache/clang/module"), None); + assert!( + !filesystem + .directories + .contains(b"/project/.cache".as_slice()) + ); + assert_eq!(filesystem.stored_bytes(), 6); + filesystem.clear_project_tree(b"/project/.cache").unwrap(); + assert_eq!( + filesystem.clear_project_tree(b"/project/../tmp"), + Err(FileSystemError::PermissionDenied) + ); + } + #[test] fn rejects_truncated_trailing_and_over_budget_snapshots() { let mut filesystem = MemoryFileSystem::new(64); diff --git a/docs/architecture.md b/docs/architecture.md index 50ae221..a0d2e7e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -76,7 +76,7 @@ Consumes a loaded process image, owns its architectural execution state and desc ### `binarrow-browser-runtime` -Provides the narrow wasm-bindgen boundary used by the Worker. It embeds deterministic C, Rust, process-replacement, and non-terminating fixtures, validates uploaded ELF bytes plus explicit arguments and environment entries, applies browser-supplied resource limits, invokes the production loader/runtime, captures the terminal host trait, and exposes output, exit status, counters, trace events, translation metrics, stable structured diagnostics, and the mutated filesystem snapshot. The process-replacement fixture loads its second ELF from the same imported OPFS-backed snapshot path used by ordinary project files. The Worker persists returned snapshots after exits, input suspension, and diagnostics, so bounded compiler runs retain completed project and cache mutations after their process state is discarded. Its Tier-1 executor compiles supported hot blocks, caches modules by guest address and instruction encodings, installs their exports in a bounded Worker-local WebAssembly function table, exchanges scalar architectural state through the backend ABI, and invokes the selected entry synchronously. The TypeScript controller starts one run at a time and implements unconditional cancellation by terminating and recreating the Worker; no AArch64 or Linux behavior is reimplemented in TypeScript. +Provides the narrow wasm-bindgen boundary used by the Worker. It embeds deterministic C, Rust, process-replacement, and non-terminating fixtures, validates uploaded ELF bytes plus explicit arguments and environment entries, applies browser-supplied resource limits, invokes the production loader/runtime, captures the terminal host trait, and exposes output, exit status, counters, trace events, translation metrics, stable structured diagnostics, and the mutated filesystem snapshot. The process-replacement fixture loads its second ELF from the same imported OPFS-backed snapshot path used by ordinary project files. The Worker persists returned snapshots after exits, input suspension, and diagnostics, so bounded compiler runs retain completed project and cache mutations after their process state is discarded. `/project/.cache` is the designated persistent build-cache namespace; a snapshot transformation clears that tree without disturbing source or installed toolchains, and the Worker saves the canonical result back to OPFS. Its Tier-1 executor compiles supported hot blocks, caches modules by guest address and instruction encodings, installs their exports in a bounded Worker-local WebAssembly function table, exchanges scalar architectural state through the backend ABI, and invokes the selected entry synchronously. The TypeScript controller starts one run at a time and implements unconditional cancellation by terminating and recreating the Worker; no AArch64 or Linux behavior is reimplemented in TypeScript. ### `binarrow-wasm-backend` @@ -104,7 +104,7 @@ Phase 4 is complete for the initial Tier-1 scope: bounded profiling, scalar basi Phase 5 is complete. A shared native/browser CPython regression independently installs the standard-library snapshot, a multi-file project image, and a checksum-pinned official `packaging` 26.2 wheel image. The CLI's repeatable image-overlay option mirrors the browser install operation, while the browser disables conflicting controls until each asynchronous image request is acknowledged and persists the result in OPFS. The Linux runtime implements the close-on-exec ioctl requests used by CPython's directory-opening path. Project and third-party filesystem imports, retained wheel metadata, deterministic output, guest-path tracebacks, exception text, and exit status are verified in both hosts. Guest-side package resolution, network indexes, and native wheels remain outside this phase; Phase 6 moves to in-browser C compilation. -Phase 6 is in progress. Static `execve` replacement provides bounded guest-vector ingestion and Linux-compatible close-on-exec handling; the packaged Clang/LLD/musl path now compiles browser-edited source, links it, and executes the resulting ELF. Compiler-format stderr records navigate to editor source positions. Bounded `O_NONBLOCK` pipes move bytes through one process, and the constrained single-child scheduler now covers clone/exit/wait plus child `execve`, inherited open-file descriptions and pipe queues, and parent restoration in native and Chromium hosts. Blocking wakeups, broader clone modes, cache policy, and interactive-performance work remain. +Phase 6 is in progress. Static `execve` replacement provides bounded guest-vector ingestion and Linux-compatible close-on-exec handling; the packaged Clang/LLD/musl path now compiles browser-edited source, links it, and executes the resulting ELF. Compiler-format stderr records navigate to editor source positions. Bounded `O_NONBLOCK` pipes move bytes through one process, and the constrained single-child scheduler now covers clone/exit/wait plus child `execve`, inherited open-file descriptions and pipe queues, and parent restoration in native and Chromium hosts. Build artifacts under `/project/.cache` persist within the existing filesystem quota and can be cleared independently from the UI. Blocking wakeups, broader clone modes, and interactive-performance work remain. An official static AArch64 Linux Zig 0.16.0 distribution serves as an ignored LLVM/LLD compatibility probe while the final Clang package is selected and pruned. It now completes its version path and advances `zig cc` through a bounded 20-million-instruction startup/compilation run using project-local caches. The trace-derived additions are floating-point width conversion, NEON `rev32`, directory-relative `*at` operations, path-only descriptors, initial guest environment entries, and precise missing-parent errno behavior. The probe bundle and caches remain under `.tmp`; they are evidence and test input, not a shipped replacement for Clang/LLD. diff --git a/web/index.html b/web/index.html index 6e34e8f..5dfe1b1 100644 --- a/web/index.html +++ b/web/index.html @@ -138,6 +138,7 @@ +

WebAssembly capabilities

diff --git a/web/src/main.ts b/web/src/main.ts index e7df6dc..0e90fae 100644 --- a/web/src/main.ts +++ b/web/src/main.ts @@ -77,6 +77,9 @@ const projectImage = requiredElement("#project-image"); const replaceImageButton = requiredElement("#replace-image"); const installImageButton = requiredElement("#install-image"); const exportImageButton = requiredElement("#export-image"); +const clearBuildCacheButton = requiredElement( + "#clear-build-cache", +); let worker: Worker; let workerReady = false; @@ -165,6 +168,7 @@ function setControls(): void { replaceImageButton.disabled = !workerReady || busy; installImageButton.disabled = !workerReady || busy; exportImageButton.disabled = !workerReady || busy; + clearBuildCacheButton.disabled = !workerReady || busy; } function updateProgramControls(): void { @@ -325,7 +329,9 @@ function createWorker(): void { ? "Project snapshot imported" : event.data.operation === "install" ? "Project image installed" - : "Project source saved"; + : event.data.operation === "clear-cache" + ? "Build cache cleared" + : "Project source saved"; } status.dataset.state = "ready"; imageOperationPending = false; @@ -529,6 +535,20 @@ exportImageButton.addEventListener("click", () => { worker.postMessage(command); }); +clearBuildCacheButton.addEventListener("click", () => { + imageOperationPending = true; + status.textContent = "Clearing build cache…"; + status.dataset.state = "running"; + setControls(); + imageRequestId += 1; + const command: WorkerCommand = { + kind: "filesystem-clear-cache", + requestId: imageRequestId, + maxFilesystemBytes: parseLimit(filesystemLimit), + }; + worker.postMessage(command); +}); + saveSourceButton.addEventListener("click", () => { imageOperationPending = true; status.textContent = "Saving project source…"; diff --git a/web/src/probe.ts b/web/src/probe.ts index a7a505b..a811d9c 100644 --- a/web/src/probe.ts +++ b/web/src/probe.ts @@ -124,6 +124,11 @@ export type WorkerCommand = path: string; contents: Uint8Array; } + | { + kind: "filesystem-clear-cache"; + requestId: number; + maxFilesystemBytes: bigint; + } | { kind: "translation-benchmark"; requestId: number; @@ -141,7 +146,7 @@ export type WorkerMessage = | { kind: "filesystem-image"; requestId: number; - operation: "export" | "replace" | "install" | "write"; + operation: "export" | "replace" | "install" | "write" | "clear-cache"; snapshot: Uint8Array; } | { diff --git a/web/src/probe.worker.ts b/web/src/probe.worker.ts index dc65c1b..f0fdf5f 100644 --- a/web/src/probe.worker.ts +++ b/web/src/probe.worker.ts @@ -7,6 +7,7 @@ import { typedFunctionReferences, } from "wasm-feature-detect"; import initBrowserRuntime, { + clear_filesystem_build_cache, install_filesystem_snapshot, normalize_filesystem_snapshot, start_fixture, @@ -316,12 +317,18 @@ async function execute( async function handleFilesystemCommand( command: Extract< WorkerCommand, - { kind: "filesystem-export" | "filesystem-import" | "filesystem-write" } + { + kind: + | "filesystem-export" + | "filesystem-import" + | "filesystem-write" + | "filesystem-clear-cache"; + } >, ): Promise { const current = await loadFilesystemSnapshot(); let snapshot: Uint8Array; - let operation: "export" | "replace" | "install" | "write"; + let operation: "export" | "replace" | "install" | "write" | "clear-cache"; if (command.kind === "filesystem-export") { snapshot = normalize_filesystem_snapshot( command.maxFilesystemBytes, @@ -337,6 +344,13 @@ async function handleFilesystemCommand( ); await saveFilesystemSnapshot(snapshot); operation = "write"; + } else if (command.kind === "filesystem-clear-cache") { + snapshot = clear_filesystem_build_cache( + command.maxFilesystemBytes, + current, + ); + await saveFilesystemSnapshot(snapshot); + operation = "clear-cache"; } else if (command.mode === "replace") { snapshot = normalize_filesystem_snapshot( command.maxFilesystemBytes, @@ -476,7 +490,8 @@ self.addEventListener("message", (event: MessageEvent) => { if ( command.kind === "filesystem-export" || command.kind === "filesystem-import" || - command.kind === "filesystem-write" + command.kind === "filesystem-write" || + command.kind === "filesystem-clear-cache" ) { await handleFilesystemCommand(command); return; diff --git a/web/tests/probe.spec.ts b/web/tests/probe.spec.ts index 6e68ed4..015108c 100644 --- a/web/tests/probe.spec.ts +++ b/web/tests/probe.spec.ts @@ -2,6 +2,24 @@ import { expect, test } from "@playwright/test"; import fs from "node:fs"; import path from "node:path"; +function projectSnapshot( + entries: Array<{ type: 1 | 2; path: string; contents?: string }>, +): Buffer { + const header = Buffer.alloc(12); + header.write("BNFS\x02\0\0\0", 0, "binary"); + header.writeUInt32LE(entries.length, 8); + const encoded = entries.map((entry) => { + const pathBytes = Buffer.from(entry.path); + const contents = Buffer.from(entry.contents ?? ""); + const metadata = Buffer.alloc(13); + metadata.writeUInt8(entry.type, 0); + metadata.writeUInt32LE(pathBytes.length, 1); + metadata.writeBigUInt64LE(BigInt(contents.length), 5); + return Buffer.concat([metadata, pathBytes, contents]); + }); + return Buffer.concat([header, ...encoded]); +} + test.beforeEach(async ({ page }) => { await page.goto("/"); await expect(page.getByRole("status")).toHaveText("Browser runtime ready"); @@ -323,6 +341,42 @@ test("saves edited C source into the persistent project filesystem", async ({ expect(fs.readFileSync(downloadPath!).includes(Buffer.from(source))).toBe(true); }); +test("clears the persistent build cache without removing project source", async ({ + page, +}) => { + const source = "int main(void) { return 42; }\n"; + const snapshot = projectSnapshot([ + { type: 1, path: "/project/.cache" }, + { type: 1, path: "/project/.cache/clang" }, + { + type: 2, + path: "/project/.cache/clang/module", + contents: "cached bytes", + }, + { type: 2, path: "/project/main.c", contents: source }, + ]); + await page.getByLabel("Snapshot or package image").setInputFiles({ + name: "cache.binarrow-project", + mimeType: "application/vnd.binarrow.project", + buffer: snapshot, + }); + await page.getByRole("button", { name: "Replace project" }).click(); + await expect(page.getByRole("status")).toHaveText("Project snapshot imported"); + + await page.getByRole("button", { name: "Clear build cache" }).click(); + await expect(page.getByRole("status")).toHaveText("Build cache cleared"); + const downloadPath = await Promise.all([ + page.waitForEvent("download"), + page.getByRole("button", { name: "Export project" }).click(), + ]).then(([download]) => download.path()); + + expect(downloadPath).not.toBeNull(); + const exported = fs.readFileSync(downloadPath!); + expect(exported.includes(Buffer.from("/project/.cache"))).toBe(false); + expect(exported.includes(Buffer.from("/project/main.c"))).toBe(true); + expect(exported.includes(Buffer.from(source))).toBe(true); +}); + test("creates, modifies, lists, renames, and deletes guest files", async ({ page, }) => { -- 2.51.2