From 9bdbdf7afd493f00422f7d2fab70736bb1e3ac6f Mon Sep 17 00:00:00 2001 From: Corbin Crutchley Date: Fri, 24 Jul 2026 19:57:34 -0700 Subject: [PATCH] feat: package the AArch64 Clang toolchain --- PLAN.md | 4 +- toolchains/clang-musl/README.md | 34 ++++ .../clang-musl/aarch64-linux-musl.cmake | 36 ++++ toolchains/clang-musl/build.sh | 188 ++++++++++++++++++ toolchains/clang-musl/project/main.c | 6 + toolchains/clang-musl/zig-ar.sh | 4 + toolchains/clang-musl/zig-ranlib.sh | 4 + 7 files changed, 275 insertions(+), 1 deletion(-) create mode 100644 toolchains/clang-musl/README.md create mode 100644 toolchains/clang-musl/aarch64-linux-musl.cmake create mode 100755 toolchains/clang-musl/build.sh create mode 100644 toolchains/clang-musl/project/main.c create mode 100755 toolchains/clang-musl/zig-ar.sh create mode 100755 toolchains/clang-musl/zig-ranlib.sh diff --git a/PLAN.md b/PLAN.md index 43c3e66..a4462b1 100644 --- a/PLAN.md +++ b/PLAN.md @@ -1,7 +1,7 @@ # AArch64 ELF-to-WebAssembly Browser Runtime ## Engineering Build Plan and Agent Handoff -**Status:** Phase 5 complete; Phase 6 in progress (`execve` checkpoint complete) +**Status:** Phase 5 complete; Phase 6 in progress (packaged Clang prototype) **Primary implementation language:** Rust **Initial browser target:** Google Chrome **Guest architecture:** AArch64, little-endian, Linux userspace @@ -1877,6 +1877,8 @@ The next Phase 6 compatibility checkpoint is driven by an ignored, checksum-veri Restartable filesystem checkpoints now cover the compiler workload in both hosts. The native CLI accepts `--filesystem-output` and exports completed project mutations even after instruction exhaustion; a regression stops a writer after 25 instructions and successfully consumes the emitted snapshot in a second process. Browser-uploaded programs accept explicit environment entries, large image imports transfer their buffers into the Worker, and an opt-in real Chromium regression imports the ignored 330 MB Zig image, completes `zig version` within three million instructions with translated blocks, drives `zig cc` to a five-million-instruction boundary, and exports a larger post-run snapshot. The measured browser probe completes in 2.2 minutes. This is filesystem/cache continuation, not suspended process-memory continuation. The remaining Phase 6 work is to prune and package the promised Clang/LLD/musl distribution, decide whether subprocess semantics beyond `execve` are required by its trace, and drive a source-to-runnable-ELF compile through completion. +The promised distribution now has a reproducible, checksum-pinned producer for LLVM 22.1.8 and musl 1.2.6. It cross-builds 52 MB Clang and 32 MB LLD AArch64 executables as static musl ELFs, packages Clang resource headers plus a 3.7 MB musl sysroot and license notices, and keeps every source, cache, build, and output under the repository's ignored `.tmp` directory. `clang --version` completes inside the native runtime after adding single-threaded `CLREX` handling. A ten-million-instruction `clang -cc1` probe reads `stdio.h` from the packaged sysroot and reaches only the configured instruction boundary inside a hot `memset`, with its filesystem snapshot exported and no further semantic or syscall trap. The next checkpoint is the higher-budget translated Chromium compile, followed by a separate LLD run over the persisted object and execution of the resulting ELF; direct child-process orchestration and pipes remain subsequent Phase 6 work. + Do not begin the full web IDE before item 30 passes. --- diff --git a/toolchains/clang-musl/README.md b/toolchains/clang-musl/README.md new file mode 100644 index 0000000..e0e6e1d --- /dev/null +++ b/toolchains/clang-musl/README.md @@ -0,0 +1,34 @@ +# AArch64 Clang/LLD/musl toolchain + +This producer builds the Phase 6 browser toolchain as statically linked +AArch64 Linux executables and a Binarrow filesystem image. Generated sources, +build trees, caches, runtimes, and images stay under the repository's ignored +`.tmp/phase6-clang` directory. + +The pinned inputs are LLVM 22.1.8 and musl 1.2.6. Both downloads are verified +by SHA-256 before extraction. The image contains: + +- static AArch64 `clang` and `lld` executables; +- Clang's resource headers; +- musl headers, startup objects, and static libraries; and +- the LLVM and musl license notices. + +Prerequisites: + +- Zig 0.16.0; +- CMake and Ninja; +- the repository Rust toolchain; and +- `curl`, `make`, `tar`, and `shasum`. + +Build from any working directory: + +```sh +toolchains/clang-musl/build.sh +``` + +The build emits `clang-musl.bnfs`, `clang-project.bnfs`, and standalone Clang +and LLD upload executables. Clang and LLD are deliberately invoked as two +guest runs during this checkpoint: `clang -cc1` writes an object into the +persisted project filesystem, then LLD consumes that object and writes a +static ELF. This exercises the existing filesystem checkpoint boundary while +child-process and pipe support are still in progress. diff --git a/toolchains/clang-musl/aarch64-linux-musl.cmake b/toolchains/clang-musl/aarch64-linux-musl.cmake new file mode 100644 index 0000000..3655916 --- /dev/null +++ b/toolchains/clang-musl/aarch64-linux-musl.cmake @@ -0,0 +1,36 @@ +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_PROCESSOR aarch64) + +if(NOT DEFINED ENV{BINARROW_ZIG}) + message(FATAL_ERROR "BINARROW_ZIG must name the pinned Zig executable") +endif() +if(NOT DEFINED ENV{BINARROW_ZIG_AR}) + message(FATAL_ERROR "BINARROW_ZIG_AR must name the Zig ar wrapper") +endif() +if(NOT DEFINED ENV{BINARROW_ZIG_RANLIB}) + message(FATAL_ERROR "BINARROW_ZIG_RANLIB must name the Zig ranlib wrapper") +endif() + +set(CMAKE_C_COMPILER "$ENV{BINARROW_ZIG}") +set(CMAKE_C_COMPILER_ARG1 cc) +set(CMAKE_CXX_COMPILER "$ENV{BINARROW_ZIG}") +set(CMAKE_CXX_COMPILER_ARG1 c++) +set(CMAKE_ASM_COMPILER "$ENV{BINARROW_ZIG}") +set(CMAKE_ASM_COMPILER_ARG1 cc) + +set(CMAKE_AR "$ENV{BINARROW_ZIG_AR}") +set(CMAKE_RANLIB "$ENV{BINARROW_ZIG_RANLIB}") +set(CMAKE_C_COMPILER_AR "$ENV{BINARROW_ZIG_AR}") +set(CMAKE_C_COMPILER_RANLIB "$ENV{BINARROW_ZIG_RANLIB}") +set(CMAKE_CXX_COMPILER_AR "$ENV{BINARROW_ZIG_AR}") +set(CMAKE_CXX_COMPILER_RANLIB "$ENV{BINARROW_ZIG_RANLIB}") + +set(CMAKE_C_FLAGS_INIT "-target aarch64-linux-musl -fno-sanitize=undefined") +set(CMAKE_CXX_FLAGS_INIT "-target aarch64-linux-musl -fno-sanitize=undefined") +set(CMAKE_ASM_FLAGS_INIT "-target aarch64-linux-musl -fno-sanitize=undefined") +set(CMAKE_EXE_LINKER_FLAGS_INIT "-target aarch64-linux-musl -static") + +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) diff --git a/toolchains/clang-musl/build.sh b/toolchains/clang-musl/build.sh new file mode 100755 index 0000000..e99804b --- /dev/null +++ b/toolchains/clang-musl/build.sh @@ -0,0 +1,188 @@ +#!/bin/sh +set -eu + +toolchain_directory=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +workspace_directory=$(CDPATH= cd -- "$toolchain_directory/../.." && pwd) +work_directory=$workspace_directory/.tmp/phase6-clang +llvm_version=22.1.8 +musl_version=1.2.6 +llvm_archive=$work_directory/llvm-project-$llvm_version.src.tar.xz +llvm_source=$work_directory/llvm-project-$llvm_version.src +llvm_build=$work_directory/build +musl_directory=$work_directory/musl +musl_archive=$musl_directory/musl-$musl_version.tar.gz +musl_source=$musl_directory/source +musl_build=$musl_directory/build +musl_sysroot=$musl_directory/sysroot +package_directory=$work_directory/package/toolchain +toolchain_image=$work_directory/clang-musl.bnfs +project_image=$work_directory/clang-project.bnfs +llvm_url=https://github.com/llvm/llvm-project/releases/download/llvmorg-$llvm_version/llvm-project-$llvm_version.src.tar.xz +llvm_sha256=922f1817a0df7b1489272d18134ee0087a8b068828f87ac63b9861b1a9965888 +musl_url=https://musl.libc.org/releases/musl-$musl_version.tar.gz +musl_sha256=d585fd3b613c66151fc3249e8ed44f77020cb5e6c1e635a616d3f9f82460512a +jobs=${JOBS:-4} + +zig=$(command -v zig || true) +cmake=$(command -v cmake || true) +ninja=$(command -v ninja || true) +if [ -z "$zig" ] || [ -z "$cmake" ] || [ -z "$ninja" ]; then + echo "Clang toolchain build requires Zig, CMake, and Ninja" >&2 + exit 1 +fi +zig_version=$($zig version) +if [ "$zig_version" != "0.16.0" ]; then + echo "Clang toolchain build requires Zig 0.16.0; found $zig_version" >&2 + exit 1 +fi + +mkdir -p \ + "$workspace_directory/.tmp/rust-tmp" \ + "$workspace_directory/.tmp/cargo-home" \ + "$workspace_directory/.tmp/cargo-target" \ + "$work_directory/zig-global-cache" \ + "$work_directory/zig-local-cache" \ + "$musl_directory" +export TMPDIR=$workspace_directory/.tmp/rust-tmp +export CARGO_HOME=$workspace_directory/.tmp/cargo-home +export CARGO_TARGET_DIR=$workspace_directory/.tmp/cargo-target +export ZIG_GLOBAL_CACHE_DIR=$work_directory/zig-global-cache +export ZIG_LOCAL_CACHE_DIR=$work_directory/zig-local-cache +export BINARROW_ZIG=$zig +export BINARROW_ZIG_AR=$toolchain_directory/zig-ar.sh +export BINARROW_ZIG_RANLIB=$toolchain_directory/zig-ranlib.sh + +verify_archive() ( + archive=$1 + expected_sha256=$2 + label=$3 + actual_sha256=$(shasum -a 256 "$archive" | awk '{print $1}') + if [ "$actual_sha256" != "$expected_sha256" ]; then + echo "unexpected $label archive checksum: $actual_sha256" >&2 + return 1 + fi +) + +download_archive() ( + destination=$1 + url=$2 + expected_sha256=$3 + label=$4 + if [ ! -f "$destination" ]; then + partial_archive=$destination.part + curl --fail --location --retry 3 --continue-at - \ + --output "$partial_archive" "$url" + verify_archive "$partial_archive" "$expected_sha256" "$label" + mv "$partial_archive" "$destination" + else + verify_archive "$destination" "$expected_sha256" "$label" + fi +) + +download_archive "$llvm_archive" "$llvm_url" "$llvm_sha256" LLVM +if [ ! -f "$llvm_source/llvm/CMakeLists.txt" ]; then + mkdir -p "$llvm_source" + tar -xJf "$llvm_archive" --strip-components=1 -C "$llvm_source" +fi + +"$cmake" -G Ninja \ + -S "$llvm_source/llvm" \ + -B "$llvm_build" \ + -DCMAKE_TOOLCHAIN_FILE="$toolchain_directory/aarch64-linux-musl.cmake" \ + -DCMAKE_BUILD_TYPE=MinSizeRel \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DCMAKE_AR="$BINARROW_ZIG_AR" \ + -DCMAKE_RANLIB="$BINARROW_ZIG_RANLIB" \ + -DCMAKE_C_COMPILER_AR="$BINARROW_ZIG_AR" \ + -DCMAKE_C_COMPILER_RANLIB="$BINARROW_ZIG_RANLIB" \ + -DCMAKE_CXX_COMPILER_AR="$BINARROW_ZIG_AR" \ + -DCMAKE_CXX_COMPILER_RANLIB="$BINARROW_ZIG_RANLIB" \ + -DCMAKE_C_FLAGS="-target aarch64-linux-musl -fno-sanitize=undefined" \ + -DCMAKE_CXX_FLAGS="-target aarch64-linux-musl -fno-sanitize=undefined" \ + -DCMAKE_ASM_FLAGS="-target aarch64-linux-musl -fno-sanitize=undefined" \ + -DCMAKE_EXE_LINKER_FLAGS="-target aarch64-linux-musl -static" \ + -DLLVM_ENABLE_PROJECTS="clang;lld" \ + -DLLVM_TARGETS_TO_BUILD=AArch64 \ + -DLLVM_HOST_TRIPLE=aarch64-linux-musl \ + -DLLVM_DEFAULT_TARGET_TRIPLE=aarch64-linux-musl \ + -DLLVM_ENABLE_ASSERTIONS=OFF \ + -DLLVM_ENABLE_THREADS=OFF \ + -DLLVM_ENABLE_ZLIB=OFF \ + -DLLVM_ENABLE_ZSTD=OFF \ + -DLLVM_ENABLE_LIBXML2=OFF \ + -DLLVM_ENABLE_CURL=OFF \ + -DLLVM_ENABLE_TERMINFO=OFF \ + -DLLVM_ENABLE_LIBEDIT=OFF \ + -DLLVM_ENABLE_FFI=OFF \ + -DLLVM_ENABLE_LIBPFM=OFF \ + -DLLVM_ENABLE_BINDINGS=OFF \ + -DLLVM_ENABLE_BACKTRACES=OFF \ + -DLLVM_ENABLE_CRASH_OVERRIDES=OFF \ + -DLLVM_ENABLE_UNWIND_TABLES=OFF \ + -DLLVM_INCLUDE_TESTS=OFF \ + -DLLVM_INCLUDE_BENCHMARKS=OFF \ + -DLLVM_INCLUDE_EXAMPLES=OFF \ + -DLLVM_INCLUDE_DOCS=OFF \ + -DCLANG_INCLUDE_TESTS=OFF \ + -DCLANG_INCLUDE_DOCS=OFF \ + -DCLANG_ENABLE_STATIC_ANALYZER=OFF \ + -DCLANG_ENABLE_ARCMT=OFF \ + -DLLD_INCLUDE_TESTS=OFF \ + -DLLVM_INSTALL_TOOLCHAIN_ONLY=ON \ + -DLLVM_PARALLEL_LINK_JOBS=1 +"$cmake" --build "$llvm_build" --target clang lld -- -j"$jobs" + +download_archive "$musl_archive" "$musl_url" "$musl_sha256" musl +if [ ! -f "$musl_source/configure" ]; then + mkdir -p "$musl_source" + tar -xzf "$musl_archive" --strip-components=1 -C "$musl_source" +fi +mkdir -p "$musl_build" "$musl_sysroot" +if [ ! -f "$musl_build/config.mak" ]; then + ( + cd "$musl_build" + CC="$zig cc -target aarch64-linux-musl -fno-sanitize=undefined" \ + AR="$zig ar" \ + RANLIB="$zig ranlib" \ + "$musl_source/configure" \ + --target=aarch64-linux-musl \ + --prefix=/usr \ + --syslibdir=/lib \ + --disable-shared \ + CFLAGS=-Os + ) +fi +make -C "$musl_build" -j"$jobs" +make -C "$musl_build" DESTDIR="$musl_sysroot" install + +mkdir -p \ + "$package_directory/bin" \ + "$package_directory/lib/clang/22" \ + "$package_directory/share/licenses/llvm" \ + "$package_directory/share/licenses/musl" \ + "$package_directory/sysroot" +cp "$llvm_build/bin/clang-22" "$package_directory/bin/clang" +cp "$llvm_build/bin/lld" "$package_directory/bin/lld" +cp -R "$llvm_build/lib/clang/22/include" "$package_directory/lib/clang/22/" +cp -R "$musl_sysroot/usr" "$package_directory/sysroot/" +cp "$llvm_source/LICENSE.TXT" "$package_directory/share/licenses/llvm/LICENSE.TXT" +cp "$musl_source/COPYRIGHT" "$package_directory/share/licenses/musl/COPYRIGHT" + +env CARGO_NET_OFFLINE=true cargo build \ + --manifest-path "$workspace_directory/Cargo.toml" \ + --release \ + -p binarrow-cli +runner=$CARGO_TARGET_DIR/release/binarrow +"$runner" image pack \ + --guest-root /project/toolchain \ + "$package_directory" \ + "$toolchain_image" +"$runner" image pack \ + --guest-root /project \ + "$toolchain_directory/project" \ + "$project_image" + +echo "Clang runtime: $package_directory/bin/clang" +echo "LLD runtime: $package_directory/bin/lld" +echo "Toolchain image: $toolchain_image" +echo "Project image: $project_image" diff --git a/toolchains/clang-musl/project/main.c b/toolchains/clang-musl/project/main.c new file mode 100644 index 0000000..9733234 --- /dev/null +++ b/toolchains/clang-musl/project/main.c @@ -0,0 +1,6 @@ +#include + +int main(void) { + puts("browser-local clang"); + return 0; +} diff --git a/toolchains/clang-musl/zig-ar.sh b/toolchains/clang-musl/zig-ar.sh new file mode 100755 index 0000000..406a4b5 --- /dev/null +++ b/toolchains/clang-musl/zig-ar.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu + +exec "$BINARROW_ZIG" ar "$@" diff --git a/toolchains/clang-musl/zig-ranlib.sh b/toolchains/clang-musl/zig-ranlib.sh new file mode 100755 index 0000000..705cc1b --- /dev/null +++ b/toolchains/clang-musl/zig-ranlib.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu + +exec "$BINARROW_ZIG" ranlib "$@" -- 2.51.2