diff --git a/oven/native-builder.mjs b/oven/native-builder.mjs --- a/oven/native-builder.mjs +++ b/oven/native-builder.mjs @@ -175,39 +175,56 @@ } async function runBuildJob(job) { try { - await setupBuildCache(); - job.status = "running"; job.startedAt = nowISO(); job.percent = 0; - // Phase 1: build vmlinuz (no --flash) - const buildScript = path.join(NATIVE_DIR, "scripts/build-and-flash.sh"); - await runPhase(job, "build", "bash", [buildScript, ...job.flags], NATIVE_DIR); + const repoDir = path.resolve(NATIVE_DIR, "../.."); + const buildName = `oven-${job.ref.slice(0, 7)}`; + const vmlinuzOut = `/tmp/oven-vmlinuz-${job.id}`; + + // Phase 1: Docker image build (cached layers = fast) + addLogLine(job, "stdout", "Phase 1: Building Docker image..."); + await runPhase(job, "docker-build", "docker", [ + "build", "-t", "ac-os-builder", + "-f", path.join(repoDir, "fedac/native/Dockerfile.builder"), + repoDir, + ], repoDir); + + job.percent = 30; - job.percent = 80; + // Phase 2: Docker run → compile binary + initramfs + kernel + addLogLine(job, "stdout", "Phase 2: Compiling kernel in Docker..."); + const cidFile = `/tmp/oven-cid-${job.id}`; + await runPhase(job, "build", "bash", ["-c", [ + `CID=$(docker create -e AC_BUILD_NAME=${buildName} ac-os-builder)`, + `echo $CID > ${cidFile}`, + `docker start -a $CID`, + ].join(" && ")], repoDir); + + job.percent = 75; - // Phase 2: QEMU smoke test (boot kernel, check serial for success/panic) - const acOs = path.join(NATIVE_DIR, "ac-os"); - try { - await runPhase(job, "smoke-test", "bash", [acOs, "test"], NATIVE_DIR); - addLogLine(job, "stdout", " SMOKE TEST: passed"); - } catch (smokeErr) { - addLogLine(job, "stderr", ` SMOKE TEST: failed — ${smokeErr.message}`); - // Non-fatal for now — log warning but continue upload - // TODO: make this fatal once QEMU + virtio-gpu is reliable - } + // Phase 3: Extract vmlinuz from container + addLogLine(job, "stdout", "Phase 3: Extracting kernel..."); + const cid = (await fs.readFile(cidFile, "utf8")).trim(); + await runPhase(job, "extract", "bash", ["-c", + `docker cp ${cid}:/tmp/ac-build/vmlinuz ${vmlinuzOut} && docker rm ${cid} >/dev/null` + ], repoDir); - job.percent = 85; + job.percent = 80; - // Phase 3: upload vmlinuz to DO Spaces CDN - const vmlinuz = path.join(CACHE_DIR, "vmlinuz"); + // Phase 4: Upload vmlinuz to DO Spaces CDN + addLogLine(job, "stdout", "Phase 4: Uploading to CDN..."); const uploadScript = path.join(NATIVE_DIR, "scripts/upload-release.sh"); - await runPhase(job, "upload", "bash", [uploadScript, vmlinuz], NATIVE_DIR, { + await runPhase(job, "upload", "bash", [uploadScript, vmlinuzOut], NATIVE_DIR, { DO_SPACES_KEY: process.env.DO_SPACES_KEY || process.env.ART_SPACES_KEY || "", DO_SPACES_SECRET: process.env.DO_SPACES_SECRET || process.env.ART_SPACES_SECRET || "", }); + + // Cleanup + try { await fs.unlink(vmlinuzOut); } catch {} + try { await fs.unlink(cidFile); } catch {} job.status = "success"; job.stage = "done"; diff --git a/plans/docker-ota-build-pipeline.md b/plans/docker-ota-build-pipeline.md new file mode 100644 --- /dev/null +++ b/plans/docker-ota-build-pipeline.md @@ -0,0 +1,85 @@ +# AC Native OS — Docker-Based OTA Build Pipeline + +## Goal +One build process everywhere: local devcontainer, oven server, any CI/CD. +Produces identical, reproducible vmlinuz kernels with no user secrets. + +## Architecture + +``` +Source (git) → Docker Build → vmlinuz (generic, no creds) + ↓ + ┌───────────┴───────────┐ + ↓ ↓ + ac-os upload ac-os flash + (OTA to CDN) (USB + user creds) +``` + +### Build (same everywhere) +```bash +docker build -t ac-os-builder -f fedac/native/Dockerfile.builder . +docker create -e AC_BUILD_NAME= ac-os-builder +docker start -a +docker cp :/tmp/ac-build/vmlinuz ./vmlinuz +``` + +### Upload (OTA — no creds in kernel) +```bash +ac-os upload vmlinuz # Signs + pushes to DO Spaces CDN +``` + +### Flash (local — adds user creds) +```bash +ac-os pull # Downloads OTA vmlinuz from CDN +ac-os flash # Writes vmlinuz + config.json (creds) to USB +``` + +## Transition Plan + +### Phase 1: Unify `ac-os build` to use Docker +- [ ] `ac-os build` detects Docker, runs `docker build + docker run` +- [ ] Falls back to native build if no Docker (e.g. direct on host) +- [ ] `ac-os build --lisp` passes `AC_BUILD_LISP=1` to Docker +- [ ] Remove `build-and-flash.sh` dependency (keep as legacy fallback) +- [ ] `ac-os flash` only flashes — never builds (downloads OTA or uses local vmlinuz) + +### Phase 2: Oven adopts Docker builds +- [ ] Update `native-builder.mjs` to use Docker instead of `build-and-flash.sh` +- [ ] Oven poller: `git pull` → `docker build` → `docker run` → `upload-release.sh` +- [ ] Remove native-cache symlinks, musl-gcc, Ubuntu package installs +- [ ] Oven only needs: Docker, git, curl (for upload) +- [ ] Deploy updated `native-builder.mjs` to oven + +### Phase 3: API + UX endpoints +- [ ] `GET /native-build` — add `dockerImage`, `dockerCached` fields to status +- [ ] `GET /native-build/:id/logs` — stream Docker build logs +- [ ] VSCode extension status bar — show build stage from Docker output +- [ ] `ac-os build --upload` — local build + automatic OTA upload +- [ ] `ac-os status` — check oven build status from CLI + +### Phase 4: Signing + verification +- [ ] `ac-os upload` signs vmlinuz with ed25519 key +- [ ] `ac-os pull` verifies signature before flashing +- [ ] OTA releases include `.sig` file on CDN +- [ ] Device verifies signature before applying OTA update + +### Phase 5: CL variant +- [ ] `ac-os build --lisp` produces CL-based vmlinuz +- [ ] Same Docker image, different entrypoint flag +- [ ] CL kernel uploaded to separate CDN path (`os/native-cl-latest.vmlinuz`) +- [ ] `ac-os pull --lisp` downloads CL variant + +## Key Principles +1. **No secrets in builds** — OTA kernels are generic, creds injected at flash +2. **Reproducible** — same Docker image = same vmlinuz, any machine +3. **One process** — local and oven use identical Docker commands +4. **Fast rebuilds** — Docker layer cache: only recompile what changed +5. **Wide compat** — Fedora 43 base, GCC 15, Linux 6.19.9, Intel i915 + simpledrm + +## Files to modify +- `fedac/native/ac-os` — unify build/flash/upload around Docker +- `oven/native-builder.mjs` — switch to Docker builds +- `oven/native-git-poller.mjs` — simplify (just detect new commits) +- `fedac/native/Dockerfile.builder` — already working +- `fedac/native/docker-build.sh` — already working +- `vscode-extension/extension.ts` — OTA status bar updates