diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml deleted file mode 100644 index 23c895e00..000000000 --- a/.github/workflows/nix.yml +++ /dev/null @@ -1,207 +0,0 @@ -# Nix CI for darling-nix -# -# This workflow builds the Darling package with Nix, runs flake checks, -# and executes integration tests. It uses Cachix to cache build artifacts -# so that subsequent runs (and contributor builds) are fast. -# -# See: plan/08-phase6-ci.md (Task 6.3) - -name: Nix CI - -on: - push: - branches: [master, main] - paths-ignore: - - "**.md" - - "plan/**" - - "LICENSE" - - ".github/ISSUE_TEMPLATE/**" - - ".github/FUNDING.yml" - pull_request: - paths-ignore: - - "**.md" - - "plan/**" - - "LICENSE" - - ".github/ISSUE_TEMPLATE/**" - - ".github/FUNDING.yml" - -concurrency: - group: nix-${{ github.ref }} - cancel-in-progress: true - -jobs: - # ── Flake check ─────────────────────────────────────────────────────────── - # Fast: evaluates the flake, checks formatting, runs lightweight checks. - flake-check: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Install Nix - uses: cachix/install-nix-action@v27 - with: - extra_nix_config: | - accept-flake-config = true - experimental-features = nix-command flakes - - - name: Set up Cachix - uses: cachix/cachix-action@v15 - with: - name: darling-nix - authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} - continue-on-error: true # Don't fail if Cachix isn't configured - - - name: nix flake check (eval only) - run: nix flake check --no-build --all-systems 2>&1 - - # ── Build packages ──────────────────────────────────────────────────────── - # Builds the main Darling package and the SDK output. - build: - runs-on: ubuntu-latest - needs: flake-check - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Free up disk space - run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc - df -h / - - - name: Install Nix - uses: cachix/install-nix-action@v27 - with: - extra_nix_config: | - accept-flake-config = true - experimental-features = nix-command flakes - - - name: Set up Cachix - uses: cachix/cachix-action@v15 - with: - name: darling-nix - authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} - continue-on-error: true - - - name: Build Darling - run: nix build .#darling -L --no-link --print-out-paths - - - name: Build Darling SDK - run: nix build .#darling-sdk -L --no-link --print-out-paths - - # ── DevShell evaluation ─────────────────────────────────────────────────── - # Ensures the devShell evaluates without error so contributors can always - # `nix develop`. This is cheap (eval-only, no build). - devshell: - runs-on: ubuntu-latest - needs: flake-check - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Install Nix - uses: cachix/install-nix-action@v27 - with: - extra_nix_config: | - accept-flake-config = true - experimental-features = nix-command flakes - - - name: Set up Cachix - uses: cachix/cachix-action@v15 - with: - name: darling-nix - authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} - continue-on-error: true - - - name: Evaluate devShell - run: nix eval .#devShells.x86_64-linux.default.name 2>&1 - - - name: Build devShell - run: nix build .#devShells.x86_64-linux.default -L --no-link --print-out-paths - - # ── Smoke test ──────────────────────────────────────────────────────────── - # If the build succeeds, run a quick smoke test to verify Darling starts - # and the sandbox-exec stub is present. - smoke-test: - runs-on: ubuntu-latest - needs: build - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Free up disk space - run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc - df -h / - - - name: Install Nix - uses: cachix/install-nix-action@v27 - with: - extra_nix_config: | - accept-flake-config = true - experimental-features = nix-command flakes - - - name: Set up Cachix - uses: cachix/cachix-action@v15 - with: - name: darling-nix - authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} - continue-on-error: true - - - name: Build Darling - id: build - run: | - out=$(nix build .#darling -L --no-link --print-out-paths) - echo "darling=$out" >> "$GITHUB_OUTPUT" - - - name: Verify Darling binary exists - run: | - test -x "${{ steps.build.outputs.darling }}/bin/darling" - echo "✓ darling binary found" - - - name: Check sandbox-exec stub is installed - run: | - test -f "${{ steps.build.outputs.darling }}/libexec/darling/usr/bin/sandbox-exec" - echo "✓ sandbox-exec stub found" - - - name: Check diskutil supports info verb - run: | - grep -q 'info' "${{ steps.build.outputs.darling }}/libexec/darling/usr/sbin/diskutil" - echo "✓ diskutil info verb found" - - # The following tests require actually running Darling, which needs - # user namespaces and overlayfs — these may not be available in all - # GitHub Actions runners. We attempt them but allow failure. - - name: Test darling shell (may need user namespaces) - run: | - timeout 30 "${{ steps.build.outputs.darling }}/bin/darling" shell echo "Hello from Darling" || { - echo "::warning::darling shell failed — runner may lack user namespace support" - exit 0 - } - continue-on-error: true - - - name: Test sandbox-exec stub inside Darling - run: | - timeout 30 "${{ steps.build.outputs.darling }}/bin/darling" shell \ - /usr/bin/sandbox-exec -f /dev/null -D _GLOBAL_TMP_DIR=/tmp /bin/echo "sandbox-exec works" || { - echo "::warning::sandbox-exec test failed — darling shell may not be functional on this runner" - exit 0 - } - continue-on-error: true - - - name: Test diskutil info inside Darling - run: | - timeout 30 "${{ steps.build.outputs.darling }}/bin/darling" shell \ - /usr/sbin/diskutil info / || { - echo "::warning::diskutil info test failed — darling shell may not be functional on this runner" - exit 0 - } - continue-on-error: true diff --git a/.tangled/workflows/ci.yml b/.tangled/workflows/ci.yml new file mode 100644 index 000000000..d43249708 --- /dev/null +++ b/.tangled/workflows/ci.yml @@ -0,0 +1,31 @@ +when: + - event: ["push", "pull_request"] + branch: main + +engine: nixery + +environment: + USER: root + CACHIX_NAME: darling-nix + +steps: + - name: "Setup Cachix" + command: | + nix-env -iA cachix -f https://cachix.org/api/v1/install + mkdir -p /tangled/home/.config/nix + echo -e "experimental-features = nix-command flakes\nmax-jobs = auto" > /tangled/home/.config/nix/nix.conf + cachix use $CACHIX_NAME + + - name: "Nix flake check" + command: | + rm -rf /homeless-shelter + ulimit -n 65536 + cachix watch-exec $CACHIX_NAME -- nix flake check --max-jobs 1 + + - name: "Build Darling" + command: | + cachix watch-exec $CACHIX_NAME -- nix build .#darling -L --no-link --print-out-paths + + - name: "Build Darling SDK" + command: | + cachix watch-exec $CACHIX_NAME -- nix build .#darling-sdk -L --no-link --print-out-paths diff --git a/PLAN.md b/PLAN.md index d3752d7ad..4ce8c7451 100644 --- a/PLAN.md +++ b/PLAN.md @@ -18,7 +18,7 @@ See the **[plan/](./plan/)** directory for all details. | Phase 3 — Nix Install | 🚧 In progress | `scripts/install-nix-in-darling.sh`, `scripts/darling-nix`, `scripts/verify-nix.sh` | | Phase 4 — Building | 🚧 Tooling ready | `scripts/build-trivial.sh` (new) | | Phase 5 — Daemon | 🚧 Stubs done | `src/dirserv/` (new), `tests/dirserv/` (new) | -| Phase 6 — CI | 🚧 In progress | `.github/workflows/nix.yml`, `tests/darling-smoke.nix`, `tests/nix-in-darling.nix` (new) | +| Phase 6 — CI | 🚧 In progress | `.tangled/workflows/ci.yml`, `tests/darling-smoke.nix`, `tests/nix-in-darling.nix` (new) | | Phase 7 — Remote Builder | 📋 Planned | — | | Phase 8 — Stretch | 📋 Planned | — | @@ -130,8 +130,8 @@ See the **[plan/](./plan/)** directory for all details. a Darling prefix in single-user mode. - **Phase 3.4**: Created `scripts/darling-nix` — host-side wrapper for running Nix commands inside Darling without manual `darling shell bash -lc` boilerplate. -- **Phase 6.3**: Created `.github/workflows/nix.yml` — Nix CI workflow with - flake check, package build, devShell evaluation, and smoke tests. +- **Phase 6.3**: Created `.tangled/workflows/ci.yml` — tangled.org CI workflow + with Cachix caching and `nix flake check`. - **Phase 1.7**: Created `plan/syscall-triage.md` — tracking table for unimplemented syscalls with categories, impact levels, and discovery log. - **Testing**: Created `tests/sandbox/test_sandbox_api.c` (C-level sandbox API @@ -151,7 +151,7 @@ See the **[plan/](./plan/)** directory for all details. | [plan/05-phase3-nix-install.md](./plan/05-phase3-nix-install.md) | Automated installer, verification, wrappers | | [plan/06-phase4-building.md](./plan/06-phase4-building.md) | Trivial derivations → stdenv → binary substitution | | [plan/07-phase5-daemon.md](./plan/07-phase5-daemon.md) | Multi-user mode, Directory Services stubs, launchd | -| [plan/08-phase6-ci.md](./plan/08-phase6-ci.md) | NixOS VM tests, regression suite, GitHub Actions | +| [plan/08-phase6-ci.md](./plan/08-phase6-ci.md) | NixOS VM tests, regression suite, tangled.org CI | | [plan/09-phase7-remote-builder.md](./plan/09-phase7-remote-builder.md) | Darling as a `nix.buildMachines` target | | [plan/10-phase8-stretch.md](./plan/10-phase8-stretch.md) | `aarch64-darwin`, GUI testing, Hydra builder | | [plan/11-architecture.md](./plan/11-architecture.md) | System diagram, key technical decisions, glossary | @@ -163,7 +163,7 @@ Files created or modified as part of this plan: ```text darling-nix/ -├── .github/workflows/nix.yml # Nix CI workflow (Phase 6) +├── .tangled/workflows/ci.yml # tangled.org CI workflow (Phase 6) ├── flake.nix # Flake with package, devShell, NixOS module (Phase 0) ├── nix/ │ ├── package.nix # Darling Nix derivation (Phase 0) @@ -309,7 +309,7 @@ The **critical path to MVP** (Nix running inside Darling) is: | 5.1 | ✅ | Directory Services stubs (`dseditgroup`, `sysadminctl`, `dscl`) | | 6.1 | ✅ | NixOS VM test (`tests/nix-in-darling.nix`) | | 6.2 | ✅ | Wired tests into `flake.nix` (checks output) | -| 6.3 | ✅ | `.github/workflows/nix.yml` CI workflow | +| 6.3 | ✅ | `.tangled/workflows/ci.yml` tangled.org CI workflow | | 6.6 | ✅ | Darling smoke test (`tests/darling-smoke.nix`) | | — | ✅ | `run-tests.sh` unified test runner (6 suites) | | — | ✅ | `getattrlist` attribute buffer ordering bug fixed | diff --git a/plan/08-phase6-ci.md b/plan/08-phase6-ci.md index fc0889771..453a80e36 100644 --- a/plan/08-phase6-ci.md +++ b/plan/08-phase6-ci.md @@ -23,8 +23,10 @@ It does not: - Verify Nix compatibility. - Test inside a NixOS VM (which is needed for namespace/overlay support). -We need to replace or supplement this with a Nix-native CI pipeline that runs -real integration tests. +We use [tangled.org](https://tangled.org) CI instead of GitHub Actions. Tangled +provides a `nixery` engine that gives us a Nix-enabled container out of the box, +eliminating the need for `install-nix-action` and simplifying the workflow. The +workflow is defined in `.tangled/workflows/ci.yml`. --- @@ -146,103 +148,62 @@ nix build .#checks.x86_64-linux.nix-in-darling --- -### 6.3 — GitHub Actions Workflow ✅ +### 6.3 — tangled.org CI Workflow ✅ -Replace or supplement the existing `.github/workflows/actions.yaml` with a -Nix-native workflow. +Replace the GitHub Actions workflow with a tangled.org CI workflow using the +`nixery` engine, which provides Nix out of the box. -**Workflow file**: `.github/workflows/nix-ci.yaml` +**Workflow file**: `.tangled/workflows/ci.yml` ```yaml -name: Nix CI - -on: - push: - branches: [main] - pull_request: - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - - uses: cachix/install-nix-action@v27 - with: - extra_nix_config: | - experimental-features = nix-command flakes - - - uses: cachix/cachix-action@v15 - with: - name: darling-nix # our Cachix cache - authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - - - name: Build Darling - run: nix build .#darling -L - - - name: Build Darling SDK - run: nix build .#darling-sdk -L - - test-syscalls: - runs-on: ubuntu-latest - needs: build - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - - uses: cachix/install-nix-action@v27 - with: - extra_nix_config: | - experimental-features = nix-command flakes - - - uses: cachix/cachix-action@v15 - with: - name: darling-nix - - - name: Run syscall regression tests - run: nix build .#checks.x86_64-linux.syscall-regression -L - - test-nix-integration: - runs-on: ubuntu-latest - needs: build - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - - uses: cachix/install-nix-action@v27 - with: - extra_nix_config: | - experimental-features = nix-command flakes - system-features = kvm - - - uses: cachix/cachix-action@v15 - with: - name: darling-nix - - - name: Run Nix-in-Darling integration test - run: nix build .#checks.x86_64-linux.nix-in-darling -L - timeout-minutes: 60 # generous timeout for VM test +when: + - event: ["push", "pull_request"] + branch: main + +engine: nixery + +environment: + USER: root + CACHIX_NAME: darling-nix + +steps: + - name: "Setup Cachix" + command: | + nix-env -iA cachix -f https://cachix.org/api/v1/install + mkdir -p /tangled/home/.config/nix + echo -e "experimental-features = nix-command flakes\nmax-jobs = auto" > /tangled/home/.config/nix/nix.conf + cachix use $CACHIX_NAME + + - name: "Nix flake check" + command: | + rm -rf /homeless-shelter + ulimit -n 65536 + cachix watch-exec $CACHIX_NAME -- nix flake check --max-jobs 1 + + - name: "Build Darling" + command: | + cachix watch-exec $CACHIX_NAME -- nix build .#darling -L --no-link --print-out-paths + + - name: "Build Darling SDK" + command: | + cachix watch-exec $CACHIX_NAME -- nix build .#darling-sdk -L --no-link --print-out-paths ``` **Notes**: -- The integration test requires KVM for the NixOS VM. GitHub's `ubuntu-latest` - runners have KVM available. Verify with `system-features = kvm` in the Nix - config. -- The build job runs first and pushes artifacts to Cachix. Subsequent test jobs - pull from the cache, avoiding redundant rebuilds. -- The `timeout-minutes: 60` is important — Darling operations inside a VM inside - CI can be very slow. Adjust as needed based on real-world timings. -- `submodules: recursive` is required because Darling has 100+ submodules. This - checkout step may itself take 5–10 minutes. - -**Alternative: use a self-hosted runner** if GitHub's runners are too slow or -lack KVM. A dedicated NixOS machine with nested virtualisation enabled would -provide the most reliable CI environment. +- tangled.org's `nixery` engine provides a Nix-enabled container, so there is + no need for `install-nix-action` or checkout actions — the repo is already + cloned and Nix is pre-installed. +- Cachix is installed at runtime and used via `cachix watch-exec` to + automatically push all build artifacts. Subsequent runs pull from the cache, + avoiding redundant rebuilds. +- `nix flake check` runs all flake checks (build smoke test, dirserv stubs, + etc.) in a single step. The `--max-jobs 1` flag prevents OOM on + memory-constrained CI runners. +- The `CACHIX_NAME` environment variable should match the Cachix cache name. + The Cachix auth token must be configured as a tangled.org secret. +- `rm -rf /homeless-shelter` works around a Nix sandbox issue in containerised + environments where `HOME` is set to a nonexistent path. --- @@ -593,13 +554,9 @@ NixOS VM tests are slow. Strategies to keep CI times reasonable: 3. **Incremental testing**: On PRs that only touch `plan/` or `docs/`, skip the expensive VM tests. Use path filters in the workflow: - ```yaml - on: - push: - paths-ignore: - - 'plan/**' - - '*.md' - ``` + In the tangled workflow, this can be handled at the application level by + checking changed paths in early steps, or by relying on Cachix cache hits + to make unchanged builds near-instant. 4. **Test VM snapshots**: If the NixOS testing framework supports it, take a snapshot after Darling initialization and restore from it for each test. This @@ -623,12 +580,12 @@ NixOS VM tests are slow. Strategies to keep CI times reasonable: After completing Phase 6, ALL of the following should be true: - [ ] `nix flake check` passes (includes build smoke test) -- [ ] `.github/workflows/nix-ci.yaml` exists and runs on PRs +- [ ] `.tangled/workflows/ci.yml` exists and runs on pushes/PRs to `main` - [ ] Syscall regression tests exist for `lchflags`, `renameatx_np`, `utimensat` (at minimum) - [ ] Sandbox stub tests verify `sandbox-exec` passthrough works - [ ] NixOS VM test installs Nix inside Darling and evaluates an expression - [ ] NixOS VM test builds a trivial derivation inside Darling -- [ ] CI results are visible on GitHub PR checks +- [ ] CI results are visible on tangled.org - [ ] Cachix cache is populated by CI and speeds up subsequent runs - [ ] Compatibility matrix script exists and produces JSON output - [ ] Adding a new syscall implementation has a clear path: implement → add test → CI verifies