diff --git a/tests/systemd-test/Dockerfile b/tests/systemd-test/Dockerfile new file mode 100644 index 000000000..f077f84f0 --- /dev/null +++ b/tests/systemd-test/Dockerfile @@ -0,0 +1,81 @@ +# Debian-based image that runs systemd --user end-to-end so the solstone +# install-integration test can verify that `journal setup` actually starts the +# user service, not just that the unit file was written. See the playbook +# at vpe/playbooks/solstone-systemd-test.md for trade-offs and limits. + +FROM debian:bookworm + +ENV container=docker +ENV LC_ALL=C +ENV DEBIAN_FRONTEND=noninteractive + +# Full systemd (not systemd-sysv-only) + dbus-user-session is required for +# per-user systemd instances to run without a GUI login. python3 is the +# install target. curl + ca-certificates fetch uv. sudo is for any setup +# step that escalates (journal setup does not, but having it avoids surprises). +RUN apt-get update && apt-get install -y --no-install-recommends \ + systemd \ + systemd-sysv \ + dbus \ + dbus-user-session \ + libsystemd0 \ + python3 \ + python3-pip \ + python3-venv \ + curl \ + ca-certificates \ + sudo \ + procps \ + less \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Units that don't apply inside a container — masking keeps `systemctl +# is-system-running` from going into degraded state for irrelevant reasons. +RUN systemctl mask \ + sys-kernel-config.mount \ + sys-kernel-debug.mount \ + sys-kernel-tracing.mount \ + dev-hugepages.mount \ + dev-mqueue.mount \ + systemd-logind.service \ + getty.target \ + console-getty.service \ + systemd-firstboot.service \ + kmod-static-nodes.service \ + systemd-modules-load.service \ + systemd-udevd.service \ + systemd-udev-trigger.service + +ARG TEST_USER=solstone +ARG TEST_UID=1000 +RUN useradd -m -u ${TEST_UID} -s /bin/bash ${TEST_USER} \ + && echo "${TEST_USER} ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/${TEST_USER} + +# loginctl enable-linger needs to run AFTER the user instance manager exists, +# which means after PID 1 boot. Pre-touching the linger marker file is the +# documented equivalent that survives a fresh `docker run`. +RUN mkdir -p /var/lib/systemd/linger \ + && touch /var/lib/systemd/linger/${TEST_USER} + +# uv as the test user, so `uv tool install solstone-journal` is the same +# host-install path the install-integration tests are meant to exercise. +USER ${TEST_USER} +WORKDIR /home/${TEST_USER} +RUN curl -LsSf https://astral.sh/uv/install.sh | sh +ENV PATH="/home/solstone/.local/bin:${PATH}" + +# /run/user/ + XDG_RUNTIME_DIR are normally provisioned by pam_systemd +# on a real login. Inside the container the user-instance manager will set +# them up itself once linger is on, but the shell that we `docker exec` into +# does NOT inherit them automatically — we set XDG_RUNTIME_DIR in profile +# so `systemctl --user` works from any exec without re-deriving it. +RUN echo 'export XDG_RUNTIME_DIR="/run/user/$(id -u)"' >> /home/${TEST_USER}/.profile \ + && echo 'export XDG_RUNTIME_DIR="/run/user/$(id -u)"' >> /home/${TEST_USER}/.bashrc + +USER root + +# systemd's documented Docker stop signal. +STOPSIGNAL SIGRTMIN+3 + +CMD ["/sbin/init"] diff --git a/tests/systemd-test/Makefile b/tests/systemd-test/Makefile new file mode 100644 index 000000000..8c0985b6a --- /dev/null +++ b/tests/systemd-test/Makefile @@ -0,0 +1,36 @@ +# Solstone install-integration test runner — see README.md for usage and +# vpe/playbooks/solstone-systemd-test.md for the trade-off discussion. + +IMAGE ?= solstone-systemd-test:latest + +.PHONY: build smoke install observer-ingest full legacy-upgrade shell clean rebuild + +build: + docker build -t $(IMAGE) . + +smoke: build + IMAGE=$(IMAGE) ./run-test.sh smoke + +install: build + IMAGE=$(IMAGE) ./run-test.sh install + +observer-ingest: build + IMAGE=$(IMAGE) ./run-test.sh observer-ingest + +# Full path: also runs the model + Claude-skill installers. Slow. +full: build + IMAGE=$(IMAGE) ./run-test.sh install full + +# Upgrade-over-legacy-state path: seed a legacy non-symlink wrapper, then +# assert journal setup self-heals it through to a healthy service_identity. +legacy-upgrade: build + IMAGE=$(IMAGE) ./run-test.sh legacy-upgrade + +shell: build + IMAGE=$(IMAGE) KEEP=1 ./run-test.sh shell + +clean: + -docker rm -f solstone-systemd-test-run 2>/dev/null + -docker rmi $(IMAGE) 2>/dev/null + +rebuild: clean build diff --git a/tests/systemd-test/README.md b/tests/systemd-test/README.md new file mode 100644 index 000000000..1a2bab585 --- /dev/null +++ b/tests/systemd-test/README.md @@ -0,0 +1,154 @@ +# solstone-systemd-test + +Docker image that runs `systemd --user` end-to-end so the solstone +install-integration suite can verify that `journal setup` actually starts the +user service — not just that the unit file got written to +`~/.config/systemd/user/solstone.service`. + +For trade-offs (privilege requirements, host-kernel assumptions, what +this image does NOT model, and the CI-runner path forward) see the +operational playbook in the sol pbc org repo, +`vpe/playbooks/solstone-systemd-test.md`. Read that first. + +## quick start + +```bash +make build # build the image +make smoke # ~30s — verifies systemd --user works end-to-end +make install # ~3-5min — uv tool host install && journal setup +make observer-ingest # ~3-5min — install + setup + observer ingest round-trip +make legacy-upgrade # ~3-5min — install over a seeded legacy non-symlink wrapper +``` + +`smoke` installs a tiny `runner-smoke.service` and confirms systemd +`--user` accepts, enables, starts, and reports it active. Use it as a +fast pre-flight before chasing solstone-specific failures. + +`install` runs the actual journal install path (`uv tool install solstone-journal && journal setup -y --skip-models --skip-skills`) and +verifies the resulting `solstone.service` reaches `active` plus `journal +service status` returns 0. `--skip-models / --skip-skills` are passed by default because +faster-whisper / Parakeet / Claude-skill installation is orthogonal to +the systemd question; use `make full` to drop those flags. + +`observer-ingest` extends `install` with one real observer round-trip. It +registers a loopback observer through `/app/observer/register`, posts a +minimal contract-covered `screen.jsonl` segment to `/app/observer/ingest`, +then asserts the segment file and `stream.json` landed under +`~/journal/chronicle/` and `/app/observer/ingest/segments/` reports +that segment with `screen.jsonl` present and `observed: true`. This is the +release gate for package-data omissions such as a missing +`solstone/think/contract/layout.json`: it installs the built wheel, not the +source checkout, and drives the route that calls the ingest contract. + +For a pre-publish local wheel gate, build the solstone artifacts and mount +that `dist/` directory. The directory must contain the root `solstone` +wheel, the `solstone-journal` leaf wheel, and the `solstone-journal-models` +wheel; the runner installs the leaf with the root pinned as a direct-URL +requirement plus `--find-links /work/dist`, so every first-party artifact +is resolved from the candidate build, not PyPI. + +```bash +cd ~/projects/solstone # repo root +rm -rf dist/ && uv build +cd tests/systemd-test +SOLSTONE_WHEEL_DIR=~/projects/solstone/dist make observer-ingest +``` + +`legacy-upgrade` is `install` with one precondition added: before +`journal setup` runs, it seeds a **legacy non-symlink regular-file wrapper** +at `~/.local/bin/sol` (the accumulated manual-materialization state a clean +install never has — `cat`s a marker-less bash wrapper over the alias after +`rm`-ing the uv symlink, so `check_alias` classifies it `FOREIGN`). It then +asserts setup self-heals the foreign wrapper — replaced by a managed wrapper +(`# managed-version:` marker), legacy content preserved at +`/tmp/sol.old-symlink-*` — and a full `journal doctor` reports +`service_identity: ok`. This guards the wrapper/identity self-heal class +(Ryan Bennett's 0.4.10→0.5.1 cutover #2) that the clean-install matrix can't +exercise, because a clean install classifies the alias `OWNED` and never +takes the `FOREIGN` heal path. Cell 6 in `vpe/playbooks/solstone-install-verify.md` (org repo). + +## the worked example + +```bash +docker build -t solstone-systemd-test . +./run-test.sh install +``` + +Internally that runs (as the non-root `solstone` user inside a +`--privileged` container that booted `/sbin/init` to PID 1): + +```bash +uv tool install solstone-journal +journal setup -y --skip-models --skip-skills +test -f ~/.config/systemd/user/solstone.service +systemctl --user is-active solstone # → active +journal service status # → exit 0 +``` + +## interactive debugging + +```bash +make shell # opens a user shell in the running container +# inside: systemctl --user status solstone, journalctl --user -u solstone, etc. +``` + +`KEEP=1 ./run-test.sh install` runs the test then leaves the container +up — useful when an install step fails and you want to poke at it. + +## environment knobs + +| Variable | Default | What it does | +|--------------|-------------------------------|-------------------------------------------------------------------------| +| `IMAGE` | `solstone-systemd-test:latest`| Image tag. | +| `CONTAINER` | `solstone-systemd-test-run` | Container name (so parallel runs need distinct names). | +| `TEST_USER` | `solstone` | Non-root user inside the image. Matches the Dockerfile `TEST_USER` arg. | +| `PRIVILEGED` | `1` | `0` switches to the less-privileged path (cgroup-v2 host namespace + `CAP_SYS_ADMIN` + apparmor=unconfined). | +| `KEEP` | `0` | `1` leaves the container up on success for inspection. | +| `SOLSTONE_WHEEL_DIR` | unset | Optional host `dist/` directory mounted at `/work/dist`; the latest `solstone_journal-*-py3-none-any.whl` is installed with the root `solstone-*.whl` as a direct-URL requirement; `solstone_journal_models-*.whl` must also be present. | +| `SOLSTONE_INSTALL_TARGET` | `solstone-journal` | Package spec used when `SOLSTONE_WHEEL_DIR` is unset. | + +## why `--privileged` + +Booting `systemd` as PID 1 inside a container needs read-write access to +the cgroup hierarchy and a few capabilities (`CAP_SYS_ADMIN`, etc.) that +the default Docker profile denies. `--privileged` is the simplest, most +portable way to grant that. On modern Docker (≥20.10) with a cgroup-v2 +host (Fedora 31+, Debian 11+, Ubuntu 21.10+, Arch), the same setup works +with just `--cgroupns=host`, `CAP_SYS_ADMIN`, and a bind-mount of +`/sys/fs/cgroup` — that's what `PRIVILEGED=0` uses. Run the less- +privileged path first on a new host; fall back to `--privileged` if you +hit cgroup write-permission errors. + +See `vpe/playbooks/solstone-systemd-test.md` (org repo) § trade-offs for the full +discussion including the rootless-podman, distrobox, and lima +alternatives. + +## what this does NOT model + +- No graphical login session (no `pam_systemd` running for real, no + `user@.service` graph populated by GUI login) +- No real journald persistence across container restarts +- No NetworkManager, no resolved as the resolver +- No hardware-backed secure enclaves, no TPM +- No real PL-networked observer client or tunnel. `observer-ingest` uses + the real register and ingest HTTP routes, but the client is a loopback + `curl` payload inside the container. +- 7657 (the mutual-TLS pairing/sync surface) is bound by the convey + secure_listener (`solstone/solstone/convey/secure_listener/accept.py:41`) + for device-to-device pairing/sync. Plain HTTP on 5015 + (`DEFAULT_SERVICE_PORT`) is the convey Flask app — login, `/init`, + `/app/today`, etc. — and is reachable from the container, but there + is no explicit `/health` route there. The authoritative readiness + probe is `journal service status`, which talks to the callosum Unix + socket at `/health/callosum.sock`. The runner uses that + probe instead of `curl http://localhost:5015/health` (the request + body's shorthand). + +## file inventory + +| File | What it is | +|---------------|---------------------------------------------------------------------------| +| `Dockerfile` | Debian 12 (bookworm) base, full systemd, dbus-user-session, pre-lingered non-root user, uv pre-installed. | +| `run-test.sh` | `smoke` / `install` / `observer-ingest` / `legacy-upgrade` / `shell` modes. Drives the boot-wait, runs the install, asserts readiness. | +| `Makefile` | `build` / `smoke` / `install` / `observer-ingest` / `full` / `legacy-upgrade` / `shell` / `clean` / `rebuild`. | +| `README.md` | This file. | diff --git a/tests/systemd-test/run-test.sh b/tests/systemd-test/run-test.sh new file mode 100755 index 000000000..f8b3ba359 --- /dev/null +++ b/tests/systemd-test/run-test.sh @@ -0,0 +1,494 @@ +#!/usr/bin/env bash +# Run a single solstone install-integration test inside the +# solstone-systemd-test container. +# +# Usage: +# ./run-test.sh # default: smoke (verify systemd --user only) +# ./run-test.sh smoke # tiny user unit, no solstone install +# ./run-test.sh install [extra-args] # full: uv tool host install && journal setup +# ./run-test.sh observer-ingest # install + setup + real observer ingest round-trip +# ./run-test.sh legacy-upgrade # install, but seed a legacy non-symlink +# # wrapper first; assert setup self-heals it +# # through to a healthy service_identity +# ./run-test.sh shell # leave container up + drop into a user shell +# +# Environment overrides: +# IMAGE — image tag (default: solstone-systemd-test:latest) +# CONTAINER — container name (default: solstone-systemd-test-run) +# TEST_USER — non-root user in the image (default: solstone) +# PRIVILEGED — "1" (default) for --privileged, "0" for the less-privileged path +# KEEP — "1" to keep the container after the run for inspection +# SOLSTONE_WHEEL_DIR — optional host dist/ dir mounted read-only at /work/dist; +# when present, install the latest solstone-journal leaf +# wheel with the matching solstone root wheel pinned as a +# direct requirement (plus solstone_journal_models from +# the same dir) instead of PyPI +# SOLSTONE_INSTALL_TARGET — package spec when SOLSTONE_WHEEL_DIR is unset +# (default: solstone-journal) +# +# Exit codes: +# 0 test passed +# 1 test failed (specific failure printed to stderr) +# 2 usage error or pre-flight failure +# +# The "install" mode passes `--skip-models --skip-skills` to `journal setup` by +# default — the systemd-runner is meant to verify the service-install path, +# not the model-installer or skill-installer. Override with `install full` +# to drop those flags. + +set -euo pipefail + +IMAGE="${IMAGE:-solstone-systemd-test:latest}" +CONTAINER="${CONTAINER:-solstone-systemd-test-run}" +TEST_USER="${TEST_USER:-solstone}" +PRIVILEGED="${PRIVILEGED:-1}" +KEEP="${KEEP:-0}" +SOLSTONE_WHEEL_DIR="${SOLSTONE_WHEEL_DIR:-}" +SOLSTONE_INSTALL_TARGET="${SOLSTONE_INSTALL_TARGET:-solstone-journal}" + +mode="${1:-smoke}" +shift || true + +die() { echo "error: $*" >&2; exit 2; } +log() { echo "[$(date -u +%H:%M:%S)] $*" >&2; } + +case "$mode" in + smoke|install|observer-ingest|legacy-upgrade|shell) ;; + *) die "unknown mode: $mode (expected: smoke | install | observer-ingest | legacy-upgrade | shell)" ;; +esac + +command -v docker >/dev/null || die "docker not found in PATH" + +# Image is built by `make build` — fail fast if it isn't there yet. +docker image inspect "$IMAGE" >/dev/null 2>&1 || \ + die "image $IMAGE not found; run 'make build' first" + +# Clean any stale container from a previous run (KEEP=1 leaves it on success). +docker rm -f "$CONTAINER" >/dev/null 2>&1 || true + +# Privilege flags. --privileged is the simple, reliable path. The less- +# privileged path (PRIVILEGED=0) uses cgroup-v2 host namespace + SYS_ADMIN, +# which works on modern Docker + cgroup-v2 hosts but is more host-sensitive. +if [ "$PRIVILEGED" = "1" ]; then + PRIV_FLAGS=(--privileged --tmpfs /tmp --tmpfs /run --tmpfs /run/lock) +else + PRIV_FLAGS=( + --cap-add SYS_ADMIN + --security-opt apparmor=unconfined + --cgroupns=host + --tmpfs /tmp + --tmpfs /run + --tmpfs /run/lock + -v /sys/fs/cgroup:/sys/fs/cgroup:rw + ) +fi + +RUN_FLAGS=("${PRIV_FLAGS[@]}" -e "SOLSTONE_INSTALL_TARGET=$SOLSTONE_INSTALL_TARGET") +if [ -n "$SOLSTONE_WHEEL_DIR" ]; then + [ -d "$SOLSTONE_WHEEL_DIR" ] || die "SOLSTONE_WHEEL_DIR is not a directory: $SOLSTONE_WHEEL_DIR" + compgen -G "$SOLSTONE_WHEEL_DIR/solstone-*-py3-none-any.whl" >/dev/null \ + || die "SOLSTONE_WHEEL_DIR has no solstone-*-py3-none-any.whl: $SOLSTONE_WHEEL_DIR" + compgen -G "$SOLSTONE_WHEEL_DIR/solstone_journal-*-py3-none-any.whl" >/dev/null \ + || die "SOLSTONE_WHEEL_DIR has no solstone_journal-*-py3-none-any.whl: $SOLSTONE_WHEEL_DIR" + compgen -G "$SOLSTONE_WHEEL_DIR/solstone_journal_models-*-py3-none-any.whl" >/dev/null \ + || die "SOLSTONE_WHEEL_DIR has no solstone_journal_models-*-py3-none-any.whl: $SOLSTONE_WHEEL_DIR" + RUN_FLAGS+=(-v "$SOLSTONE_WHEEL_DIR:/work/dist:ro") +fi + +install_solstone_cmd=' +set -euo pipefail +if compgen -G "/work/dist/solstone_journal-*-py3-none-any.whl" >/dev/null 2>&1; then + compgen -G "/work/dist/solstone-*-py3-none-any.whl" >/dev/null 2>&1 || { + echo "SOLSTONE_WHEEL_DIR must include solstone-*-py3-none-any.whl" >&2 + exit 2 + } + leaf=$(ls /work/dist/solstone_journal-*-py3-none-any.whl | sort -V | tail -1) + root=$(ls /work/dist/solstone-*-py3-none-any.whl | sort -V | tail -1) + leaf_uri=$(python3 -c "import pathlib,sys; print(pathlib.Path(sys.argv[1]).resolve().as_uri())" "$leaf") + root_uri=$(python3 -c "import pathlib,sys; print(pathlib.Path(sys.argv[1]).resolve().as_uri())" "$root") + # The leaf wheel is the install target; the root rides as a direct-URL + # requirement so an equal-version PyPI solstone can never shadow the local + # build. solstone_journal_models resolves from /work/dist via --find-links. + target="solstone-journal @ ${leaf_uri}" + echo "install target: ${target} (root: ${root_uri})" + uv tool install --python 3.12 \ + --find-links /work/dist \ + --with "solstone[journal-host] @ ${root_uri}" \ + "$target" +else + echo "install target: ${SOLSTONE_INSTALL_TARGET}" + uv tool install --python 3.12 \ + "$SOLSTONE_INSTALL_TARGET" +fi +' + +log "starting $CONTAINER from $IMAGE (privileged=$PRIVILEGED, mode=$mode)" +docker run -d --rm --name "$CONTAINER" "${RUN_FLAGS[@]}" "$IMAGE" >/dev/null + +cleanup() { + rc=$? + if [ "$KEEP" = "1" ] && [ "$rc" = "0" ]; then + log "KEEP=1: leaving $CONTAINER running for inspection" + log " docker exec -u $TEST_USER -it $CONTAINER bash -l" + log " docker rm -f $CONTAINER" + else + docker rm -f "$CONTAINER" >/dev/null 2>&1 || true + fi + exit $rc +} +trap cleanup EXIT + +# Wait for PID 1 systemd to reach a running/degraded steady state. We +# accept "degraded" because some masked units may be reported as failed-by- +# masking; the user-level path doesn't depend on them. +log "waiting for system manager to reach steady state..." +for _ in $(seq 1 60); do + state=$(docker exec "$CONTAINER" systemctl is-system-running 2>/dev/null || true) + case "$state" in + running|degraded) break ;; + esac + sleep 1 +done +if [ "$state" != "running" ] && [ "$state" != "degraded" ]; then + log "system manager never became ready (last state: ${state:-unknown})" + docker exec "$CONTAINER" systemctl --failed --no-pager || true + exit 1 +fi +log "system manager: $state" + +# Start the user instance manager. systemd-logind would normally read the +# linger marker file we baked into the image and fire user@.service +# automatically, but logind is one of the units we mask (it pulls in seat +# / TTY / utmp logic that doesn't apply in a container). So we start the +# user manager explicitly — same end state, just deterministic. +USER_UID=$(docker exec "$CONTAINER" id -u "$TEST_USER") +log "starting user@${USER_UID}.service..." +docker exec "$CONTAINER" systemctl start "user@${USER_UID}.service" \ + || { log "failed to start user@${USER_UID}.service"; exit 1; } + +# Wait for the user instance to accept commands. +log "waiting for user@${USER_UID}..." +for _ in $(seq 1 30); do + if docker exec -u "$TEST_USER" "$CONTAINER" bash -lc 'systemctl --user list-units --no-pager >/dev/null 2>&1'; then + break + fi + sleep 1 +done +docker exec -u "$TEST_USER" "$CONTAINER" bash -lc 'systemctl --user list-units --no-pager >/dev/null' \ + || { log "user systemd never became responsive"; exit 1; } +log "user systemd: ready" + +case "$mode" in + smoke) + log "smoke test: install a tiny --user unit and verify it starts" + docker exec -u "$TEST_USER" "$CONTAINER" bash -lc ' + set -euo pipefail + mkdir -p ~/.config/systemd/user + cat > ~/.config/systemd/user/runner-smoke.service < /tmp/runner-smoke.out" +RemainAfterExit=yes + +[Install] +WantedBy=default.target +UNIT + systemctl --user daemon-reload + systemctl --user enable --now runner-smoke.service + test "$(cat /tmp/runner-smoke.out)" = "runner-smoke-ok" + systemctl --user is-active runner-smoke.service + ' + log "smoke: PASS" + ;; + + install) + # Heavy: pulls solstone wheel + transitive deps. --skip-models / + # --skip-skills cuts faster-whisper / Parakeet / Claude-skill + # downloads — those are orthogonal to the systemd integration we're + # testing here. Pass "full" as an extra arg to drop both flags. + extra=("$@") + skip_flags=(--skip-models --skip-skills) + if [ "${1:-}" = "full" ]; then + skip_flags=() + extra=("${extra[@]:1}") + fi + + log "install: uv tool install --python 3.12" + # Debian bookworm ships python 3.11; solstone 0.4.0+ requires >=3.12. + # uv downloads a standalone 3.12 on the fly when requested explicitly. + docker exec -u "$TEST_USER" "$CONTAINER" bash -lc "$install_solstone_cmd" + + log "install: journal setup -y ${skip_flags[*]} ${extra[*]:-}" + docker exec -u "$TEST_USER" "$CONTAINER" bash -lc \ + "journal setup -y ${skip_flags[*]} ${extra[*]:-}" + + log "verify: unit file exists and was loaded by user systemd" + docker exec -u "$TEST_USER" "$CONTAINER" bash -lc ' + set -euo pipefail + test -f ~/.config/systemd/user/solstone.service + systemctl --user cat solstone.service >/dev/null + ' + + log "verify: systemctl --user is-active solstone" + # 30s budget for Type=notify READY=1. + for _ in $(seq 1 30); do + state=$(docker exec -u "$TEST_USER" "$CONTAINER" \ + bash -lc 'systemctl --user is-active solstone' 2>/dev/null || true) + [ "$state" = "active" ] && break + sleep 1 + done + if [ "$state" != "active" ]; then + log "solstone.service did not reach active (last: ${state:-unknown})" + docker exec -u "$TEST_USER" "$CONTAINER" \ + bash -lc 'systemctl --user status solstone --no-pager -l || true' >&2 + exit 1 + fi + + log "verify: journal service status" + # `journal service status` returns 0 only when the callosum health probe + # succeeds — this is the authoritative readiness signal. Port 5015 + # is the plain-HTTP convey Flask app (login, /init, /app/today); + # 7657 is the mutual-TLS pairing/sync surface. Neither exposes an + # explicit /health route — `journal service status` (callosum.sock) is + # the canonical probe. + docker exec -u "$TEST_USER" "$CONTAINER" bash -lc 'journal service status' + + log "install: PASS" + ;; + + observer-ingest) + log "observer-ingest: uv tool install --python 3.12" + docker exec -u "$TEST_USER" "$CONTAINER" bash -lc "$install_solstone_cmd" + + log "observer-ingest: journal setup -y --skip-models --skip-skills" + docker exec -u "$TEST_USER" "$CONTAINER" bash -lc \ + "journal setup -y --skip-models --skip-skills" + + log "verify: systemctl --user is-active solstone" + for _ in $(seq 1 30); do + state=$(docker exec -u "$TEST_USER" "$CONTAINER" \ + bash -lc 'systemctl --user is-active solstone' 2>/dev/null || true) + [ "$state" = "active" ] && break + sleep 1 + done + if [ "$state" != "active" ]; then + log "solstone.service did not reach active (last: ${state:-unknown})" + docker exec -u "$TEST_USER" "$CONTAINER" \ + bash -lc 'systemctl --user status solstone --no-pager -l || true' >&2 + exit 1 + fi + + log "verify: journal service status" + docker exec -u "$TEST_USER" "$CONTAINER" bash -lc 'journal service status' + + log "observer-ingest: register loopback observer and post one segment" + docker exec -u "$TEST_USER" "$CONTAINER" bash -lc ' + set -euo pipefail + + base_url="http://127.0.0.1:5015" + day="$(date -u +%Y%m%d)" + segment="120000_1" + host="systemd-gate" + + register_body=$(curl -fsS \ + -H "Content-Type: application/json" \ + -d "{\"platform\":\"linux\",\"hostname\":\"${host}\",\"stream_type\":\"tmux\",\"version\":\"systemd-test\"}" \ + "${base_url}/app/observer/register") + printf "%s\n" "$register_body" > /tmp/observer-register.json + + key=$(python3 - < "$payload" <&2 + cat /tmp/observer-ingest-response.json >&2 || true + exit 1 + fi + + saved_segment=$(python3 - < /tmp/observer-segments.json + if python3 - "$saved_segment" <&2 + echo "register:" >&2 + cat /tmp/observer-register.json >&2 || true + echo "ingest:" >&2 + cat /tmp/observer-ingest-response.json >&2 || true + echo "segments:" >&2 + cat /tmp/observer-segments.json >&2 || true + echo "service log tail:" >&2 + tail -120 "${HOME}/journal/health/service.log" >&2 || true + exit 1 + fi + + echo "observer-ingest: ${day}/${stream}/${saved_segment} observed=true" + ' + + log "observer-ingest: PASS" + ;; + + legacy-upgrade) + # Upgrade-over-legacy-state cell (the path that hid Ryan Bennett's + # 0.4.10->0.5.1 cutover bugs). Same as `install`, but BEFORE journal + # setup we seed a LEGACY non-symlink regular-file wrapper at + # ~/.local/bin/sol — the accumulated manual-materialization state a + # clean install never has. Then we assert setup self-heals the foreign + # wrapper (managed wrapper + /tmp backup) through to a healthy + # service_identity. A clean install classifies the alias OWNED and + # never exercises the FOREIGN heal path, which is why the post-0.5.2 + # 5-cell clean matrix couldn't catch the wrapper/identity class. + log "legacy-upgrade: uv tool install --python 3.12" + docker exec -u "$TEST_USER" "$CONTAINER" bash -lc "$install_solstone_cmd" + + log "legacy-upgrade: seed legacy non-symlink wrapper at ~/.local/bin/sol" + # rm the uv symlink first — `cat >` through a symlink writes the target, + # not the alias. parse_wrapper has no managed-version marker to find, so + # check_alias classifies this regular file FOREIGN. The exec target is a + # deliberately-defunct path; provision_wrappers only READS the wrapper. + docker exec -u "$TEST_USER" "$CONTAINER" bash -lc ' + set -euo pipefail + rm -f ~/.local/bin/sol + cat > ~/.local/bin/sol </dev/null || true) + [ "$state" = "active" ] && break + sleep 1 + done + if [ "$state" != "active" ]; then + log "solstone.service did not reach active (last: ${state:-unknown})" + docker exec -u "$TEST_USER" "$CONTAINER" \ + bash -lc 'systemctl --user status solstone --no-pager -l || true' >&2 + exit 1 + fi + + log "verify: foreign wrapper self-healed (managed wrapper + /tmp backup)" + docker exec -u "$TEST_USER" "$CONTAINER" bash -lc ' + set -euo pipefail + # the foreign wrapper has been replaced by a managed solstone wrapper + grep -q "^# managed-version:" ~/.local/bin/sol + # and the legacy wrapper was preserved, not destroyed + ls /tmp/sol.old-symlink-* >/dev/null 2>&1 + ' + + log "verify: full journal doctor reports service_identity ok" + # journal doctor runs JOURNAL_CHECKS (service_identity lives only there, + # not in the setup --readiness battery — which is exactly why Ryan's + # FAIL service_identity never gated setup). After the heal the service + # target resolves to the current install, so it must report ok. + docker exec -u "$TEST_USER" "$CONTAINER" bash -lc ' + set -euo pipefail + journal doctor --json > /tmp/legacy-upgrade-doctor.json + python3 - <