Something went wrong. Try again.
A community based topic aggregation platform built on atproto
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535#!/usr/bin/env bash# The hermetic stack, as a library. Sourced — never executed.## WHY THIS FILE EXISTS## Two host-side entry points need the same stack: scripts/ci.sh (the merge gate,# which runs every tier) and scripts/test-e2e.sh (T2 alone, for an agent or a# human iterating on pipeline contracts). docs/TEST_ARCHITECTURE.md §3.5 is# explicit that there must be exactly ONE way the pipeline tier executes — the# stack publishes no host ports, so a host-run `go test -tags e2e` cannot reach# it at all, and a second, subtly different bring-up path is how "it passes for# me" starts.## So the bring-up lives here once, and both callers get the identical stack:# same images built from the working tree, same staging (the AppView cannot# start until its PDS account exists), same egress-blocked network, same cache# volumes.## CONTRACT FOR CALLERS## source "$(dirname "$0")/lib/ci-stack.sh" # sets REPO_ROOT and cds to it# stack_preflight # docker reachable, caches exist# stack_up # build, prefetch modules, start# ... do work through `compose run --rm ...` ...## Callers own their own EXIT trap. stack_teardown is provided; when it runs (or# whether it runs at all) is the caller's decision, because "leave the stack for# debugging" means different things to a gate and to an iteration loop.## Environment overrides (shared by both callers):# COVES_CI_PROJECT Compose project name (default coves-ci)# COVES_CI_KEEP_STACK 1 to leave the stack running for debugging# COVES_CI_GOARCH override the detected build architecture
REPO_ROOT=$(git rev-parse --show-toplevel)cd "$REPO_ROOT"
COMPOSE_FILE=docker-compose.ci.yml# Overlay for the ONE scenario that needs a differently-configured AppView; see# the file's own header and stack_control_run_verb below.COMPOSE_FILE_TWO_FEED=docker-compose.ci.two-feed.ymlPROJECT=${COVES_CI_PROJECT:-coves-ci}OUT_DIR="$REPO_ROOT/.ci-out"
# Named so the volumes match docker-compose.ci.yml's external declarations.CACHE_VOLUMES=(coves-ci-go-mod-cache coves-ci-go-build-cache)
# The services whose logs are captured when a run fails: every hop of the# ingest path, in the order events travel it. See stack_capture_logs.CAPTURED_SERVICES=(appview pds pds2 relay jetstream postgres-relay)
CYAN='\033[36m'GREEN='\033[32m'YELLOW='\033[33m'RED='\033[31m'RESET='\033[0m'
step() { printf "\n${CYAN}▶ %s${RESET}\n" "$1"; }ok() { printf "${GREEN} ✓ %s${RESET}\n" "$1"; }warn() { printf "${YELLOW} ⚠ %s${RESET}\n" "$1"; }fail() { printf "${RED} ✗ %s${RESET}\n" "$1" >&2; }
compose() { docker compose -f "$COMPOSE_FILE" -p "$PROJECT" "$@"; }
# ---------------------------------------------------------------------------# Architecture# ---------------------------------------------------------------------------# The Dockerfile defaults GOARCH to amd64 for the production deploy target.# Building that under emulation on an arm64 machine makes every run# substantially slower, so CI builds natively. Derived from uname rather than# `go env` so this needs no Go toolchain on the host — Docker is the only# prerequisite.## A value already in the environment is honoured rather than overwritten — the# documented escape hatch for a host whose uname the case below does not know,# and for reproducing an amd64 build on an arm64 machine.stack_detect_arch() { if [[ -n ${COVES_CI_GOARCH:-} ]]; then export COVES_CI_GOARCH return 0 fi case "$(uname -m)" in arm64 | aarch64) COVES_CI_GOARCH=arm64 ;; x86_64 | amd64) COVES_CI_GOARCH=amd64 ;; *) fail "unsupported host architecture $(uname -m); set COVES_CI_GOARCH manually" return 1 ;; esac export COVES_CI_GOARCH}
# ---------------------------------------------------------------------------# Teardown# ---------------------------------------------------------------------------# `down -v` removes this project's state volumes; the Go caches are declared# external in the compose file precisely so they survive this.stack_teardown() { step "Tearing down the CI stack..." compose down -v --remove-orphans --timeout 10 >/dev/null 2>&1 || true ok "removed containers, network, and state volumes (Go caches kept)"}
# compose_cmdline renders a `docker compose` command a human can paste.## The COVES_CI_GOARCH prefix is not decoration. docker-compose.ci.yml declares# GOARCH as a required build argument, and Compose interpolates the whole file# on EVERY subcommand — so a bare `docker compose ... down -v` fails before it# does anything, with "required variable COVES_CI_GOARCH is missing a value".# Printing the bare form left a trail of copy-pasteable commands that all error,# in exactly the situation (a stack left up after a failure) where the reader is# least inclined to debug the instructions.compose_cmdline() { # Resolved rather than defaulted to `uname -m`: on x86_64 hosts uname prints # "x86_64", which is not a GOARCH, so the fallback would emit a command that # fails differently. Every caller has already run stack_preflight, so this is # belt-and-braces — but a belt that prints a broken command is worse than no # belt. stack_detect_arch >/dev/null 2>&1 || true printf "COVES_CI_GOARCH=%s docker compose -f %s -p %s %s" \ "${COVES_CI_GOARCH:-amd64}" "$COMPOSE_FILE" "$PROJECT" "$*"}
# stack_keep_notice prints how to inspect a stack that was deliberately left up.## The AppView caveat is not hypothetical. The pipeline tier's reliability# scenarios stop it and reconfigure it with a second Jetstream feed, restoring it# through t.Cleanup — which does not run if the run dies without unwinding (a# -timeout kill, a panic off the test goroutine, docker being stopped). What# survives is either a stopped AppView or, worse because it looks fine, a healthy# TWO-FEED one that the next run happily reuses and whose every consumer-health# delta is then doubled. The tier refuses to start against it# (requireSingleFeedTopology in tests/e2e/contracts_test.go), so this is a# recovery hint rather than a warning about silent corruption.stack_keep_notice() { printf "\n${YELLOW}Stack left running (COVES_CI_KEEP_STACK=1).${RESET}\n" printf " Inspect: %s\n" "$(compose_cmdline ps)" printf " Logs: %s\n" "$(compose_cmdline logs appview)" printf " Tear down: %s\n" "$(compose_cmdline down -v --remove-orphans)" printf " If a run died mid-reliability-suite the AppView may be stopped or left on two feeds:\n" printf " %s\n" "$(compose_cmdline up -d --force-recreate appview)"}
# ---------------------------------------------------------------------------# Preflight# ---------------------------------------------------------------------------stack_preflight() { step "Preflight"
if ! docker info >/dev/null 2>&1; then fail "the Docker daemon is not reachable — start Docker and retry" return 1 fi ok "docker daemon reachable"
stack_detect_arch || return 1
local volume for volume in "${CACHE_VOLUMES[@]}"; do if ! docker volume inspect "$volume" >/dev/null 2>&1; then docker volume create "$volume" >/dev/null ok "created cache volume $volume" fi done ok "go module and build caches present"
mkdir -p "$OUT_DIR"}
# ---------------------------------------------------------------------------# Is a usable stack already up?# ---------------------------------------------------------------------------# Only ever true for a stack this project name owns and whose AppView is# actually serving — a half-started leftover is worse than none, because the# suite would grade whatever fraction of it happens to answer.## The probe runs INSIDE the namespace, for the same reason the runner's# readiness checks do: every service healthchecks itself over its own loopback,# so container health says nothing about whether the shared namespace is wired.stack_is_up() { local state state=$(compose ps --status running --format '{{.Service}}' 2>/dev/null || true) grep -q '^appview$' <<<"$state" || return 1 compose exec -T appview wget -q -O /dev/null http://localhost:8081/xrpc/_health 2>/dev/null}
# ---------------------------------------------------------------------------# Bring-up# ---------------------------------------------------------------------------
stack_discard_previous() { # A stack left behind by an interrupted previous run would otherwise be # reused with its state intact, which is the one thing the gate must never # do. step "Discarding any previous CI stack" compose down -v --remove-orphans --timeout 10 >/dev/null 2>&1 || true
# VERIFIED, not assumed. `down` is best-effort here on purpose — there may # be nothing to remove, and its complaints in that case are noise — but # suppressing its status meant this step printed "clean slate" over a stack # it had failed to remove, and the run then died forty seconds later inside # `up`, on a container-name conflict that named a container this function # had just claimed to delete. Observed after a reliability run left a kept # stack behind: `down` reported nothing and left the namespace container. # # So the postcondition is checked. A survivor is removed by force (these are # throwaway CI containers; nothing in them is worth preserving), and a # survivor that outlives THAT is fatal — proceeding would mean grading a # stack with a previous run's state in it. local leftovers leftovers=$(compose ps -aq 2>/dev/null) if [[ -n $leftovers ]]; then warn "compose down left containers behind; removing them by force" # Word-splitting is intended: one container id per line. # shellcheck disable=SC2086 docker rm -f $leftovers >/dev/null 2>&1 || true leftovers=$(compose ps -aq 2>/dev/null) fi if [[ -n $leftovers ]]; then fail "could not remove the previous CI stack; these containers survived:" printf ' %s\n' $leftovers >&2 fail "remove them by hand and retry: $(compose_cmdline down -v --remove-orphans)" return 1 fi ok "clean slate"}
stack_build() { step "Building images (AppView from the current working tree)" compose build ok "images built"}
# The stack's network is `internal: true` (see docker-compose.ci.yml), so once# it is up nothing in it can reach the internet — including the Go toolchain,# which downloads modules at test time rather than at image-build time. On a# cold cache that would leave the runner unable to compile anything.## So the modules are fetched here, in a throwaway container on Docker's default# bridge, writing into the same external volume the runner mounts. Deliberately# `docker run` and not `compose run`: every service in the compose file is# pinned to the egress-blocked namespace, which is the whole point.## A warm cache makes this a no-op costing a second or two. A failure here is# fatal — continuing would produce a confusing "cannot find module" failure# inside the suite instead of "the network was down before we started".stack_prefetch_modules() { step "Populating the Go module cache (needs network; the stack itself has none)" if ! docker run --rm \ -v "$REPO_ROOT:/src" -w /src \ -v coves-ci-go-mod-cache:/go/pkg/mod \ -v coves-ci-go-build-cache:/go-build-cache \ --entrypoint go \ "$PROJECT-runner" mod download; then fail "could not download Go modules — check network access and go.sum" fail "if the network is fine, the cache volume may be corrupt: run 'make ci-clean' and retry" return 1 fi ok "module cache populated"}
# stack_start brings the services up in the order their dependencies force.stack_start() { # Staged deliberately: the AppView is NOT started with the rest. It # authenticates to the PDS as PDS_INSTANCE_HANDLE, and in a fresh PDS that # account does not exist yet, so it has to be created in between. step "Starting infrastructure (Postgres ×4, PLC, PDS ×2, relay, Turnstile stub)" # Jetstream is deliberately absent from this --wait list. `up --wait` fails # outright — "has no healthcheck configured" — for any service it is asked # to wait on that cannot report health, and the Jetstream image ships no # HTTP client to probe itself with (docker-compose.dev.yml disables its # healthcheck for the same reason). # # The relay IS on the list, and waiting for it here rather than letting # Jetstream's depends_on handle it is what makes the next step possible: # the crawl announcement is an HTTP call against a relay that has to be # serving already. compose up -d --wait \ netns postgres postgres-test postgres-plc postgres-relay \ plc-directory pds pds2 relay turnstile-stub ok "infrastructure healthy"
# Started only after the relay is healthy, since it consumes the relay's # merged firehose rather than a PDS' directly (see docker-compose.ci.yml's # header). Readiness is then gated on its metrics endpoint from inside the # namespace — see scripts/lib/runner-ready.sh. step "Starting Jetstream" compose up -d jetstream ok "jetstream started (readiness gated by the runner)"
step "Seeding the stack (relay crawl announcements, instance PDS account)" # --no-deps so this does not drag the AppView up before its account exists. compose run --rm --no-deps --entrypoint bash runner /src/scripts/ci-bootstrap.sh
step "Starting the AppView" compose up -d --wait appview ok "appview healthy on :8081"}
# stack_capture_logs writes the logs of every service in the ingest path to# .ci-out/, and echoes the filenames it wrote.## WHICH SERVICES, AND WHY IT IS NO LONGER THREE## It used to be appview, pds and jetstream, which was the whole path when# Jetstream read the PDS directly. The federation topology put three more# services INSIDE that path — pds2, the relay, and the relay's Postgres — and a# fault in any of them reddens contracts that mention none of them: a relay that# stopped crawling pds2 looks exactly like a consumer that stopped indexing.# Capturing them costs nothing on a green run, because this only runs on# failure, and re-running a whole gate to collect a log nobody thought to keep# costs several minutes.## Every capture is `|| true`: a service that never started has no logs, and a# diagnostics step must not be the thing that fails the diagnosis.stack_capture_logs() { local service tail_lines for service in "${CAPTURED_SERVICES[@]}"; do # The AppView's own log is the one that usually answers the question, so # it gets the deeper tail. tail_lines=200 [[ $service == appview ]] && tail_lines=400 compose logs --no-color --tail "$tail_lines" "$service" \ >"$OUT_DIR/$service.log" 2>&1 || true done}
# stack_captured_logs is that same list as a printable string, so a message# pointing at the artifacts cannot name a file nobody wrote.stack_captured_logs() { local service names="" for service in "${CAPTURED_SERVICES[@]}"; do names+="${names:+, }$service.log" done echo "$names"}
# stack_up is the whole bring-up: the sequence both callers need, in one call.stack_up() { stack_discard_previous stack_build stack_prefetch_modules stack_start}
# ---------------------------------------------------------------------------# The stack control channel# ---------------------------------------------------------------------------## WHY THE PIPELINE TIER NEEDS ONE## docs/TEST_ARCHITECTURE.md §3.4c asks for reliability scenarios that CRUD# contracts cannot reach: resume from a persisted cursor after a restart, replay# a rewound stream exactly once, refuse a stale event after a delete, recover a# dead letter, consume two overlapping feeds. Every one of those needs the# AppView to stop, start, or come back with a different configuration — and the# tests run INSIDE the stack, in a container with no Docker socket and, by# §3.7's `internal: true` network, no route to anything but the stack itself.# Only the host can act on containers.## THE SHAPE, AND THE ALTERNATIVES IT WAS CHOSEN OVER## A directory in .ci-out (already bind-mounted into the runner, already# gitignored) is the channel. The container writes a one-word request file; this# watcher, running on the host beside `compose run`, executes the matching# command and writes a response file back. That keeps each scenario a single# readable Go function — stop, write, start, assert, all in one place, with# t.Cleanup restoring the stack however the test ends.## * Splitting the suite into host-orchestrated phases (`go test -run X_part1`,# compose restart, `go test -run X_part2`) needs no channel at all, but it# spreads one scenario across bash and three processes and has to hand state# between them through a file anyway — the same plumbing, minus the# readability.# * Mounting /var/run/docker.sock into a sidecar the tests could call is the# obvious "control plane" answer and hands host root to anything running in# the stack. This trades that for a bash `case` with five arms.## WHAT IT WILL AND WILL NOT DO## The verbs are a closed set, take NO arguments, and are dispatched by exact# string match. A request naming anything else is refused and logged. So the# most a compromised or buggy test can do is stop and start the AppView of a# throwaway CI stack — which it could achieve anyway by making the AppView# crash. Nothing here evaluates a string from the container as a command, and# the two-feed topology lives in a compose overlay in the repo rather than in# the request, precisely so that no request can name a configuration.## LIFECYCLE## Started by the host entry point before `compose run` and stopped after, so it# exists exactly as long as a suite is running. Both callers register it in# their EXIT trap: an orphaned watcher would sit polling a directory forever.
# PER-PROJECT, not per-repository. COVES_CI_PROJECT exists so two stacks can run# from one checkout; a single shared channel directory would let one run's# watcher answer the other run's requests and restart the wrong AppView. The# container side is kept in step by docker-compose.ci.yml, which interpolates# the same project name into COVES_STACK_CONTROL_DIR.STACK_CONTROL_DIR="$OUT_DIR/stack-control-$PROJECT"STACK_CONTROL_PIDFILE="$STACK_CONTROL_DIR/watcher.pid"STACK_CONTROL_PID=""
# stack_control_start wipes the channel and begins serving it in the background.## The wipe matters: a response left by a previous run under a filename a new run# happened to reuse would be read as an instant answer to a request nobody# executed. Run-scoped request ids make that improbable; starting empty makes it# impossible.## The pidfile check happens BEFORE the wipe, and guards the case the wipe would# otherwise make worse: a watcher orphaned by a killed run is still polling this# directory, so a second watcher would race it for each *.req — the `[[ -e# $response ]]` guard in the loop is a check-then-act, not an atomic claim, and# both could serve the same request. Refusing to start is right rather than# harsh: two watchers on one channel means container operations firing twice# with no way to tell which run asked.stack_control_start() { step "Starting the stack control channel"
if [[ -f $STACK_CONTROL_PIDFILE ]]; then local existing existing=$(cat "$STACK_CONTROL_PIDFILE" 2>/dev/null || true) if [[ -n $existing ]] && kill -0 "$existing" 2>/dev/null; then fail "a stack-control watcher (pid $existing) is already serving ${STACK_CONTROL_DIR#"$REPO_ROOT"/}" printf " Another run is using project %q, or a previous run was killed and left it behind.\n" "$PROJECT" printf " Kill it: kill %s\n" "$existing" printf " Or use a project: COVES_CI_PROJECT=<name> make ci\n" return 1 fi printf "${YELLOW} stale watcher pidfile (pid %s is gone) — taking over the channel${RESET}\n" "${existing:-unknown}" fi
rm -rf "$STACK_CONTROL_DIR" mkdir -p "$STACK_CONTROL_DIR" stack_control_loop & STACK_CONTROL_PID=$! printf '%s\n' "$STACK_CONTROL_PID" >"$STACK_CONTROL_PIDFILE" ok "watcher (pid $STACK_CONTROL_PID) serving ${STACK_CONTROL_DIR#"$REPO_ROOT"/}"}
stack_control_stop() { [[ -n $STACK_CONTROL_PID ]] || return 0 kill "$STACK_CONTROL_PID" 2>/dev/null || true wait "$STACK_CONTROL_PID" 2>/dev/null || true rm -f "$STACK_CONTROL_PIDFILE" STACK_CONTROL_PID=""}
# stack_control_loop polls for requests. Polling rather than inotify/fswatch# because neither is installed everywhere this runs, and because the latency# that matters here is measured against a container restart.stack_control_loop() { # This runs as a background job of a `set -e` script: without this, a single # non-zero command anywhere below would kill the watcher silently and every # subsequent request would time out with no explanation. set +e while true; do local request response for request in "$STACK_CONTROL_DIR"/*.req; do [[ -e $request ]] || continue response="${request%.req}.res" [[ -e $response ]] && continue stack_control_serve "$request" "$response" done sleep 0.2 done}
# stack_control_serve executes one request and answers it.## The answer is written to a temporary name and renamed, because the reader# polls for the response file's existence: a partially-written response would# otherwise be read as a complete one. Rename within a directory is atomic, and# the reader never sees the temporary name (it does not end in .res).stack_control_serve() { local request=$1 response=$2 verb output status IFS= read -r verb <"$request"
printf "${CYAN} ⟲ stack-control: %s${RESET}\n" "${verb:-<empty>}" output=$(stack_control_run_verb "$verb" 2>&1) status=$? if [[ $status -eq 0 ]]; then ok "stack-control: $verb" else fail "stack-control: $verb exited $status" fi
{ printf 'exit %d\n' "$status" printf '%s\n' "$output" } >"$response.tmp" mv "$response.tmp" "$response"}
# stack_control_run_verb is the closed set. Adding an arm here is the only way# to widen what the pipeline tier can do to the stack, which is the point.stack_control_run_verb() { case $1 in ping) # Answers the one question a test must ask before it depends on any of # this: is a watcher listening at all? Without it the first restart # would hang for its full timeout and report a stack problem, when the # truth is that the suite was launched a way that has no host beside it # (`make test-e2e-dev`, which grades a host-run AppView and therefore # skips this suite by name). printf 'pong\n' ;; appview-stop) # -t 20 rather than Compose's default 10s: SIGTERM starts a graceful # drain in which every connector flushes its cursor # (connector.go's flushCursorOnShutdown), and a cursor lost to SIGKILL # would make the resume scenario measure the wrong thing. compose stop -t 20 appview ;; appview-start) compose start appview ;; appview-two-feed) docker compose -f "$COMPOSE_FILE" -f "$COMPOSE_FILE_TWO_FEED" -p "$PROJECT" \ up -d --force-recreate appview ;; appview-single-feed) # The restore, and it is a --force-recreate rather than a restart on # purpose: the container has to be rebuilt from the base compose file # alone for the overlay's environment to actually go away. compose up -d --force-recreate appview ;; *) printf 'refused: %q is not a stack-control verb\n' "$1" return 64 ;; esac}