#!/usr/bin/env bash # scripts/arena.sh starts containers, and the test stage has no docker daemon. # So what is tested here is every decision it makes around the daemon: the # resource caps and their clamping, which containers count as arena's, what the # argument parser refuses, and what happens when a start never becomes ready. # # docker and curl are stubbed on PATH. The stub records every call and answers # from files this script writes, so the assertions can be about the arguments # docker was given - which is the only place the caps actually exist. # DOCKER_HOST is pointed at nothing as a second line of defence: a call that # ever escaped the stub fails rather than reaching a real daemon. # # Not covered: anything that needs a container to really run. Whether the image # serves its client paths, how long it takes, and whether 2 CPUs and 8GB are enough for # three JVMs are questions only a real start answers. set -uo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" TMP="$(mktemp -d)" trap 'rm -rf "$TMP"' EXIT pass=0; fail=0 check() { if "${@:2}"; then echo "ok $1"; pass=$((pass+1)) else echo "FAIL $1"; fail=$((fail+1)); fi } command -v jq >/dev/null || { echo "SKIP: jq is not installed"; exit 0; } BIN="$TMP/bin" mkdir -p "$BIN" cat > "$BIN/docker" <<'STUB' #!/usr/bin/env bash # Stands in for the docker CLI. Records the call, then answers from whatever # the test set up. Nothing here reaches a daemon. printf '%s\n' "$*" >> "$STUB_LOG" case "${1:-}" in ps) shift # The name-only listing ends in that format and nothing else; the fuller # one contains {{.Names}} too, so this has to be an exact tail match. if [[ "$*" == *"--format {{.Names}}" ]]; then cat "${STUB_NAMES:-/dev/null}" elif [[ " $* " == *" -a "* ]]; then cat "${STUB_PS_ALL:-/dev/null}" else cat "${STUB_PS:-/dev/null}" fi ;; image) shift 2 grep -qxF "$1" "${STUB_IMAGES:-/dev/null}" ;; run) shift # A detached start is the match. Anything else is the throwaway container # that reads the scenario's faction list out of the image. Matching on the # first argument only: the faction reader's shell snippet contains a "-d" # of its own, in `tr -d`. if [ "${1:-}" = -d ]; then printf '%s\n' "${STUB_ID:-c0ffee}" else printf '%s\n' "${STUB_FACTIONS:-TraineeA,TraineeB}" fi ;; inspect) shift case "$*" in *State.Running*) printf '%s\n' "${STUB_RUNNING:-true}" ;; # One published port is two bindings on a dual-stack host, 0.0.0.0 and # [::], so the range in the template runs once per binding. Answering # with a single line hid the bug where the two ran together into # 80808080. STUB_PORT_BINDINGS sets how many come back. *HostPort*) for _ in $(seq "${STUB_PORT_BINDINGS:-2}"); do printf '%s\n' "${STUB_PORT:-8080}" done ;; *) cat "${STUB_INSPECT:-/dev/null}" ;; esac ;; logs) printf 'stub log line\n' ;; # `docker cp` out of a container that is about to be removed. STUB_CP_DIR is # a directory to copy in place of the container's state dir; unset means the # copy fails, which is what an unprofiled or already-gone container looks # like. cp) shift [ -n "${STUB_CP_DIR:-}" ] || exit 1 cp -a "$STUB_CP_DIR/." "${2%/}/" ;; *) ;; esac STUB cat > "$BIN/curl" <<'STUB' #!/usr/bin/env bash printf 'curl %s\n' "$*" >> "$STUB_LOG" exit "${STUB_READY:-0}" STUB chmod +x "$BIN/docker" "$BIN/curl" export PATH="$BIN:$PATH" export DOCKER_HOST="unix://$TMP/no-such-docker.sock" # Pin the session rather than letting it come from the checkout's name, so the # fixtures and the script agree and the test does not change meaning when run # from a differently named worktree. export ARENA_SESSION="testsession" export STUB_LOG="$TMP/calls" export ARENA_STATE_DIR="$TMP/state" # versions.env would otherwise name the image after this checkout's commit, and # the stub daemon has no such image. Pinning the tag keeps every case about the # thing it is testing. export IMAGE_TAG=latest # Hard stop rather than a skip. If the stub is not what `docker` resolves to, # the cases below would start real containers on whatever machine is running # the tests, which is the one thing this whole feature exists to prevent. [ "$(command -v docker)" = "$BIN/docker" ] || { echo "ERROR: docker is not stubbed" >&2; exit 1; } [ "$(command -v curl)" = "$BIN/curl" ] || { echo "ERROR: curl is not stubbed" >&2; exit 1; } export STUB_IMAGES="$TMP/images" printf 'lance-blue/arena:latest\nlance-blue/arena:dev\n' > "$STUB_IMAGES" : > "$TMP/empty" export STUB_PS="$TMP/empty" STUB_PS_ALL="$TMP/empty" STUB_NAMES="$TMP/empty" # arena -> its exit status; output lands in $TMP/out, calls in the log arena() { : > "$STUB_LOG" "$ROOT/scripts/arena.sh" "$@" > "$TMP/out" 2>&1 } saw() { grep -q -- "$1" "$TMP/out"; } called() { grep -q -- "$1" "$STUB_LOG"; } json() { jq -e "$1" "$TMP/out" >/dev/null; } # One docker ps row: id, name, image, state, image title label, our label, and # the session that owns it. The session column is what stops one session's # `down` removing another's match. row() { printf '%s|%s|%s|%s|%s|%s|%s\n' "$@" "${7:-$ARENA_SESSION}"; } # --- caps ------------------------------------------------------------------- # # The caps are the point of the script, so they get the most cases. Each one is # about a value that must not silently turn into something else. caps() { arena caps; } caps check "the default caps are the production shape" \ saw "caps: 2 CPUs, 8192m memory (no swap), 512 pids" ARENA_CPUS=4 ARENA_MEMORY=4g caps check "an in-range override is taken" saw "4 CPUs, 4096m memory" ARENA_CPUS=1.5 caps check "a fractional CPU count survives" saw "1.500 CPUs" ARENA_CPUS=32 caps check "too many CPUs is clamped, not obeyed" saw "8 CPUs" check " and the clamp is announced" saw "out of range" ARENA_CPUS=0 caps check "too few CPUs is clamped up" saw "1 CPUs" ARENA_MEMORY=64g caps check "too much memory is clamped to the host's headroom" saw "12288m memory" ARENA_MEMORY=256m caps check "too little memory is clamped up" saw "2048m memory" ARENA_MEMORY=4294967296 caps check "a bare byte count is read the way docker reads it" saw "4096m memory" ARENA_PIDS=99999 caps check "too high a pid limit is clamped" saw "4096 pids" # A typo in an override must not quietly become the default: the reason to set # one is that the default was not what you wanted. ARENA_CPUS=banana caps check "a CPU count that is not a number is fatal" test $? -ne 0 ARENA_MEMORY=8gb caps check "a memory size in a unit docker does not take is fatal" test $? -ne 0 ARENA_PIDS=lots caps check "a pid limit that is not a number is fatal" test $? -ne 0 # --- which containers count as arena's -------------------------------------- # # The guard's decision, given the rows docker ps would have produced. This is # what catches a second match running under a name nobody guessed. filter() { bash -c '. "$1"; arena_filter_containers' _ "$ROOT/scripts/lib/common.sh"; } matched() { [ -n "$(row "$@" | filter)" ]; } unmatched() { [ -z "$(row "$@" | filter)" ]; } check "an image tagged lance-blue/arena counts" \ matched abc arena-dev lance-blue/arena:latest running "" "" check "any tag of it counts, not only the one just built" \ matched abc other lance-blue/arena:mm0.51.0-sur26.4.7-main-deadbeef running "" "" check "the ECR copy of the same image counts" \ matched abc other 1.dkr.ecr.us-east-1.amazonaws.com/lance-blue/arena:t running "" "" check "a renamed copy still counts, by the image's own title label" \ matched abc mine somebody/renamed:1 running "lance.blue arena" "" check "a container these scripts started counts, whatever it is called" \ matched abc weird sha256:0123 running "" "1" check "somebody else's container does not" \ unmatched abc db postgres:16 running "" "" check "a registry port is not mistaken for a tag" \ unmatched abc x reg:5000/other/thing running "" "" check " and the same registry serving arena still counts" \ matched abc x reg:5000/lance-blue/arena running "" "" # --- the singleton guard ---------------------------------------------------- row deadbeef some-other-match lance-blue/arena:dev running "" "" > "$TMP/ps-running" STUB_PS="$TMP/ps-running" arena up check "up refuses when a differently named match is already running" test $? -eq 2 check " and names the one in the way" saw "some-other-match" check " and says how to stop it" saw "arena.sh down" check " and starts nothing" bash -c '! grep -q "^run -d" "$1"' _ "$STUB_LOG" printf '%s\n' "arena-$ARENA_SESSION" > "$TMP/names" STUB_NAMES="$TMP/names" arena up check "up refuses a name collision" test $? -eq 2 check " and points at down rather than at docker rm" saw "arena.sh down" # --- up, past the guards ---------------------------------------------------- arena up check "up returns 0 once the endpoint answers" test $? -eq 0 check " it starts detached" called "run -d" check " the CPU cap is on the command line" called "--cpus 2" check " the memory cap is on the command line" called "--memory 8192m" check " swap is capped with it, so the host cannot be swapped to death" \ called "--memory-swap 8192m" check " the pid limit is on the command line" called "--pids-limit 512" check " the caps are printed, not only applied" saw "caps: 2 CPUs" # Nothing reached the container before this: up passed only # ARENA_MANIFEST_FILE, so ARENA_PROFILE=1 in the caller's shell was silently # dropped and the profiler could never be switched on from here. check " an unprofiled start passes no profiling variables" \ bash -c '! grep -q "ARENA_PROFILE" "$1"' _ "$STUB_LOG" ARENA_PROFILE=1 ARENA_PROFILE_INTERVAL=5ms arena up check "ARENA_PROFILE reaches the container" called "-e ARENA_PROFILE" check " along with the interval" called "-e ARENA_PROFILE_INTERVAL" check " and the start says profiling is on" saw "profile: on" # -e NAME, not -e NAME=value: an unset variable has to stay unset inside the # container, because 40-render-config.sh's defaults do not survive arriving as # an empty string. check " a variable the caller did not set is not invented" \ bash -c '! grep -q "ARENA_PROFILE_HOST_EVENT" "$1"' _ "$STUB_LOG" # The same bug as the comment above, twice more: a switch the container reads # and this list does not carry is set on the command line, absent inside, and # silent in both places. ARENA_LAZY_PNG and ARENA_PROFILE_TIMEOUT each shipped # that way. Anything new that 40-render-config.sh reads belongs here too. ARENA_PROFILE=1 ARENA_PROFILE_TIMEOUT=45 arena up check "the client profile window reaches the container" called "-e ARENA_PROFILE_TIMEOUT" ARENA_LAZY_PNG=1 arena up check "the lazy-png switch reaches the container" called "-e ARENA_LAZY_PNG" check " and it does not drag the profiler in with it" \ bash -c '! grep -q -- "-e ARENA_PROFILE\b" "$1"' _ "$STUB_LOG" check " the manifest it wrote has a slot for every faction" \ bash -c 'jq -e ".players | length == 2" "$1/manifest.json" >/dev/null' _ "$ARENA_STATE_DIR" check " the manifest is readable by the container's uid" \ bash -c '[ "$(stat -c %a "$1/manifest.json")" = 644 ]' _ "$ARENA_STATE_DIR" # 0 is a person clicking Done, which is what a local match should default to. check " humans are left to ready themselves by default" \ bash -c 'jq -e ".limits.autoReadySeconds == 0" "$1/manifest.json" >/dev/null' _ "$ARENA_STATE_DIR" # The throwaway container that reads the faction list is a container start too, # and it is capped for the same reason. check " even the scenario-reading container is capped" called "--cpus 1 --memory 256m" arena up -H TraineeB check "up takes the faction to play" test $? -eq 0 check " and the manifest gives it to the human" \ bash -c 'jq -e ".players[] | select(.control == \"human\") | .slot == \"TraineeB\"" "$1/manifest.json" >/dev/null' \ _ "$ARENA_STATE_DIR" arena up -H Nobody check "up refuses a faction the scenario does not have" test $? -ne 0 # Something other than a person driving the match needs the seat readied for # it, or the match never leaves the lounge - and a lounge is what made the # first client profile misleading. See PERFORMANCE.md. ARENA_AUTO_READY=5 arena up check "ARENA_AUTO_READY reaches the manifest" \ bash -c 'jq -e ".limits.autoReadySeconds == 5" "$1/manifest.json" >/dev/null' _ "$ARENA_STATE_DIR" ARENA_AUTO_READY=soon arena up check "a non-numeric ARENA_AUTO_READY is refused" test $? -ne 0 arena up --image dev check "up takes an image selector" test $? -eq 0 check " and starts that image" called "lance-blue/arena:dev" arena up --image nope check "up refuses a selector with no image behind it" test $? -eq 1 check " and says where one comes from" saw "build.sh" arena up --nonsense check "up refuses an option it does not know" test $? -eq 1 arena up one.mms two.mms check "up refuses two scenarios" test $? -eq 1 # --- a start that never serves ---------------------------------------------- # # A silent hang is the worst outcome for an automated caller, so this is the # case worth being sure about. row c0ffee arena-dev lance-blue/arena:latest running "" "" > "$TMP/ps-all" export STUB_PS_ALL="$TMP/ps-all" STUB_READY=7 arena up -t 1 check "up fails when the endpoint never answers" test $? -eq 3 check " it says how long it waited" saw "not serving after 1s" check " it prints the container's logs" saw "stub log line" check " and it cleans up, rather than blocking the next start" called "rm -f -v" STUB_READY=7 STUB_RUNNING=false arena up -t 30 check "up fails at once when the container exits, without waiting out the timeout" \ test $? -eq 3 check " and says the container exited" saw "exited before it served" STUB_READY=7 ARENA_KEEP_ON_FAIL=1 arena up -t 1 check "ARENA_KEEP_ON_FAIL leaves the container to be looked at" \ bash -c '! grep -q "rm -f -v" "$1"' _ "$STUB_LOG" # --- down ------------------------------------------------------------------- row aaa arena-dev lance-blue/arena:latest running "" "" > "$TMP/ps-all" row bbb stray-match lance-blue/arena:dev exited "" "" >> "$TMP/ps-all" arena down check "down removes every arena container, not only the expected name" test $? -eq 0 check " the named one" saw "removed arena-dev" check " and the stray one" saw "removed stray-match" check " including one that had already exited" called "rm -f -v bbb" check " and it takes the manifest with it" bash -c '[ ! -d "$1" ]' _ "$ARENA_STATE_DIR" # The exit scripts hang off SIGTERM: on a profiled run they stop the JVMs so # async-profiler flushes, then finalize collects. rm -f SIGKILLs and all of # that is lost, which is why the first profiled match produced nothing. check " the running one is stopped first, so the exit scripts run" \ called "stop -t 60 aaa" check " one that had already exited is not stopped again" \ bash -c '! grep -q "stop -t 60 bbb" "$1"' _ "$STUB_LOG" # --- down collects what finalize left behind -------------------------------- # # finalize has no upload target on a local match, so it keeps the profiles and # the stats table inside a container that down is about to remove. mkdir -p "$TMP/cp-src" printf 'a;b 1\n' > "$TMP/cp-src/profile-client.collapsed" printf 'metric n\n' > "$TMP/cp-src/stats-summary.txt" : > "$TMP/cp-src/host.status" export STUB_CP_DIR="$TMP/cp-src" export ARENA_ARTIFACTS_DIR="$TMP/artifacts" row aaa arena-dev lance-blue/arena:latest running "" "" > "$TMP/ps-all" arena down check "down keeps the profile finalize could not upload" \ bash -c '[ -s "$1/arena-dev/profile-client.collapsed" ]' _ "$ARENA_ARTIFACTS_DIR" check " and the stats table" \ bash -c '[ -s "$1/arena-dev/stats-summary.txt" ]' _ "$ARENA_ARTIFACTS_DIR" check " but not the run-state bookkeeping" \ bash -c '[ ! -e "$1/arena-dev/host.status" ]' _ "$ARENA_ARTIFACTS_DIR" check " and it says where they went" saw "kept 2 artefact(s)" unset STUB_CP_DIR rm -rf "$TMP/artifacts" row aaa arena-dev lance-blue/arena:latest running "" "" > "$TMP/ps-all" arena down check "an unprofiled match leaves no artefact directory behind" \ bash -c '[ ! -d "$1" ]' _ "$ARENA_ARTIFACTS_DIR" check " and down still succeeds" test $? -eq 0 export STUB_PS_ALL="$TMP/empty" export STUB_PS="$TMP/empty" arena down check "down is safe with nothing running" test $? -eq 0 check " and says so" saw "nothing of this session's is running" # --- one session must not remove another's match ---------------------------- # # This is the bug that made the A/B sweep produce nothing: `down` took every # arena container it could see, so two sessions tore down each other's matches # mid-run and finalize never wrote. row aaa arena-testsession lance-blue/arena:latest running "" "" testsession \ > "$TMP/ps-all" row zzz arena-other lance-blue/arena:latest running "" "" someoneelse \ >> "$TMP/ps-all" cp "$TMP/ps-all" "$TMP/ps-running" export STUB_PS_ALL="$TMP/ps-all" STUB_PS="$TMP/ps-running" arena down check "down removes this session's match" called "rm -f -v aaa" check " and leaves another session's alone" \ bash -c '! grep -q "rm -f -v zzz" "$1"' _ "$STUB_LOG" check " but says the other one is there" saw "belongs to someoneelse" check " and how to take it too" saw "down --all" : > "$STUB_LOG" arena down --all check "down --all takes every arena container" called "rm -f -v zzz" export STUB_NAMES="$TMP/empty" # no name collision; that is a different guard # Another session's match no longer blocks a start; the cap protects the # laptop instead. Refusing outright is what stopped cooperating sessions - # and jmm - from working at the same time. : > "$STUB_LOG" row zzz arena-other lance-blue/arena:latest running "" "" someoneelse > "$TMP/ps-running" export STUB_PS="$TMP/ps-running" STUB_PS_ALL="$TMP/ps-running" arena up check "another session's match does not block a start" test $? -eq 0 check " but it is mentioned" saw "sharing the box" ARENA_MAX_MATCHES=1 arena up check "the cap refuses when the machine is already full" test $? -eq 2 check " and names who has them" saw "someoneelse" check " without offering to stop them" saw "ask before stopping" # --- status ----------------------------------------------------------------- arena status check "status with nothing running is still JSON" json '.running == false' cat > "$TMP/inspect" <<'FIELDS' /arena-dev lance-blue/arena:latest running 2020-01-01T00:00:00.000000000Z 2000000000 8589934592 512 FIELDS export STUB_INSPECT="$TMP/inspect" export STUB_PS="$TMP/ps-running" row c0ffee arena-dev lance-blue/arena:latest running "" "" > "$TMP/ps-running" arena status check "status of a running match parses" json '.' check " it reports the caps docker is enforcing, not the ones asked for" \ json '.caps.cpus == 2 and .caps.memory_mb == 8192 and .caps.pids_limit == 512' check " it reports the image" json '.image == "lance-blue/arena:latest"' check " it reports the port and a url to open" \ json '.port == 8080 and (.url == "http://localhost:8080/")' # The stub answers with both bindings a dual-stack host publishes, because the # real failure was reading them as one number: 8080 and 8080 arrived as # 80808080, status printed that as the port, and wait-ready then probed a port # nothing was listening on and timed out against a container that was serving. STUB_PORT_BINDINGS=3 arena status check " extra port bindings do not run together into one number" \ json '.port == 8080' STUB_PORT_BINDINGS=1 arena status check " a single binding still reports the port" json '.port == 8080' arena status check " it reports an uptime" json '.uptime_seconds > 0' check " ready is reported separately from running" json '.running and .ready' STUB_READY=7 arena status check " a running match that is not serving reports ready false" \ json '.running and (.ready | not)' # --- wait-ready and logs ---------------------------------------------------- arena wait-ready check "wait-ready returns once it answers" test $? -eq 0 STUB_READY=7 arena wait-ready -t 1 check "wait-ready gives up loudly" test $? -eq 3 check " with the logs" saw "stub log line" export STUB_PS="$TMP/empty" arena wait-ready check "wait-ready says so when nothing is running" test $? -eq 4 arena logs check "logs says so when nothing is running" test $? -eq 4 # --- dispatch --------------------------------------------------------------- arena check "no command is a usage error" test $? -eq 1 arena frobnicate check "an unknown command is a usage error" test $? -eq 1 check " and the usage is printed" saw "usage: arena.sh" arena --help check "--help works" test $? -eq 0 echo echo "$pass passed, $fail failed" [ "$fail" -eq 0 ]