Something went wrong. Try again.
Backend environment for match hosting for lance.blue
Something went wrong. Try again.
Shell
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480#!/usr/bin/env bash# scripts/arena.sh starts containers, and the test stage has no docker daemon.# So what is tested here is every decision it makes around the daemon: the# resource caps and their clamping, which containers count as arena's, what the# argument parser refuses, and what happens when a start never becomes ready.## docker and curl are stubbed on PATH. The stub records every call and answers# from files this script writes, so the assertions can be about the arguments# docker was given - which is the only place the caps actually exist.# DOCKER_HOST is pointed at nothing as a second line of defence: a call that# ever escaped the stub fails rather than reaching a real daemon.## Not covered: anything that needs a container to really run. Whether the image# serves its client paths, how long it takes, and whether 2 CPUs and 8GB are enough for# three JVMs are questions only a real start answers.set -uo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"TMP="$(mktemp -d)"trap 'rm -rf "$TMP"' EXIT
pass=0; fail=0check() { if "${@:2}"; then echo "ok $1"; pass=$((pass+1)) else echo "FAIL $1"; fail=$((fail+1)); fi}
command -v jq >/dev/null || { echo "SKIP: jq is not installed"; exit 0; }
BIN="$TMP/bin"mkdir -p "$BIN"
cat > "$BIN/docker" <<'STUB'#!/usr/bin/env bash# Stands in for the docker CLI. Records the call, then answers from whatever# the test set up. Nothing here reaches a daemon.printf '%s\n' "$*" >> "$STUB_LOG"case "${1:-}" in ps) shift # The name-only listing ends in that format and nothing else; the fuller # one contains {{.Names}} too, so this has to be an exact tail match. if [[ "$*" == *"--format {{.Names}}" ]]; then cat "${STUB_NAMES:-/dev/null}" elif [[ " $* " == *" -a "* ]]; then cat "${STUB_PS_ALL:-/dev/null}" else cat "${STUB_PS:-/dev/null}" fi ;; image) shift 2 grep -qxF "$1" "${STUB_IMAGES:-/dev/null}" ;; run) shift # A detached start is the match. Anything else is the throwaway container # that reads the scenario's faction list out of the image. Matching on the # first argument only: the faction reader's shell snippet contains a "-d" # of its own, in `tr -d`. if [ "${1:-}" = -d ]; then printf '%s\n' "${STUB_ID:-c0ffee}" else printf '%s\n' "${STUB_FACTIONS:-TraineeA,TraineeB}" fi ;; inspect) shift case "$*" in *State.Running*) printf '%s\n' "${STUB_RUNNING:-true}" ;; # One published port is two bindings on a dual-stack host, 0.0.0.0 and # [::], so the range in the template runs once per binding. Answering # with a single line hid the bug where the two ran together into # 80808080. STUB_PORT_BINDINGS sets how many come back. *HostPort*) for _ in $(seq "${STUB_PORT_BINDINGS:-2}"); do printf '%s\n' "${STUB_PORT:-8080}" done ;; *) cat "${STUB_INSPECT:-/dev/null}" ;; esac ;; logs) printf 'stub log line\n' ;; # `docker cp` out of a container that is about to be removed. STUB_CP_DIR is # a directory to copy in place of the container's state dir; unset means the # copy fails, which is what an unprofiled or already-gone container looks # like. cp) shift [ -n "${STUB_CP_DIR:-}" ] || exit 1 cp -a "$STUB_CP_DIR/." "${2%/}/" ;; *) ;;esacSTUB
cat > "$BIN/curl" <<'STUB'#!/usr/bin/env bashprintf 'curl %s\n' "$*" >> "$STUB_LOG"exit "${STUB_READY:-0}"STUB
chmod +x "$BIN/docker" "$BIN/curl"
export PATH="$BIN:$PATH"export DOCKER_HOST="unix://$TMP/no-such-docker.sock"# Pin the session rather than letting it come from the checkout's name, so the# fixtures and the script agree and the test does not change meaning when run# from a differently named worktree.export ARENA_SESSION="testsession"export STUB_LOG="$TMP/calls"export ARENA_STATE_DIR="$TMP/state"
# versions.env would otherwise name the image after this checkout's commit, and# the stub daemon has no such image. Pinning the tag keeps every case about the# thing it is testing.export IMAGE_TAG=latest
# Hard stop rather than a skip. If the stub is not what `docker` resolves to,# the cases below would start real containers on whatever machine is running# the tests, which is the one thing this whole feature exists to prevent.[ "$(command -v docker)" = "$BIN/docker" ] || { echo "ERROR: docker is not stubbed" >&2; exit 1; }[ "$(command -v curl)" = "$BIN/curl" ] || { echo "ERROR: curl is not stubbed" >&2; exit 1; }
export STUB_IMAGES="$TMP/images"printf 'lance-blue/arena:latest\nlance-blue/arena:dev\n' > "$STUB_IMAGES"
: > "$TMP/empty"export STUB_PS="$TMP/empty" STUB_PS_ALL="$TMP/empty" STUB_NAMES="$TMP/empty"
# arena <args...> -> its exit status; output lands in $TMP/out, calls in the logarena() { : > "$STUB_LOG" "$ROOT/scripts/arena.sh" "$@" > "$TMP/out" 2>&1}saw() { grep -q -- "$1" "$TMP/out"; }called() { grep -q -- "$1" "$STUB_LOG"; }json() { jq -e "$1" "$TMP/out" >/dev/null; }
# One docker ps row: id, name, image, state, image title label, our label, and# the session that owns it. The session column is what stops one session's# `down` removing another's match.row() { printf '%s|%s|%s|%s|%s|%s|%s\n' "$@" "${7:-$ARENA_SESSION}"; }
# --- caps -------------------------------------------------------------------## The caps are the point of the script, so they get the most cases. Each one is# about a value that must not silently turn into something else.
caps() { arena caps; }
capscheck "the default caps are the production shape" \ saw "caps: 2 CPUs, 8192m memory (no swap), 512 pids"
ARENA_CPUS=4 ARENA_MEMORY=4g capscheck "an in-range override is taken" saw "4 CPUs, 4096m memory"ARENA_CPUS=1.5 capscheck "a fractional CPU count survives" saw "1.500 CPUs"
ARENA_CPUS=32 capscheck "too many CPUs is clamped, not obeyed" saw "8 CPUs"check " and the clamp is announced" saw "out of range"ARENA_CPUS=0 capscheck "too few CPUs is clamped up" saw "1 CPUs"ARENA_MEMORY=64g capscheck "too much memory is clamped to the host's headroom" saw "12288m memory"ARENA_MEMORY=256m capscheck "too little memory is clamped up" saw "2048m memory"ARENA_MEMORY=4294967296 capscheck "a bare byte count is read the way docker reads it" saw "4096m memory"ARENA_PIDS=99999 capscheck "too high a pid limit is clamped" saw "4096 pids"
# A typo in an override must not quietly become the default: the reason to set# one is that the default was not what you wanted.ARENA_CPUS=banana capscheck "a CPU count that is not a number is fatal" test $? -ne 0ARENA_MEMORY=8gb capscheck "a memory size in a unit docker does not take is fatal" test $? -ne 0ARENA_PIDS=lots capscheck "a pid limit that is not a number is fatal" test $? -ne 0
# --- which containers count as arena's --------------------------------------## The guard's decision, given the rows docker ps would have produced. This is# what catches a second match running under a name nobody guessed.
filter() { bash -c '. "$1"; arena_filter_containers' _ "$ROOT/scripts/lib/common.sh"; }matched() { [ -n "$(row "$@" | filter)" ]; }unmatched() { [ -z "$(row "$@" | filter)" ]; }
check "an image tagged lance-blue/arena counts" \ matched abc arena-dev lance-blue/arena:latest running "" ""check "any tag of it counts, not only the one just built" \ matched abc other lance-blue/arena:mm0.51.0-sur26.4.7-main-deadbeef running "" ""check "the ECR copy of the same image counts" \ matched abc other 1.dkr.ecr.us-east-1.amazonaws.com/lance-blue/arena:t running "" ""check "a renamed copy still counts, by the image's own title label" \ matched abc mine somebody/renamed:1 running "lance.blue arena" ""check "a container these scripts started counts, whatever it is called" \ matched abc weird sha256:0123 running "" "1"check "somebody else's container does not" \ unmatched abc db postgres:16 running "" ""check "a registry port is not mistaken for a tag" \ unmatched abc x reg:5000/other/thing running "" ""check " and the same registry serving arena still counts" \ matched abc x reg:5000/lance-blue/arena running "" ""
# --- the singleton guard ----------------------------------------------------
row deadbeef some-other-match lance-blue/arena:dev running "" "" > "$TMP/ps-running"
STUB_PS="$TMP/ps-running" arena upcheck "up refuses when a differently named match is already running" test $? -eq 2check " and names the one in the way" saw "some-other-match"check " and says how to stop it" saw "arena.sh down"check " and starts nothing" bash -c '! grep -q "^run -d" "$1"' _ "$STUB_LOG"
printf '%s\n' "arena-$ARENA_SESSION" > "$TMP/names"STUB_NAMES="$TMP/names" arena upcheck "up refuses a name collision" test $? -eq 2check " and points at down rather than at docker rm" saw "arena.sh down"
# --- up, past the guards ----------------------------------------------------
arena upcheck "up returns 0 once the endpoint answers" test $? -eq 0check " it starts detached" called "run -d"check " the CPU cap is on the command line" called "--cpus 2"check " the memory cap is on the command line" called "--memory 8192m"check " swap is capped with it, so the host cannot be swapped to death" \ called "--memory-swap 8192m"check " the pid limit is on the command line" called "--pids-limit 512"check " the caps are printed, not only applied" saw "caps: 2 CPUs"# Nothing reached the container before this: up passed only# ARENA_MANIFEST_FILE, so ARENA_PROFILE=1 in the caller's shell was silently# dropped and the profiler could never be switched on from here.check " an unprofiled start passes no profiling variables" \ bash -c '! grep -q "ARENA_PROFILE" "$1"' _ "$STUB_LOG"
ARENA_PROFILE=1 ARENA_PROFILE_INTERVAL=5ms arena upcheck "ARENA_PROFILE reaches the container" called "-e ARENA_PROFILE"check " along with the interval" called "-e ARENA_PROFILE_INTERVAL"check " and the start says profiling is on" saw "profile: on"# -e NAME, not -e NAME=value: an unset variable has to stay unset inside the# container, because 40-render-config.sh's defaults do not survive arriving as# an empty string.check " a variable the caller did not set is not invented" \ bash -c '! grep -q "ARENA_PROFILE_HOST_EVENT" "$1"' _ "$STUB_LOG"
# The same bug as the comment above, twice more: a switch the container reads# and this list does not carry is set on the command line, absent inside, and# silent in both places. ARENA_LAZY_PNG and ARENA_PROFILE_TIMEOUT each shipped# that way. Anything new that 40-render-config.sh reads belongs here too.ARENA_PROFILE=1 ARENA_PROFILE_TIMEOUT=45 arena upcheck "the client profile window reaches the container" called "-e ARENA_PROFILE_TIMEOUT"
ARENA_LAZY_PNG=1 arena upcheck "the lazy-png switch reaches the container" called "-e ARENA_LAZY_PNG"check " and it does not drag the profiler in with it" \ bash -c '! grep -q -- "-e ARENA_PROFILE\b" "$1"' _ "$STUB_LOG"check " the manifest it wrote has a slot for every faction" \ bash -c 'jq -e ".players | length == 2" "$1/manifest.json" >/dev/null' _ "$ARENA_STATE_DIR"check " the manifest is readable by the container's uid" \ bash -c '[ "$(stat -c %a "$1/manifest.json")" = 644 ]' _ "$ARENA_STATE_DIR"# 0 is a person clicking Done, which is what a local match should default to.check " humans are left to ready themselves by default" \ bash -c 'jq -e ".limits.autoReadySeconds == 0" "$1/manifest.json" >/dev/null' _ "$ARENA_STATE_DIR"# The throwaway container that reads the faction list is a container start too,# and it is capped for the same reason.check " even the scenario-reading container is capped" called "--cpus 1 --memory 256m"
arena up -H TraineeBcheck "up takes the faction to play" test $? -eq 0check " and the manifest gives it to the human" \ bash -c 'jq -e ".players[] | select(.control == \"human\") | .slot == \"TraineeB\"" "$1/manifest.json" >/dev/null' \ _ "$ARENA_STATE_DIR"
arena up -H Nobodycheck "up refuses a faction the scenario does not have" test $? -ne 0
# Something other than a person driving the match needs the seat readied for# it, or the match never leaves the lounge - and a lounge is what made the# first client profile misleading. See PERFORMANCE.md.ARENA_AUTO_READY=5 arena upcheck "ARENA_AUTO_READY reaches the manifest" \ bash -c 'jq -e ".limits.autoReadySeconds == 5" "$1/manifest.json" >/dev/null' _ "$ARENA_STATE_DIR"
ARENA_AUTO_READY=soon arena upcheck "a non-numeric ARENA_AUTO_READY is refused" test $? -ne 0
arena up --image devcheck "up takes an image selector" test $? -eq 0check " and starts that image" called "lance-blue/arena:dev"
arena up --image nopecheck "up refuses a selector with no image behind it" test $? -eq 1check " and says where one comes from" saw "build.sh"
arena up --nonsensecheck "up refuses an option it does not know" test $? -eq 1arena up one.mms two.mmscheck "up refuses two scenarios" test $? -eq 1
# --- a start that never serves ----------------------------------------------## A silent hang is the worst outcome for an automated caller, so this is the# case worth being sure about.
row c0ffee arena-dev lance-blue/arena:latest running "" "" > "$TMP/ps-all"export STUB_PS_ALL="$TMP/ps-all"
STUB_READY=7 arena up -t 1check "up fails when the endpoint never answers" test $? -eq 3check " it says how long it waited" saw "not serving after 1s"check " it prints the container's logs" saw "stub log line"check " and it cleans up, rather than blocking the next start" called "rm -f -v"
STUB_READY=7 STUB_RUNNING=false arena up -t 30check "up fails at once when the container exits, without waiting out the timeout" \ test $? -eq 3check " and says the container exited" saw "exited before it served"
STUB_READY=7 ARENA_KEEP_ON_FAIL=1 arena up -t 1check "ARENA_KEEP_ON_FAIL leaves the container to be looked at" \ bash -c '! grep -q "rm -f -v" "$1"' _ "$STUB_LOG"
# --- down -------------------------------------------------------------------
row aaa arena-dev lance-blue/arena:latest running "" "" > "$TMP/ps-all"row bbb stray-match lance-blue/arena:dev exited "" "" >> "$TMP/ps-all"arena downcheck "down removes every arena container, not only the expected name" test $? -eq 0check " the named one" saw "removed arena-dev"check " and the stray one" saw "removed stray-match"check " including one that had already exited" called "rm -f -v bbb"check " and it takes the manifest with it" bash -c '[ ! -d "$1" ]' _ "$ARENA_STATE_DIR"# The exit scripts hang off SIGTERM: on a profiled run they stop the JVMs so# async-profiler flushes, then finalize collects. rm -f SIGKILLs and all of# that is lost, which is why the first profiled match produced nothing.check " the running one is stopped first, so the exit scripts run" \ called "stop -t 60 aaa"check " one that had already exited is not stopped again" \ bash -c '! grep -q "stop -t 60 bbb" "$1"' _ "$STUB_LOG"
# --- down collects what finalize left behind --------------------------------## finalize has no upload target on a local match, so it keeps the profiles and# the stats table inside a container that down is about to remove.
mkdir -p "$TMP/cp-src"printf 'a;b 1\n' > "$TMP/cp-src/profile-client.collapsed"printf 'metric n\n' > "$TMP/cp-src/stats-summary.txt": > "$TMP/cp-src/host.status"export STUB_CP_DIR="$TMP/cp-src"export ARENA_ARTIFACTS_DIR="$TMP/artifacts"row aaa arena-dev lance-blue/arena:latest running "" "" > "$TMP/ps-all"arena downcheck "down keeps the profile finalize could not upload" \ bash -c '[ -s "$1/arena-dev/profile-client.collapsed" ]' _ "$ARENA_ARTIFACTS_DIR"check " and the stats table" \ bash -c '[ -s "$1/arena-dev/stats-summary.txt" ]' _ "$ARENA_ARTIFACTS_DIR"check " but not the run-state bookkeeping" \ bash -c '[ ! -e "$1/arena-dev/host.status" ]' _ "$ARENA_ARTIFACTS_DIR"check " and it says where they went" saw "kept 2 artefact(s)"
unset STUB_CP_DIRrm -rf "$TMP/artifacts"row aaa arena-dev lance-blue/arena:latest running "" "" > "$TMP/ps-all"arena downcheck "an unprofiled match leaves no artefact directory behind" \ bash -c '[ ! -d "$1" ]' _ "$ARENA_ARTIFACTS_DIR"check " and down still succeeds" test $? -eq 0
export STUB_PS_ALL="$TMP/empty"export STUB_PS="$TMP/empty"arena downcheck "down is safe with nothing running" test $? -eq 0check " and says so" saw "nothing of this session's is running"
# --- one session must not remove another's match ----------------------------## This is the bug that made the A/B sweep produce nothing: `down` took every# arena container it could see, so two sessions tore down each other's matches# mid-run and finalize never wrote.
row aaa arena-testsession lance-blue/arena:latest running "" "" testsession \ > "$TMP/ps-all"row zzz arena-other lance-blue/arena:latest running "" "" someoneelse \ >> "$TMP/ps-all"cp "$TMP/ps-all" "$TMP/ps-running"export STUB_PS_ALL="$TMP/ps-all" STUB_PS="$TMP/ps-running"
arena downcheck "down removes this session's match" called "rm -f -v aaa"check " and leaves another session's alone" \ bash -c '! grep -q "rm -f -v zzz" "$1"' _ "$STUB_LOG"check " but says the other one is there" saw "belongs to someoneelse"check " and how to take it too" saw "down --all"
: > "$STUB_LOG"arena down --allcheck "down --all takes every arena container" called "rm -f -v zzz"
export STUB_NAMES="$TMP/empty" # no name collision; that is a different guard# Another session's match no longer blocks a start; the cap protects the# laptop instead. Refusing outright is what stopped cooperating sessions -# and jmm - from working at the same time.: > "$STUB_LOG"row zzz arena-other lance-blue/arena:latest running "" "" someoneelse > "$TMP/ps-running"export STUB_PS="$TMP/ps-running" STUB_PS_ALL="$TMP/ps-running"arena upcheck "another session's match does not block a start" test $? -eq 0check " but it is mentioned" saw "sharing the box"
ARENA_MAX_MATCHES=1 arena upcheck "the cap refuses when the machine is already full" test $? -eq 2check " and names who has them" saw "someoneelse"check " without offering to stop them" saw "ask before stopping"
# --- status -----------------------------------------------------------------
arena statuscheck "status with nothing running is still JSON" json '.running == false'
cat > "$TMP/inspect" <<'FIELDS'/arena-devlance-blue/arena:latestrunning2020-01-01T00:00:00.000000000Z20000000008589934592512FIELDSexport STUB_INSPECT="$TMP/inspect"export STUB_PS="$TMP/ps-running"row c0ffee arena-dev lance-blue/arena:latest running "" "" > "$TMP/ps-running"
arena statuscheck "status of a running match parses" json '.'check " it reports the caps docker is enforcing, not the ones asked for" \ json '.caps.cpus == 2 and .caps.memory_mb == 8192 and .caps.pids_limit == 512'check " it reports the image" json '.image == "lance-blue/arena:latest"'check " it reports the port and a url to open" \ json '.port == 8080 and (.url == "http://localhost:8080/")'# The stub answers with both bindings a dual-stack host publishes, because the# real failure was reading them as one number: 8080 and 8080 arrived as# 80808080, status printed that as the port, and wait-ready then probed a port# nothing was listening on and timed out against a container that was serving.STUB_PORT_BINDINGS=3 arena statuscheck " extra port bindings do not run together into one number" \ json '.port == 8080'STUB_PORT_BINDINGS=1 arena statuscheck " a single binding still reports the port" json '.port == 8080'arena statuscheck " it reports an uptime" json '.uptime_seconds > 0'check " ready is reported separately from running" json '.running and .ready'
STUB_READY=7 arena statuscheck " a running match that is not serving reports ready false" \ json '.running and (.ready | not)'
# --- wait-ready and logs ----------------------------------------------------
arena wait-readycheck "wait-ready returns once it answers" test $? -eq 0
STUB_READY=7 arena wait-ready -t 1check "wait-ready gives up loudly" test $? -eq 3check " with the logs" saw "stub log line"
export STUB_PS="$TMP/empty"arena wait-readycheck "wait-ready says so when nothing is running" test $? -eq 4arena logscheck "logs says so when nothing is running" test $? -eq 4
# --- dispatch ---------------------------------------------------------------
arenacheck "no command is a usage error" test $? -eq 1arena frobnicatecheck "an unknown command is a usage error" test $? -eq 1check " and the usage is printed" saw "usage: arena.sh"arena --helpcheck "--help works" test $? -eq 0
echoecho "$pass passed, $fail failed"[ "$fail" -eq 0 ]