diff --git a/docs/runbook.md b/docs/runbook.md index f1e9635..b732810 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -226,7 +226,9 @@ and nothing else, and the distribution takes `/assets/megamek` off at the edge. We upload the unit art and not the rest of the tree; a subset of the same layout costs nothing. -Run this when the pinned MegaMek moves, not on a deploy. Take both arguments +Run this when the pinned MegaMek moves, not on a deploy. The release tarball is +639MB against the 39MB of art it carries, so it is cached under +`~/.cache/lance-blue/megamek` and reused whenever its digest still matches. Take both arguments from arena's `versions.env` — `MEGAMEK_VERSION` and `MEGAMEK_RELEASE_SHA256` — so the art in the bucket is provably the release the container runs: diff --git a/scripts/assets-megamek.sh b/scripts/assets-megamek.sh index 001778f..a7880d9 100755 --- a/scripts/assets-megamek.sh +++ b/scripts/assets-megamek.sh @@ -31,6 +31,9 @@ # capital in them. S3 is case-sensitive; normalising any of it breaks a fifth # of the lookups, silently, months later. # +# The tarball is cached under ~/.cache/lance-blue/megamek (MEGAMEK_CACHE +# overrides), and a cached copy is used only when its digest still matches. +# # Credentials are the caller's, as everywhere else here. set -euo pipefail @@ -74,17 +77,35 @@ echo "bucket: $bucket" # the same trap. url="https://github.com/MegaMek/megamek/releases/download/v${version}/MegaMek-${version}.tar.gz" +# The release tarball is 639MB and this run uploads 39MB of it, so it is kept +# rather than fetched again. The digest is what says a kept copy is the right +# one - it is checked on a cached file exactly as on a fresh download, so a +# truncated or tampered cache is caught rather than trusted. +cache="${MEGAMEK_CACHE:-${XDG_CACHE_HOME:-$HOME/.cache}/lance-blue/megamek}" +mkdir -p "$cache" +tarball="$cache/MegaMek-${version}.tar.gz" + +if [ -f "$tarball" ] && echo "$sha256 $tarball" | sha256sum -c --status -; then + echo "using $tarball (digest matches)" +else + echo "fetching $url" + # To a partial name, moved into place only once the digest is checked, so an + # interrupted download is never mistaken for a cached release. + curl -fSL --retry 3 --retry-connrefused -o "$tarball.part" "$url" + echo "$sha256 $tarball.part" | sha256sum -c --status - || { + rm -f "$tarball.part" + echo "digest mismatch: refusing to publish art from a tarball that is not the pinned release" >&2 + exit 1 + } + mv "$tarball.part" "$tarball" +fi + work="$(mktemp -d)" trap 'rm -rf "$work"' EXIT -echo "fetching $url" -curl -fsSL "$url" -o "$work/megamek.tar.gz" -echo "$sha256 $work/megamek.tar.gz" | sha256sum -c - || { - echo "digest mismatch: refusing to publish art from a tarball that is not the pinned release" >&2 - exit 1 -} - -tar -xzf "$work/megamek.tar.gz" -C "$work" +# Only the tree that is published. Reading the whole stream is unavoidable - +# gzip is not seekable - but writing 590MB to unpack 39MB is not. +tar -xzf "$tarball" -C "$work" --wildcards 'MegaMek-*/data/images/units/*' 'MegaMek-*/LICENSE*' root="$(find "$work" -maxdepth 1 -type d -name 'MegaMek-*' | head -1)" units="$root/data/images/units" [ -d "$units" ] || { @@ -102,8 +123,14 @@ echo "syncing $files files to s3://$bucket/$prefix/$tree/" # Immutable, and said so: the key carries the version that produced it, so a # viewer that has one never needs to ask again and a bump is a new path rather # than an invalidation. +# +# --size-only for the same reason. A key under a version is written once and +# never changes, so size is a sufficient comparison; the default also weighs +# modification times, and a fresh extract whose mtimes land after the objects' +# would re-upload all 6,995 files to no effect. sync_args=( --no-progress + --size-only --cache-control "public, max-age=31536000, immutable" ) [ -n "$dry_run" ] && sync_args+=(--dryrun)