Something went wrong. Try again.
Tangled infrastructure definitions in Nix
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449#!/usr/bin/env nu# build, deploy, preview and roll back the tangled gcp envs. dev also deploys itself from ci.## usage:# nix run .#deploy-gcp -- --target dev|prod (--ref REF | --path DIR) [--build-on HOST]# build it, push, pin the new digests# nix run .#deploy-gcp -- --target dev|prod --preview what the tree would change, applies nothing# nix run .#deploy-gcp -- --target dev|prod --no-build apply the tree, keeping the images running now# nix run .#deploy-gcp -- --target dev|prod --rollback [--to SHA8|r-N] [--list]
# the control root knows the env, and the blueprint's last applied revision knows the restdef env-of [name: string, infra: string] { let root = $"($infra)/terraform/control/($name)" if not ($root | path exists) { let have = (ls $"($infra)/terraform/control" | where type == dir | get name | path basename | str join ', ') error make { msg: $"unknown --target ($name), want one of: ($have)" } } if not ($"($root)/.terraform" | path exists) { ^terraform $"-chdir=($root)" init -input=false | ignore } let e = (^terraform $"-chdir=($root)" output -json control | from json | get env) let outputs = (gc infra-manager revisions list --deployment $e.deployment --format json | from json | where state == "APPLIED" | sort-by createTime --reverse | get 0.applyResults.outputs) # a hostname wins, then a bare lb address, then cloud run's own url let urls = if ($e.hostnames? | is-not-empty) { $e.hostnames | items {|svc, host| { $svc: $"https://($host)" } } | into record } else if ($outputs.lb_ips?.value? | is-not-empty) { $outputs.lb_ips.value | items {|svc, ip| { $svc: $"http://($ip)" } } | into record } else { $outputs.services.value | values | first | reject region } $e | merge { name: $name root: $root location: ($e.deployment | path dirname | path dirname) registry: $outputs.registry.value urls: $urls }}
# gcloud, failing loudly with its stderr instead of handing back an empty stringdef --wrapped gc [...args: string] { let res = (^gcloud ...$args | complete) if $res.exit_code != 0 { error make { msg: $"gcloud ($args | first 3 | str join ' ') failed: ($res.stderr | str trim)" } } $res.stdout}
# anything that queues behind a running apply can deadlock it at unlock, so wait for ci insteaddef idle-deployment [e: record] { let dep = (gc infra-manager deployments describe $e.deployment --format json | from json) if $dep.state != "ACTIVE" or $dep.lockState? == "LOCKED" { error make { msg: $"($e.deployment | path basename) is ($dep.state), ($dep.lockState? | default 'unlocked'), try again once it's done" } } $dep}
def running-refs [dep: record] { $dep.terraformBlueprint?.inputValues?.image_refs?.inputValue? | default {}}
def refs-var [refs: record] { let pairs = ($refs | transpose svc ref | each {|r| $"($r.svc) = \"($r.ref)\"" } | str join ", ") $"image_refs={ ($pairs) }"}
# every mode applies the control root through here, so the image pins always go along. an empty# map would put the env back on its floating tagdef apply-control [e: record, refs: record, --yes] { let root = $"-chdir=($e.root)" let vars = if ($refs | is-empty) { [] } else { [-var (refs-var $refs)] } if $yes { ^terraform $root apply -input=false -auto-approve -no-color ...$vars } else { ^terraform $root apply ...$vars }}
# ---- building
const REPO = "https://tangled.org/tangled.org/core"
# cargo profile per env; anything unlisted builds releaseconst BOBBIN_PROFILE = { dev: "dev-deploy" }def bobbin-profile [e: record] { $BOBBIN_PROFILE | get -o $e.name | default "release" }
def src-tgz [e: record] { $"/tmp/tangled-($e.name)-src.tgz" }def work-dir [e: record] { $"/tmp/tangled-($e.name)-build" }
# the source as a tarball with everything under src/, and the commit it isdef fetch-source [e: record, ref: string, path: string] { let tgz = (src-tgz $e) if ($path | is-not-empty) { return { tgz: (pack-tree $path $tgz), sha: null } } let headers = $"($tgz).headers" let got = (^curl -fsSL -D $headers -o $tgz $"($REPO)/archive/($ref).tar.gz?prefix=src" | complete) if $got.exit_code != 0 { error make { msg: $"no archive of ($ref) at ($REPO): ($got.stderr | str trim)" } } # the appview answers with an immutable link to the commit the ref resolved to let sha = (open --raw $headers | parse -r 'archive/(?<sha>[0-9a-f]{40})' | get -o 0.sha) rm -f $headers if $sha == null { error make { msg: $"couldn't tell which commit ($ref) is" } } { tgz: $tgz, sha: $sha }}
# the checkout as it is, uncommitted changes included, minus whatever the vcs ignores. tar can't# read .gitignore properly, so git or jj says which files countdef pack-tree [path: string, tgz: string] { cd $path let listed = if (".git" | path exists) { ^git ls-files --cached --others --exclude-standard -z | split row (char nul) } else if (".jj" | path exists) { ^jj file list | lines } else { error make { msg: $"($path) is neither a git nor a jj checkout, so there's no telling what to leave out" } } let list = $"($tgz).files" $listed | where {|f| $f | path exists } | str join (char nul) | save -f $list let prefix = if (^tar --version | str contains "GNU") { [--transform "s,^,src/,"] } else { [-s ",^,src/,"] } # macos tar would add a ._ file next to every file for its extended attributes with-env { COPYFILE_DISABLE: "1" } { ^tar -czf $tgz --null -T $list ...$prefix } rm -f $list $tgz}
def src-tree [e: record, tgz: string] { let work = (work-dir $e) rm -rf $work mkdir $work tar -xzf $tgz -C $work --strip-components=1 cd $work # flakes only see files git knows about git init -q git add -A git -c user.email=deploy@local -c user.name=deploy commit -qm src $work}
def cleanup [e: record] { let work = (work-dir $e) if ($work | path exists) { ^chmod -R u+w $work; rm -rf $work } rm -f (src-tgz $e)}
def image [e: record, svc: string] { $"($e.registry)/($svc)-($e.suffix)" }
def build-local [e: record, src: record, origin: string] { # a working tree isn't a commit, so it only gets the env tag let tags = ([$e.name] | append (if $src.sha == null { [] } else { [$"($e.name)-($src.sha | str substring 0..7)"] })) let work = (src-tree $e $src.tgz) # bobbin's Containerfile copies from the repo root, so the root is the context docker buildx build ...[ "--platform=linux/amd64" $"--file=($work | path join bobbin containerfiles bobbin.Containerfile)" $"--build-arg=BOBBIN_PROFILE=(bobbin-profile $e)" ...($tags | each {|t| $"--tag=(image $e bobbin):($t)" }) "--provenance=false" $"--cache-from=type=registry,ref=($e.registry)/bobbin-cache:buildcache" $"--cache-to=type=registry,ref=($e.registry)/bobbin-cache:buildcache,mode=max" "--push" "--quiet" $"--metadata-file=($work)/bobbin.json" ] $work
cd $work let out = (nix build .#web-static-files --no-link --print-out-paths | complete | get stdout | lines | last) mkdir $"($work)/web/static" rm -rf $"($work)/web/static/fonts" $"($work)/web/static/logos" ^cp -fr $"($out)/." $"($work)/web/static/"
docker buildx build ...[ "--platform=linux/amd64" $"--file=($work | path join web Containerfile)" $"--build-arg=VITE_OAUTH_CLIENT_ID=($origin)/oauth-client-metadata.json" $"--build-arg=VITE_OAUTH_REDIRECT_URI=($origin)/oauth/callback" ...($tags | each {|t| $"--tag=(image $e svfe):($t)" }) "--provenance=false" $"--cache-from=type=registry,ref=($e.registry)/svfe-cache:buildcache" $"--cache-to=type=registry,ref=($e.registry)/svfe-cache:buildcache,mode=max" "--push" "--quiet" $"--metadata-file=($work)/svfe.json" ] $"($work)/web"
{ bobbin: (open --raw $"($work)/bobbin.json" | from json | get containerimage.digest) svfe: (open --raw $"($work)/svfe.json" | from json | get containerimage.digest) }}
const REMOTE_BUILD = r#' set -euo pipefail WORK="$HOME/$WORK_NAME" export DOCKER_CONFIG="$HOME/$WORK_NAME-docker" cleanup() { chmod -R u+w "$WORK" 2>/dev/null || true; rm -rf "$WORK" "$TGZ" "$DOCKER_CONFIG"; } trap cleanup EXIT chmod -R u+w "$WORK" 2>/dev/null || true rm -rf "$WORK" "$DOCKER_CONFIG" mkdir -p "$WORK" mkdir -m 700 "$DOCKER_CONFIG" tar -xzf "$TGZ" -C "$WORK" --strip-components=1 cd "$WORK" git init -q git add -A git -c user.email=deploy@local -c user.name=deploy commit -qm src auth=$(printf 'oauth2accesstoken:%s' "$REG_TOKEN" | base64 -w0) printf '{"auths":{"%s":{"auth":"%s"}}}' "$REG_HOST" "$auth" > "$DOCKER_CONFIG/config.json"
# buildkit rather than podman, so builds read the registry cache ci writes. # the daemon outlives the deploy, keeping its own cache for the next one; # nothing is exported back, the builder's uplink is too slow for that BX="$(nix build --no-link --print-out-paths nixpkgs#docker-buildx)/bin/docker-buildx" podman container exists tangled-buildkit || podman run -d --name tangled-buildkit --privileged \ -p 127.0.0.1:18234:1234 docker.io/moby/buildkit:v0.33.0 --addr tcp://0.0.0.0:1234 >/dev/null podman start tangled-buildkit >/dev/null "$BX" create --name tangled --driver remote tcp://127.0.0.1:18234 >/dev/null for _ in $(seq 30); do "$BX" --builder tangled inspect --bootstrap >/dev/null 2>&1 && break; sleep 1; done
tags() { echo "--tag=$1:$TAG"; if [ -n "$SHA" ]; then echo "--tag=$1:$TAG-$SHA"; fi; } digest() { sed -n 's/.*"containerimage\.digest": *"\([^"]*\)".*/\1/p' "$1"; }
"$BX" --builder tangled build --platform=linux/amd64 \ --file=bobbin/containerfiles/bobbin.Containerfile \ --build-arg "BOBBIN_PROFILE=$PROFILE" \ $(tags "$BOBBIN") --provenance=false \ --cache-from "type=registry,ref=$CACHE_REPO/bobbin-cache:buildcache" \ --push --quiet --metadata-file "$DOCKER_CONFIG/bobbin.json" .
out=$(nix build .#web-static-files --no-link --print-out-paths) static=$(echo "$out" | tail -1) mkdir -p web/static rm -rf web/static/fonts web/static/logos cp -fr "$static"/* web/static chmod -R u+w web/static
"$BX" --builder tangled build --platform=linux/amd64 \ --file=web/Containerfile \ --build-arg "VITE_OAUTH_CLIENT_ID=$ORIGIN/oauth-client-metadata.json" \ --build-arg "VITE_OAUTH_REDIRECT_URI=$ORIGIN/oauth/callback" \ $(tags "$SVFE") --provenance=false \ --cache-from "type=registry,ref=$CACHE_REPO/svfe-cache:buildcache" \ --push --quiet --metadata-file "$DOCKER_CONFIG/svfe.json" web/
echo "DIGEST_BOBBIN=$(digest "$DOCKER_CONFIG/bobbin.json")" echo "DIGEST_SVFE=$(digest "$DOCKER_CONFIG/svfe.json")"'#
def build-remote [e: record, src: record, origin: string, host: string] { let sha = if $src.sha == null { "" } else { $src.sha | str substring 0..7 } let tgz = $src.tgz let sent = (^scp -q $tgz $"($host):($tgz)" | complete) if $sent.exit_code != 0 { error make { msg: $"copying the source to ($host) failed: ($sent.stderr | str trim)" } } let token = (gc auth print-access-token | str trim) # stdin, not the remote command line: there the token would show up in ps on the builder let vars = { REG_TOKEN: $token, REG_HOST: ($e.registry | split row '/' | first), TGZ: $tgz, WORK_NAME: $"tangled-($e.name)-build" BOBBIN: (image $e bobbin), SVFE: (image $e svfe), TAG: $e.name, SHA: $sha, ORIGIN: $origin PROFILE: (bobbin-profile $e), CACHE_REPO: $e.registry } let env_block = ($vars | transpose k v | each {|r| $"($r.k)='($r.v)'" } | str join (char nl)) let res = ([$env_block $REMOTE_BUILD] | str join (char nl) | ssh $host "bash -s" | complete) if $res.exit_code != 0 { error make { msg: $"remote build failed: ($res.stderr | str substring 0..600)" } } let digest = {|key| $res.stdout | lines | where {|l| $l starts-with $"($key)=" } | first | str replace $"($key)=" "" } { bobbin: (do $digest DIGEST_BOBBIN), svfe: (do $digest DIGEST_SVFE) }}
# ---- rolling back
def images [e: record, svc: string] { gc artifacts docker images list (image $e $svc) --include-tags --format json | from json}
# every <env>-<sha8> build of one servicedef tagged [e: record, imgs: list] { $imgs | each {|img| $img.tags | where {|t| $t =~ $'^($e.name)-[0-9a-f]{8}$' } | each {|t| { build: ($t | str substring (($e.name | str length) + 1)..), digest: $img.version, pushed: $img.createTime } } } | flatten}
# a deploy that pinned digests keeps them in its revision's inputs, tag or no tagdef pinned [e: record] { gc infra-manager revisions list --deployment $e.deployment --format json | from json | where state == "APPLIED" | where {|r| $r.terraformBlueprint?.inputValues?.image_refs?.inputValue? | is-not-empty } | each {|r| let refs = $r.terraformBlueprint.inputValues.image_refs.inputValue { build: ($r.name | path basename) svfe: ($refs.svfe | str trim --left --char "@") pushed: $r.createTime bobbin: ($refs.bobbin | str trim --left --char "@") } }}
def running [e: record, svc: string] { let rev = (gc run services describe $"($svc)-($e.suffix)" --region $e.region --project $e.project --format "value(status.latestReadyRevisionName)" | str trim) gc run revisions describe $rev --region $e.region --project $e.project --format "value(status.imageDigest)" | str trim | split row "@" | last}
def candidates [e: record] { let now = { bobbin: (running $e bobbin), svfe: (running $e svfe) } let bob = (images $e bobbin) let svf = (images $e svfe) let builds = (tagged $e $svf | rename build svfe pushed | join (tagged $e $bob | select build digest | rename build bobbin) build) let revisions = (pinned $e | where {|r| $r.bobbin in $bob.version and $r.svfe in $svf.version and not ($builds | any {|b| $b.bobbin == $r.bobbin and $b.svfe == $r.svfe }) }) # a commit that doesn't touch either service rebuilds the same images, so one pair can carry several builds let all = ($builds | append $revisions | group-by {|c| $"($c.bobbin) ($c.svfe)" } | values | each {|g| { builds: ($g.build | uniq), pushed: ($g.pushed | math max), bobbin: $g.0.bobbin, svfe: $g.0.svfe } } | sort-by pushed --reverse | insert build {|c| $c.builds | str join " " } | insert running {|c| $c.bobbin == $now.bobbin and $c.svfe == $now.svfe }) if ($all | is-empty) { error make { msg: $"no build or pinned revision of ($e.name) has both images left in the registry" } } $all}
def pick [all: list, to: string] { if ($to | is-not-empty) { let hit = ($all | where {|c| $to in $c.builds }) if ($hit | is-empty) { error make { msg: $"no build ($to) with both images; have: ($all.builds | flatten | str join ', ')" } } return ($hit | first) } let labels = ($all | each {|c| let when = ($c.pushed | into datetime | format date "%Y-%m-%d %H:%M") $"($c.build) ($when)(if $c.running { ' (running)' } else { '' })" }) let i = ($labels | input list --index "roll back to") if $i == null { error make { msg: "nothing picked" } } $all | get $i}
# ---- previewing
def preview [e: record, dep: record] { let refs = (running-refs $dep) let id = $"preview-(random chars --length 8 | str downcase)" let out = $"/tmp/($id)"
# the control plan also zips the tree, which the infra manager preview below needs let root = $"-chdir=($e.root)" let planned = (^terraform $root plan -input=false -lock=false -no-color ...(if ($refs | is-empty) { [] } else { [-var (refs-var $refs)] }) $"-out=($out)-control.tfplan" | complete) if $planned.exit_code != 0 { error make { msg: $"control plan failed: ($planned.stdout | lines | last 30 | str join (char nl))" } } let control = (^terraform $root show -json $"($out)-control.tfplan" | from json | get resource_changes)
let preview = $"($e.location)/previews/($id)" let obj = $"gs://($e.project)-blueprints/($id).zip" let vars = $"($out).tfvars" let pairs = ($refs | transpose svc ref | each {|r| $"($r.svc) = \"($r.ref)\"" } | str join ", ") $"env_name = \"($e.name)\"\nimage_refs = { ($pairs) }\n" | save -f $vars gc storage cp --quiet $"($e.root)/../../.build/($e.name).zip" $obj let result = (try { (gc infra-manager previews create $preview --deployment $e.deployment --gcs-source $obj --service-account $e.im_sa --inputs-file $vars --tf-version-constraint $e.tf_version --quiet) gc infra-manager previews export $preview --file $out { ok: true } } catch {|err| { ok: false, msg: $err.msg } }) do -i { ^gcloud infra-manager previews delete $preview --quiet | complete } do -i { ^gcloud storage rm --quiet $obj | complete } rm -f $vars if not $result.ok { error make { msg: $result.msg } }
let changes = {|rcs, layer| $rcs | where {|c| $c.change.actions != ["no-op"] } | each {|c| { layer: $layer, resource: $c.address, action: ($c.change.actions | str join "/") } } } let all = ((do $changes $control control) | append (do $changes (open $"($out).json" | get resource_changes) blueprint)) print $"== full plans: ($out)-control.tfplan, ($out).json" if ($all | is-empty) { print "== no changes" } else { $all }}
# ----
def main [ --target: string = "" # dev or prod --build-on: string = "" # ssh host with podman to build on; empty builds locally --preview # plan the tree against the live env, apply nothing --no-build # apply the tree, keeping the images running now --rollback # pick an earlier build and pin it --to: string = "" # with --rollback: the build (8-char commit) or revision (r-N), skipping the menu --list # with --rollback: print the builds and exit --ref: string = "" # what to build: a branch, tag or commit of tangled.org/core --path: string = "" # or: build this local checkout as it is --infra: string = "" # this infra repo (default ~/proj/infra)] { if ($target | is-empty) { error make { msg: "pass --target dev or --target prod" } } let rollback = ($rollback or ($to | is-not-empty) or $list) if ([$preview $no_build $rollback] | where {|m| $m } | length) > 1 { error make { msg: "--preview, --no-build and --rollback don't go together" } } let infra = if ($infra | is-empty) { $"($env.HOME)/proj/infra" } else { $infra } let e = (env-of $target $infra)
if $rollback { let all = (candidates $e) if $list { return ($all | select build pushed running) } let dep = (idle-deployment $e) let it = (pick $all $to) print $"== bobbin-($e.suffix)@($it.bobbin)" print $"== svfe-($e.suffix)@($it.svfe)" apply-control $e { bobbin: $"@($it.bobbin)", svfe: $"@($it.svfe)" } if $e.name == "dev" { print "== dev moves on again with the next push to sv-fe" } return }
let dep = (idle-deployment $e) if $preview { return (preview $e $dep) } if $no_build { let refs = (running-refs $dep) print $"== keeping (if ($refs | is-empty) { $':($e.name)' } else { $refs | values | str join ', ' })" return (apply-control $e $refs) }
if ($ref | is-empty) == ($path | is-empty) { error make { msg: "pass either --ref or --path, to say what to build" } } let at = $e.urls let src = (fetch-source $e $ref $path) print $"== building (if ($ref | is-empty) { $'the working tree of ($path)' } else { $'($ref) = ($src.sha)' }) for ($e.name)" let built = (try { { ok: true, digests: (if ($build_on | is-empty) { build-local $e $src $at.svfe } else { build-remote $e $src $at.svfe $build_on }) } } catch {|err| { ok: false, msg: $err.msg } }) cleanup $e if not $built.ok { error make { msg: $built.msg } } let digests = $built.digests print $"== bobbin-($e.suffix)@($digests.bobbin)" print $"== svfe-($e.suffix)@($digests.svfe)" apply-control $e { bobbin: $"@($digests.bobbin)", svfe: $"@($digests.svfe)" } --yes
let after = (gc infra-manager deployments describe $e.deployment --format json | from json) print $"== deployment: ($after.state) ($after.latestRevision | path basename)" if $after.state != "ACTIVE" { error make { msg: $"($e.deployment | path basename) is ($after.state), not ACTIVE: ($after.stateDetail?)" } } print $"== bobbin: ($at.bobbin)/xrpc/sh.tangled.bobbin.getCoverage" print $"== svfe: ($at.svfe)/"}