#!/usr/bin/env nu
# build, deploy, preview and roll back the tangled gcp envs, and apply their hub nodes and
# cloudflare workers. dev's services also deploy themselves from ci.
#
# usage:
#   nix run .#deploy -- --target ENV           apply the tree, keeping the images running now
#   nix run .#deploy -- --target ENV (--ref REF | --path DIR) [--only SVC,..] [--rebuild] [--build-on HOST]
#                                     build what changed, push, pin the new digests, deploy the workers
#   nix run .#deploy -- --target ENV --preview  what the tree would change, applies nothing
#   nix run .#deploy -- --target ENV --rollback [--to SHA8|r-N] [--list]
#                                     images only, the nodes stay as they are
#   --no-nodes with any of the first three leaves the nodes alone
#   an env that was never applied comes up with its first --ref or --path deploy

# the control root knows the env, and the blueprint's last applied revision knows the rest
def env-of [name: string, infra: string] {
  let root = $"($infra)/terraform/control/($name)"
  if not ($root | path exists) {
    let have = (ls $"($infra)/terraform/control" | where type == dir | get name | path basename | str join ', ')
    error make { msg: $"unknown --target ($name), want one of: ($have)" }
  }
  if not ($"($root)/.terraform" | path exists) { ^terraform $"-chdir=($root)" init -input=false | ignore }
  let out = (^terraform $"-chdir=($root)" output -json control | complete)
  if $out.exit_code != 0 {
    if ($out.stderr | str contains 'Output "control" not found') { return (fresh-env $name $root $infra) }
    error make { msg: $"reading control/($name) failed: ($out.stderr | str trim)" }
  }
  let e = ($out.stdout | from json | get env)
  let outputs = (applied-outputs $e)
  # a hostname wins, then a bare lb address, then cloud run's own url
  let urls = if ($e.hostnames? | is-not-empty) {
    $e.hostnames | items {|svc, host| { $svc: $"https://($host)" } } | into record
  } else if ($outputs.lb_ips?.value? | is-not-empty) {
    $outputs.lb_ips.value | items {|svc, ip| { $svc: $"http://($ip)" } } | into record
  } else {
    $outputs.services.value | values | first | reject region
  }
  $e | merge {
    name: $name
    root: $root
    location: ($e.deployment | path dirname | path dirname)
    registry: $outputs.registry.value
    urls: $urls
  }
}

def applied-outputs [e: record] {
  gc infra-manager revisions list --deployment $e.deployment --format json
    | from json
    | where state == "APPLIED"
    | sort-by createTime --reverse
    | get 0.applyResults.outputs
}

# gcloud, failing loudly with its stderr instead of handing back an empty string
def --wrapped gc [...args: string] {
  let res = (^gcloud ...$args | complete)
  if $res.exit_code != 0 {
    error make { msg: $"gcloud ($args | first 3 | str join ' ') failed: ($res.stderr | str trim)" }
  }
  $res.stdout
}

# anything that queues behind a running apply can deadlock it at unlock, so wait for ci instead
def idle-deployment [e: record] {
  let dep = (gc infra-manager deployments describe $e.deployment --format json | from json)
  if $dep.state != "ACTIVE" or $dep.lockState? == "LOCKED" {
    error make { msg: $"($e.deployment | path basename) is ($dep.state), ($dep.lockState? | default 'unlocked'), try again once it's done" }
  }
  $dep
}

def running-refs [dep: record] {
  $dep.terraformBlueprint?.inputValues?.image_refs?.inputValue? | default {}
}

def ref-pairs [refs: record] {
  $refs | transpose svc ref | each {|r| $"($r.svc) = \"($r.ref)\"" } | str join ", "
}

def refs-var [refs: record] {
  $"image_refs={ (ref-pairs $refs) }"
}

# what infra manager takes as the blueprint's inputs when we go around the control root. like
# the control root, no pins means no image_refs at all, not an empty map
def write-inputs [e: record, refs: record, path: string] {
  let pins = if ($refs | is-empty) { "" } else { $"image_refs = { (ref-pairs $refs) }\n" }
  $"env_name = \"($e.name)\"\n($pins)" | save -f $path
}

# every mode applies the control root through here, so the image pins always go along. an empty
# map would put the env back on its floating tag
def apply-control [e: record, refs: record, --yes] {
  let root = $"-chdir=($e.root)"
  let vars = if ($refs | is-empty) { [] } else { [-var (refs-var $refs)] }
  if $yes {
    ^terraform $root apply -input=false -auto-approve -no-color ...$vars
  } else {
    ^terraform $root apply ...$vars
  }
}

# ---- nodes

# no pins to lose here, but every plan builds the vm systems, so this needs nix and an
# x86_64-linux builder
def nodes-root [e: record] {
  let root = ($e.root | path dirname | path dirname | path join nodes $e.name)
  if not ($root | path exists) { return null }
  if not ($"($root)/.terraform" | path exists) { ^terraform $"-chdir=($root)" init -input=false | ignore }
  $root
}

# the plan goes ahead when it destroys nothing, other than the redeploy a system change replaces.
# anything else asks first, because a replaced vm comes back empty and has to backfill from scratch
def apply-nodes [e: record] {
  let root = (nodes-root $e)
  if $root == null { return }
  print $"== nodes/($e.name)"
  let chdir = $"-chdir=($root)"
  # holds the install key, so it doesn't outlive the apply
  let plan = (mktemp -t nodes-plan.XXXXXX)
  let result = (try {
    ^terraform $chdir plan -input=false $"-out=($plan)"
    let destroys = (^terraform $chdir show -json $plan
      | from json
      | get -o resource_changes
      | default []
      | where {|c| "delete" in $c.change.actions and not ($c.type == "null_resource" and $c.name == "nixos-rebuild") })
    if ($destroys | is-empty) {
      ^terraform $chdir apply -input=false $plan
    } else {
      print $"== this destroys ($destroys.address | str join ', '), so it asks"
      ^terraform $chdir apply
    }
    { ok: true }
  } catch {|err| { ok: false, msg: $err.msg } })
  rm -f $plan
  if not $result.ok { error make { msg: $result.msg } }
  finish-cells $e
}

# the blueprint leaves bobbin out of a cell until the hub nodes it needs exist, so once the nodes
# made them, roll the blueprint again the way ci does. the control root wouldn't, its plan has
# nothing new
def finish-cells [e: record] {
  let waiting = (applied-outputs $e | get -o waiting.value | default {})
  if ($waiting | is-empty) { return }
  let made = (gc compute addresses list --project $e.project --format "value(name)" | lines)
  let ready = ($waiting
    | transpose cell nodes
    | where {|w| $w.nodes | all {|n| $"($n)-internal" in $made } })
  if ($ready | is-empty) { return }
  print $"== bobbin can go into ($ready.cell | str join ', ') now"
  roll $e (idle-deployment $e)
}

# a new revision of what the deployment already has, the way ci rolls one
def roll [e: record, dep: record] {
  let vars = $"/tmp/roll-($e.name).tfvars"
  write-inputs $e (running-refs $dep) $vars
  (gc infra-manager deployments apply $e.deployment --service-account $e.im_sa
    --gcs-source $dep.terraformBlueprint.gcsSource --inputs-file $vars
    --tf-version-constraint $e.tf_version --quiet)
  rm -f $vars
}

# ---- workers

def workers-root [e: record] {
  let root = ($e.root | path dirname | path dirname | path join workers $e.name)
  if not ($root | path exists) { return null }
  if not ($"($root)/.terraform" | path exists) { ^terraform $"-chdir=($root)" init -input=false | ignore }
  $root
}

def bundle-dir [e: record] { $"/tmp/tangled-($e.name)-workers" }

def --wrapped run-in [dir: string, ...cmd: string] {
  let res = (do { cd $dir; ^($cmd | first) ...($cmd | skip 1) } | complete)
  if $res.exit_code != 0 {
    let out = ($res.stdout + $res.stderr | lines | last 20 | str join (char nl))
    error make { msg: $"($cmd | str join ' ') in ($dir) failed: ($out)" }
  }
}

# all of them or none, because the workers root reads a missing bundle as a worker to delete.
# they land outside the work dir so they outlive cleanup
def build-workers [e: record, work: string] {
  let names = ($e.workers? | default [])
  let missing = ($names | where {|w| not ($work | path join $w package.json | path exists) })
  if ($missing | is-not-empty) {
    print $"== the workers stay as they are, this tree has no ($missing | str join ', ')"
    return {}
  }
  let out = (bundle-dir $e)
  rm -rf $out
  mkdir $out
  $names | each {|w|
    let dir = ($work | path join $w)
    run-in $dir pnpm install --frozen-lockfile
    run-in $dir pnpm run build
    let bundle = ($out | path join $"($w).js")
    cp ($dir | path join dist index.js) $bundle
    [$w $bundle]
  } | into record
}

# auto-approved like the control root. the bundles build the same from the same source, so an
# unchanged worker plans as nothing
def apply-workers [e: record, bundles: record] {
  let root = (workers-root $e)
  if $root == null or ($bundles | is-empty) { return }
  print $"== workers/($e.name)"
  let pairs = ($bundles | transpose w path | each {|r| $"\"($r.w)\" = \"($r.path)\"" } | str join ", ")
  ^terraform $"-chdir=($root)" apply -input=false -auto-approve -no-color -var $"bundles={ ($pairs) }"
  rm -rf (bundle-dir $e)
}

# ---- building

const REPO = "https://tangled.org/tangled.org/core"

# cargo profile per env; anything unlisted builds release
const BOBBIN_PROFILE = { dev: "dev-deploy", next-dev: "dev-deploy" }
def bobbin-profile [e: record] { $BOBBIN_PROFILE | get -o $e.name | default "release" }

def src-tgz [e: record] { $"/tmp/tangled-($e.name)-src.tgz" }
def work-dir [e: record] { $"/tmp/tangled-($e.name)-build" }

# the source as a tarball with everything under src/, and the commit it is
def fetch-source [e: record, ref: string, path: string] {
  let tgz = (src-tgz $e)
  if ($path | is-not-empty) {
    return { tgz: (pack-tree $path $tgz), sha: null }
  }
  let headers = $"($tgz).headers"
  let got = (^curl -fsSL -D $headers -o $tgz $"($REPO)/archive/($ref).tar.gz?prefix=src" | complete)
  if $got.exit_code != 0 { error make { msg: $"no archive of ($ref) at ($REPO): ($got.stderr | str trim)" } }
  # the appview answers with an immutable link to the commit the ref resolved to
  let sha = (open --raw $headers | parse -r 'archive/(?<sha>[0-9a-f]{40})' | get -o 0.sha)
  rm -f $headers
  if $sha == null { error make { msg: $"couldn't tell which commit ($ref) is" } }
  { tgz: $tgz, sha: $sha }
}

# the checkout as it is, uncommitted changes included, minus whatever the vcs ignores. tar can't
# read .gitignore properly, so git or jj says which files count
def pack-tree [path: string, tgz: string] {
  cd $path
  let listed = if (".git" | path exists) {
    ^git ls-files --cached --others --exclude-standard -z | split row (char nul)
  } else if (".jj" | path exists) {
    ^jj file list | lines
  } else {
    error make { msg: $"($path) is neither a git nor a jj checkout, so there's no telling what to leave out" }
  }
  let list = $"($tgz).files"
  $listed | where {|f| $f | path exists } | str join (char nul) | save -f $list
  let prefix = if (^tar --version | str contains "GNU") { [--transform "s,^,src/,"] } else { [-s ",^,src/,"] }
  # macos tar would add a ._ file next to every file for its extended attributes
  with-env { COPYFILE_DISABLE: "1" } { ^tar -czf $tgz --null -T $list ...$prefix }
  rm -f $list
  $tgz
}

def src-tree [e: record, tgz: string] {
  let work = (work-dir $e)
  rm -rf $work
  mkdir $work
  tar -xzf $tgz -C $work --strip-components=1
  cd $work
  # flakes only see files git knows about
  git init -q
  git add -A
  git -c user.email=deploy@local -c user.name=deploy commit -qm src
  $work
}

def cleanup [e: record] {
  let work = (work-dir $e)
  if ($work | path exists) { ^chmod -R u+w $work; rm -rf $work }
  rm -f (src-tgz $e)
}

def image [e: record, svc: string] { $"($e.registry)/($svc)-($e.suffix)" }

const SERVICES = [bobbin svfe]
# --only takes this too, for the env's cloudflare workers as one
const DEPLOYABLES = [bobbin svfe workers]

def build-tags [e: record, src: record] {
  [$e.name] | append (if $src.sha == null { [] } else { [$"($e.name)-($src.sha | str substring 0..7)"] })
}

def stage-static-files [work: string] {
  cd $work
  let out = (nix build .#web-static-files --no-link --print-out-paths | complete | get stdout | lines | last)
  mkdir $"($work)/web/static"
  rm -rf $"($work)/web/static/fonts" $"($work)/web/static/logos"
  ^cp -fr $"($out)/." $"($work)/web/static/"
}

def build-local [e: record, work: string, todo: list, tags: list, origin: string] {
  # same as ci, so the same source gives the same digest
  $env.SOURCE_DATE_EPOCH = "0"
  $todo | each {|t|
    let svc = $t.svc
    # bobbin's Containerfile copies from the repo root, so the root is the context
    let context = if $svc == "svfe" { stage-static-files $work; $"($work)/web" } else { $work }
    let args = match $svc {
      bobbin => [$"--file=($work | path join bobbin containerfiles bobbin.Containerfile)" $"--build-arg=BOBBIN_PROFILE=(bobbin-profile $e)"]
      svfe => [
        $"--file=($work | path join web Containerfile)"
        $"--build-arg=VITE_OAUTH_CLIENT_ID=($origin)/oauth-client-metadata.json"
        $"--build-arg=VITE_OAUTH_REDIRECT_URI=($origin)/oauth/callback"
        $"--build-arg=APP_VERSION=(^git -C $work rev-parse HEAD:web | str trim)"
      ]
    }
    docker buildx build ...[
        "--platform=linux/amd64"
        ...$args
        ...($tags | append $"src-($t.key)" | each {|tag| $"--tag=(image $e $svc):($tag)" })
        "--provenance=false"
        $"--cache-from=type=registry,ref=($e.registry)/($svc)-cache:buildcache"
        $"--cache-to=type=registry,ref=($e.registry)/($svc)-cache:buildcache,mode=max"
        "--output=type=registry,rewrite-timestamp=true"
        "--quiet"
        $"--metadata-file=($work)/($svc).json"
      ] $context
    [$svc (open --raw $"($work)/($svc).json" | from json | get containerimage.digest)]
  } | into record
}

const REMOTE_BUILD = r#'
  set -euo pipefail
  WORK="$HOME/$WORK_NAME"
  export DOCKER_CONFIG="$HOME/$WORK_NAME-docker"
  cleanup() { chmod -R u+w "$WORK" 2>/dev/null || true; rm -rf "$WORK" "$TGZ" "$DOCKER_CONFIG"; }
  trap cleanup EXIT
  chmod -R u+w "$WORK" 2>/dev/null || true
  rm -rf "$WORK" "$DOCKER_CONFIG"
  mkdir -p "$WORK"
  mkdir -m 700 "$DOCKER_CONFIG"
  tar -xzf "$TGZ" -C "$WORK" --strip-components=1
  cd "$WORK"
  git init -q
  git add -A
  git -c user.email=deploy@local -c user.name=deploy commit -qm src
  auth=$(printf 'oauth2accesstoken:%s' "$REG_TOKEN" | base64 -w0)
  printf '{"auths":{"%s":{"auth":"%s"}}}' "$REG_HOST" "$auth" > "$DOCKER_CONFIG/config.json"

  # buildkit rather than podman, so builds read the registry cache ci writes.
  # the daemon outlives the deploy, keeping its own cache for the next one;
  # nothing is exported back, the builder's uplink is too slow for that
  BX="$(nix build --no-link --print-out-paths nixpkgs#docker-buildx)/bin/docker-buildx"
  podman container exists tangled-buildkit || podman run -d --name tangled-buildkit --privileged \
    -p 127.0.0.1:18234:1234 docker.io/moby/buildkit:v0.33.0 --addr tcp://0.0.0.0:1234 >/dev/null
  podman start tangled-buildkit >/dev/null
  "$BX" create --name tangled --driver remote tcp://127.0.0.1:18234 >/dev/null
  for _ in $(seq 30); do "$BX" --builder tangled inspect --bootstrap >/dev/null 2>&1 && break; sleep 1; done

  export SOURCE_DATE_EPOCH=0
  wants() { [[ " $BUILD " == *" $1 "* ]]; }
  tags() { echo "--tag=$1:$TAG"; if [ -n "$SHA" ]; then echo "--tag=$1:$TAG-$SHA"; fi; echo "--tag=$1:src-$2"; }
  digest() { sed -n 's/.*"containerimage\.digest": *"\([^"]*\)".*/\1/p' "$1"; }

  if wants bobbin; then
    "$BX" --builder tangled build --platform=linux/amd64 \
      --file=bobbin/containerfiles/bobbin.Containerfile \
      --build-arg "BOBBIN_PROFILE=$PROFILE" \
      $(tags "$BOBBIN" "$KEY_BOBBIN") --provenance=false \
      --cache-from "type=registry,ref=$CACHE_REPO/bobbin-cache:buildcache" \
      --output type=registry,rewrite-timestamp=true --quiet --metadata-file "$DOCKER_CONFIG/bobbin.json" .
    echo "DIGEST_BOBBIN=$(digest "$DOCKER_CONFIG/bobbin.json")"
  fi

  if wants svfe; then
    out=$(nix build .#web-static-files --no-link --print-out-paths)
    static=$(echo "$out" | tail -1)
    mkdir -p web/static
    rm -rf web/static/fonts web/static/logos
    cp -fr "$static"/* web/static

    "$BX" --builder tangled build --platform=linux/amd64 \
      --file=web/Containerfile \
      --build-arg "VITE_OAUTH_CLIENT_ID=$ORIGIN/oauth-client-metadata.json" \
      --build-arg "VITE_OAUTH_REDIRECT_URI=$ORIGIN/oauth/callback" \
      --build-arg "APP_VERSION=$(git rev-parse HEAD:web)" \
      $(tags "$SVFE" "$KEY_SVFE") --provenance=false \
      --cache-from "type=registry,ref=$CACHE_REPO/svfe-cache:buildcache" \
      --output type=registry,rewrite-timestamp=true --quiet --metadata-file "$DOCKER_CONFIG/svfe.json" web/
    echo "DIGEST_SVFE=$(digest "$DOCKER_CONFIG/svfe.json")"
  fi
'#

def build-remote [e: record, src: record, todo: list, origin: string, host: string] {
  let sha = if $src.sha == null { "" } else { $src.sha | str substring 0..7 }
  let tgz = $src.tgz
  let sent = (^scp -q $tgz $"($host):($tgz)" | complete)
  if $sent.exit_code != 0 { error make { msg: $"copying the source to ($host) failed: ($sent.stderr | str trim)" } }
  let token = (gc auth print-access-token | str trim)
  let key = {|svc| $todo | where svc == $svc | get -o 0.key | default "" }
  # stdin, not the remote command line: there the token would show up in ps on the builder
  let vars = {
    REG_TOKEN: $token, REG_HOST: ($e.registry | split row '/' | first), TGZ: $tgz, WORK_NAME: $"tangled-($e.name)-build"
    BOBBIN: (image $e bobbin), SVFE: (image $e svfe), TAG: $e.name, SHA: $sha, ORIGIN: $origin
    PROFILE: (bobbin-profile $e), CACHE_REPO: $e.registry
    BUILD: ($todo.svc | str join " "), KEY_BOBBIN: (do $key bobbin), KEY_SVFE: (do $key svfe)
  }
  let env_block = ($vars | transpose k v | each {|r| $"($r.k)='($r.v)'" } | str join (char nl))
  let res = ([$env_block $REMOTE_BUILD] | str join (char nl) | ssh $host "bash -s" | complete)
  if $res.exit_code != 0 {
    error make { msg: $"remote build failed: ($res.stderr | str substring 0..600)" }
  }
  $todo.svc | each {|svc|
    let prefix = $"DIGEST_($svc | str upcase)="
    [$svc ($res.stdout | lines | where {|l| $l starts-with $prefix } | first | str replace $prefix "")]
  } | into record
}

# ---- skipping unchanged images

# the key is the source (flake store paths and git hashes) plus the build args. an image
# already tagged src-<key> came from exactly that, so it gets reused
def source-key [e: record, work: string, svc: string, origin: string] {
  let tree = {|p| ^git -C $work rev-parse $"HEAD:($p)" | complete | get stdout | str trim }
  let store = {|attr| ^nix eval --raw $"git+file://($work)#packages.x86_64-linux.($attr).outPath" }
  let parts = match $svc {
    # the flake's bobbin source is only the crates bobbin depends on, same as what the
    # Containerfile's sed narrows the workspace to
    bobbin => ([bobbin/containerfiles/bobbin.Containerfile .dockerignore rust-toolchain.toml]
      | each $tree | prepend (do $store bobbin.src) | append (bobbin-profile $e))
    # the static files come from the flake, so their store path already covers everything in them
    svfe => [(do $tree web) (do $store web-static-files) $origin]
  }
  $parts | prepend $svc | str join (char nl) | hash sha256 | str substring 0..31
}

def built-from [e: record, svc: string, key: string] {
  let res = (^gcloud artifacts docker images describe $"(image $e $svc):src-($key)" --format "value(image_summary.digest)" | complete)
  if $res.exit_code == 0 { $res.stdout | str trim } else { null }
}

# a reused image gets this deploy's tags too, so rollback still lists it under the commit
def retag [e: record, svc: string, digest: string, tags: list] {
  for t in $tags { gc artifacts docker tags add $"(image $e $svc)@($digest)" $"(image $e $svc):($t)" --quiet | ignore }
}

# ---- bootstrapping

# an env as modules/envs has it, which needs no state
def env-config [name: string, infra: string] {
  let res = ('jsonencode(local.envs["' + $name + '"])' | ^terraform $"-chdir=($infra)/terraform/modules/envs" console | complete)
  if $res.exit_code != 0 { error make { msg: $"modules/envs has no ($name): ($res.stderr | str trim)" } }
  $res.stdout | from json | from json
}

# a target whose control root was never applied has nothing for env-of to read, so this names
# what a build needs the way the control root and the blueprint are going to
def fresh-env [name: string, root: string, infra: string] {
  let c = (env-config $name $infra)
  let urls = if ($c.hostnames? | is-not-empty) {
    $c.hostnames | items {|svc, host| { $svc: $"https://($host)" } } | into record
  } else if $c.lb {
    error make { msg: $"($name) has an lb but no hostnames, so svfe's address, which its build needs, only exists after the first apply" }
  } else {
    let number = (gc projects describe $c.project --format "value(projectNumber)" | str trim)
    let region = ($c.regions | values | first | get region)
    $SERVICES | each {|svc| { $svc: $"https://($svc)-($c.env_suffix)-($number).($region).run.app" } } | into record
  }
  let location = $"projects/($c.project)/locations/($c.region)"
  {
    name: $name
    root: $root
    infra: $infra
    fresh: true
    project: $c.project
    region: $c.region
    suffix: $c.env_suffix
    location: $location
    deployment: $"($location)/deployments/tangled-($name)"
    registry: $"($c.region)-docker.pkg.dev/($c.project)/($c.registry_name)"
    workers: ($c.workers? | default {} | columns)
    urls: $urls
  }
}

# a new project's registry comes from its blueprint, which runs after the images are pushed, so
# it's made here first, and infra manager adopts it like the runtime accounts
def ensure-registry [e: record] {
  let name = ($e.registry | path basename)
  if (^gcloud artifacts repositories describe $name --location $e.region --project $e.project | complete).exit_code != 0 {
    gc artifacts repositories create $name --repository-format docker --location $e.region --project $e.project | ignore
  }
}

# what the first revision runs on: the deployment identity's roles and read of its blueprint, and
# the runtime accounts' reads of their secrets
const FIRST_GRANTS = [
  google_project_iam_member.deploy
  google_storage_bucket_iam_member.deployment_reads_blueprint
  google_secret_manager_secret_iam_member.readers
]

# the first apply of a new env, once its images exist. a first revision that fails leaves the
# deployment out of the control root's state, so it gets every grant it runs on beforehand. the
# runtime accounts the blueprint makes are made here first so they can be granted, which is fine
# because infra manager adopts what already exists while its state is clean
def bootstrap [e: record, refs: record] {
  for svc in $SERVICES {
    let sa = $"($svc)-($e.suffix)"
    if (^gcloud iam service-accounts describe $"($sa)@($e.project).iam.gserviceaccount.com" --project $e.project | complete).exit_code != 0 {
      gc iam service-accounts create $sa --project $e.project --display-name $"($sa) cloud run runtime" | ignore
    }
  }
  let root = $"-chdir=($e.root)"
  print $"== ($e.name) is new, so its identities and grants go first"
  ^terraform $root apply ...($FIRST_GRANTS | each {|g| $"-target=module.control.($g)" })
  print "== giving the grants two minutes to land"
  sleep 2min

  let applied = (try { apply-control $e $refs --yes; true } catch { false })
  let dep = (^gcloud infra-manager deployments describe $e.deployment --format json | complete)
  if $dep.exit_code != 0 { error make { msg: $"the control apply failed before it made ($e.deployment | path basename)" } }
  if $applied and ($dep.stdout | from json).state == "ACTIVE" { return }

  print "== the first revision failed, taking the deployment back and rolling it again"
  let vars = if ($refs | is-empty) { [] } else { [-var (refs-var $refs)] }
  if "module.control.google_config_deployment.this" not-in (^terraform $root state list | lines) {
    ^terraform $root import -input=false ...$vars module.control.google_config_deployment.this $e.deployment
  }
  apply-control $e $refs --yes
  let e = (env-of $e.name $e.infra)
  let dep = (gc infra-manager deployments describe $e.deployment --format json | from json)
  if $dep.state != "ACTIVE" { roll $e $dep }
}

# ---- rolling back

def images [e: record, svc: string] {
  gc artifacts docker images list (image $e $svc) --include-tags --format json | from json
}

# every <env>-<sha8> build of one service
def tagged [e: record, imgs: list] {
  $imgs | each {|img|
    $img.tags
    | where {|t| $t =~ $'^($e.name)-[0-9a-f]{8}$' }
    | each {|t| { build: ($t | str substring (($e.name | str length) + 1)..), digest: $img.version, pushed: $img.createTime } }
  } | flatten
}

# a deploy that pinned digests keeps them in its revision's inputs, tag or no tag
def pinned [e: record] {
  gc infra-manager revisions list --deployment $e.deployment --format json
  | from json
  | where state == "APPLIED"
  | where {|r| $r.terraformBlueprint?.inputValues?.image_refs?.inputValue? | is-not-empty }
  | each {|r|
    let refs = $r.terraformBlueprint.inputValues.image_refs.inputValue
    {
      build: ($r.name | path basename)
      svfe: ($refs.svfe | str trim --left --char "@")
      pushed: $r.createTime
      bobbin: ($refs.bobbin | str trim --left --char "@")
    }
  }
}

def running [e: record, svc: string] {
  let rev = (gc run services describe $"($svc)-($e.suffix)" --region $e.region --project $e.project --format "value(status.latestReadyRevisionName)" | str trim)
  gc run revisions describe $rev --region $e.region --project $e.project --format "value(status.imageDigest)" | str trim | split row "@" | last
}

def candidates [e: record] {
  let now = { bobbin: (running $e bobbin), svfe: (running $e svfe) }
  let bob = (images $e bobbin)
  let svf = (images $e svfe)
  let builds = (tagged $e $svf | rename build svfe pushed
    | join (tagged $e $bob | select build digest | rename build bobbin) build)
  let revisions = (pinned $e | where {|r|
    $r.bobbin in $bob.version and $r.svfe in $svf.version and not ($builds | any {|b| $b.bobbin == $r.bobbin and $b.svfe == $r.svfe })
  })
  # a commit that doesn't touch either service rebuilds the same images, so one pair can carry several builds
  let all = ($builds | append $revisions
    | group-by {|c| $"($c.bobbin) ($c.svfe)" }
    | values
    | each {|g| { builds: ($g.build | uniq), pushed: ($g.pushed | math max), bobbin: $g.0.bobbin, svfe: $g.0.svfe } }
    | sort-by pushed --reverse
    | insert build {|c| $c.builds | str join " " }
    | insert running {|c| $c.bobbin == $now.bobbin and $c.svfe == $now.svfe })
  if ($all | is-empty) {
    error make { msg: $"no build or pinned revision of ($e.name) has both images left in the registry" }
  }
  $all
}

def pick [all: list, to: string] {
  if ($to | is-not-empty) {
    let hit = ($all | where {|c| $to in $c.builds })
    if ($hit | is-empty) { error make { msg: $"no build ($to) with both images; have: ($all.builds | flatten | str join ', ')" } }
    return ($hit | first)
  }
  let labels = ($all | each {|c|
    let when = ($c.pushed | into datetime | format date "%Y-%m-%d %H:%M")
    $"($c.build)  ($when)(if $c.running { '  (running)' } else { '' })"
  })
  let i = ($labels | input list --index "roll back to")
  if $i == null { error make { msg: "nothing picked" } }
  $all | get $i
}

# ---- previewing

def preview [e: record, dep: record, --no-nodes] {
  let refs = (running-refs $dep)
  let id = $"preview-(random chars --length 8 | str downcase)"
  let out = $"/tmp/($id)"

  # the control plan also zips the tree, which the infra manager preview below needs
  let root = $"-chdir=($e.root)"
  let planned = (^terraform $root plan -input=false -lock=false -no-color
    ...(if ($refs | is-empty) { [] } else { [-var (refs-var $refs)] }) $"-out=($out)-control.tfplan" | complete)
  if $planned.exit_code != 0 { error make { msg: $"control plan failed: ($planned.stdout | lines | last 30 | str join (char nl))" } }
  let control = (^terraform $root show -json $"($out)-control.tfplan" | from json | get resource_changes)

  let nodes_root = if $no_nodes { null } else { nodes-root $e }
  let nodes = if $nodes_root == null { [] } else {
    let nroot = $"-chdir=($nodes_root)"
    let planned = (^terraform $nroot plan -input=false -lock=false -no-color $"-out=($out)-nodes.tfplan" | complete)
    if $planned.exit_code != 0 { error make { msg: $"nodes plan failed: ($planned.stdout | lines | last 30 | str join (char nl))" } }
    ^terraform $nroot show -json $"($out)-nodes.tfplan" | from json | get -o resource_changes | default []
  }

  let preview = $"($e.location)/previews/($id)"
  let obj = $"gs://($e.project)-blueprints/($id).zip"
  let vars = $"($out).tfvars"
  write-inputs $e $refs $vars
  gc storage cp --quiet $"($e.root)/../../.build/($e.name).zip" $obj
  let result = (try {
    (gc infra-manager previews create $preview --deployment $e.deployment --gcs-source $obj
      --service-account $e.im_sa --inputs-file $vars --tf-version-constraint $e.tf_version --quiet)
    gc infra-manager previews export $preview --file $out
    { ok: true }
  } catch {|err| { ok: false, msg: $err.msg } })
  do -i { ^gcloud infra-manager previews delete $preview --quiet | complete }
  do -i { ^gcloud storage rm --quiet $obj | complete }
  rm -f $vars
  if not $result.ok { error make { msg: $result.msg } }

  let changes = {|rcs, layer| $rcs
    | where {|c| $c.change.actions != ["no-op"] }
    | each {|c| { layer: $layer, resource: $c.address, action: ($c.change.actions | str join "/") } } }
  let all = ((do $changes $control control)
    | append (do $changes (open $"($out).json" | get resource_changes) blueprint)
    | append (do $changes $nodes nodes))
  print $"== full plans: ($out)-control.tfplan, ($out).json(if $nodes_root == null { '' } else { $', ($out)-nodes.tfplan' })"
  if ($all | is-empty) { print "== no changes" } else { $all }
}

# ----

def main [
  --target: string = ""   # an env with a control root, like dev or prod
  --ref: string = ""      # a branch, tag or commit of tangled.org/core to build
  --path: string = ""     # or a local checkout to build as it is. with neither, the images stay
  --only: string = ""     # with --ref/--path, the services to deploy (comma-separated, workers counts). the rest keep theirs
  --rebuild               # with --ref/--path, build even when an image of the same source exists
  --build-on: string = "" # ssh host with podman to build on; empty builds locally
  --preview               # plan the tree against the live env, apply nothing
  --rollback              # pick an earlier build and pin it
  --to: string = ""       # with --rollback: the build (8-char commit) or revision (r-N), skipping the menu
  --list                  # with --rollback: print the builds and exit
  --no-nodes              # leave the env's hub nodes alone
  --infra: string = ""    # this infra repo (default ~/proj/infra)
] {
  if ($target | is-empty) { error make { msg: "pass --target <env>, like dev or prod" } }
  let rollback = ($rollback or ($to | is-not-empty) or $list)
  if $preview and $rollback { error make { msg: "--preview and --rollback don't go together" } }
  if ($ref | is-not-empty) and ($path | is-not-empty) { error make { msg: "pass --ref or --path, not both" } }
  let building = ($ref | is-not-empty) or ($path | is-not-empty)
  if not $building and (($only | is-not-empty) or $rebuild) {
    error make { msg: "--only and --rebuild pick what to build, so they need --ref or --path" }
  }
  let chosen = if ($only | is-empty) { $DEPLOYABLES } else { $only | split row "," | str trim }
  let unknown = ($chosen | where {|s| $s not-in $DEPLOYABLES })
  if ($unknown | is-not-empty) { error make { msg: $"no service ($unknown | str join ', '), have: ($DEPLOYABLES | str join ', ')" } }
  let infra = if ($infra | is-empty) { $"($env.HOME)/proj/infra" } else { $infra }
  let e = (env-of $target $infra)
  let fresh = ($e.fresh? | default false)
  if $fresh and ($rollback or $preview or not $building or ($only | is-not-empty)) {
    error make { msg: $"($target) was never applied, so its first deploy builds everything, with --ref or --path and no --only" }
  }

  if $rollback {
    let all = (candidates $e)
    if $list { return ($all | select build pushed running) }
    let dep = (idle-deployment $e)
    let it = (pick $all $to)
    print $"== bobbin-($e.suffix)@($it.bobbin)"
    print $"== svfe-($e.suffix)@($it.svfe)"
    apply-control $e { bobbin: $"@($it.bobbin)", svfe: $"@($it.svfe)" }
    if $e.name == "dev" { print "== dev moves on again with the next push to sv-fe" }
    return
  }

  let dep = if $fresh { {} } else { idle-deployment $e }
  if $preview { return (preview $e $dep --no-nodes=$no_nodes) }
  let running = (running-refs $dep)
  if not $building {
    print $"== keeping (if ($running | is-empty) { $':($e.name)' } else { $running | values | str join ', ' })"
    apply-control $e $running
    if not $no_nodes { apply-nodes $e }
    return
  }

  let at = $e.urls
  if $fresh { ensure-registry $e }
  let src = (fetch-source $e $ref $path)
  print $"== building (if ($ref | is-empty) { $'the working tree of ($path)' } else { $'($ref) = ($src.sha)' }) for ($e.name)"
  let built = (try {
    let work = (src-tree $e $src.tgz)
    let tags = (build-tags $e $src)
    let wanted = ($chosen | where {|s| $s in $SERVICES } | each {|svc|
      let key = (source-key $e $work $svc $at.svfe)
      { svc: $svc, key: $key, digest: (if $rebuild { null } else { built-from $e $svc $key }) }
    })
    let reused = ($wanted | where digest != null)
    for r in $reused {
      print $"== ($r.svc) is unchanged, reusing ($r.digest)"
      retag $e $r.svc $r.digest $tags
    }
    let todo = ($wanted | where digest == null)
    let fresh = if ($todo | is-empty) { {} } else if ($build_on | is-empty) {
      build-local $e $work $todo $tags $at.svfe
    } else {
      build-remote $e $src $todo $at.svfe $build_on
    }
    let bundles = if "workers" in $chosen { build-workers $e $work } else { {} }
    { ok: true, digests: ($reused | each {|r| [$r.svc $r.digest] } | into record | merge $fresh), bundles: $bundles }
  } catch {|err| { ok: false, msg: $err.msg } })
  cleanup $e
  if not $built.ok { error make { msg: $built.msg } }
  let refs = ($running | merge ($built.digests | items {|svc, d| [$svc $"@($d)"] } | into record))
  for r in ($refs | transpose svc ref) { print $"== ($r.svc)-($e.suffix)($r.ref)" }
  if $fresh { bootstrap $e $refs } else { apply-control $e $refs --yes }

  let after = (gc infra-manager deployments describe $e.deployment --format json | from json)
  print $"== deployment: ($after.state) ($after.latestRevision | path basename)"
  if $after.state != "ACTIVE" {
    error make { msg: $"($e.deployment | path basename) is ($after.state), not ACTIVE: ($after.stateDetail?)" }
  }
  print $"== bobbin: ($at.bobbin)/xrpc/sh.tangled.bobbin.getCoverage"
  print $"== svfe:   ($at.svfe)/"
  apply-workers $e $built.bundles
  if not $no_nodes { apply-nodes $e }
}