diff --git a/deploy-gcp.nu b/deploy-gcp.nu new file mode 100644 index 0000000..f71ad66 --- /dev/null +++ b/deploy-gcp.nu @@ -0,0 +1,423 @@ +#!/usr/bin/env nu +# build, deploy, preview and roll back the tangled gcp envs. dev also deploys itself from ci. +# +# usage: +# nix run .#deploy-gcp -- --target dev|prod (--ref REF | --path DIR) [--build-on HOST] +# build it, push, pin the new digests +# nix run .#deploy-gcp -- --target dev|prod --preview what the tree would change, applies nothing +# nix run .#deploy-gcp -- --target dev|prod --no-build apply the tree, keeping the images running now +# nix run .#deploy-gcp -- --target dev|prod --rollback [--to SHA8|r-N] [--list] + +# the control root knows the env, and the blueprint's last applied revision knows the rest +def env-of [name: string, infra: string] { + let root = $"($infra)/terraform/control/($name)" + if not ($root | path exists) { + let have = (ls $"($infra)/terraform/control" | where type == dir | get name | path basename | str join ', ') + error make { msg: $"unknown --target ($name), want one of: ($have)" } + } + if not ($"($root)/.terraform" | path exists) { ^terraform $"-chdir=($root)" init -input=false | ignore } + let e = (^terraform $"-chdir=($root)" output -json control | from json | get env) + let outputs = (gc infra-manager revisions list --deployment $e.deployment --format json + | from json + | where state == "APPLIED" + | sort-by createTime --reverse + | get 0.applyResults.outputs) + # prod is reached through its lbs, dev straight through cloud run + let urls = if ($outputs.lb_ips?.value? | is-not-empty) { + $outputs.lb_ips.value | items {|svc, ip| { $svc: $"http://($ip)" } } | into record + } else { + $outputs.services.value | values | first | reject region + } + $e | merge { + name: $name + root: $root + location: ($e.deployment | path dirname | path dirname) + registry: $outputs.registry.value + urls: $urls + } +} + +# gcloud, failing loudly with its stderr instead of handing back an empty string +def --wrapped gc [...args: string] { + let res = (^gcloud ...$args | complete) + if $res.exit_code != 0 { + error make { msg: $"gcloud ($args | first 3 | str join ' ') failed: ($res.stderr | str trim)" } + } + $res.stdout +} + +# anything that queues behind a running apply can deadlock it at unlock, so wait for ci instead +def idle-deployment [e: record] { + let dep = (gc infra-manager deployments describe $e.deployment --format json | from json) + if $dep.state != "ACTIVE" or $dep.lockState? == "LOCKED" { + error make { msg: $"($e.deployment | path basename) is ($dep.state), ($dep.lockState? | default 'unlocked'), try again once it's done" } + } + $dep +} + +def running-refs [dep: record] { + $dep.terraformBlueprint?.inputValues?.image_refs?.inputValue? | default {} +} + +def refs-var [refs: record] { + let pairs = ($refs | transpose svc ref | each {|r| $"($r.svc) = \"($r.ref)\"" } | str join ", ") + $"image_refs={ ($pairs) }" +} + +# every mode applies the control root through here, so the image pins always go along. an empty +# map would put the env back on its floating tag +def apply-control [e: record, refs: record, --yes] { + let root = $"-chdir=($e.root)" + let vars = if ($refs | is-empty) { [] } else { [-var (refs-var $refs)] } + if $yes { + ^terraform $root apply -input=false -auto-approve -no-color ...$vars + } else { + ^terraform $root apply ...$vars + } +} + +# ---- building + +const REPO = "https://tangled.org/tangled.org/core" + +def src-tgz [e: record] { $"/tmp/tangled-($e.name)-src.tgz" } +def work-dir [e: record] { $"/tmp/tangled-($e.name)-build" } + +# the source as a tarball with everything under src/, and the commit it is +def fetch-source [e: record, ref: string, path: string] { + let tgz = (src-tgz $e) + if ($path | is-not-empty) { + return { tgz: (pack-tree $path $tgz), sha: null } + } + let headers = $"($tgz).headers" + let got = (^curl -fsSL -D $headers -o $tgz $"($REPO)/archive/($ref).tar.gz?prefix=src" | complete) + if $got.exit_code != 0 { error make { msg: $"no archive of ($ref) at ($REPO): ($got.stderr | str trim)" } } + # the appview answers with an immutable link to the commit the ref resolved to + let sha = (open --raw $headers | parse -r 'archive/(?[0-9a-f]{40})' | get -o 0.sha) + rm -f $headers + if $sha == null { error make { msg: $"couldn't tell which commit ($ref) is" } } + { tgz: $tgz, sha: $sha } +} + +# the checkout as it is, uncommitted changes included, minus whatever the vcs ignores. tar can't +# read .gitignore properly, so git or jj says which files count +def pack-tree [path: string, tgz: string] { + cd $path + let listed = if (".git" | path exists) { + ^git ls-files --cached --others --exclude-standard -z | split row (char nul) + } else if (".jj" | path exists) { + ^jj file list | lines + } else { + error make { msg: $"($path) is neither a git nor a jj checkout, so there's no telling what to leave out" } + } + let list = $"($tgz).files" + $listed | where {|f| $f | path exists } | str join (char nul) | save -f $list + let prefix = if (^tar --version | str contains "GNU") { [--transform "s,^,src/,"] } else { [-s ",^,src/,"] } + # macos tar would add a ._ file next to every file for its extended attributes + with-env { COPYFILE_DISABLE: "1" } { ^tar -czf $tgz --null -T $list ...$prefix } + rm -f $list + $tgz +} + +def src-tree [e: record, tgz: string] { + let work = (work-dir $e) + rm -rf $work + mkdir $work + tar -xzf $tgz -C $work --strip-components=1 + cd $work + # flakes only see files git knows about + git init -q + git add -A + git -c user.email=deploy@local -c user.name=deploy commit -qm src + $work +} + +def cleanup [e: record] { + let work = (work-dir $e) + if ($work | path exists) { ^chmod -R u+w $work; rm -rf $work } + rm -f (src-tgz $e) +} + +def image [e: record, svc: string] { $"($e.registry)/($svc)-($e.suffix)" } + +def build-local [e: record, src: record, origin: string] { + # a working tree isn't a commit, so it only gets the env tag + let tags = ([$e.name] | append (if $src.sha == null { [] } else { [$"($e.name)-($src.sha | str substring 0..7)"] })) + let work = (src-tree $e $src.tgz) + # bobbin's Containerfile copies from the repo root, so the root is the context + docker buildx build ...[ + "--platform=linux/amd64" + $"--file=($work | path join bobbin containerfiles bobbin.Containerfile)" + ...($tags | each {|t| $"--tag=(image $e bobbin):($t)" }) + "--provenance=false" + $"--cache-from=type=registry,ref=($e.registry)/bobbin-cache:buildcache" + $"--cache-to=type=registry,ref=($e.registry)/bobbin-cache:buildcache,mode=max" + "--push" + "--quiet" + $"--metadata-file=($work)/bobbin.json" + ] $work + + cd $work + let out = (nix build .#web-static-files --no-link --print-out-paths | complete | get stdout | lines | last) + mkdir $"($work)/web/static" + rm -rf $"($work)/web/static/fonts" $"($work)/web/static/logos" + ^cp -fr $"($out)/." $"($work)/web/static/" + + docker buildx build ...[ + "--platform=linux/amd64" + $"--file=($work | path join web Containerfile)" + $"--build-arg=VITE_OAUTH_CLIENT_ID=($origin)/oauth-client-metadata.json" + $"--build-arg=VITE_OAUTH_REDIRECT_URI=($origin)/oauth/callback" + ...($tags | each {|t| $"--tag=(image $e svfe):($t)" }) + "--provenance=false" + $"--cache-from=type=registry,ref=($e.registry)/svfe-cache:buildcache" + $"--cache-to=type=registry,ref=($e.registry)/svfe-cache:buildcache,mode=max" + "--push" + "--quiet" + $"--metadata-file=($work)/svfe.json" + ] $"($work)/web" + + { + bobbin: (open --raw $"($work)/bobbin.json" | from json | get containerimage.digest) + svfe: (open --raw $"($work)/svfe.json" | from json | get containerimage.digest) + } +} + +const REMOTE_BUILD = r#' + set -euo pipefail + WORK="$HOME/$WORK_NAME" + cleanup() { chmod -R u+w "$WORK" 2>/dev/null || true; rm -rf "$WORK" "$TGZ"; } + trap cleanup EXIT + chmod -R u+w "$WORK" 2>/dev/null || true + rm -rf "$WORK" + mkdir -p "$WORK" + tar -xzf "$TGZ" -C "$WORK" --strip-components=1 + cd "$WORK" + git init -q + git add -A + git -c user.email=deploy@local -c user.name=deploy commit -qm src + printf '%s' "$REG_TOKEN" | podman login "$REG_HOST" -u oauth2accesstoken --password-stdin + + podman build --platform=linux/amd64 \ + -f bobbin/containerfiles/bobbin.Containerfile \ + -t "$BOBBIN:$TAG" . + podman push --digestfile /tmp/d-bobbin "$BOBBIN:$TAG" + if [ -n "$SHA" ]; then podman push "$BOBBIN:$TAG" "$BOBBIN:$TAG-$SHA"; fi + + out=$(nix build .#web-static-files --no-link --print-out-paths) + static=$(echo "$out" | tail -1) + mkdir -p web/static + rm -rf web/static/fonts web/static/logos + cp -fr "$static"/* web/static + chmod -R u+w web/static + + podman build --platform=linux/amd64 \ + -f web/Containerfile \ + --build-arg "VITE_OAUTH_CLIENT_ID=$ORIGIN/oauth-client-metadata.json" \ + --build-arg "VITE_OAUTH_REDIRECT_URI=$ORIGIN/oauth/callback" \ + -t "$SVFE:$TAG" web/ + podman push --digestfile /tmp/d-svfe "$SVFE:$TAG" + if [ -n "$SHA" ]; then podman push "$SVFE:$TAG" "$SVFE:$TAG-$SHA"; fi + + echo "DIGEST_BOBBIN=$(cat /tmp/d-bobbin)" + echo "DIGEST_SVFE=$(cat /tmp/d-svfe)" +'# + +def build-remote [e: record, src: record, origin: string, host: string] { + let sha = if $src.sha == null { "" } else { $src.sha | str substring 0..7 } + let tgz = $src.tgz + let sent = (^scp -q $tgz $"($host):($tgz)" | complete) + if $sent.exit_code != 0 { error make { msg: $"copying the source to ($host) failed: ($sent.stderr | str trim)" } } + let token = (gc auth print-access-token | str trim) + # stdin, not the remote command line: there the token would show up in ps on the builder + let vars = { + REG_TOKEN: $token, REG_HOST: ($e.registry | split row '/' | first), TGZ: $tgz, WORK_NAME: $"tangled-($e.name)-build" + BOBBIN: (image $e bobbin), SVFE: (image $e svfe), TAG: $e.name, SHA: $sha, ORIGIN: $origin + } + let env_block = ($vars | transpose k v | each {|r| $"($r.k)='($r.v)'" } | str join (char nl)) + let res = ([$env_block $REMOTE_BUILD] | str join (char nl) | ssh $host "bash -s" | complete) + if $res.exit_code != 0 { + error make { msg: $"remote build failed: ($res.stderr | str substring 0..600)" } + } + let digest = {|key| $res.stdout | lines | where {|l| $l starts-with $"($key)=" } | first | str replace $"($key)=" "" } + { bobbin: (do $digest DIGEST_BOBBIN), svfe: (do $digest DIGEST_SVFE) } +} + +# ---- rolling back + +def images [e: record, svc: string] { + gc artifacts docker images list (image $e $svc) --include-tags --format json | from json +} + +# every - build of one service +def tagged [e: record, imgs: list] { + $imgs | each {|img| + $img.tags + | where {|t| $t =~ $'^($e.name)-[0-9a-f]{8}$' } + | each {|t| { build: ($t | str substring (($e.name | str length) + 1)..), digest: $img.version, pushed: $img.createTime } } + } | flatten +} + +# a deploy that pinned digests keeps them in its revision's inputs, tag or no tag +def pinned [e: record] { + gc infra-manager revisions list --deployment $e.deployment --format json + | from json + | where state == "APPLIED" + | where {|r| $r.terraformBlueprint?.inputValues?.image_refs?.inputValue? | is-not-empty } + | each {|r| + let refs = $r.terraformBlueprint.inputValues.image_refs.inputValue + { + build: ($r.name | path basename) + svfe: ($refs.svfe | str trim --left --char "@") + pushed: $r.createTime + bobbin: ($refs.bobbin | str trim --left --char "@") + } + } +} + +def running [e: record, svc: string] { + let rev = (gc run services describe $"($svc)-($e.suffix)" --region $e.region --project $e.project --format "value(status.latestReadyRevisionName)" | str trim) + gc run revisions describe $rev --region $e.region --project $e.project --format "value(status.imageDigest)" | str trim | split row "@" | last +} + +def candidates [e: record] { + let now = { bobbin: (running $e bobbin), svfe: (running $e svfe) } + let bob = (images $e bobbin) + let svf = (images $e svfe) + let builds = (tagged $e $svf | rename build svfe pushed + | join (tagged $e $bob | select build digest | rename build bobbin) build) + let revisions = (pinned $e | where {|r| + $r.bobbin in $bob.version and $r.svfe in $svf.version and not ($builds | any {|b| $b.bobbin == $r.bobbin and $b.svfe == $r.svfe }) + }) + # a commit that doesn't touch either service rebuilds the same images, so one pair can carry several builds + let all = ($builds | append $revisions + | group-by {|c| $"($c.bobbin) ($c.svfe)" } + | values + | each {|g| { builds: ($g.build | uniq), pushed: ($g.pushed | math max), bobbin: $g.0.bobbin, svfe: $g.0.svfe } } + | sort-by pushed --reverse + | insert build {|c| $c.builds | str join " " } + | insert running {|c| $c.bobbin == $now.bobbin and $c.svfe == $now.svfe }) + if ($all | is-empty) { + error make { msg: $"no build or pinned revision of ($e.name) has both images left in the registry" } + } + $all +} + +def pick [all: list, to: string] { + if ($to | is-not-empty) { + let hit = ($all | where {|c| $to in $c.builds }) + if ($hit | is-empty) { error make { msg: $"no build ($to) with both images; have: ($all.builds | flatten | str join ', ')" } } + return ($hit | first) + } + let labels = ($all | each {|c| + let when = ($c.pushed | into datetime | format date "%Y-%m-%d %H:%M") + $"($c.build) ($when)(if $c.running { ' (running)' } else { '' })" + }) + let i = ($labels | input list --index "roll back to") + if $i == null { error make { msg: "nothing picked" } } + $all | get $i +} + +# ---- previewing + +def preview [e: record, dep: record] { + let refs = (running-refs $dep) + let id = $"preview-(random chars --length 8 | str downcase)" + let out = $"/tmp/($id)" + + # the control plan also zips the tree, which the infra manager preview below needs + let root = $"-chdir=($e.root)" + let planned = (^terraform $root plan -input=false -lock=false -no-color + ...(if ($refs | is-empty) { [] } else { [-var (refs-var $refs)] }) $"-out=($out)-control.tfplan" | complete) + if $planned.exit_code != 0 { error make { msg: $"control plan failed: ($planned.stdout | lines | last 30 | str join (char nl))" } } + let control = (^terraform $root show -json $"($out)-control.tfplan" | from json | get resource_changes) + + let preview = $"($e.location)/previews/($id)" + let obj = $"gs://($e.project)-blueprints/($id).zip" + let vars = $"($out).tfvars" + let pairs = ($refs | transpose svc ref | each {|r| $"($r.svc) = \"($r.ref)\"" } | str join ", ") + $"env_name = \"($e.name)\"\nimage_refs = { ($pairs) }\n" | save -f $vars + gc storage cp --quiet $"($e.root)/../../.build/($e.name).zip" $obj + let result = (try { + (gc infra-manager previews create $preview --deployment $e.deployment --gcs-source $obj + --service-account $e.im_sa --inputs-file $vars --tf-version-constraint $e.tf_version --quiet) + gc infra-manager previews export $preview --file $out + { ok: true } + } catch {|err| { ok: false, msg: $err.msg } }) + do -i { ^gcloud infra-manager previews delete $preview --quiet | complete } + do -i { ^gcloud storage rm --quiet $obj | complete } + rm -f $vars + if not $result.ok { error make { msg: $result.msg } } + + let changes = {|rcs, layer| $rcs + | where {|c| $c.change.actions != ["no-op"] } + | each {|c| { layer: $layer, resource: $c.address, action: ($c.change.actions | str join "/") } } } + let all = ((do $changes $control control) | append (do $changes (open $"($out).json" | get resource_changes) blueprint)) + print $"== full plans: ($out)-control.tfplan, ($out).json" + if ($all | is-empty) { print "== no changes" } else { $all } +} + +# ---- + +def main [ + --target: string = "" # dev or prod + --build-on: string = "" # ssh host with podman to build on; empty builds locally + --preview # plan the tree against the live env, apply nothing + --no-build # apply the tree, keeping the images running now + --rollback # pick an earlier build and pin it + --to: string = "" # with --rollback: the build (8-char commit) or revision (r-N), skipping the menu + --list # with --rollback: print the builds and exit + --ref: string = "" # what to build: a branch, tag or commit of tangled.org/core + --path: string = "" # or: build this local checkout as it is + --infra: string = "" # this infra repo (default ~/proj/infra) +] { + if ($target | is-empty) { error make { msg: "pass --target dev or --target prod" } } + let rollback = ($rollback or ($to | is-not-empty) or $list) + if ([$preview $no_build $rollback] | where {|m| $m } | length) > 1 { + error make { msg: "--preview, --no-build and --rollback don't go together" } + } + let infra = if ($infra | is-empty) { $"($env.HOME)/proj/infra" } else { $infra } + let e = (env-of $target $infra) + + if $rollback { + let all = (candidates $e) + if $list { return ($all | select build pushed running) } + let dep = (idle-deployment $e) + let it = (pick $all $to) + print $"== bobbin-($e.suffix)@($it.bobbin)" + print $"== svfe-($e.suffix)@($it.svfe)" + apply-control $e { bobbin: $"@($it.bobbin)", svfe: $"@($it.svfe)" } + if $e.name == "dev" { print "== dev moves on again with the next push to sv-fe" } + return + } + + let dep = (idle-deployment $e) + if $preview { return (preview $e $dep) } + if $no_build { + let refs = (running-refs $dep) + print $"== keeping (if ($refs | is-empty) { $':($e.name)' } else { $refs | values | str join ', ' })" + return (apply-control $e $refs) + } + + if ($ref | is-empty) == ($path | is-empty) { error make { msg: "pass either --ref or --path, to say what to build" } } + let at = $e.urls + let src = (fetch-source $e $ref $path) + print $"== building (if ($ref | is-empty) { $'the working tree of ($path)' } else { $'($ref) = ($src.sha)' }) for ($e.name)" + let built = (try { + { ok: true, digests: (if ($build_on | is-empty) { build-local $e $src $at.svfe } else { build-remote $e $src $at.svfe $build_on }) } + } catch {|err| { ok: false, msg: $err.msg } }) + cleanup $e + if not $built.ok { error make { msg: $built.msg } } + let digests = $built.digests + print $"== bobbin-($e.suffix)@($digests.bobbin)" + print $"== svfe-($e.suffix)@($digests.svfe)" + apply-control $e { bobbin: $"@($digests.bobbin)", svfe: $"@($digests.svfe)" } --yes + + let after = (gc infra-manager deployments describe $e.deployment --format json | from json) + print $"== deployment: ($after.state) ($after.latestRevision | path basename)" + if $after.state != "ACTIVE" { + error make { msg: $"($e.deployment | path basename) is ($after.state), not ACTIVE: ($after.stateDetail?)" } + } + print $"== bobbin: ($at.bobbin)/xrpc/sh.tangled.bobbin.getCoverage" + print $"== svfe: ($at.svfe)/" +} diff --git a/flake.nix b/flake.nix index 4e55081..e24993d 100644 --- a/flake.nix +++ b/flake.nix @@ -80,6 +80,9 @@ }: let lib = nixpkgs.lib; system = "x86_64-linux"; + supportedSystems = ["x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin"]; + forAllSystems = lib.genAttrs supportedSystems; + nixpkgsFor = forAllSystems (system: nixpkgs.legacyPackages.${system}); commonArgs = import ./common/ssh.nix; cloudflareRanges = lib.filter (line: line != "") (lib.concatMap (input: lib.splitString "\n" (builtins.readFile input.outPath)) @@ -274,52 +277,32 @@ (name: host: mkColmenaHost name host.target (host.port or 22) host.modules) hosts ); - formatter = let - supportedSystems = ["x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin"]; - forAllSystems = nixpkgs.lib.genAttrs supportedSystems; - nixpkgsFor = forAllSystems (system: nixpkgs.legacyPackages.${system}); - in - forAllSystems (system: nixpkgsFor."${system}".alejandra); + apps = lib.genAttrs ["x86_64-darwin" "aarch64-darwin"] (system: let + pkgs = import nixpkgs { + inherit system; + config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) ["terraform"]; + }; + # deploy-gcp.nu shells out to all of these + deploy-gcp = pkgs.writeShellScriptBin "deploy-gcp" '' + export PATH="${lib.makeBinPath [ + pkgs.nushell + pkgs.google-cloud-sdk + pkgs.terraform + pkgs.git + pkgs.jujutsu + pkgs.openssh + pkgs.curl + ]}:$PATH" + exec ${pkgs.nushell}/bin/nu ${./deploy-gcp.nu} "$@" + ''; + in { + deploy-gcp = { + type = "app"; + program = "${deploy-gcp}/bin/deploy-gcp"; + }; + }); + + formatter = forAllSystems (system: nixpkgsFor.${system}.alejandra); - apps = let - supportedSystems = ["x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin"]; - forAllSystems = nixpkgs.lib.genAttrs supportedSystems; - nixpkgsFor = forAllSystems (system: nixpkgs.legacyPackages.${system}); - in - forAllSystems (system: let - pkgs = nixpkgsFor.${system}; - fleetBin = pkgs.writeShellScriptBin "fleet" '' - set -eu - FLEET_FILE="" - if [ -n "''${FLEET_DIR:-}" ] && [ -f "$FLEET_DIR/fleet.nu" ]; then - FLEET_FILE="$FLEET_DIR/fleet.nu" - else - dir="$PWD" - while [ "$dir" != "/" ] && [ -n "$dir" ]; do - if [ -f "$dir/fleet/fleet.nu" ]; then - FLEET_FILE="$dir/fleet/fleet.nu" - break - elif [ -f "$dir/fleet.nu" ]; then - FLEET_FILE="$dir/fleet.nu" - break - fi - dir="$(dirname "$dir")" - done - fi - if [ -z "$FLEET_FILE" ] && [ -f "/Users/dawn/proj/infra/fleet/fleet.nu" ]; then - FLEET_FILE="/Users/dawn/proj/infra/fleet/fleet.nu" - fi - if [ -z "$FLEET_FILE" ]; then - echo "error: could not locate fleet/fleet.nu from $PWD" >&2 - exit 1 - fi - exec ${pkgs.nushell}/bin/nu "$FLEET_FILE" "$@" - ''; - in { - fleet = { - type = "app"; - program = "${fleetBin}/bin/fleet"; - }; - }); }; } diff --git a/terraform/.gitignore b/terraform/.gitignore new file mode 100644 index 0000000..8ba3ce3 --- /dev/null +++ b/terraform/.gitignore @@ -0,0 +1,9 @@ +.terraform/ +*.tfstate +*.tfstate.* +*.tfplan +.build/ +*-plan.json* +.terraform.tfstate.lock.info +crash.log +/terraform.tfvars diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl new file mode 100644 index 0000000..7366d90 --- /dev/null +++ b/terraform/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/google" { + version = "8.3.0" + constraints = "~> 8.0" + hashes = [ + "h1:FM+ALGQgCJENpD1iiu48oVmE/ZQgZp0jN9iNlb+XQw4=", + "zh:0c637557b34d55bd63b328766b1c1511f8bcd2d067f3659b7aa199778f7b4ac4", + "zh:343fc2f46f3a03f465b9ba06e1ac6599cb43be4d5249914054eaad9202a9598a", + "zh:76819e2cec24197ee1a0c28e7dbfb30cb9fb68247371fbac0b42c2a75774b1bb", + "zh:7f1f468401a7c1bb94cd6fde593f2683a41ce444c5edb05fcd0cde350cee5219", + "zh:810ca7443a84fb8136ccb862e9aa4ed41596c8f042d4d113c2dc96b9889acee3", + "zh:aca28aa63bdbef5f1dbe701f644ec7bcfd4169fa67cf681efa87213a1d29693f", + "zh:bf0c1c8e24dc848d3a7f38a6c20a8d57955e5a8cc516c9e0459873a9ed81b40f", + "zh:d0669d078dd755b47f4f3ffde5e73835cdb264b52f5def8a55a8a396e7991e0c", + "zh:dd54566867c438b9b10621f575de52fe6233102240f2f07ba3d08e09dc29d5a8", + "zh:f3d38a334c60a5eaa38db6084b2aeede0f40c23b711b117b170eb6949bacadf2", + "zh:f49033adf0c48a35c9ad37259d60881746f9c1834a76589774623065d298be56", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/terraform/README.md b/terraform/README.md new file mode 100644 index 0000000..8c5ff51 --- /dev/null +++ b/terraform/README.md @@ -0,0 +1,113 @@ +services in here: +- bobbin +- svfe + +dev is configured as a pair of bobbin and svfe without anything in front. +they get automatically deployed from next branch, see more on that below. + +prod is configured as multiple bobbins and svfe's. bobbins have an LB in +front and so do the svfe's. its configured such that `tangled.org` will point +to nearest svfe, and `api.tangled.org` will point to nearest bobbin. this +way all the svfe configs are the same because they just take `api.tangled.org`. +prod is deployed via the `deploy-gcp` app on the nix flake, so just run +`nix run .#deploy-gcp -- --target prod --ref ` (or +`--path ` to deploy a local tree as it is), optionally passing +`--build-on user@host` to use a specific host for building the images on. + +to be able to work with terraform here, outside of `gcloud auth login` run +`gcloud auth application-default login` as well, because tf wants ADC. + +## layout + +- `main.tf` is what infra manager applies. it picks an env out of + `modules/envs` and builds it with `modules/environment`. +- `modules/envs` has one file per env with all the values. everything else + reads from here, so adding an env or changing a region is one file. +- `modules/control` and `control/` are the part we apply ourselves, + see below for why. +- the `deploy-gcp` app is `../deploy-gcp.nu`. + + +## why there are two layers + +we want ci to deploy dev without handing it the keys to the whole project. +so ci never runs terraform itself. it pushes images and asks infra manager to +roll a new revision, and infra manager does the actual work with its own +service account. + +that service account can manage the services, network, registry and lb, but +it can't touch iam. if it could, anything that lands in the tree (and so +anything that can push to the branch) could grant itself more. so all the +identities and grants live in `control/`, which only a project owner +applies, by hand: + +```sh +nix run .#deploy-gcp -- --target dev --preview # shows what applying would change on live deployment +nix run .#deploy-gcp -- --target dev --no-build # applies it +``` + +don't apply `main.tf` manually. infra manager only adopts resources that +already exist while its state is clean, so anything you create behind its +back will become an annoyance for you later :p + + +## deploying + +dev deploys on every push to sv-fe that touches bobbin or the web app +(`.tangled/workflows/deploy-gcp-dev.yml`). + +prod runs pinned digests, not the `:prod` tag. a tag can move under you, +a digest is exactly what we built, and it makes a rollback just "pin the old +digests again". this is also why the control roots go through `deploy-gcp` +and not bare terraform. `--no-build` passes along the pins that are already +there, a bare apply would drop them and the env would go back to its tag. + +to roll back, run `nix run .#deploy-gcp -- --target prod --rollback` (or +`dev`) and pick a build. + +rollback can only reach images the registry still has. prod keeps the last 16 +builds, and anything older gets deleted after 90 days. dev keeps the last 8, and +deletes the rest after 3 days. + + +### how ci gets into gcp + +the spindle signs a short-lived token for each workflow run that asks +for one (with `audience:` in the workflow), and gcp trusts tokens from +`https://tokens.spindle.tangled.sh` through workload identity federation. +nothing long-lived to leak, rotate, or paste into a secret. + +gcp only lets a token act as the ci deployer if it's for one exact repo +*and* branch. note that checking just the repo isn't enough, a manually +triggered run has no branch, so it would be able to deploy otherwise. + +this is only turned on for dev (`ci_oidc` in `modules/envs`). + + +## setting up a new env + +1. enable these apis: config, run, compute, artifactregistry, iam, + iamcredentials, storage, cloudresourcemanager. infra manager needs the + last one to write iam, and the error you get without it doesn't say so. +2. get an org admin to allow public members on the project. the org blocks + `allUsers` by default and project owners can't change that. prod needs + this too, the lb can't authenticate to cloud run, so the services are + public but only accept traffic from the lb. +3. make a versioned `gs://-tfstate` bucket for the control state. +4. add the env to `modules/envs`, copy `control/dev` to `control/` + with its own bucket and env name, and apply it. that creates the + deployment and infra manager builds everything else. if the first + revision fails on permissions, wait a few minutes, new grants take a + while to land. + + +## gotchas + +- infra manager runs terraform 1.5.7, so don't put anything newer in `main.tf` + and its modules. +- the lock file needs linux hashes too, since infra manager runs on linux: + `terraform providers lock -platform=linux_amd64 -platform=darwin_arm64`. +- only one apply can run per deployment at a time. a second one fails with + "unable to queue the operation". +- svfe builds aren't reproducible, so every prod deploy rolls svfe even if + nothing changed (should probably attempt to fix this, nbd though) diff --git a/terraform/control/dev/.terraform.lock.hcl b/terraform/control/dev/.terraform.lock.hcl new file mode 100644 index 0000000..f8bf241 --- /dev/null +++ b/terraform/control/dev/.terraform.lock.hcl @@ -0,0 +1,43 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/archive" { + version = "2.8.1" + constraints = "~> 2.0" + hashes = [ + "h1:aLNmq6dc3cDcqZc8s/8eKtn0I+UQXyJMGrmo4rRFtNw=", + "zh:03de290604114a89fcd45c2e5bc7787d5a1ebfc5f964fb5989306bea7a4c79ec", + "zh:0a7d69dc9fbbc48960bc2f04588c8fb1bd92c78a8f306566b7fb17fc4a4f2058", + "zh:4df1f3981379c35f1757da957470f7f7724496b57219da3485177cf1647bdf59", + "zh:50f0e72ba53bfe6e11b03b7fc899e1f72536354381d302a743a274ae2a3f45f4", + "zh:5c4e15a04c98e2a8cafb1cd9632b48ad318051e8462d105b1153006277985c35", + "zh:66069e604bcf5c4af0278e15997d9e6bd755c54fb3801d78885838b889729c5f", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:979765db3f42601870ab377104ba70befb029547b278337ec4ada980e3582de6", + "zh:b165254da774f49945a73fbccc3ef1b63d70ea00a98fa9e14716665ba80ecaad", + "zh:c0bb2697b525da9fec4511f569ed2bd2b42f25d5c1f9fa00f8f3645b50cfc2e9", + "zh:c48f6695d12df0d0fa5231f7c1b8d518a4feae91a733fdd35a5804af3a303831", + "zh:d589954c93f075180c9f4e2ce91780d5edcb56dfd0d3cda8ba08e13c10b52249", + "zh:f5792ed06da65d0daf7ca3711f5399ff78c7cb4400afe53fbce9a926fbde4477", + ] +} + +provider "registry.terraform.io/hashicorp/google" { + version = "8.3.0" + constraints = "~> 8.0" + hashes = [ + "h1:FM+ALGQgCJENpD1iiu48oVmE/ZQgZp0jN9iNlb+XQw4=", + "zh:0c637557b34d55bd63b328766b1c1511f8bcd2d067f3659b7aa199778f7b4ac4", + "zh:343fc2f46f3a03f465b9ba06e1ac6599cb43be4d5249914054eaad9202a9598a", + "zh:76819e2cec24197ee1a0c28e7dbfb30cb9fb68247371fbac0b42c2a75774b1bb", + "zh:7f1f468401a7c1bb94cd6fde593f2683a41ce444c5edb05fcd0cde350cee5219", + "zh:810ca7443a84fb8136ccb862e9aa4ed41596c8f042d4d113c2dc96b9889acee3", + "zh:aca28aa63bdbef5f1dbe701f644ec7bcfd4169fa67cf681efa87213a1d29693f", + "zh:bf0c1c8e24dc848d3a7f38a6c20a8d57955e5a8cc516c9e0459873a9ed81b40f", + "zh:d0669d078dd755b47f4f3ffde5e73835cdb264b52f5def8a55a8a396e7991e0c", + "zh:dd54566867c438b9b10621f575de52fe6233102240f2f07ba3d08e09dc29d5a8", + "zh:f3d38a334c60a5eaa38db6084b2aeede0f40c23b711b117b170eb6949bacadf2", + "zh:f49033adf0c48a35c9ad37259d60881746f9c1834a76589774623065d298be56", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/terraform/control/dev/main.tf b/terraform/control/dev/main.tf new file mode 100644 index 0000000..29a15c8 --- /dev/null +++ b/terraform/control/dev/main.tf @@ -0,0 +1,20 @@ +terraform { + backend "gcs" { + bucket = "exemplary-proxy-507408-d6-tfstate" + prefix = "control/dev" + } +} + +variable "image_refs" { + type = map(string) + default = {} +} + +module "control" { + source = "../../modules/control" + + env = "dev" + image_refs = var.image_refs +} + +output "control" { value = module.control } diff --git a/terraform/control/prod/.terraform.lock.hcl b/terraform/control/prod/.terraform.lock.hcl new file mode 100644 index 0000000..f8bf241 --- /dev/null +++ b/terraform/control/prod/.terraform.lock.hcl @@ -0,0 +1,43 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/archive" { + version = "2.8.1" + constraints = "~> 2.0" + hashes = [ + "h1:aLNmq6dc3cDcqZc8s/8eKtn0I+UQXyJMGrmo4rRFtNw=", + "zh:03de290604114a89fcd45c2e5bc7787d5a1ebfc5f964fb5989306bea7a4c79ec", + "zh:0a7d69dc9fbbc48960bc2f04588c8fb1bd92c78a8f306566b7fb17fc4a4f2058", + "zh:4df1f3981379c35f1757da957470f7f7724496b57219da3485177cf1647bdf59", + "zh:50f0e72ba53bfe6e11b03b7fc899e1f72536354381d302a743a274ae2a3f45f4", + "zh:5c4e15a04c98e2a8cafb1cd9632b48ad318051e8462d105b1153006277985c35", + "zh:66069e604bcf5c4af0278e15997d9e6bd755c54fb3801d78885838b889729c5f", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:979765db3f42601870ab377104ba70befb029547b278337ec4ada980e3582de6", + "zh:b165254da774f49945a73fbccc3ef1b63d70ea00a98fa9e14716665ba80ecaad", + "zh:c0bb2697b525da9fec4511f569ed2bd2b42f25d5c1f9fa00f8f3645b50cfc2e9", + "zh:c48f6695d12df0d0fa5231f7c1b8d518a4feae91a733fdd35a5804af3a303831", + "zh:d589954c93f075180c9f4e2ce91780d5edcb56dfd0d3cda8ba08e13c10b52249", + "zh:f5792ed06da65d0daf7ca3711f5399ff78c7cb4400afe53fbce9a926fbde4477", + ] +} + +provider "registry.terraform.io/hashicorp/google" { + version = "8.3.0" + constraints = "~> 8.0" + hashes = [ + "h1:FM+ALGQgCJENpD1iiu48oVmE/ZQgZp0jN9iNlb+XQw4=", + "zh:0c637557b34d55bd63b328766b1c1511f8bcd2d067f3659b7aa199778f7b4ac4", + "zh:343fc2f46f3a03f465b9ba06e1ac6599cb43be4d5249914054eaad9202a9598a", + "zh:76819e2cec24197ee1a0c28e7dbfb30cb9fb68247371fbac0b42c2a75774b1bb", + "zh:7f1f468401a7c1bb94cd6fde593f2683a41ce444c5edb05fcd0cde350cee5219", + "zh:810ca7443a84fb8136ccb862e9aa4ed41596c8f042d4d113c2dc96b9889acee3", + "zh:aca28aa63bdbef5f1dbe701f644ec7bcfd4169fa67cf681efa87213a1d29693f", + "zh:bf0c1c8e24dc848d3a7f38a6c20a8d57955e5a8cc516c9e0459873a9ed81b40f", + "zh:d0669d078dd755b47f4f3ffde5e73835cdb264b52f5def8a55a8a396e7991e0c", + "zh:dd54566867c438b9b10621f575de52fe6233102240f2f07ba3d08e09dc29d5a8", + "zh:f3d38a334c60a5eaa38db6084b2aeede0f40c23b711b117b170eb6949bacadf2", + "zh:f49033adf0c48a35c9ad37259d60881746f9c1834a76589774623065d298be56", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/terraform/control/prod/main.tf b/terraform/control/prod/main.tf new file mode 100644 index 0000000..503cc54 --- /dev/null +++ b/terraform/control/prod/main.tf @@ -0,0 +1,20 @@ +terraform { + backend "gcs" { + bucket = "tangled-prod-tfstate" + prefix = "control/prod" + } +} + +variable "image_refs" { + type = map(string) + default = {} +} + +module "control" { + source = "../../modules/control" + + env = "prod" + image_refs = var.image_refs +} + +output "control" { value = module.control } diff --git a/terraform/main.tf b/terraform/main.tf new file mode 100644 index 0000000..880c42c --- /dev/null +++ b/terraform/main.tf @@ -0,0 +1,20 @@ +module "envs" { + source = "./modules/envs" +} + +locals { + env = module.envs.envs[var.env_name] +} + +provider "google" { + project = local.env.project + region = local.env.region +} + +module "environment" { + source = "./modules/environment" + + env = merge(local.env, { + services = { for svc, s in local.env.services : svc => merge(s, { image = lookup(var.image_refs, svc, s.image) }) } + }) +} diff --git a/terraform/modules/cloud-run-app/main.tf b/terraform/modules/cloud-run-app/main.tf new file mode 100644 index 0000000..b081cb0 --- /dev/null +++ b/terraform/modules/cloud-run-app/main.tf @@ -0,0 +1,95 @@ +resource "google_cloud_run_v2_service" "this" { + project = var.project + name = var.service.name + location = var.region + ingress = var.service.ingress + deletion_protection = true + + template { + service_account = var.service.sa + timeout = "${var.service.timeout}s" + max_instance_request_concurrency = 80 + + scaling { + min_instance_count = var.service.min + max_instance_count = var.service.max + } + + dynamic "vpc_access" { + for_each = var.service.vpc == null ? [] : [var.service.vpc] + content { + egress = vpc_access.value.egress + network_interfaces { + network = vpc_access.value.network + subnetwork = vpc_access.value.subnetwork + } + } + } + + containers { + image = var.service.image + + ports { + container_port = 8080 + } + + resources { + limits = { + cpu = var.service.cpu + memory = var.service.memory + } + cpu_idle = var.service.cpu_idle + startup_cpu_boost = true + } + + dynamic "env" { + for_each = var.service.env + content { + name = env.key + value = env.value + } + } + + startup_probe { + http_get { + path = var.service.probe + port = 8080 + } + period_seconds = 5 + timeout_seconds = 3 + failure_threshold = 24 + } + } + } + + # the control plane reports these zeros anyway; declaring them keeps plans empty. + scaling { + min_instance_count = 0 + manual_instance_count = 0 + } + + traffic { + type = "TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST" + percent = 100 + } + + lifecycle { + ignore_changes = [ + annotations, + client, + client_version, + template[0].annotations, + template[0].labels, + ] + } +} + +resource "google_cloud_run_v2_service_iam_member" "invokers" { + for_each = toset(concat(var.service.invokers, var.service.public ? ["allUsers"] : [])) + + project = var.project + location = var.region + name = google_cloud_run_v2_service.this.name + role = "roles/run.invoker" + member = each.value +} diff --git a/terraform/modules/cloud-run-app/outputs.tf b/terraform/modules/cloud-run-app/outputs.tf new file mode 100644 index 0000000..93d7a8a --- /dev/null +++ b/terraform/modules/cloud-run-app/outputs.tf @@ -0,0 +1,4 @@ +# the -..run.app form, which stays put and is what svfe's oauth config points at +output "uri" { value = one([for u in google_cloud_run_v2_service.this.urls : u if endswith(u, ".${var.region}.run.app")]) } +output "latest_revision" { value = google_cloud_run_v2_service.this.latest_ready_revision } +output "name" { value = google_cloud_run_v2_service.this.name } diff --git a/terraform/modules/cloud-run-app/variables.tf b/terraform/modules/cloud-run-app/variables.tf new file mode 100644 index 0000000..c2fde48 --- /dev/null +++ b/terraform/modules/cloud-run-app/variables.tf @@ -0,0 +1,27 @@ +variable "project" { type = string } +variable "region" { type = string } + +variable "service" { + type = object({ + name = string + image = string + sa = string + invokers = list(string) + public = bool + probe = string + cpu = string + memory = string + min = number + max = number + timeout = number + cpu_idle = bool + vpc = optional(object({ + network = string + subnetwork = string + egress = optional(string, "PRIVATE_RANGES_ONLY") + })) + env = map(string) + ingress = string + }) + description = "one cloud run service, as described by modules/environment local.services" +} diff --git a/terraform/modules/control/main.tf b/terraform/modules/control/main.tf new file mode 100644 index 0000000..6f38386 --- /dev/null +++ b/terraform/modules/control/main.tf @@ -0,0 +1,200 @@ +locals { + env = module.envs.envs[var.env] + project = local.env.project + region = local.env.region + + # this module sits inside the tree it uploads + blueprint_dir = "${path.module}/../.." + + # infra manager's terraform, which the blueprint has to stay compatible with + tf_version = "=1.5.7" +} + +module "envs" { + source = "../envs" +} + +data "archive_file" "blueprint" { + type = "zip" + source_dir = local.blueprint_dir + output_path = "${local.blueprint_dir}/.build/${var.env}.zip" + + excludes = [ + ".build", + ".gitignore", + ".terraform", + ".terraform.lock.hcl", + "*.tfplan", + "README.md", + "control", + "modules/control", + "terraform.tfstate", + "terraform.tfstate.backup", + ] +} + +resource "google_storage_bucket" "blueprints" { + project = local.project + name = "${local.project}-blueprints" + location = local.region + uniform_bucket_level_access = true + public_access_prevention = "enforced" + + versioning { + enabled = true + } + + # revisions fetch their blueprint by name, so superseded generations only + # need to outlive the audit window, not forever + lifecycle_rule { + condition { + num_newer_versions = 10 + } + action { + type = "Delete" + } + } +} + +resource "google_storage_bucket_object" "blueprint" { + bucket = google_storage_bucket.blueprints.name + name = "${var.env}-${data.archive_file.blueprint.output_sha}.zip" + source = data.archive_file.blueprint.output_path +} + +# the deployment identity is created and granted HERE, outside the revisions +# that run as it - the blueprint never declares its own powers +resource "google_service_account" "deploy" { + project = local.project + account_id = "im-deployer-${local.env.env_suffix}" + display_name = "infra manager deployment identity for ${var.env}" +} + +resource "google_service_account" "ci" { + project = local.project + account_id = "ci-deployer-${local.env.env_suffix}" + display_name = "ci deploy identity for ${var.env}" +} + +resource "google_project_iam_member" "ci" { + for_each = toset([ + "roles/config.admin", + "roles/config.agent", + "roles/artifactregistry.writer", + ]) + + project = local.project + role = each.value + member = google_service_account.ci.member +} + +resource "google_service_account_iam_member" "ci_acts_as_deployment" { + service_account_id = google_service_account.deploy.name + role = "roles/iam.serviceAccountUser" + member = google_service_account.ci.member +} + +locals { + ci_oidc = try(local.env.ci_oidc, null) + ci_wif = local.ci_oidc == null ? toset([]) : toset(["spindle"]) +} + +resource "google_project_service" "sts" { + for_each = local.ci_wif + + project = local.project + service = "sts.googleapis.com" + disable_on_destroy = false +} + +resource "google_iam_workload_identity_pool" "ci" { + for_each = local.ci_wif + + project = local.project + workload_identity_pool_id = "tangled-ci" + display_name = "tangled ci" +} + +resource "google_iam_workload_identity_pool_provider" "spindle" { + for_each = local.ci_wif + + project = local.project + workload_identity_pool_id = google_iam_workload_identity_pool.ci[each.key].workload_identity_pool_id + workload_identity_pool_provider_id = "spindle" + display_name = "tangled spindle" + + attribute_mapping = { + "google.subject" = "assertion.sub" + "attribute.repository" = "assertion.repository" + "attribute.ref" = "assertion.ref" + } + + # manual runs carry an empty ref, so repository alone would admit them + attribute_condition = "assertion.repository == '${local.ci_oidc.repository}' && assertion.ref == '${local.ci_oidc.ref}'" + + oidc { + issuer_uri = local.ci_oidc.issuer + } + + depends_on = [google_project_service.sts] +} + +resource "google_service_account_iam_member" "ci_federated" { + for_each = local.ci_wif + + service_account_id = google_service_account.ci.name + role = "roles/iam.workloadIdentityUser" + member = "principal://iam.googleapis.com/${google_iam_workload_identity_pool.ci[each.key].name}/subject/${local.ci_oidc.repository}:${local.ci_oidc.ref}" +} + +resource "google_project_iam_member" "deploy" { + for_each = toset([ + "roles/config.agent", + "roles/run.admin", + "roles/compute.networkAdmin", + "roles/artifactregistry.admin", + "roles/iam.serviceAccountUser", + ]) + + project = local.project + role = each.value + member = google_service_account.deploy.member +} + +resource "google_config_deployment" "this" { + project = local.project + name = "tangled-${var.env}" + location = local.region + service_account = google_service_account.deploy.id + tf_version_constraint = local.tf_version + + deletion_policy = "ABANDON" + + terraform_blueprint { + gcs_source = "gs://${google_storage_bucket.blueprints.name}/${google_storage_bucket_object.blueprint.name}" + + input_values { + variable_name = "env_name" + input_value = jsonencode(var.env) + } + + # an empty map comes back from infra manager as "", which would plan as a change forever + dynamic "input_values" { + for_each = length(var.image_refs) == 0 ? [] : [var.image_refs] + content { + variable_name = "image_refs" + input_value = jsonencode(input_values.value) + } + } + } + + labels = { + env = local.env.env_suffix + } +} + +resource "google_storage_bucket_iam_member" "deployment_reads_blueprint" { + bucket = google_storage_bucket.blueprints.name + role = "roles/storage.objectViewer" + member = google_service_account.deploy.member +} diff --git a/terraform/modules/control/outputs.tf b/terraform/modules/control/outputs.tf new file mode 100644 index 0000000..afd89a9 --- /dev/null +++ b/terraform/modules/control/outputs.tf @@ -0,0 +1,20 @@ +output "blueprint" { + value = "gs://${google_storage_bucket_object.blueprint.bucket}/${google_storage_bucket_object.blueprint.name}" +} + +output "ci_oidc_audience" { + description = "the audience a workflow declares to federate into ci-deployer" + value = one([for p in google_iam_workload_identity_pool_provider.spindle : "//iam.googleapis.com/${p.name}"]) +} + +# deploy-gcp.nu finds everything about an env through this +output "env" { + value = { + project = local.project + region = local.region + suffix = local.env.env_suffix + deployment = google_config_deployment.this.id + im_sa = google_service_account.deploy.id + tf_version = local.tf_version + } +} diff --git a/terraform/modules/control/variables.tf b/terraform/modules/control/variables.tf new file mode 100644 index 0000000..68d4bcb --- /dev/null +++ b/terraform/modules/control/variables.tf @@ -0,0 +1,10 @@ +variable "env" { + type = string + description = "env key in modules/envs; one root deploys exactly one env into one project" +} + +variable "image_refs" { + type = map(string) + description = "service to image ref suffix, passed through to the blueprint" + default = {} +} diff --git a/terraform/modules/control/versions.tf b/terraform/modules/control/versions.tf new file mode 100644 index 0000000..18db165 --- /dev/null +++ b/terraform/modules/control/versions.tf @@ -0,0 +1,12 @@ +terraform { + required_providers { + archive = { + source = "hashicorp/archive" + version = "~> 2.0" + } + google = { + source = "hashicorp/google" + version = "~> 8.0" + } + } +} diff --git a/terraform/modules/environment/main.tf b/terraform/modules/environment/main.tf new file mode 100644 index 0000000..ab728ed --- /dev/null +++ b/terraform/modules/environment/main.tf @@ -0,0 +1,171 @@ +locals { + bobbin_name = "bobbin-${var.env.env_suffix}" + svfe_name = "svfe-${var.env.env_suffix}" + + registry = "${var.env.region}-docker.pkg.dev/${var.env.project}/${var.env.registry_name}" + + service_accounts = { for svc, s in local.services : svc => s.sa } + + services = { + bobbin = { + name = local.bobbin_name + image = "${local.registry}/${local.bobbin_name}${var.env.services.bobbin.image}" + sa = google_service_account.bobbin.email + invokers = ["serviceAccount:${google_service_account.svfe.email}"] + probe = "/xrpc/sh.tangled.bobbin.getCoverage" + cpu = "2" + memory = "2Gi" + min = 1 + max = 1 + timeout = 3600 + cpu_idle = false + vpc = true + env = tomap(merge({ + BOBBIN_BIND = "0.0.0.0:8080" + BOBBIN_LOG = "info" + BOBBIN_LOG_FORMAT = "json" + }, var.env.services.bobbin.env)) + public = var.env.services.bobbin.public + } + svfe = { + name = local.svfe_name + image = "${local.registry}/${local.svfe_name}${var.env.services.svfe.image}" + sa = google_service_account.svfe.email + invokers = [] + probe = "/oauth-client-metadata.json" + cpu = "1" + memory = "1Gi" + min = 0 + max = 4 + timeout = 300 + cpu_idle = true + vpc = false + env = tomap(var.env.services.svfe.env) + public = var.env.services.svfe.public + } + } + + robot_invoker = "serviceAccount:service-${data.google_project.this.number}@serverless-robot-prod.iam.gserviceaccount.com" + + apps = merge([ + for svc, s in local.services : { + for cell, c in var.env.regions : "${svc}-${cell}" => { + region = c.region + service = merge(s, { + ingress = var.env.lb ? "INGRESS_TRAFFIC_INTERNAL_LOAD_BALANCER" : "INGRESS_TRAFFIC_ALL" + invokers = var.env.lb ? concat(s.invokers, [local.robot_invoker]) : s.invokers + vpc = s.vpc ? { + network = google_compute_network.this.name + subnetwork = google_compute_subnetwork.this[cell].name + } : null + }) + } + } + ]...) +} + +data "google_project" "this" { + project_id = var.env.project +} + +resource "google_compute_network" "this" { + project = var.env.project + name = var.env.network_name + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "this" { + for_each = var.env.regions + + project = var.env.project + name = each.value.subnet + region = each.value.region + network = google_compute_network.this.id + ip_cidr_range = each.value.cidr + + stack_type = each.value.ipv6_access_type == null ? null : "IPV4_IPV6" + ipv6_access_type = each.value.ipv6_access_type +} + +resource "google_artifact_registry_repository" "this" { + project = var.env.project + location = var.env.region + repository_id = var.env.registry_name + format = "DOCKER" + description = var.env.registry_description + + cleanup_policies { + id = "keep-recent" + action = "KEEP" + + most_recent_versions { + keep_count = var.env.registry_retention.keep + } + } + + cleanup_policies { + id = "delete-old-untagged" + action = "DELETE" + + condition { + tag_state = "UNTAGGED" + older_than = "${var.env.registry_retention.days * 86400}s" + } + } + + # builds are tagged - and those tags never move, so without this they'd stay forever + cleanup_policies { + id = "delete-old-builds" + action = "DELETE" + + condition { + tag_state = "TAGGED" + tag_prefixes = ["dev-", "prod-"] + older_than = "${var.env.registry_retention.days * 86400}s" + } + } +} + +resource "google_service_account" "bobbin" { + project = var.env.project + account_id = local.bobbin_name + display_name = "${local.bobbin_name} cloud run runtime" +} + +resource "google_service_account" "svfe" { + project = var.env.project + account_id = local.svfe_name + display_name = "${local.svfe_name} cloud run runtime" +} + +resource "google_artifact_registry_repository_iam_member" "reader" { + for_each = local.service_accounts + + project = var.env.project + location = google_artifact_registry_repository.this.location + repository = google_artifact_registry_repository.this.repository_id + role = "roles/artifactregistry.reader" + member = "serviceAccount:${each.value}" +} + +module "lb" { + count = var.env.lb ? 1 : 0 + source = "../lb" + + project = var.env.project + name = "tangled-lb" + + regions = { for cell, r in var.env.regions : cell => r.region } + + # the app name is cell-independent, so one cell carries it into the load balancer's service map + services = { for k, s in local.services : k => module.app["${k}-${keys(var.env.regions)[0]}"].name } +} + +module "app" { + for_each = local.apps + source = "../cloud-run-app" + + project = var.env.project + region = each.value.region + service = each.value.service +} diff --git a/terraform/modules/environment/outputs.tf b/terraform/modules/environment/outputs.tf new file mode 100644 index 0000000..6b3fdef --- /dev/null +++ b/terraform/modules/environment/outputs.tf @@ -0,0 +1,20 @@ +output "lb_ips" { value = one(module.lb[*].ips) } +output "service_accounts" { value = local.service_accounts } +output "registry" { value = local.registry } + +output "services" { + value = { + for cell, c in var.env.regions : cell => merge( + { region = c.region }, + { for svc in keys(local.services) : svc => module.app["${svc}-${cell}"].uri } + ) + } +} + +output "revisions" { + value = { + for cell, c in var.env.regions : cell => { + for svc in keys(local.services) : svc => module.app["${svc}-${cell}"].latest_revision + } + } +} diff --git a/terraform/modules/environment/variables.tf b/terraform/modules/environment/variables.tf new file mode 100644 index 0000000..70f6919 --- /dev/null +++ b/terraform/modules/environment/variables.tf @@ -0,0 +1,26 @@ +variable "env" { + type = object({ + project = string + region = string + env_suffix = string + lb = bool + network_name = string + registry_name = string + registry_description = string + registry_retention = object({ keep = number, days = number }) + + regions = map(object({ + region = string + subnet = string + cidr = string + ipv6_access_type = optional(string) + })) + + services = map(object({ + image = string + public = bool + env = map(string) + })) + }) + description = "one modules/envs entry, its services carrying the image ref to run; the regions key set is the cell set" +} diff --git a/terraform/modules/envs/dev.tf b/terraform/modules/envs/dev.tf new file mode 100644 index 0000000..c7e8670 --- /dev/null +++ b/terraform/modules/envs/dev.tf @@ -0,0 +1,62 @@ +locals { + dev = { + env_suffix = "next" + project = "exemplary-proxy-507408-d6" + + lb = false + + region = "europe-north1" + network_name = "tangled" + registry_name = "tangled" + + registry_retention = { keep = 8, days = 3 } + registry_description = "dev images for the next.* services" + + # the spindle runs allowed to federate into ci-deployer: the core repo's sv-fe pushes + ci_oidc = { + issuer = "https://tokens.spindle.tangled.sh" + repository = "did:plc:j5hmlfdrwkvtxm7cjmu7j2is" + ref = "refs/heads/sv-fe" + } + + regions = { + eu = { + cidr = "10.166.0.0/20" + ipv6_access_type = "EXTERNAL" + region = "europe-north1" + subnet = "tangled-eu-north" + } + } + + services = { + bobbin = { + image = ":dev" + public = true + + env = { + BOBBIN_HYDRANT_URL = "https://index.tangled.network" + BOBBIN_MIRROR_URL = "https://mirror-fsn.tangled.network" + BOBBIN_SERVICE_DID = "did:web:svfe-next-264757501569.europe-north1.run.app" + BOBBIN_SLINGSHOT_URL = "https://index.tangled.network" + } + } + + svfe = { + image = ":dev" + public = false + + env = { + AVATAR_URL = "https://avatar-dev.tangled.org" + BOBBIN_UPSTREAM_URL = "https://bobbin-next-264757501569.europe-north1.run.app" + BOBBIN_URL = "https://svfe-next-264757501569.europe-north1.run.app" + CAMO_URL = "https://camo-dev.tangled.org" + DELIBERI_URL = "https://notifs-dev.tangled.network" + KNOTMIRROR_URL = "https://mirror-fsn.tangled.network" + MODERATION_SERVICE_DID = "did:plc:3t4yxvxwjrl55odxe2rwwhfr" + OGRE_URL = "https://ogre-dev.tangled.network" + TANGLED_API_URL = "https://tangled.org" + } + } + } + } +} diff --git a/terraform/modules/envs/outputs.tf b/terraform/modules/envs/outputs.tf new file mode 100644 index 0000000..3d72089 --- /dev/null +++ b/terraform/modules/envs/outputs.tf @@ -0,0 +1 @@ +output "envs" { value = { dev = local.dev, prod = local.prod } } diff --git a/terraform/modules/envs/prod.tf b/terraform/modules/envs/prod.tf new file mode 100644 index 0000000..732f48c --- /dev/null +++ b/terraform/modules/envs/prod.tf @@ -0,0 +1,50 @@ +locals { + prod = { + env_suffix = "prod" + project = "tangled-prod" + + lb = true + + region = "europe-north1" + network_name = "tangled" + registry_name = "tangled" + + registry_retention = { keep = 16, days = 90 } + registry_description = "prod images for the tangled.org services" + + regions = { + eu = { + cidr = "10.166.0.0/20" + ipv6_access_type = "EXTERNAL" + region = "europe-north1" + subnet = "tangled-eu-north" + } + } + + services = { + bobbin = { + image = ":prod" + public = true + + env = { + BOBBIN_HYDRANT_URL = "https://api.tangled.org" + BOBBIN_MIRROR_URL = "https://mirror-fsn.tangled.network" + BOBBIN_SERVICE_DID = "did:web:api.tangled.org" + BOBBIN_SLINGSHOT_URL = "https://api.tangled.org" + } + } + + svfe = { + image = ":prod" + public = true + + env = { + BOBBIN_UPSTREAM_URL = "https://api.tangled.org" + BOBBIN_URL = "https://next.tangled.org" + KNOTMIRROR_URL = "https://mirror-fsn.tangled.network" + TANGLED_API_URL = "https://tangled.org" + } + } + } + } +} diff --git a/terraform/modules/lb/main.tf b/terraform/modules/lb/main.tf new file mode 100644 index 0000000..f348caa --- /dev/null +++ b/terraform/modules/lb/main.tf @@ -0,0 +1,76 @@ +locals { + endpoints = merge([ + for svc, name in var.services : { + for cell, region in var.regions : "${svc}-${cell}" => { + service = name + region = region + } + } + ]...) +} + +resource "google_compute_region_network_endpoint_group" "this" { + for_each = local.endpoints + + project = var.project + name = "${each.key}-neg" + region = each.value.region + network_endpoint_type = "SERVERLESS" + + cloud_run { + service = each.value.service + } +} + +resource "google_compute_backend_service" "this" { + for_each = var.services + + project = var.project + name = each.value + load_balancing_scheme = "EXTERNAL_MANAGED" + + dynamic "backend" { + for_each = var.regions + content { + group = google_compute_region_network_endpoint_group.this["${each.key}-${backend.key}"].id + } + } +} + +resource "google_compute_url_map" "this" { + for_each = var.services + + project = var.project + name = "${var.name}-${each.key}" + default_service = google_compute_backend_service.this[each.key].id +} + +resource "google_compute_target_http_proxy" "this" { + for_each = var.services + + project = var.project + name = "${var.name}-${each.key}" + url_map = google_compute_url_map.this[each.key].id +} + +resource "google_compute_global_address" "this" { + for_each = var.services + + project = var.project + name = "${var.name}-${each.key}" +} + +resource "google_compute_global_forwarding_rule" "this" { + for_each = var.services + + project = var.project + name = "${var.name}-${each.key}" + ip_address = google_compute_global_address.this[each.key].id + port_range = "80" + target = google_compute_target_http_proxy.this[each.key].id + load_balancing_scheme = "EXTERNAL_MANAGED" +} + +output "ips" { + value = { for k, a in google_compute_global_address.this : k => a.address } +} diff --git a/terraform/modules/lb/variables.tf b/terraform/modules/lb/variables.tf new file mode 100644 index 0000000..add9d0d --- /dev/null +++ b/terraform/modules/lb/variables.tf @@ -0,0 +1,16 @@ +variable "project" { type = string } + +variable "name" { + type = string + description = "resource name prefix for the load balancer components" +} + +variable "regions" { + type = map(string) + description = "cell name to its gcp region" +} + +variable "services" { + type = map(string) + description = "service key to the cloud run service name the load balancer fronts" +} diff --git a/terraform/outputs.tf b/terraform/outputs.tf new file mode 100644 index 0000000..b1813d1 --- /dev/null +++ b/terraform/outputs.tf @@ -0,0 +1,5 @@ +output "lb_ips" { value = module.environment.lb_ips } +output "services" { value = module.environment.services } +output "revisions" { value = module.environment.revisions } +output "service_accounts" { value = module.environment.service_accounts } +output "registry" { value = module.environment.registry } diff --git a/terraform/variables.tf b/terraform/variables.tf new file mode 100644 index 0000000..a76bfec --- /dev/null +++ b/terraform/variables.tf @@ -0,0 +1,14 @@ +variable "env_name" { type = string } + +variable "image_refs" { + type = map(string) + description = "service to image ref suffix (@sha256:digest or :tag); overrides the env default when set" + default = {} + # infra manager sends an unset input as an explicit null + nullable = false + + validation { + condition = alltrue([for ref in values(var.image_refs) : can(regex("^(:[^:@]+|@sha256:[0-9a-f]{64})$", ref))]) + error_message = "image_refs values must be a ':tag' or '@sha256:' suffix; leave the map empty to use the env default." + } +} diff --git a/terraform/versions.tf b/terraform/versions.tf new file mode 100644 index 0000000..38336ed --- /dev/null +++ b/terraform/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.5.0" + + required_providers { + google = { + source = "hashicorp/google" + version = "~> 8.0" + } + } +}