diff --git a/terraform/README.md b/terraform/README.md index dc61fc6..8c167cd 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -127,7 +127,10 @@ subnets, registry, secrets, ci pool, blueprints bucket and hubs, so its cells sit in the host's subnets and read the host's nodes. with `hub = ""` it runs one of each service, in that hub's cell. it only makes its own services and service accounts, its lb if it has one (`tangled-lb-`), its -deployment and a `spindle-` ci provider, and it has no nodes root. +deployment and a `spindle-` ci provider. what it declares for a hub goes over the +host's hub, node by node, and it runs only the hub nodes it declares itself. those and its own +`nodes` (next-dev's knot) get a nodes root that skips the firewalls and secrets the host's +makes for the same kinds. ## gotchas @@ -169,6 +172,9 @@ is that one config with its name passed in, and none of them are in the colmena debian image is ignored after creation. - `hubs` in `modules/envs` groups the cells around the `nodes` their bobbins read, one of each kind. a node with a `url` is one we don't run, the rest are vms made here. + the vms nothing reads (migrator, knot) are the env's own `nodes` instead, one of each + kind, with a `cell` for where they sit. a node's `system` picks another + `nixosConfigurations` entry than its kind's, e.g. a knot built from another core branch. `kinds.tf` has what a kind's vm runs and which of bobbin's env vars get its address. a new region goes into the hub whose hydrant it should use. a vm lives in one of its hub's cells, and is named after its `name`. diff --git a/terraform/modules/environment/main.tf b/terraform/modules/environment/main.tf index 082314f..ff550b8 100644 --- a/terraform/modules/environment/main.tf +++ b/terraform/modules/environment/main.tf @@ -100,7 +100,7 @@ locals { # the nodes root makes this address before the node, and keeps it across a replaced vm data "google_compute_addresses" "node" { - for_each = var.env.nodes + for_each = var.env.hub_nodes project = var.env.project region = each.value.region diff --git a/terraform/modules/environment/variables.tf b/terraform/modules/environment/variables.tf index aca27f1..08ba3ce 100644 --- a/terraform/modules/environment/variables.tf +++ b/terraform/modules/environment/variables.tf @@ -20,13 +20,13 @@ variable "env" { ipv6_access_type = optional(string) # the addresses of what its hub gives it, as bobbin's env bobbin_env = map(string) - # the `nodes` keys bobbin can't get ready without + # the `hub_nodes` keys bobbin can't get ready without bobbin_needs = list(string) })) hostnames = optional(map(string), {}) - nodes = map(object({ + hub_nodes = map(object({ region = string })) diff --git a/terraform/modules/envs/next-dev.tf b/terraform/modules/envs/next-dev.tf index 3bebbd7..b60b81f 100644 --- a/terraform/modules/envs/next-dev.tf +++ b/terraform/modules/envs/next-dev.tf @@ -8,6 +8,18 @@ locals { hub = "eu" lb = false + # the hubs are dev's, so declare `hubs.eu.nodes.hydrant` here to run a hydrant of its own + nodes = { + knot = { + name = "knot-next-dev-eu-1" + cell = "eu" + zone = "europe-north1-a" + internal_ip = "10.166.15.11" + hostname = "knot.next-dev.tangled.org" + system = "knot-gcp-next-dev" + } + } + ci_oidc = merge(local.dev.ci_oidc, { ref = "refs/heads/next-dev" }) services = { @@ -21,8 +33,9 @@ locals { svfe = merge(local.dev.services.svfe, { image = ":next-dev" env = merge(local.dev.services.svfe.env, { - BOBBIN_URL = "https://bobbin-next-dev-${local.next_dev_run_app}" - ORIGIN = "https://svfe-next-dev-${local.next_dev_run_app}" + BOBBIN_URL = "https://bobbin-next-dev-${local.next_dev_run_app}" + ORIGIN = "https://svfe-next-dev-${local.next_dev_run_app}" + PRIMARY_KNOT = "knot.next-dev.tangled.org" }) }) } diff --git a/terraform/modules/envs/outputs.tf b/terraform/modules/envs/outputs.tf index ea7a7a7..7a9a2c7 100644 --- a/terraform/modules/envs/outputs.tf +++ b/terraform/modules/envs/outputs.tf @@ -2,12 +2,27 @@ locals { declared = { dev = local.dev, prod = local.prod, next-dev = local.next_dev } # what a guest runs on from its host env, which makes all of it in the project they share - hosted = ["project", "region", "network_name", "registry_name", "registry_description", "registry_retention", "hubs"] + hosted = ["project", "region", "network_name", "registry_name", "registry_description", "registry_retention"] + # and its hubs, with the env's own laid over them so a guest only restates the nodes it changes + # HACK: merged by hand to the depth hubs go, since infra manager runs terraform 1.5.7. once it + # has 1.8, make it use provider::deepmerge::mergo shared = { - for name, e in local.declared : name => merge(e, { - for k in local.hosted : k => local.declared[try(e.guest_of, name)][k] - }) + for name, e in local.declared : name => merge(e, + { for k in local.hosted : k => local.declared[try(e.guest_of, name)][k] }, + { + hubs = { + for h in distinct(concat(keys(local.declared[try(e.guest_of, name)].hubs), keys(try(e.hubs, {})))) : h => merge( + try(local.declared[try(e.guest_of, name)].hubs[h], {}), + try(e.hubs[h], {}), + { + nodes = merge(try(local.declared[try(e.guest_of, name)].hubs[h].nodes, {}), try(e.hubs[h].nodes, {})) + cells = merge(try(local.declared[try(e.guest_of, name)].hubs[h].cells, {}), try(e.hubs[h].cells, {})) + }, + ) + } + }, + ) } } @@ -40,23 +55,29 @@ locals { cell => local.cells[name][cell] } - # the ones the nodes root makes: the hubs' own, then the env's standalone `nodes` that no - # bobbin reads. the region comes from the env's cells, so a node placed outside them fails - # the plan + # every node its hubs have, the host's too, since a bobbin waits on those + hub_nodes = merge([ + for hub in values(e.hubs) : { + for kind, n in hub.nodes : n.name => { region = hub.cells[n.cell].region } if !can(n.url) + } + ]...) + + # only what the env declares itself, since the host runs the hubs it inherits. a node placed + # outside the env's cells fails the plan nodes = { for p in concat( - flatten([for hub in values(e.hubs) : [for kind, n in hub.nodes : { kind = kind, n = n } if !can(n.url)]]), + flatten([for hub in values(try(local.declared[name].hubs, {})) : [for kind, n in try(hub.nodes, {}) : { kind = kind, n = n } if !can(n.url)]]), [for kind, n in try(e.nodes, {}) : { kind = kind, n = n }], ) : p.n.name => merge( { machine_type = local.kinds[p.kind].machine_type disk_gb = local.kinds[p.kind].disk_gb + system = local.kinds[p.kind].system hostname = null tunnel = false }, p.n, { kind = p.kind - system = local.kinds[p.kind].system tag = local.kinds[p.kind].tag region = local.cells[name][p.n.cell].region public = try(local.kinds[p.kind].public, null) diff --git a/terraform/nodes/next-dev/.terraform.lock.hcl b/terraform/nodes/next-dev/.terraform.lock.hcl new file mode 100644 index 0000000..10a3fc9 --- /dev/null +++ b/terraform/nodes/next-dev/.terraform.lock.hcl @@ -0,0 +1,87 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/external" { + version = "2.4.2" + hashes = [ + "h1:4UInMFuK4GNw4uf2vkUwwDtc0CajvJ88BkAE6xLKOa4=", + "h1:QP724n6VWM/iitpILCwO079UOlzx6I0Ylh7g8Gwv1Y0=", + "zh:0b51793be4f66934a3666339e44c01fd56e1c6a56256dfc66d1cb391584b4c2f", + "zh:31cdd9b30e4ec63d130befc89471757ef3937b99a8f6cc006769d215365c5ba8", + "zh:61f86de4a3166cfa5da6800eeba8e6a2e4ab6403fc3d7b396508260b45d3de7d", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:817e8d5946aed6ca692e0bb2f6463c28774ef8fb2fdd3922543a495a249229ea", + "zh:b35f1bd1be09ed1a1620b43ab8cb43fe93407cf419586d53fe965691cc1b4a8e", + "zh:bcb170063ec8b5728bc2a4568bc48f539a1991cdaaf31667aa35568aebc34725", + "zh:c0d2c824cc7c047f26ce793bb0cbb6746f9745d1fc6e630d34a0afb5b92850d1", + "zh:cde68f51089b02db50e2c5a17f6e132dc1ead2fc08d3dd267b8dd54ec58133fa", + "zh:d1f3c497aa41f17e8d61067122f6d94e51ef942ab08be5465489864d900854ab", + "zh:e62568bfc0934b63e14f3547c823b01ed60d7475ff16bf12c0e55d5ac8d98ba7", + "zh:ed8890c29dba2b0ac27afcefa75e7395e27253b7025aaaa4258345fc59dad23e", + "zh:f220c56c7e487f01fd158126066f6525178bbd82805b27205354bc523cc7c413", + ] +} + +provider "registry.terraform.io/hashicorp/google" { + version = "8.4.0" + constraints = "~> 8.0" + hashes = [ + "h1:vxMhPGBXNTrgoK8PpiDUHwboHUuiUhb2HPG8pYZpySE=", + "h1:wrTEDMONJgXiuysbdlnTKr9LbD7uL8Q8eMA0lwX+mVE=", + "zh:0a5eb94e558ae40cc0d992ce1c9bd717decda1a109fd743c5e0872be858f2c79", + "zh:1b0c6708fadba06883f56e5de2b7c199450642a1d1af932d09a95983935f10bf", + "zh:28bd7dbce76d2925a3257c4095fd120404e35f4722f48aef771a3cba281d748b", + "zh:623f853b5f3d3883dd40bd6f9bb75680f89ffa83591d8089f61462d25e98a011", + "zh:8299aac08c13c611107a7c8d8faaed5a8f0abfb802e40bea401c147bcf0bf87c", + "zh:8ba4e7b9e103a93a9d8fcf0d059cf5dabacba14497792b470ead517b72723360", + "zh:92478110a52b2ba78d5e3f190f40bb94c9d87ec1bda28419bb42124dc6afbae1", + "zh:9e0ba6564ef6c0ae46d60a73aaf9cb9e81c80d141fd7a1503fe15850adc5d96a", + "zh:bf300965bb76e1681a4688d0113d502d833217c1c8dd81f428229b54234b73a9", + "zh:c6a05a8577a8f7da78cba98d665fcc925aa9e20c8614d31e7d64a7573ceeee15", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:f7f724b63400239face928c8205708eaa46e78ee8c1684912e3067e82a6e3556", + ] +} + +provider "registry.terraform.io/hashicorp/null" { + version = "3.3.2" + hashes = [ + "h1:/Wbz7DHFO92KAqyDBMxGA8nwOJYECM8iLGhR7+wVWhQ=", + "h1:IQ1qrkht1sC1nibUR+AJ3ulryyhVDHfCHZhoJi0sg2Y=", + "zh:10ec43b8b7b18d5639238c7fb9e111f6a4b038523dd66c7a426bf27b25fa4c08", + "zh:60beb9cc2ad5b871c710860cee75b42850cc6acd43db0d77cb5e00fda7288b55", + "zh:62538582d0a4a2f10ad8a8d9a6c3cd3f05af6c6d91c6641ffc78d4f0e8e69b27", + "zh:64a8f9ce7852d9efc5b464c12306c946366d59f5e2757def97969c9fd64bd1d6", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:92a374fb736a52f465283326d0a5bf4f495132eb99be209dfb4c75ec803fe8db", + "zh:98da9c42785d27a50f0604758bcb61a30f6278b9f2acd92bb3b2046e0e71916c", + "zh:b0f7896fae554729cdf4a24ac06359a050cff5817e6cd8597cba8a4ae01a7409", + "zh:bc8179ee35d67c72fb03012e7023b9f9816f033a7ec4109c001dd6d29752e812", + "zh:d23a598f713bfb6098bc003571d7de90b5a33b78f9be240488252fe5f3c2a60d", + "zh:d2855b922ea345dbd89ea287e4c6c4757e38bc0aaffeb2b79aa0b8004f9c53ff", + "zh:d3a60422bc6a2f9244d076c5222c07060c826ef91bdbaf4634cb752b86057473", + "zh:faa01928c25d2a6ecd9c7eb8b88134cb08de55a6b11ca6c703ac0092845344ba", + ] +} + +provider "registry.terraform.io/hashicorp/tls" { + version = "4.4.1" + constraints = "~> 4.0" + hashes = [ + "h1:1Zk4TqZwWmUZHfGTtM/vw8EKoWecYboMRqwhNaoTopc=", + "h1:x0sq38+DvTpCH55KdhERVDgNjHL036y4i+DITul45TA=", + "zh:0855f9a980426a0af2c4415fb23cccbbb88c2d1dd40d83da6f45622c075b398b", + "zh:189227fdae28e547b1d988f5d83c4280fce02e3152d8b62aa810047e2f9f7e59", + "zh:2557db4f493aa90b3683905d9eeb103e55a3facc4ccda763d6de6025116ae121", + "zh:2cfd8d6a543ac61c6e7c0461839ab89bb8da1ba6c8090ab946b1cf47f100d7fa", + "zh:5c04cf2a683ea60a0072c59bbcf415f232c382c891b3aaeb9bbbb28213e0d878", + "zh:6681e35d50328bc11a49bb61c002121353f6a844cc6d63a0c3db57745ec38a56", + "zh:693b810e438bb1dfbc13a42629597284ac1eb08eb30f46f18738d7a9bb768d34", + "zh:77c4425f75346b23817837f3947b6ab7e765308abbf573fec5d2cafd4b82f142", + "zh:b3fd64ec1cecea98c88d2332dbeac40498ce2506e3235bdbd1afdfbaa47bb758", + "zh:c06224b5797b129a84e2514be43f2b140622928fe9c92b55d7941be3296ea289", + "zh:debc830ec123f27944c6170d76cb4ea3542845896960bb9fc530702774df328e", + "zh:f300fcc909014550fd74b2b751665dcc049f874e7a475903f65671baf2c2d86d", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/terraform/nodes/next-dev/main.tf b/terraform/nodes/next-dev/main.tf new file mode 100644 index 0000000..0e2463d --- /dev/null +++ b/terraform/nodes/next-dev/main.tf @@ -0,0 +1,16 @@ +terraform { + backend "gcs" { + bucket = "exemplary-proxy-507408-d6-tfstate" + prefix = "nodes/next-dev" + } +} + +module "nodes" { + source = "../../modules/nodes" + + env = "next-dev" + lb = false + flake = abspath("${path.root}/../../..") +} + +output "nodes" { value = module.nodes }