diff --git a/.github/workflows/check-config.yml b/.github/workflows/check-config.yml index ce969add..1cf8abc9 100644 --- a/.github/workflows/check-config.yml +++ b/.github/workflows/check-config.yml @@ -26,33 +26,16 @@ jobs: - run: bun scripts/check.ts chart-templates: - name: helm template each chart + name: render declared Helm releases runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 with: { fetch-depth: 1 } + - uses: oven-sh/setup-bun@v2 - uses: azure/setup-helm@v5 - - uses: azure/setup-kubectl@v5 - - name: helm template each local chart with the generated global values - run: | - set -euo pipefail - # Source the same cute-haus-global values Flux feeds charts via - # valuesFrom, straight from the generated ConfigMap — no plain values - # file to drift. Full valuesFrom-resolving render lives in local - # `just flux-test`; it can't run in public CI because decrypting the - # SOPS secrets layer risks leaking plaintext into public logs. - values="$(mktemp)" - kubectl kustomize k8s/flux/sources \ - | yq e 'select(.kind == "ConfigMap" and .metadata.name == "cute-haus-global") | .data["values.yaml"]' - \ - > "$values" - for chart in k8s/charts/*/; do - name="$(basename "$chart")" - [[ "$name" == "common" ]] && continue - echo "::group::$name" - helm dependency update "$chart" >/dev/null - helm template "$chart" -f "$values" >/dev/null - echo "::endgroup::" - done + - uses: yannh/kubeconform-action@v1.7.0 + - name: lint, render, and schema-check declared local releases + run: bun scripts/render-releases.ts flux-kustomize: name: flux kustomize build diff --git a/.justfile b/.justfile index 8a14cc15..e3bcc790 100644 --- a/.justfile +++ b/.justfile @@ -178,6 +178,7 @@ sops-edit FILE: [group('kubes')] check: bun scripts/check.ts + bun scripts/render-releases.ts for dir in k8s/flux/system k8s/flux/sources k8s/flux/secrets k8s/flux/infra-crds k8s/flux/infra-core k8s/flux/platform k8s/flux/apps k8s/flux/external-routes; do kubectl kustomize "$dir" >/dev/null; done nix flake check --impure @@ -188,15 +189,20 @@ check: flux-test: uvx flux-local@8.3.0 test --enable-helm --path k8s/flux +# Concise live Flux, workload, storage, certificate, and warning-event health. +[group('kubes')] +k8s-status: + bun scripts/k8s-status.ts + # Bump a digest-pinned chart image to its current upstream digest. # Usage: just bump | just bump --all | just bump --check [group('kubes')] bump TARGET: bun scripts/bump-image.ts {{ TARGET }} -# Real-release helper for local in-tree charts only. Flux HelmRelease objects -# live in flux-system; Helm installs into the target namespace. -# Usage: just k8s {apply|diff|suspend|resume|reconcile} [target-namespace] [helmrelease-namespace] +# Flux release helper. Git is the normal deployment path; `diff` previews a +# local in-tree chart, and the other actions operate on its HelmRelease. +# Usage: just k8s {diff|suspend|resume|reconcile} [target-namespace] [helmrelease-namespace] [group('kubes')] k8s action release namespace='default' hr_namespace='flux-system': #!/usr/bin/env bash @@ -221,17 +227,6 @@ k8s action release namespace='default' hr_namespace='flux-system': } case "{{action}}" in - apply) - if [[ ! -f "$chart/Chart.yaml" ]]; then - echo "just k8s apply supports local charts only: $chart/Chart.yaml not found" >&2 - exit 2 - fi - values="$(global_values)" - trap 'status=$?; rm -f "$values"; flux_cmd resume helmrelease "{{release}}" -n "{{hr_namespace}}" || true; exit "$status"' EXIT - flux_cmd suspend helmrelease "{{release}}" -n "{{hr_namespace}}" || true - helm upgrade --install "{{release}}" "$chart" \ - -n "{{namespace}}" -f "$values" - ;; diff) if [[ ! -f "$chart/Chart.yaml" ]]; then echo "just k8s diff supports local charts only: $chart/Chart.yaml not found" >&2 @@ -251,7 +246,7 @@ k8s action release namespace='default' hr_namespace='flux-system': flux_cmd reconcile helmrelease "{{release}}" -n "{{hr_namespace}}" --with-source ;; *) - echo "usage: just k8s {apply|diff|suspend|resume|reconcile} [target-namespace] [helmrelease-namespace]" >&2 + echo "usage: just k8s {diff|suspend|resume|reconcile} [target-namespace] [helmrelease-namespace]" >&2 exit 2 ;; esac diff --git a/ansible/playbooks/vars/oidc-clients.yml b/ansible/playbooks/vars/oidc-clients.yml index f6607dcd..1c0dd636 100644 --- a/ansible/playbooks/vars/oidc-clients.yml +++ b/ansible/playbooks/vars/oidc-clients.yml @@ -11,6 +11,7 @@ oidc_clients: # Slug must match the forward-auth- Deployment name. - slug: forward-auth-navidrome name: Navidrome + namespace: identity logoUrl: https://cdn.jsdelivr.net/gh/selfhst/icons@main/png/navidrome.png launchURL: https://navidrome.cute.haus/ callbackURLs: diff --git a/k8s/OPERATIONS.md b/k8s/OPERATIONS.md new file mode 100644 index 00000000..4e5e5d16 --- /dev/null +++ b/k8s/OPERATIONS.md @@ -0,0 +1,128 @@ +# Kubernetes operations + +Flux is the deployment controller. Git is the desired state: make and review a +change here, merge it, then let Flux reconcile it. Use `just k8s-status` first +when the cluster looks unhealthy; it prints only actionable failures. + +## Normal changes and incidents + +```bash +just check +git push +flux reconcile kustomization apps -n flux-system --with-source +just k8s-status +``` + +Preview a local chart with `just k8s diff `. Use +`just k8s suspend`, `resume`, and `reconcile` for a HelmRelease; do not use +`helm upgrade` as a deployment path. + +For an emergency live recovery: suspend the affected HelmRelease, make the +minimum live recovery change, commit the desired fix, then resume and reconcile +the release. Flux must regain Git authority before the incident is closed. + +Useful incident commands: + +```bash +flux get kustomizations -A +flux get helmreleases -A +kubectl get pods,pvc,certificate -A +flux suspend helmrelease -n flux-system +flux reconcile helmrelease -n flux-system --with-source +kubectl rollout undo deployment/ -n +``` + +`kubectl rollout undo` is temporary recovery only: capture the resulting desired +configuration in Git, then reconcile Flux. Helm rollback is likewise an +emergency measure (`helm history` then `helm rollback`) while the release is +suspended. + +## Platform and identity namespace migration + +Apply the Tika, Gotenberg, and forward-auth namespace move as a controlled +maintenance operation. The new HelmRelease targets must not overlap the old +release state. + +```bash +flux suspend helmrelease tika -n flux-system +flux suspend helmrelease gotenberg -n flux-system +flux suspend helmrelease forward-auth -n flux-system + +# Merge the desired state, then create namespaces and the identity Secret. +flux reconcile kustomization secrets -n flux-system --with-source + +# Once the identity Secret exists, remove the old release records only. +helm uninstall tika -n default +helm uninstall gotenberg -n default +helm uninstall forward-auth -n default + +flux resume helmrelease tika -n flux-system +flux resume helmrelease gotenberg -n flux-system +flux resume helmrelease forward-auth -n flux-system +flux reconcile kustomization platform -n flux-system --with-source +flux reconcile kustomization apps -n flux-system --with-source +``` + +Then verify `paperless` starts with `tika.platform.svc` and +`gotenberg.platform.svc`, and open Navidrome through +`identity-forward-auth-navidrome`. This intentionally causes a short outage for +the three moved services. + +## Service inventory + +This table is maintained with the HelmRelease files. `local-path` means +node-local or host-mounted data and must be recovered on its owning node. + +| Release | Chart | Namespace | URL | Dependencies | Persistence | Backup / restore owner | Data-loss expectation | +| ---------------- | ------------ | ----------- | ---------------------------- | ---------------------------------- | ------------------------------------------- | ------------------------------ | --------------------------------------- | +| aly-codes | aly-codes | websites | https://aly.codes | — | none | Git/site build | rebuildable | +| collabora | collabora | default | https://collabora.cute.haus | nextcloud | none | configuration only | rebuildable | +| error-pages | error-pages | default | internal | traefik | none | Git | rebuildable | +| forward-auth | forward-auth | identity | internal | Pocket ID | SOPS secret | SOPS / identity operator | reconfigure clients | +| gotenberg | gotenberg | platform | internal | — | none | Git | rebuildable | +| immich | immich | default | https://immich.cute.haus | valkey | local-path uploads/Postgres, Longhorn cache | node owner / Immich export | photo loss possible without node data | +| morsels | morsels | websites | https://morsels.blue | — | Longhorn | Longhorn backup owner | restore from volume backup | +| navidrome | navidrome | default | https://navidrome.cute.haus | forward-auth | Longhorn config, host media | Longhorn / media host owner | media is external; config may be lost | +| nextcloud | nextcloud | nextcloud | https://nextcloud.cute.haus | pg-shared, valkey-nextcloud | local-path data, Longhorn config | node owner / CNPG backup owner | file loss possible without node data | +| ombi | ombi | default | https://ombi.cute.haus | — | Longhorn | Longhorn backup owner | restore config/database | +| paperless | paperless | default | https://paperless.cute.haus | pg-shared, valkey, tika, gotenberg | local-path data, Longhorn export | node owner / CNPG backup owner | documents may be lost without node data | +| pg-shared | pg-shared | cnpg-system | internal | CNPG | Longhorn database volumes, B2 backups | CNPG backup owner | point-in-time limited to backups | +| plex | plex | default | https://plex.cute.haus | media host | local-path config and media mounts | node/media owner | config or metadata loss possible | +| pocket-id | pocket-id | default | https://id.cute.haus | pg-shared | CNPG | CNPG backup owner | identity records depend on DB backup | +| seerr | seerr | default | https://seerr.cute.haus | pg-shared | Longhorn config, CNPG | Longhorn / CNPG backup owner | restore app config and DB | +| slingshot | slingshot | microcosm | https://slingshot.cute.haus | — | ephemeral cache | Git | rebuildable | +| switchyard | switchyard | websites | https://switchyard.aly.codes | — | none | Git/site build | rebuildable | +| tika | tika | platform | internal | — | none | Git | rebuildable | +| tranquil-pds | tranquil-pds | default | https://pds.cute.haus | pg-shared, valkey | CNPG, B2 repository data | CNPG/B2 owner | account data loss without backups | +| uptime-kuma | uptime-kuma | default | https://kuma.cute.haus | — | Longhorn | Longhorn backup owner | monitor history/config loss possible | +| vaultwarden | vaultwarden | default | https://vault.cute.haus | — | Longhorn | Longhorn backup owner | vault loss is critical | +| valkey | valkey | default | internal | — | Longhorn | Longhorn backup owner | cache/session loss tolerated | +| valkey-nextcloud | valkey | nextcloud | internal | — | Longhorn | Longhorn backup owner | cache/session loss tolerated | +| watsup | watsup | websites | https://cute.haus | — | ConfigMap | Git | rebuildable | + +## Stateful recovery + +- Longhorn: inspect replicas and backups with `kubectl -n longhorn-system get +volumes,longhornbackups`; restore to a new volume/PVC, verify it, then point + the workload at it. Do not delete the original volume while diagnosing. +- local-path and host-path: identify the PVC's node with `kubectl get pv` and + recover the data on that node before rescheduling. The storage class does not + provide replication. +- CNPG: use the `pg-shared` ScheduledBackup/B2 path, restore to a separate + Cluster, validate databases and roles, then plan the controlled cutover. + Never overwrite the live cluster as the first recovery action. + +## Uptime Kuma checklist (v1) + +Kuma is configured in its UI; this checklist is the versioned source of +monitoring intent. After every deployment, configure HTTPS monitors for every +public URL in the inventory, including all `pds.cute.haus`, `aly.social`, +`status.cute.haus`, `status.aly.codes`, and `status.aly.social` routes. + +- Confirm each monitor runs externally, follows the intended HTTP success rule, + and validates TLS expiry/certificate hostname. +- Put public customer-facing endpoints on the correct Kuma status page. +- Open each status page from outside the cluster and confirm its monitor state + and TLS certificate are visible. +- Record new URLs here before adding their UI monitor; remove UI monitors only + after removing the URL from this checklist. diff --git a/k8s/charts/README.md b/k8s/charts/README.md index bbde4f36..3bcc66eb 100644 --- a/k8s/charts/README.md +++ b/k8s/charts/README.md @@ -12,11 +12,9 @@ visible in the app chart. ```text charts/ ├── aly-codes/ # Static site (aly.codes) -├── audiobookshelf/ # Audiobook library with rclone-mounted media ├── tranquil-pds/ # Reference atproto Personal Data Server ├── cert-manager-issuers/ # Let's Encrypt ClusterIssuer + wildcard Certificates ├── external-routes/ # Ingress + Service + EndpointSlice for off-cluster targets -├── forgejo/ # Git hosting (git.aly.codes) ├── forward-auth/ # Per-app traefik-forward-auth frontends ├── immich/ # Photo library + ML + app-specific Postgres ├── longhorn-creds/ # B2 backup Secret + recurring backup job + UI ingress @@ -25,6 +23,9 @@ charts/ └── ... ``` +Undeployed charts live in [`../drafts`](../drafts), outside the Flux chart +inventory. See its README before promoting a draft to production. + ## Chart Style Prefer direct manifests: diff --git a/k8s/charts/navidrome/templates/ingress.yaml b/k8s/charts/navidrome/templates/ingress.yaml index 2547edc9..5abe40f6 100644 --- a/k8s/charts/navidrome/templates/ingress.yaml +++ b/k8s/charts/navidrome/templates/ingress.yaml @@ -5,7 +5,7 @@ metadata: labels: app: {{ .Chart.Name }} annotations: - traefik.ingress.kubernetes.io/router.middlewares: {{ .Release.Namespace }}-forward-auth-navidrome@kubernetescrd + traefik.ingress.kubernetes.io/router.middlewares: identity-forward-auth-navidrome@kubernetescrd spec: ingressClassName: traefik tls: diff --git a/k8s/charts/paperless/templates/deployment.yaml b/k8s/charts/paperless/templates/deployment.yaml index 6f5a1ecb..1dd37ecd 100644 --- a/k8s/charts/paperless/templates/deployment.yaml +++ b/k8s/charts/paperless/templates/deployment.yaml @@ -50,9 +50,9 @@ spec: - -c - | echo "waiting for tika..." - until wget -qO- http://tika:9998/; do sleep 2; done + until wget -qO- http://tika.platform.svc:9998/; do sleep 2; done echo "waiting for gotenberg..." - until wget -qO- http://gotenberg:3000/health; do sleep 2; done + until wget -qO- http://gotenberg.platform.svc:3000/health; do sleep 2; done echo "dependencies ready" resources: requests: @@ -101,9 +101,9 @@ spec: - name: PAPERLESS_TIKA_ENABLED value: "1" - name: PAPERLESS_TIKA_ENDPOINT - value: "http://tika:9998" + value: "http://tika.platform.svc:9998" - name: PAPERLESS_TIKA_GOTENBERG_ENDPOINT - value: "http://gotenberg:3000" + value: "http://gotenberg.platform.svc:3000" - name: PAPERLESS_TIME_ZONE value: "America/New_York" - name: PAPERLESS_URL diff --git a/k8s/drafts/README.md b/k8s/drafts/README.md new file mode 100644 index 00000000..23cb4f0a --- /dev/null +++ b/k8s/drafts/README.md @@ -0,0 +1,9 @@ +# Draft charts + +These charts are deliberately not reconciled by Flux. They are retained as +work-in-progress manifests, not deployable production state. Move a chart back +to `k8s/charts/` and add exactly one HelmRelease before deploying it. + +- `audiobookshelf` +- `forgejo` +- `jellyfin` diff --git a/k8s/charts/audiobookshelf/Chart.yaml b/k8s/drafts/audiobookshelf/Chart.yaml similarity index 100% rename from k8s/charts/audiobookshelf/Chart.yaml rename to k8s/drafts/audiobookshelf/Chart.yaml diff --git a/k8s/charts/audiobookshelf/templates/deployment.yaml b/k8s/drafts/audiobookshelf/templates/deployment.yaml similarity index 100% rename from k8s/charts/audiobookshelf/templates/deployment.yaml rename to k8s/drafts/audiobookshelf/templates/deployment.yaml diff --git a/k8s/charts/audiobookshelf/templates/ingress.yaml b/k8s/drafts/audiobookshelf/templates/ingress.yaml similarity index 80% rename from k8s/charts/audiobookshelf/templates/ingress.yaml rename to k8s/drafts/audiobookshelf/templates/ingress.yaml index 6561e497..f3b2ca57 100644 --- a/k8s/charts/audiobookshelf/templates/ingress.yaml +++ b/k8s/drafts/audiobookshelf/templates/ingress.yaml @@ -1,9 +1,9 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: {{ .Chart.Name }} + name: { { .Chart.Name } } labels: - app: {{ .Chart.Name }} + app: { { .Chart.Name } } spec: ingressClassName: traefik tls: @@ -18,6 +18,6 @@ spec: pathType: Prefix backend: service: - name: {{ .Chart.Name }} + name: { { .Chart.Name } } port: number: 80 diff --git a/k8s/charts/audiobookshelf/templates/pvc.yaml b/k8s/drafts/audiobookshelf/templates/pvc.yaml similarity index 100% rename from k8s/charts/audiobookshelf/templates/pvc.yaml rename to k8s/drafts/audiobookshelf/templates/pvc.yaml diff --git a/k8s/charts/audiobookshelf/templates/service.yaml b/k8s/drafts/audiobookshelf/templates/service.yaml similarity index 63% rename from k8s/charts/audiobookshelf/templates/service.yaml rename to k8s/drafts/audiobookshelf/templates/service.yaml index f8cc341d..c68a5eab 100644 --- a/k8s/charts/audiobookshelf/templates/service.yaml +++ b/k8s/drafts/audiobookshelf/templates/service.yaml @@ -1,13 +1,13 @@ apiVersion: v1 kind: Service metadata: - name: {{ .Chart.Name }} + name: { { .Chart.Name } } labels: - app: {{ .Chart.Name }} + app: { { .Chart.Name } } spec: type: ClusterIP selector: - app: {{ .Chart.Name }} + app: { { .Chart.Name } } ports: - name: http port: 80 diff --git a/k8s/charts/forgejo/Chart.yaml b/k8s/drafts/forgejo/Chart.yaml similarity index 100% rename from k8s/charts/forgejo/Chart.yaml rename to k8s/drafts/forgejo/Chart.yaml diff --git a/k8s/charts/forgejo/templates/deployment.yaml b/k8s/drafts/forgejo/templates/deployment.yaml similarity index 100% rename from k8s/charts/forgejo/templates/deployment.yaml rename to k8s/drafts/forgejo/templates/deployment.yaml diff --git a/k8s/charts/forgejo/templates/ingress.yaml b/k8s/drafts/forgejo/templates/ingress.yaml similarity index 100% rename from k8s/charts/forgejo/templates/ingress.yaml rename to k8s/drafts/forgejo/templates/ingress.yaml diff --git a/k8s/charts/forgejo/templates/ingressroute-ssh.yaml b/k8s/drafts/forgejo/templates/ingressroute-ssh.yaml similarity index 100% rename from k8s/charts/forgejo/templates/ingressroute-ssh.yaml rename to k8s/drafts/forgejo/templates/ingressroute-ssh.yaml diff --git a/k8s/charts/forgejo/templates/middleware.yaml b/k8s/drafts/forgejo/templates/middleware.yaml similarity index 100% rename from k8s/charts/forgejo/templates/middleware.yaml rename to k8s/drafts/forgejo/templates/middleware.yaml diff --git a/k8s/charts/forgejo/templates/pvc.yaml b/k8s/drafts/forgejo/templates/pvc.yaml similarity index 100% rename from k8s/charts/forgejo/templates/pvc.yaml rename to k8s/drafts/forgejo/templates/pvc.yaml diff --git a/k8s/charts/forgejo/templates/service.yaml b/k8s/drafts/forgejo/templates/service.yaml similarity index 70% rename from k8s/charts/forgejo/templates/service.yaml rename to k8s/drafts/forgejo/templates/service.yaml index cd2916fa..a60969a6 100644 --- a/k8s/charts/forgejo/templates/service.yaml +++ b/k8s/drafts/forgejo/templates/service.yaml @@ -1,13 +1,13 @@ apiVersion: v1 kind: Service metadata: - name: {{ .Chart.Name }} + name: { { .Chart.Name } } labels: - app: {{ .Chart.Name }} + app: { { .Chart.Name } } spec: type: ClusterIP selector: - app: {{ .Chart.Name }} + app: { { .Chart.Name } } ports: - name: http port: 80 diff --git a/k8s/charts/jellyfin/Chart.yaml b/k8s/drafts/jellyfin/Chart.yaml similarity index 100% rename from k8s/charts/jellyfin/Chart.yaml rename to k8s/drafts/jellyfin/Chart.yaml diff --git a/k8s/charts/jellyfin/templates/deployment.yaml b/k8s/drafts/jellyfin/templates/deployment.yaml similarity index 100% rename from k8s/charts/jellyfin/templates/deployment.yaml rename to k8s/drafts/jellyfin/templates/deployment.yaml diff --git a/k8s/charts/jellyfin/templates/ingress.yaml b/k8s/drafts/jellyfin/templates/ingress.yaml similarity index 80% rename from k8s/charts/jellyfin/templates/ingress.yaml rename to k8s/drafts/jellyfin/templates/ingress.yaml index ac865793..e7eb203c 100644 --- a/k8s/charts/jellyfin/templates/ingress.yaml +++ b/k8s/drafts/jellyfin/templates/ingress.yaml @@ -1,9 +1,9 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: {{ .Chart.Name }} + name: { { .Chart.Name } } labels: - app: {{ .Chart.Name }} + app: { { .Chart.Name } } spec: ingressClassName: traefik tls: @@ -18,6 +18,6 @@ spec: pathType: Prefix backend: service: - name: {{ .Chart.Name }} + name: { { .Chart.Name } } port: number: 80 diff --git a/k8s/charts/jellyfin/templates/pvc.yaml b/k8s/drafts/jellyfin/templates/pvc.yaml similarity index 100% rename from k8s/charts/jellyfin/templates/pvc.yaml rename to k8s/drafts/jellyfin/templates/pvc.yaml diff --git a/k8s/charts/jellyfin/templates/service.yaml b/k8s/drafts/jellyfin/templates/service.yaml similarity index 63% rename from k8s/charts/jellyfin/templates/service.yaml rename to k8s/drafts/jellyfin/templates/service.yaml index 566130ae..e44fcd47 100644 --- a/k8s/charts/jellyfin/templates/service.yaml +++ b/k8s/drafts/jellyfin/templates/service.yaml @@ -1,13 +1,13 @@ apiVersion: v1 kind: Service metadata: - name: {{ .Chart.Name }} + name: { { .Chart.Name } } labels: - app: {{ .Chart.Name }} + app: { { .Chart.Name } } spec: type: ClusterIP selector: - app: {{ .Chart.Name }} + app: { { .Chart.Name } } ports: - name: http port: 80 diff --git a/k8s/flux/apps/aly-codes.yaml b/k8s/flux/apps/aly-codes.yaml new file mode 100644 index 00000000..fc4e0635 --- /dev/null +++ b/k8s/flux/apps/aly-codes.yaml @@ -0,0 +1,28 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: aly-codes + namespace: flux-system +spec: + interval: 15m + releaseName: aly-codes + targetNamespace: websites + install: + remediation: + retries: 3 + upgrade: + remediation: + retries: 3 + chart: + spec: + chart: ./k8s/charts/aly-codes + interval: 15m + reconcileStrategy: Revision + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + valuesFrom: + - kind: ConfigMap + name: cute-haus-global + valuesKey: values.yaml diff --git a/k8s/flux/apps/collabora.yaml b/k8s/flux/apps/collabora.yaml new file mode 100644 index 00000000..be80650c --- /dev/null +++ b/k8s/flux/apps/collabora.yaml @@ -0,0 +1,22 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: collabora + namespace: flux-system +spec: + interval: 15m + releaseName: collabora + targetNamespace: default + install: + remediation: { retries: 3 } + upgrade: + remediation: { retries: 3 } + chart: + spec: + chart: ./k8s/charts/collabora + interval: 15m + reconcileStrategy: Revision + sourceRef: + { kind: GitRepository, name: flux-system, namespace: flux-system } + valuesFrom: + - { kind: ConfigMap, name: cute-haus-global, valuesKey: values.yaml } diff --git a/k8s/flux/apps/error-pages.yaml b/k8s/flux/apps/error-pages.yaml new file mode 100644 index 00000000..390152cf --- /dev/null +++ b/k8s/flux/apps/error-pages.yaml @@ -0,0 +1,20 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: error-pages + namespace: flux-system +spec: + interval: 15m + releaseName: error-pages + targetNamespace: default + install: { remediation: { retries: 3 } } + upgrade: { remediation: { retries: 3 } } + chart: + spec: + chart: ./k8s/charts/error-pages + interval: 15m + reconcileStrategy: Revision + sourceRef: + { kind: GitRepository, name: flux-system, namespace: flux-system } + valuesFrom: + - { kind: ConfigMap, name: cute-haus-global, valuesKey: values.yaml } diff --git a/k8s/flux/apps/forward-auth.yaml b/k8s/flux/apps/forward-auth.yaml new file mode 100644 index 00000000..84f1e209 --- /dev/null +++ b/k8s/flux/apps/forward-auth.yaml @@ -0,0 +1,23 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: forward-auth + namespace: flux-system +spec: + interval: 15m + releaseName: forward-auth + targetNamespace: identity + install: { remediation: { retries: 3 } } + upgrade: { remediation: { retries: 3 } } + chart: + spec: + chart: ./k8s/charts/forward-auth + interval: 15m + reconcileStrategy: Revision + sourceRef: + { kind: GitRepository, name: flux-system, namespace: flux-system } + valuesFrom: + - { kind: ConfigMap, name: cute-haus-global, valuesKey: values.yaml } + values: + apps: + navidrome: {} diff --git a/k8s/flux/apps/immich.yaml b/k8s/flux/apps/immich.yaml new file mode 100644 index 00000000..66e3767a --- /dev/null +++ b/k8s/flux/apps/immich.yaml @@ -0,0 +1,22 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: immich + namespace: flux-system +spec: + dependsOn: + - name: valkey + interval: 15m + releaseName: immich + targetNamespace: default + install: { remediation: { retries: 3 } } + upgrade: { remediation: { retries: 3 } } + chart: + spec: + chart: ./k8s/charts/immich + interval: 15m + reconcileStrategy: Revision + sourceRef: + { kind: GitRepository, name: flux-system, namespace: flux-system } + valuesFrom: + - { kind: ConfigMap, name: cute-haus-global, valuesKey: values.yaml } diff --git a/k8s/flux/apps/kustomization.yaml b/k8s/flux/apps/kustomization.yaml index 0493e6c6..e70a9d94 100644 --- a/k8s/flux/apps/kustomization.yaml +++ b/k8s/flux/apps/kustomization.yaml @@ -1,4 +1,22 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - releases.yaml + - aly-codes.yaml + - collabora.yaml + - error-pages.yaml + - forward-auth.yaml + - immich.yaml + - morsels.yaml + - navidrome.yaml + - nextcloud.yaml + - ombi.yaml + - paperless.yaml + - plex.yaml + - pocket-id.yaml + - seerr.yaml + - slingshot.yaml + - switchyard.yaml + - tranquil-pds.yaml + - uptime-kuma.yaml + - vaultwarden.yaml + - watsup.yaml diff --git a/k8s/flux/apps/morsels.yaml b/k8s/flux/apps/morsels.yaml new file mode 100644 index 00000000..95222c2c --- /dev/null +++ b/k8s/flux/apps/morsels.yaml @@ -0,0 +1,20 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: morsels + namespace: flux-system +spec: + interval: 15m + releaseName: morsels + targetNamespace: websites + install: { remediation: { retries: 3 } } + upgrade: { remediation: { retries: 3 } } + chart: + spec: + chart: ./k8s/charts/morsels + interval: 15m + reconcileStrategy: Revision + sourceRef: + { kind: GitRepository, name: flux-system, namespace: flux-system } + valuesFrom: + - { kind: ConfigMap, name: cute-haus-global, valuesKey: values.yaml } diff --git a/k8s/flux/apps/navidrome.yaml b/k8s/flux/apps/navidrome.yaml new file mode 100644 index 00000000..a53eb094 --- /dev/null +++ b/k8s/flux/apps/navidrome.yaml @@ -0,0 +1,22 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: navidrome + namespace: flux-system +spec: + dependsOn: + - name: forward-auth + interval: 15m + releaseName: navidrome + targetNamespace: default + install: { remediation: { retries: 3 } } + upgrade: { remediation: { retries: 3 } } + chart: + spec: + chart: ./k8s/charts/navidrome + interval: 15m + reconcileStrategy: Revision + sourceRef: + { kind: GitRepository, name: flux-system, namespace: flux-system } + valuesFrom: + - { kind: ConfigMap, name: cute-haus-global, valuesKey: values.yaml } diff --git a/k8s/flux/apps/nextcloud.yaml b/k8s/flux/apps/nextcloud.yaml new file mode 100644 index 00000000..ca2b469f --- /dev/null +++ b/k8s/flux/apps/nextcloud.yaml @@ -0,0 +1,23 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: nextcloud + namespace: flux-system +spec: + dependsOn: + - name: pg-shared + - name: valkey-nextcloud + interval: 15m + releaseName: nextcloud + targetNamespace: nextcloud + install: { remediation: { retries: 3 } } + upgrade: { remediation: { retries: 3 } } + chart: + spec: + chart: ./k8s/charts/nextcloud + interval: 15m + reconcileStrategy: Revision + sourceRef: + { kind: GitRepository, name: flux-system, namespace: flux-system } + valuesFrom: + - { kind: ConfigMap, name: cute-haus-global, valuesKey: values.yaml } diff --git a/k8s/flux/apps/ombi.yaml b/k8s/flux/apps/ombi.yaml new file mode 100644 index 00000000..14e26346 --- /dev/null +++ b/k8s/flux/apps/ombi.yaml @@ -0,0 +1,20 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: ombi + namespace: flux-system +spec: + interval: 15m + releaseName: ombi + targetNamespace: default + install: { remediation: { retries: 3 } } + upgrade: { remediation: { retries: 3 } } + chart: + spec: + chart: ./k8s/charts/ombi + interval: 15m + reconcileStrategy: Revision + sourceRef: + { kind: GitRepository, name: flux-system, namespace: flux-system } + valuesFrom: + - { kind: ConfigMap, name: cute-haus-global, valuesKey: values.yaml } diff --git a/k8s/flux/apps/paperless.yaml b/k8s/flux/apps/paperless.yaml new file mode 100644 index 00000000..1fa52362 --- /dev/null +++ b/k8s/flux/apps/paperless.yaml @@ -0,0 +1,25 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: paperless + namespace: flux-system +spec: + dependsOn: + - name: pg-shared + - name: valkey + - name: tika + - name: gotenberg + interval: 15m + releaseName: paperless + targetNamespace: default + install: { remediation: { retries: 3 } } + upgrade: { remediation: { retries: 3 } } + chart: + spec: + chart: ./k8s/charts/paperless + interval: 15m + reconcileStrategy: Revision + sourceRef: + { kind: GitRepository, name: flux-system, namespace: flux-system } + valuesFrom: + - { kind: ConfigMap, name: cute-haus-global, valuesKey: values.yaml } diff --git a/k8s/flux/apps/plex.yaml b/k8s/flux/apps/plex.yaml new file mode 100644 index 00000000..8aec150d --- /dev/null +++ b/k8s/flux/apps/plex.yaml @@ -0,0 +1,20 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: plex + namespace: flux-system +spec: + interval: 15m + releaseName: plex + targetNamespace: default + install: { remediation: { retries: 3 } } + upgrade: { remediation: { retries: 3 } } + chart: + spec: + chart: ./k8s/charts/plex + interval: 15m + reconcileStrategy: Revision + sourceRef: + { kind: GitRepository, name: flux-system, namespace: flux-system } + valuesFrom: + - { kind: ConfigMap, name: cute-haus-global, valuesKey: values.yaml } diff --git a/k8s/flux/apps/pocket-id.yaml b/k8s/flux/apps/pocket-id.yaml new file mode 100644 index 00000000..7517a712 --- /dev/null +++ b/k8s/flux/apps/pocket-id.yaml @@ -0,0 +1,22 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: pocket-id + namespace: flux-system +spec: + dependsOn: + - name: pg-shared + interval: 15m + releaseName: pocket-id + targetNamespace: default + install: { remediation: { retries: 3 } } + upgrade: { remediation: { retries: 3 } } + chart: + spec: + chart: ./k8s/charts/pocket-id + interval: 15m + reconcileStrategy: Revision + sourceRef: + { kind: GitRepository, name: flux-system, namespace: flux-system } + valuesFrom: + - { kind: ConfigMap, name: cute-haus-global, valuesKey: values.yaml } diff --git a/k8s/flux/apps/releases.yaml b/k8s/flux/apps/releases.yaml deleted file mode 100644 index 533915d8..00000000 --- a/k8s/flux/apps/releases.yaml +++ /dev/null @@ -1,556 +0,0 @@ -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: error-pages - namespace: flux-system -spec: - interval: 15m - releaseName: error-pages - targetNamespace: default - install: - remediation: - retries: 3 - upgrade: - remediation: - retries: 3 - chart: - spec: - chart: ./k8s/charts/error-pages - interval: 15m - reconcileStrategy: Revision - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - valuesFrom: - - kind: ConfigMap - name: cute-haus-global - valuesKey: values.yaml ---- -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: aly-codes - namespace: flux-system -spec: - interval: 15m - releaseName: aly-codes - targetNamespace: websites - install: - remediation: - retries: 3 - upgrade: - remediation: - retries: 3 - chart: - spec: - chart: ./k8s/charts/aly-codes - interval: 15m - reconcileStrategy: Revision - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - valuesFrom: - - kind: ConfigMap - name: cute-haus-global - valuesKey: values.yaml ---- -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: switchyard - namespace: flux-system -spec: - interval: 15m - releaseName: switchyard - targetNamespace: websites - install: - remediation: - retries: 3 - upgrade: - remediation: - retries: 3 - chart: - spec: - chart: ./k8s/charts/switchyard - interval: 15m - reconcileStrategy: Revision - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - valuesFrom: - - kind: ConfigMap - name: cute-haus-global - valuesKey: values.yaml ---- -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: morsels - namespace: flux-system -spec: - interval: 15m - releaseName: morsels - targetNamespace: websites - install: - remediation: - retries: 3 - upgrade: - remediation: - retries: 3 - chart: - spec: - chart: ./k8s/charts/morsels - interval: 15m - reconcileStrategy: Revision - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - valuesFrom: - - kind: ConfigMap - name: cute-haus-global - valuesKey: values.yaml ---- -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: ombi - namespace: flux-system -spec: - interval: 15m - releaseName: ombi - targetNamespace: default - install: - remediation: - retries: 3 - upgrade: - remediation: - retries: 3 - chart: - spec: - chart: ./k8s/charts/ombi - interval: 15m - reconcileStrategy: Revision - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - valuesFrom: - - kind: ConfigMap - name: cute-haus-global - valuesKey: values.yaml ---- -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: seerr - namespace: flux-system -spec: - interval: 15m - releaseName: seerr - targetNamespace: default - install: - remediation: - retries: 3 - upgrade: - remediation: - retries: 3 - chart: - spec: - chart: ./k8s/charts/seerr - interval: 15m - reconcileStrategy: Revision - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - valuesFrom: - - kind: ConfigMap - name: cute-haus-global - valuesKey: values.yaml ---- -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: pocket-id - namespace: flux-system -spec: - interval: 15m - releaseName: pocket-id - targetNamespace: default - install: - remediation: - retries: 3 - upgrade: - remediation: - retries: 3 - chart: - spec: - chart: ./k8s/charts/pocket-id - interval: 15m - reconcileStrategy: Revision - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - valuesFrom: - - kind: ConfigMap - name: cute-haus-global - valuesKey: values.yaml ---- -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: forward-auth - namespace: flux-system -spec: - interval: 15m - releaseName: forward-auth - targetNamespace: default - install: - remediation: - retries: 3 - upgrade: - remediation: - retries: 3 - chart: - spec: - chart: ./k8s/charts/forward-auth - interval: 15m - reconcileStrategy: Revision - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - valuesFrom: - - kind: ConfigMap - name: cute-haus-global - valuesKey: values.yaml - values: - apps: - navidrome: {} ---- -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: immich - namespace: flux-system -spec: - interval: 15m - releaseName: immich - targetNamespace: default - install: - remediation: - retries: 3 - upgrade: - remediation: - retries: 3 - chart: - spec: - chart: ./k8s/charts/immich - interval: 15m - reconcileStrategy: Revision - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - valuesFrom: - - kind: ConfigMap - name: cute-haus-global - valuesKey: values.yaml ---- -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: plex - namespace: flux-system -spec: - interval: 15m - releaseName: plex - targetNamespace: default - install: - remediation: - retries: 3 - upgrade: - remediation: - retries: 3 - chart: - spec: - chart: ./k8s/charts/plex - interval: 15m - reconcileStrategy: Revision - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - valuesFrom: - - kind: ConfigMap - name: cute-haus-global - valuesKey: values.yaml ---- -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: navidrome - namespace: flux-system -spec: - dependsOn: - - name: forward-auth - interval: 15m - releaseName: navidrome - targetNamespace: default - install: - remediation: - retries: 3 - upgrade: - remediation: - retries: 3 - chart: - spec: - chart: ./k8s/charts/navidrome - interval: 15m - reconcileStrategy: Revision - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - valuesFrom: - - kind: ConfigMap - name: cute-haus-global - valuesKey: values.yaml ---- -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: paperless - namespace: flux-system -spec: - interval: 15m - releaseName: paperless - targetNamespace: default - install: - remediation: - retries: 3 - upgrade: - timeout: 30m - remediation: - retries: 3 - chart: - spec: - chart: ./k8s/charts/paperless - interval: 15m - reconcileStrategy: Revision - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - valuesFrom: - - kind: ConfigMap - name: cute-haus-global - valuesKey: values.yaml ---- -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: nextcloud - namespace: flux-system -spec: - interval: 15m - releaseName: nextcloud - targetNamespace: nextcloud - install: - remediation: - retries: 3 - upgrade: - remediation: - retries: 3 - chart: - spec: - chart: ./k8s/charts/nextcloud - interval: 15m - reconcileStrategy: Revision - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - valuesFrom: - - kind: ConfigMap - name: cute-haus-global - valuesKey: values.yaml ---- -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: collabora - namespace: flux-system -spec: - interval: 15m - releaseName: collabora - targetNamespace: default - install: - remediation: - retries: 3 - upgrade: - remediation: - retries: 3 - chart: - spec: - chart: ./k8s/charts/collabora - interval: 15m - reconcileStrategy: Revision - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - valuesFrom: - - kind: ConfigMap - name: cute-haus-global - valuesKey: values.yaml ---- -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: uptime-kuma - namespace: flux-system -spec: - interval: 15m - releaseName: uptime-kuma - targetNamespace: default - install: - remediation: - retries: 3 - upgrade: - remediation: - retries: 3 - chart: - spec: - chart: ./k8s/charts/uptime-kuma - interval: 15m - reconcileStrategy: Revision - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - valuesFrom: - - kind: ConfigMap - name: cute-haus-global - valuesKey: values.yaml ---- -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: vaultwarden - namespace: flux-system -spec: - interval: 15m - releaseName: vaultwarden - targetNamespace: default - install: - remediation: - retries: 3 - upgrade: - remediation: - retries: 3 - chart: - spec: - chart: ./k8s/charts/vaultwarden - interval: 15m - reconcileStrategy: Revision - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - valuesFrom: - - kind: ConfigMap - name: cute-haus-global - valuesKey: values.yaml ---- -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: tranquil-pds - namespace: flux-system -spec: - interval: 15m - releaseName: tranquil-pds - targetNamespace: default - install: - remediation: - retries: 3 - upgrade: - remediation: - retries: 3 - chart: - spec: - chart: ./k8s/charts/tranquil-pds - interval: 15m - reconcileStrategy: Revision - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - valuesFrom: - - kind: ConfigMap - name: cute-haus-global - valuesKey: values.yaml ---- -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: watsup - namespace: flux-system -spec: - interval: 15m - releaseName: watsup - targetNamespace: websites - install: - remediation: - retries: 3 - upgrade: - remediation: - retries: 3 - chart: - spec: - chart: ./k8s/charts/watsup - interval: 15m - reconcileStrategy: Revision - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - valuesFrom: - - kind: ConfigMap - name: cute-haus-global - valuesKey: values.yaml ---- -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: slingshot - namespace: flux-system -spec: - interval: 15m - releaseName: slingshot - targetNamespace: microcosm - install: - remediation: - retries: 3 - upgrade: - remediation: - retries: 3 - chart: - spec: - chart: ./k8s/charts/slingshot - interval: 15m - reconcileStrategy: Revision - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - valuesFrom: - - kind: ConfigMap - name: cute-haus-global - valuesKey: values.yaml diff --git a/k8s/flux/apps/seerr.yaml b/k8s/flux/apps/seerr.yaml new file mode 100644 index 00000000..ed32aca6 --- /dev/null +++ b/k8s/flux/apps/seerr.yaml @@ -0,0 +1,22 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: seerr + namespace: flux-system +spec: + dependsOn: + - name: pg-shared + interval: 15m + releaseName: seerr + targetNamespace: default + install: { remediation: { retries: 3 } } + upgrade: { remediation: { retries: 3 } } + chart: + spec: + chart: ./k8s/charts/seerr + interval: 15m + reconcileStrategy: Revision + sourceRef: + { kind: GitRepository, name: flux-system, namespace: flux-system } + valuesFrom: + - { kind: ConfigMap, name: cute-haus-global, valuesKey: values.yaml } diff --git a/k8s/flux/apps/slingshot.yaml b/k8s/flux/apps/slingshot.yaml new file mode 100644 index 00000000..555173d8 --- /dev/null +++ b/k8s/flux/apps/slingshot.yaml @@ -0,0 +1,20 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: slingshot + namespace: flux-system +spec: + interval: 15m + releaseName: slingshot + targetNamespace: microcosm + install: { remediation: { retries: 3 } } + upgrade: { remediation: { retries: 3 } } + chart: + spec: + chart: ./k8s/charts/slingshot + interval: 15m + reconcileStrategy: Revision + sourceRef: + { kind: GitRepository, name: flux-system, namespace: flux-system } + valuesFrom: + - { kind: ConfigMap, name: cute-haus-global, valuesKey: values.yaml } diff --git a/k8s/flux/apps/switchyard.yaml b/k8s/flux/apps/switchyard.yaml new file mode 100644 index 00000000..0c389a4b --- /dev/null +++ b/k8s/flux/apps/switchyard.yaml @@ -0,0 +1,20 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: switchyard + namespace: flux-system +spec: + interval: 15m + releaseName: switchyard + targetNamespace: websites + install: { remediation: { retries: 3 } } + upgrade: { remediation: { retries: 3 } } + chart: + spec: + chart: ./k8s/charts/switchyard + interval: 15m + reconcileStrategy: Revision + sourceRef: + { kind: GitRepository, name: flux-system, namespace: flux-system } + valuesFrom: + - { kind: ConfigMap, name: cute-haus-global, valuesKey: values.yaml } diff --git a/k8s/flux/apps/tranquil-pds.yaml b/k8s/flux/apps/tranquil-pds.yaml new file mode 100644 index 00000000..49949da9 --- /dev/null +++ b/k8s/flux/apps/tranquil-pds.yaml @@ -0,0 +1,23 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: tranquil-pds + namespace: flux-system +spec: + dependsOn: + - name: pg-shared + - name: valkey + interval: 15m + releaseName: tranquil-pds + targetNamespace: default + install: { remediation: { retries: 3 } } + upgrade: { remediation: { retries: 3 } } + chart: + spec: + chart: ./k8s/charts/tranquil-pds + interval: 15m + reconcileStrategy: Revision + sourceRef: + { kind: GitRepository, name: flux-system, namespace: flux-system } + valuesFrom: + - { kind: ConfigMap, name: cute-haus-global, valuesKey: values.yaml } diff --git a/k8s/flux/apps/uptime-kuma.yaml b/k8s/flux/apps/uptime-kuma.yaml new file mode 100644 index 00000000..b0ad92da --- /dev/null +++ b/k8s/flux/apps/uptime-kuma.yaml @@ -0,0 +1,20 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: uptime-kuma + namespace: flux-system +spec: + interval: 15m + releaseName: uptime-kuma + targetNamespace: default + install: { remediation: { retries: 3 } } + upgrade: { remediation: { retries: 3 } } + chart: + spec: + chart: ./k8s/charts/uptime-kuma + interval: 15m + reconcileStrategy: Revision + sourceRef: + { kind: GitRepository, name: flux-system, namespace: flux-system } + valuesFrom: + - { kind: ConfigMap, name: cute-haus-global, valuesKey: values.yaml } diff --git a/k8s/flux/apps/vaultwarden.yaml b/k8s/flux/apps/vaultwarden.yaml new file mode 100644 index 00000000..bd04195b --- /dev/null +++ b/k8s/flux/apps/vaultwarden.yaml @@ -0,0 +1,20 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: vaultwarden + namespace: flux-system +spec: + interval: 15m + releaseName: vaultwarden + targetNamespace: default + install: { remediation: { retries: 3 } } + upgrade: { remediation: { retries: 3 } } + chart: + spec: + chart: ./k8s/charts/vaultwarden + interval: 15m + reconcileStrategy: Revision + sourceRef: + { kind: GitRepository, name: flux-system, namespace: flux-system } + valuesFrom: + - { kind: ConfigMap, name: cute-haus-global, valuesKey: values.yaml } diff --git a/k8s/flux/apps/watsup.yaml b/k8s/flux/apps/watsup.yaml new file mode 100644 index 00000000..a0cf1c99 --- /dev/null +++ b/k8s/flux/apps/watsup.yaml @@ -0,0 +1,20 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: watsup + namespace: flux-system +spec: + interval: 15m + releaseName: watsup + targetNamespace: websites + install: { remediation: { retries: 3 } } + upgrade: { remediation: { retries: 3 } } + chart: + spec: + chart: ./k8s/charts/watsup + interval: 15m + reconcileStrategy: Revision + sourceRef: + { kind: GitRepository, name: flux-system, namespace: flux-system } + valuesFrom: + - { kind: ConfigMap, name: cute-haus-global, valuesKey: values.yaml } diff --git a/k8s/flux/platform/gotenberg.yaml b/k8s/flux/platform/gotenberg.yaml index 32f5bc83..18f220cc 100644 --- a/k8s/flux/platform/gotenberg.yaml +++ b/k8s/flux/platform/gotenberg.yaml @@ -6,7 +6,7 @@ metadata: spec: interval: 15m releaseName: gotenberg - targetNamespace: default + targetNamespace: platform install: remediation: retries: 3 diff --git a/k8s/flux/platform/kubernetes-prometheus.yaml b/k8s/flux/platform/kubernetes-prometheus.yaml index 24e0a4a2..6d50f25b 100644 --- a/k8s/flux/platform/kubernetes-prometheus.yaml +++ b/k8s/flux/platform/kubernetes-prometheus.yaml @@ -40,6 +40,15 @@ spec: service: type: NodePort nodePort: 30220 + extraScrapeConfigs: + - job_name: flux-controllers + metrics_path: /metrics + static_configs: + - targets: + - helm-controller.flux-system.svc.cluster.local:8080 + - kustomize-controller.flux-system.svc.cluster.local:8080 + - notification-controller.flux-system.svc.cluster.local:8080 + - source-controller.flux-system.svc.cluster.local:8080 kube-state-metrics: prometheusScrape: true prometheus-node-exporter: diff --git a/k8s/flux/platform/tika.yaml b/k8s/flux/platform/tika.yaml index 61fb1770..fc5ce5ff 100644 --- a/k8s/flux/platform/tika.yaml +++ b/k8s/flux/platform/tika.yaml @@ -6,7 +6,7 @@ metadata: spec: interval: 15m releaseName: tika - targetNamespace: default + targetNamespace: platform install: remediation: retries: 3 diff --git a/k8s/flux/secrets/forward-auth-navidrome-env.sops.yaml b/k8s/flux/secrets/forward-auth-navidrome-env.sops.yaml index 1a301b34..c42fb0c4 100644 --- a/k8s/flux/secrets/forward-auth-navidrome-env.sops.yaml +++ b/k8s/flux/secrets/forward-auth-navidrome-env.sops.yaml @@ -4,7 +4,7 @@ data: kind: ENC[AES256_GCM,data:EANfDqqH,iv:V9Sipa6L2vnnoCi+kxHb9RrN6b7eB4mbp3RXhPznmqc=,tag:hdid30Jy+k6LEUG87U7ajA==,type:str] metadata: name: ENC[AES256_GCM,data:uOwJNjlXkYgbxr3E+0Pm+diFURLblyNjCTo=,iv:q7Ey4CZ7e2FXFq3ogesTIG2NdOwGz/0QOX+c0XF6p8Y=,tag:Ulan54qF/Msj8+gKSfYD3g==,type:str] - namespace: ENC[AES256_GCM,data:P12mZGWhAg==,iv:N88MVemCEPbkyF3My8N07QQSr0G31jNEtdbrFLMvSoQ=,tag:BOVwFemkamnhHGop7sF59w==,type:str] + namespace: ENC[AES256_GCM,data:0v1l+xFObYY=,iv:VsQ7usEjAveTSY3YWhQYgCMmUJvKnXdNHVuszkb6vjU=,tag:kHa0bFNTnzYlXB1jrKT76g==,type:str] sops: age: - enc: | @@ -214,7 +214,7 @@ sops: erAs+zpy6707rroGPG3IW+srOWuFZyXsV37EJLGW3ahLqHI5Zv3G/w== -----END AGE ENCRYPTED FILE----- recipient: age1yq47x0cyf95jg372v84ttptgnppsrfmxshsua6p58ecmgzxc5c0s5stfth - lastmodified: "2026-07-01T19:42:05Z" - mac: ENC[AES256_GCM,data:7OieJ4teg61brYusNmRxsKxk2/2Hdke9wnbiEa+BJBcxIvQUxuWTc7XmpBBXab+Bud4hXtsN3Tw7BBKMUn4oWHNeoIuno+vWMQlnE1wuhWA9Ux6U2PyC9iQThGYa4PLYe7KdsKYXp6zIDuKnhbgAvoUIBOWVrTdwOXTrKMO/z+c=,iv:VDHrN45g7sXDPLN6v/ljmVN8TrViGvIyQWFWjXcPn88=,tag:hWHRPlzEy4pjV6QETvOPIg==,type:str] + lastmodified: "2026-08-06T03:07:04Z" + mac: ENC[AES256_GCM,data:Qmxo/ooz8Mk5eU8YAmDGiuGujU0SH8fGHDTCWbEG6z/e5Jb3oPUF3PX/EP6shYREQOTYozGlAI3OwFUD55KEBoNprCT4V6G5UMKEb/BcLcQTChE8TvCkLGvksN4UU9F29e96FvRUenfiYaL3zAFflyU7tMrT4+k+5FC4YYxj3q4=,iv:dDhT0/+Ud0dBLMGNv4jwvDMCE5GOfU8lZfUa1YftqXQ=,tag:os1IPe9Yn0wVR+N2NQOHOQ==,type:str] unencrypted_suffix: _unencrypted version: 3.13.1 diff --git a/k8s/flux/secrets/namespaces.yaml b/k8s/flux/secrets/namespaces.yaml index 4b21e038..331afe84 100644 --- a/k8s/flux/secrets/namespaces.yaml +++ b/k8s/flux/secrets/namespaces.yaml @@ -32,3 +32,13 @@ apiVersion: v1 kind: Namespace metadata: name: microcosm +--- +apiVersion: v1 +kind: Namespace +metadata: + name: identity +--- +apiVersion: v1 +kind: Namespace +metadata: + name: platform diff --git a/k8s/flux/system/layers.yaml b/k8s/flux/system/layers.yaml index 1565b215..16f7580a 100644 --- a/k8s/flux/system/layers.yaml +++ b/k8s/flux/system/layers.yaml @@ -41,6 +41,20 @@ spec: interval: 15m path: ./k8s/flux/infra-crds prune: true + wait: true + healthChecks: + - apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: cert-manager-issuers + namespace: flux-system + - apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: cert-manager + namespace: flux-system + - apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: cnpg + namespace: flux-system sourceRef: kind: GitRepository name: flux-system @@ -56,6 +70,40 @@ spec: interval: 15m path: ./k8s/flux/infra-core prune: true + wait: true + healthChecks: + - apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: intel-device-plugins-operator + namespace: flux-system + - apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: intel-gpu-plugin + namespace: flux-system + - apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: longhorn + namespace: flux-system + - apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: longhorn-creds + namespace: flux-system + - apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: node-feature-discovery + namespace: flux-system + - apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: reflector + namespace: flux-system + - apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: tailscale-operator + namespace: flux-system + - apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: traefik + namespace: flux-system sourceRef: kind: GitRepository name: flux-system @@ -71,6 +119,36 @@ spec: interval: 15m path: ./k8s/flux/platform prune: true + wait: true + healthChecks: + - apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: gotenberg + namespace: flux-system + - apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: kubernetes-alloy + namespace: flux-system + - apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: kubernetes-prometheus + namespace: flux-system + - apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: pg-shared + namespace: flux-system + - apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: tika + namespace: flux-system + - apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: valkey + namespace: flux-system + - apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: valkey-nextcloud + namespace: flux-system sourceRef: kind: GitRepository name: flux-system @@ -86,6 +164,8 @@ spec: interval: 15m path: ./k8s/flux/apps prune: true + timeout: 30m + wait: true sourceRef: kind: GitRepository name: flux-system diff --git a/nix/devShells.nix b/nix/devShells.nix index af91e6ca..42116e7e 100644 --- a/nix/devShells.nix +++ b/nix/devShells.nix @@ -29,6 +29,7 @@ just kustomize kubectl + kubeconform helmWithDiff nh skopeo diff --git a/nix/hosts/eterna.nix b/nix/hosts/eterna.nix index 0b5e29fd..b02d0c95 100644 --- a/nix/hosts/eterna.nix +++ b/nix/hosts/eterna.nix @@ -4,6 +4,147 @@ ... }: let tnet = "narwhal-snapper.ts.net"; + kubernetesOperationsDashboard = builtins.toJSON { + annotations.list = []; + editable = true; + panels = [ + { + datasource = "Kubernetes Prometheus"; + fieldConfig.defaults.unit = "short"; + gridPos = { + h = 8; + w = 8; + x = 0; + y = 0; + }; + targets = [ + { + expr = "sum(gotk_reconcile_condition{type=\"Ready\",status=\"True\"})"; + refId = "A"; + } + ]; + title = "Flux resources ready"; + type = "stat"; + } + { + datasource = "Kubernetes Prometheus"; + fieldConfig.defaults.unit = "short"; + gridPos = { + h = 8; + w = 8; + x = 8; + y = 0; + }; + targets = [ + { + expr = "sum(gotk_reconcile_condition{type=\"Ready\",status=\"False\"})"; + refId = "A"; + } + ]; + title = "Flux reconciliation failures"; + type = "stat"; + } + { + datasource = "Kubernetes Prometheus"; + fieldConfig.defaults.unit = "short"; + gridPos = { + h = 8; + w = 8; + x = 16; + y = 0; + }; + targets = [ + { + expr = "sum(increase(kube_pod_container_status_restarts_total[1h]))"; + refId = "A"; + } + ]; + title = "Container restarts (1h)"; + type = "stat"; + } + { + datasource = "Kubernetes Prometheus"; + fieldConfig.defaults.unit = "short"; + gridPos = { + h = 8; + w = 8; + x = 0; + y = 8; + }; + targets = [ + { + expr = "sum(kube_pod_status_ready{condition=\"true\"})"; + refId = "A"; + } + ]; + title = "Ready pods"; + type = "stat"; + } + { + datasource = "Kubernetes Prometheus"; + fieldConfig.defaults.unit = "short"; + gridPos = { + h = 8; + w = 8; + x = 8; + y = 8; + }; + targets = [ + { + expr = "sum(kube_persistentvolumeclaim_status_phase{phase=\"Pending\"})"; + refId = "A"; + } + ]; + title = "Pending PVCs"; + type = "stat"; + } + { + datasource = "Kubernetes Prometheus"; + fieldConfig.defaults.unit = "percent"; + gridPos = { + h = 8; + w = 8; + x = 16; + y = 8; + }; + targets = [ + { + expr = "sum(kube_pod_container_resource_requests{resource=\"cpu\"}) / sum(kube_node_status_allocatable{resource=\"cpu\"})"; + refId = "A"; + } + ]; + title = "Requested node CPU"; + type = "gauge"; + } + { + datasource = "Loki"; + gridPos = { + h = 10; + w = 24; + x = 0; + y = 16; + }; + targets = [ + { + expr = "{job=\"kubernetes-pods\"} |~ \"(?i)error\""; + refId = "A"; + } + ]; + title = "Recent Kubernetes error logs"; + type = "logs"; + } + ]; + refresh = "30s"; + schemaVersion = 39; + tags = ["kubernetes" "operations"]; + templating.list = []; + time = { + from = "now-6h"; + to = "now"; + }; + title = "Kubernetes Operations"; + uid = "kubernetes-operations"; + }; in { flake.nixosConfigurations.eterna = inputs.nixpkgs.lib.nixosSystem { specialArgs = {inherit inputs self;}; @@ -204,7 +345,7 @@ in { } # observability - { + ({pkgs, ...}: { services = { grafana = { enable = true; @@ -242,6 +383,17 @@ in { url = "https://loki.${tnet}"; } ]; + + dashboards.settings = { + apiVersion = 1; + providers = [ + { + name = "Kubernetes Operations"; + options.path = pkgs.writeText "kubernetes-operations.json" kubernetesOperationsDashboard; + type = "file"; + } + ]; + }; }; }; @@ -382,7 +534,7 @@ in { ]; }; }; - } + }) # services { diff --git a/nix/treefmt.nix b/nix/treefmt.nix index 400922a1..3475085e 100644 --- a/nix/treefmt.nix +++ b/nix/treefmt.nix @@ -2,6 +2,9 @@ perSystem.treefmt.config = { settings.global.excludes = [ "k8s/flux/secrets/*.sops.yaml" + # Draft Helm templates are intentionally retained without being rendered + # or validated as Kubernetes YAML. + "k8s/drafts/**" # Flux-generated upstream manifest (flux install output); Renovate bumps # it verbatim, so don't let prettier reformat/diverge it from upstream. "k8s/flux/system/gotk-components.yaml" diff --git a/scripts/check-chart-inventory.ts b/scripts/check-chart-inventory.ts new file mode 100644 index 00000000..69a387f7 --- /dev/null +++ b/scripts/check-chart-inventory.ts @@ -0,0 +1,92 @@ +// Every local chart is intentionally declared: production charts have at least +// one HelmRelease, while undeployed charts live in k8s/drafts. A chart may be +// used by more than one release (for example, the two Valkey instances). + +const FLUX_DIRS = [ + "k8s/flux/infra-crds", + "k8s/flux/infra-core", + "k8s/flux/platform", + "k8s/flux/apps", + "k8s/flux/external-routes", +]; + +type HelmRelease = { + kind?: string; + spec?: { chart?: { spec?: { chart?: string } } }; +}; + +function yamlDocuments(text: string): string[] { + return text + .split(/^---\s*$/m) + .map((document) => document.trim()) + .filter(Boolean); +} + +async function declaredChartCounts(): Promise> { + const counts = new Map(); + + for (const directory of FLUX_DIRS) { + const files = new Bun.Glob(`${directory}/**/*.yaml`); + for await (const path of files.scan(".")) { + for (const document of yamlDocuments(await Bun.file(path).text())) { + const release = Bun.YAML.parse(document) as HelmRelease | null; + const chart = release?.spec?.chart?.spec?.chart; + if ( + release?.kind !== "HelmRelease" || + !chart?.startsWith("./k8s/charts/") + ) + continue; + + const name = chart.slice("./k8s/charts/".length); + counts.set(name, (counts.get(name) ?? 0) + 1); + } + } + } + + return counts; +} + +async function chartDirectories(root: string): Promise> { + const charts = new Set(); + const files = new Bun.Glob(`${root}/*/Chart.yaml`); + for await (const path of files.scan(".")) { + charts.add(path.split("/")[2]); + } + return charts; +} + +export async function checkChartInventory(): Promise { + const [declared, production, drafts] = await Promise.all([ + declaredChartCounts(), + chartDirectories("k8s/charts"), + chartDirectories("k8s/drafts"), + ]); + const errors: string[] = []; + + for (const chart of production) { + const count = declared.get(chart) ?? 0; + if (count === 0) + errors.push( + `production chart '${chart}' has no HelmRelease; move it to k8s/drafts or deploy it`, + ); + } + + for (const chart of drafts) { + const count = declared.get(chart) ?? 0; + if (count !== 0) + errors.push( + `draft chart '${chart}' is also deployed by ${count} HelmRelease(s)`, + ); + if (production.has(chart)) + errors.push(`chart '${chart}' exists in both k8s/charts and k8s/drafts`); + } + + for (const [chart, count] of declared) { + if (!production.has(chart)) + errors.push( + `HelmRelease references '${chart}', which is not a production chart (${count} release(s))`, + ); + } + + return errors; +} diff --git a/scripts/check-forward-auth.ts b/scripts/check-forward-auth.ts index ffac6b00..83762532 100755 --- a/scripts/check-forward-auth.ts +++ b/scripts/check-forward-auth.ts @@ -2,7 +2,7 @@ // the forward-auth chart, and vice versa. const OIDC_CLIENTS_FILE = "ansible/playbooks/vars/oidc-clients.yml"; -const FORWARD_AUTH_VALUES_FILE = "k8s/flux/apps/releases.yaml"; +const FORWARD_AUTH_VALUES_FILE = "k8s/flux/apps/forward-auth.yaml"; const SLUG_PREFIX = "forward-auth-"; type OIDCClient = { slug: string; integration?: string }; diff --git a/scripts/check.ts b/scripts/check.ts index a2302143..97bee47d 100755 --- a/scripts/check.ts +++ b/scripts/check.ts @@ -3,10 +3,12 @@ // check doesn't hide another). Exits non-zero if any check fails. import { checkForwardAuth } from "./check-forward-auth.ts"; +import { checkChartInventory } from "./check-chart-inventory.ts"; import { checkPinnedImages } from "./check-pinned-images.ts"; import { checkReleaseNames } from "./check-release-names.ts"; const checks = [ + { name: "chart-inventory", run: checkChartInventory }, { name: "forward-auth", run: checkForwardAuth }, { name: "release-names", run: checkReleaseNames }, { name: "pinned-images", run: checkPinnedImages }, diff --git a/scripts/k8s-status.ts b/scripts/k8s-status.ts new file mode 100644 index 00000000..49c7753f --- /dev/null +++ b/scripts/k8s-status.ts @@ -0,0 +1,165 @@ +#!/usr/bin/env bun +// Print only Kubernetes and Flux failures. Each item includes the shortest +// useful follow-up command for the single-operator incident workflow. + +type KubernetesObject = { + kind?: string; + metadata?: { name?: string; namespace?: string; creationTimestamp?: string }; + reason?: string; + message?: string; + spec?: { replicas?: number }; + status?: { + phase?: string; + readyReplicas?: number; + conditions?: Array<{ type?: string; status?: string; message?: string }>; + }; +}; + +type KubernetesList = { items?: KubernetesObject[] }; + +async function kubectlJson(arguments_: string[]): Promise { + const process = Bun.spawn(["kubectl", ...arguments_, "-o", "json"], { + stdout: "pipe", + stderr: "pipe", + }); + const output = await new Response(process.stdout).text(); + const error = await new Response(process.stderr).text(); + if ((await process.exited) !== 0) throw new Error(error.trim()); + return JSON.parse(output) as KubernetesList; +} + +function objectName(object: KubernetesObject): string { + return `${object.metadata?.namespace ?? "default"}/${object.metadata?.name ?? "unknown"}`; +} + +function readyCondition(object: KubernetesObject): boolean { + return ( + object.status?.conditions?.some( + (condition) => condition.type === "Ready" && condition.status === "True", + ) ?? false + ); +} + +function failedObjects(objects: KubernetesObject[]): KubernetesObject[] { + return objects.filter((object) => !readyCondition(object)); +} + +function unhealthyWorkloads(objects: KubernetesObject[]): KubernetesObject[] { + return objects.filter((object) => { + if (object.kind === "StatefulSet") { + return ( + (object.status?.readyReplicas ?? 0) !== (object.spec?.replicas ?? 1) + ); + } + return !object.status?.conditions?.some( + (condition) => + condition.type === "Available" && condition.status === "True", + ); + }); +} + +const findings: Array<{ title: string; lines: string[] }> = []; + +function collectFindings(title: string, lines: string[]): boolean { + if (lines.length === 0) return false; + findings.push({ title, lines }); + return true; +} + +let degraded = false; +try { + const [ + kustomizations, + releases, + deployments, + statefulSets, + pvcs, + certificates, + events, + ] = await Promise.all([ + kubectlJson(["get", "kustomizations.kustomize.toolkit.fluxcd.io", "-A"]), + kubectlJson(["get", "helmreleases.helm.toolkit.fluxcd.io", "-A"]), + kubectlJson(["get", "deployments", "-A"]), + kubectlJson(["get", "statefulsets", "-A"]), + kubectlJson(["get", "pvc", "-A"]), + kubectlJson(["get", "certificates.cert-manager.io", "-A"]), + kubectlJson([ + "get", + "events", + "-A", + "--field-selector=type=Warning", + "--sort-by=.lastTimestamp", + ]), + ]); + + degraded = + collectFindings( + "Flux Kustomizations", + failedObjects(kustomizations.items ?? []).map( + (object) => + `${objectName(object)} — flux reconcile kustomization ${object.metadata?.name} -n ${object.metadata?.namespace} --with-source`, + ), + ) || degraded; + degraded = + collectFindings( + "Flux HelmReleases", + failedObjects(releases.items ?? []).map( + (object) => + `${objectName(object)} — flux reconcile helmrelease ${object.metadata?.name} -n ${object.metadata?.namespace} --with-source`, + ), + ) || degraded; + const workloads = [ + ...(deployments.items ?? []), + ...(statefulSets.items ?? []), + ]; + degraded = + collectFindings( + "Unhealthy workloads", + unhealthyWorkloads(workloads).map( + (object) => + `${object.kind?.toLowerCase()} ${objectName(object)} — kubectl describe ${object.kind?.toLowerCase()} ${object.metadata?.name} -n ${object.metadata?.namespace}`, + ), + ) || degraded; + degraded = + collectFindings( + "Unbound PVCs", + (pvcs.items ?? []) + .filter((pvc) => pvc.status?.phase !== "Bound") + .map( + (pvc) => + `${objectName(pvc)} — kubectl describe pvc ${pvc.metadata?.name} -n ${pvc.metadata?.namespace}`, + ), + ) || degraded; + degraded = + collectFindings( + "Certificate problems", + failedObjects(certificates.items ?? []).map( + (certificate) => + `${objectName(certificate)} — kubectl describe certificate ${certificate.metadata?.name} -n ${certificate.metadata?.namespace}`, + ), + ) || degraded; + collectFindings( + "Recent warning events (do not imply current degradation)", + (events.items ?? []) + .slice(-20) + .map( + (event) => + `${objectName(event)} ${event.reason ?? "Warning"}: ${event.message ?? ""} — kubectl get events -A --field-selector=type=Warning --sort-by=.lastTimestamp`, + ), + ); +} catch (error) { + degraded = true; + findings.push({ + title: "Kubernetes status unavailable", + lines: [ + `${error instanceof Error ? error.message : error} — kubectl cluster-info`, + ], + }); +} + +console.log(`Kubernetes status: ${degraded ? "DEGRADED" : "HEALTHY"}`); +for (const finding of findings) { + console.log(`\n${finding.title}`); + for (const line of finding.lines) console.log(`- ${line}`); +} +if (degraded) process.exit(1); diff --git a/scripts/render-releases.ts b/scripts/render-releases.ts new file mode 100644 index 00000000..c7daaac5 --- /dev/null +++ b/scripts/render-releases.ts @@ -0,0 +1,150 @@ +#!/usr/bin/env bun +// Render every local HelmRelease with the same global values and precedence +// Flux uses. SOPS-backed values are intentionally not read in CI. + +import { join } from "node:path"; + +const FLUX_DIRS = [ + "k8s/flux/infra-crds", + "k8s/flux/infra-core", + "k8s/flux/platform", + "k8s/flux/apps", + "k8s/flux/external-routes", +]; + +type HelmRelease = { + kind?: string; + metadata?: { name?: string }; + spec?: { + releaseName?: string; + targetNamespace?: string; + chart?: { spec?: { chart?: string } }; + values?: Record; + valuesFrom?: Array<{ kind?: string; name?: string; valuesKey?: string }>; + }; +}; + +function yamlDocuments(text: string): string[] { + return text + .split(/^---\s*$/m) + .map((document) => document.trim()) + .filter(Boolean); +} + +function mergeValues( + base: Record, + override: Record, +): Record { + const merged = { ...base }; + for (const [key, value] of Object.entries(override)) { + const existing = merged[key]; + if ( + value !== null && + typeof value === "object" && + !Array.isArray(value) && + existing !== null && + typeof existing === "object" && + !Array.isArray(existing) + ) { + merged[key] = mergeValues( + existing as Record, + value as Record, + ); + } else { + merged[key] = value; + } + } + return merged; +} + +async function localReleases(): Promise { + const releases: HelmRelease[] = []; + for (const directory of FLUX_DIRS) { + const files = new Bun.Glob(`${directory}/**/*.yaml`); + for await (const path of files.scan(".")) { + for (const document of yamlDocuments(await Bun.file(path).text())) { + const release = Bun.YAML.parse(document) as HelmRelease | null; + const chart = release?.spec?.chart?.spec?.chart; + if ( + release?.kind === "HelmRelease" && + chart?.startsWith("./k8s/charts/") + ) + releases.push(release); + } + } + } + return releases.sort((left, right) => + (left.metadata?.name ?? "").localeCompare(right.metadata?.name ?? ""), + ); +} + +async function run(command: string[], input?: string): Promise { + const process = Bun.spawn(command, { + stdin: input ? new Blob([input]) : undefined, + stdout: "inherit", + stderr: "inherit", + }); + if ((await process.exited) !== 0) throw new Error(command.join(" ")); +} + +async function renderRelease( + release: HelmRelease, + globalValues: Record, +): Promise { + const name = release.metadata?.name; + const chart = release.spec?.chart?.spec?.chart; + if (!name || !chart) + throw new Error("HelmRelease is missing a name or chart"); + + for (const source of release.spec?.valuesFrom ?? []) { + if ( + source.kind !== "ConfigMap" || + source.name !== "cute-haus-global" || + source.valuesKey !== "values.yaml" + ) { + throw new Error( + `${name}: renderer only supports the non-secret cute-haus-global valuesFrom source`, + ); + } + } + + const values = mergeValues(globalValues, release.spec?.values ?? {}); + const valuesPath = join( + process.env.TMPDIR ?? "/tmp", + `cute-haus-${name}-${crypto.randomUUID()}.yaml`, + ); + await Bun.write(valuesPath, Bun.YAML.stringify(values)); + + try { + console.log(`${name}: lint and render`); + await run(["helm", "dependency", "build", chart]); + await run(["helm", "lint", chart, "--values", valuesPath]); + + const rendered = Bun.spawn([ + "helm", + "template", + release.spec?.releaseName ?? name, + chart, + "--namespace", + release.spec?.targetNamespace ?? "default", + "--values", + valuesPath, + "--include-crds", + ]); + const manifest = await new Response(rendered.stdout).text(); + if ((await rendered.exited) !== 0) + throw new Error(`${name}: helm template`); + + await run(["kubeconform", "-ignore-missing-schemas", "-summary"], manifest); + } finally { + await run(["rm", "-f", valuesPath]); + } +} + +const globalValues = Bun.YAML.parse( + await Bun.file("k8s/flux/sources/global.values.yaml").text(), +) as Record; + +for (const release of await localReleases()) { + await renderRelease(release, globalValues); +}