diff --git a/Makefile b/Makefile index dcd27e0..52d4cdb 100644 --- a/Makefile +++ b/Makefile @@ -32,13 +32,11 @@ secrets: --host kube-1 apps: - # TODO multiple env - @temporal workflow start \ - --workflow-id apps-manual \ - --task-queue cloudlab \ - --type Apps \ - --input '{ "url": "/usr/local/src/cloudlab", "revision": "master", "registry": "registry.127.0.0.1.sslip.io", "cluster": "local" }' - @temporal workflow result --workflow-id apps-manual + toolbox apps \ + --env ${env} \ + --path apps \ + --hosts-file infra/_modules/nixos/hosts.json \ + --host kube-1 test: cd test && CLOUDLAB_ENV=${env} go test diff --git a/README.md b/README.md index 5ca64e1..6d929ef 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,8 @@ - However, the runtime doesn’t have access to Git - all manifests are pulled from an OCI registry - Apps aka SaaS: - Strict and standardized - - Uses the rendered manifests pattern, essentially `helm template && oras push` + - Generated from `apps/$NAMESPACE/$APP/$ENV.yaml` + - Published to the cluster as a Flux OCI artifact ## Estimated cost diff --git a/platform/staging/apps.yaml b/platform/staging/apps.yaml new file mode 100644 index 0000000..f87d428 --- /dev/null +++ b/platform/staging/apps.yaml @@ -0,0 +1,26 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: OCIRepository +metadata: + name: apps + namespace: flux-system +spec: + interval: 30s + url: oci://registry.registry.svc.cluster.local:5000/apps + insecure: true + ref: + tag: staging +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: apps + namespace: flux-system +spec: + interval: 1m + dependsOn: + - name: platform + path: . + prune: true + sourceRef: + kind: OCIRepository + name: apps diff --git a/toolbox/cmd/apps.go b/toolbox/cmd/apps.go new file mode 100644 index 0000000..bf3999b --- /dev/null +++ b/toolbox/cmd/apps.go @@ -0,0 +1,74 @@ +package cmd + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + + "github.com/charmbracelet/log" + "github.com/spf13/cobra" + + appbundle "github.com/khuedoan/cloudlab/toolbox/internal/apps" +) + +const appsRepository = "apps" + +var ( + appsEnv string + appsPath string +) + +func init() { + appsCmd.Flags().StringVar(&appsEnv, "env", "", "Environment to generate (for example: staging)") + appsCmd.Flags().StringVar(&appsPath, "path", "apps", "Path to the app values tree") + _ = appsCmd.MarkFlagRequired("env") +} + +var appsCmd = &cobra.Command{ + Use: "apps", + Args: cobra.NoArgs, + Short: "Generate and push the apps manifest bundle", + PreRunE: func(_ *cobra.Command, _ []string) error { + if err := validateClusterFlags(); err != nil { + return err + } + if _, err := exec.LookPath("flux"); err != nil { + return fmt.Errorf("find flux CLI: %w", err) + } + return nil + }, + RunE: runApps, +} + +func runApps(cmd *cobra.Command, _ []string) error { + sourcePath, err := filepath.Abs(appsPath) + if err != nil { + return fmt.Errorf("resolve path %q: %w", appsPath, err) + } + + releases, err := appbundle.Discover(sourcePath, appsEnv) + if err != nil { + return fmt.Errorf("discover apps: %w", err) + } + + bundleDir, err := os.MkdirTemp("", "toolbox-apps-*") + if err != nil { + return fmt.Errorf("create temp dir: %w", err) + } + defer os.RemoveAll(bundleDir) + + if err := appbundle.WriteBundle(bundleDir, releases); err != nil { + return fmt.Errorf("write bundle: %w", err) + } + + log.Infof("generated %d app HelmRelease(s) for %s", len(releases), appsEnv) + + return pushArtifact(cmd.Context(), bundleDir, artifactRef{ + Repository: appsRepository, + Tag: appsEnv, + Source: appsRepository, + Revision: appsEnv, + Kustomization: appsRepository, + }) +} diff --git a/toolbox/cmd/flux.go b/toolbox/cmd/flux.go new file mode 100644 index 0000000..4c57439 --- /dev/null +++ b/toolbox/cmd/flux.go @@ -0,0 +1,109 @@ +package cmd + +import ( + "context" + "fmt" + "os/exec" + "strings" + "time" + + "github.com/charmbracelet/log" + + "github.com/khuedoan/cloudlab/toolbox/internal/cluster" +) + +const ( + registryNamespace = "registry" + registryService = "svc/registry" + registryPort = 5000 + fluxNamespace = "flux-system" +) + +type artifactRef struct { + Repository string + Tag string + Source string + Revision string + Kustomization string +} + +func pushArtifact(ctx context.Context, manifestPath string, artifact artifactRef) error { + connectCtx, cancel := context.WithTimeout(ctx, connectTimeout) + defer cancel() + + hostAddr, err := cluster.LoadHost(hostsFile, host) + if err != nil { + return fmt.Errorf("load host: %w", err) + } + + conn, err := cluster.Connect(cluster.SSHConfig{ + Host: hostAddr, + User: sshUser, + KeyPath: sshKey, + KnownHostsPath: sshKnownHosts, + Timeout: connectTimeout, + }) + if err != nil { + return fmt.Errorf("connect to cluster: %w", err) + } + defer conn.Close() + + tunnel, err := conn.Forward(connectCtx, cluster.ServiceConfig{ + Namespace: registryNamespace, + Name: registryService, + Port: registryPort, + }) + if err != nil { + return fmt.Errorf("forward registry: %w", err) + } + + artifactURL := fmt.Sprintf("oci://%s/%s:%s", tunnel.LocalAddr, artifact.Repository, artifact.Tag) + args := []string{ + "push", + "artifact", + artifactURL, + "--path", manifestPath, + "--source", artifact.Source, + "--revision", artifact.Revision, + "--insecure-registry", + } + + log.Infof("pushing %s from %s", artifactURL, manifestPath) + + output, err := fluxOutput(ctx, args...) + if err != nil { + return fmt.Errorf("push artifact: %w\n%s", err, strings.TrimSpace(string(output))) + } + + if trimmed := strings.TrimSpace(string(output)); trimmed != "" { + log.Info(trimmed) + } + + requestedAt := time.Now().UTC().Format(time.RFC3339Nano) + log.Infof("triggering Flux sync for %s/%s", fluxNamespace, artifact.Kustomization) + + output, err = conn.RunCommandContext( + ctx, + fmt.Sprintf( + "kubectl annotate --overwrite -n %s ocirepository.source.toolkit.fluxcd.io/%s reconcile.fluxcd.io/requestedAt=%q && kubectl annotate --overwrite -n %s kustomization.kustomize.toolkit.fluxcd.io/%s reconcile.fluxcd.io/requestedAt=%q", + fluxNamespace, artifact.Source, + requestedAt, + fluxNamespace, artifact.Kustomization, + requestedAt, + ), + ) + if err != nil { + return fmt.Errorf("trigger flux sync: %w", err) + } + + if trimmed := strings.TrimSpace(string(output)); trimmed != "" { + log.Info(trimmed) + } + + return nil +} + +func fluxOutput(ctx context.Context, args ...string) ([]byte, error) { + fluxCmd := exec.CommandContext(ctx, "flux", args...) + return fluxCmd.CombinedOutput() +} diff --git a/toolbox/cmd/gitops.go b/toolbox/cmd/gitops.go index 1dfbdc3..14ce484 100644 --- a/toolbox/cmd/gitops.go +++ b/toolbox/cmd/gitops.go @@ -1,28 +1,18 @@ package cmd import ( - "context" "fmt" "os/exec" "path/filepath" - "strings" - "time" - "github.com/charmbracelet/log" "github.com/spf13/cobra" - - "github.com/khuedoan/cloudlab/toolbox/internal/cluster" ) const ( - registryNamespace = "registry" - registryService = "svc/registry" - registryPort = 5000 - gitopsRepository = "platform" - gitopsTag = "latest" - fluxNamespace = "flux-system" - fluxSource = "platform" - fluxKustomization = "platform" + gitopsRepository = "platform" + gitopsTag = "latest" + gitopsSource = "platform" + gitopsBundle = "platform" ) var ( @@ -55,81 +45,11 @@ func runGitopsPush(cmd *cobra.Command, args []string) error { return fmt.Errorf("resolve path %q: %w", gitopsPath, err) } - connectCtx, cancel := context.WithTimeout(cmd.Context(), connectTimeout) - defer cancel() - - hostAddr, err := cluster.LoadHost(hostsFile, host) - if err != nil { - return fmt.Errorf("load host: %w", err) - } - - conn, err := cluster.Connect(cluster.SSHConfig{ - Host: hostAddr, - User: sshUser, - KeyPath: sshKey, - KnownHostsPath: sshKnownHosts, - Timeout: connectTimeout, - }) - if err != nil { - return fmt.Errorf("connect to cluster: %w", err) - } - defer conn.Close() - - tunnel, err := conn.Forward(connectCtx, cluster.ServiceConfig{ - Namespace: registryNamespace, - Name: registryService, - Port: registryPort, + return pushArtifact(cmd.Context(), manifestPath, artifactRef{ + Repository: gitopsRepository, + Tag: gitopsTag, + Source: gitopsSource, + Revision: gitopsTag, + Kustomization: gitopsBundle, }) - if err != nil { - return fmt.Errorf("forward registry: %w", err) - } - - artifactURL := fmt.Sprintf("oci://%s/%s:%s", tunnel.LocalAddr, gitopsRepository, gitopsTag) - args = []string{ - "push", - "artifact", - artifactURL, - "--path", manifestPath, - // TODO should be actual source and revision - "--source", "platform", - "--revision", "latest", - "--insecure-registry", - } - - log.Infof("pushing %s from %s", artifactURL, manifestPath) - - output, err := fluxOutput(cmd.Context(), args...) - if err != nil { - return fmt.Errorf("push artifact: %w\n%s", err, strings.TrimSpace(string(output))) - } - - if trimmed := strings.TrimSpace(string(output)); trimmed != "" { - log.Info(trimmed) - } - - requestedAt := time.Now().UTC().Format(time.RFC3339Nano) - log.Infof("triggering Flux sync for %s/%s", fluxNamespace, fluxKustomization) - - output, err = conn.RunCommandContext( - cmd.Context(), - fmt.Sprintf( - "kubectl annotate --overwrite -n %s ocirepository.source.toolkit.fluxcd.io/%s reconcile.fluxcd.io/requestedAt=%q && kubectl annotate --overwrite -n %s kustomization.kustomize.toolkit.fluxcd.io/%s reconcile.fluxcd.io/requestedAt=%q", - fluxNamespace, fluxSource, requestedAt, - fluxNamespace, fluxKustomization, requestedAt, - ), - ) - if err != nil { - return fmt.Errorf("trigger flux sync: %w", err) - } - - if trimmed := strings.TrimSpace(string(output)); trimmed != "" { - log.Info(trimmed) - } - - return nil -} - -func fluxOutput(ctx context.Context, args ...string) ([]byte, error) { - fluxCmd := exec.CommandContext(ctx, "flux", args...) - return fluxCmd.CombinedOutput() } diff --git a/toolbox/cmd/root.go b/toolbox/cmd/root.go index bbbee18..020e63c 100644 --- a/toolbox/cmd/root.go +++ b/toolbox/cmd/root.go @@ -27,6 +27,7 @@ func init() { rootCmd.PersistentFlags().StringVar(&sshKnownHosts, "ssh-known-hosts", defaultKnownHostsFile(), "Path to SSH known_hosts file") rootCmd.AddCommand(gitopsCmd) + rootCmd.AddCommand(appsCmd) rootCmd.AddCommand(secretsCmd) rootCmd.AddCommand(vendorCmd) } diff --git a/toolbox/internal/apps/bundle.go b/toolbox/internal/apps/bundle.go new file mode 100644 index 0000000..7dd2517 --- /dev/null +++ b/toolbox/internal/apps/bundle.go @@ -0,0 +1,187 @@ +package apps + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "gopkg.in/yaml.v3" +) + +const ( + chartName = "app-template" + chartVersion = "4.6.0" + fluxNamespace = "flux-system" + helmRepositoryKind = "HelmRepository" + helmRepositoryName = "app-template" + helmReleaseKind = "HelmRelease" + helmReleaseVersion = "helm.toolkit.fluxcd.io/v2" + kustomizationKind = "Kustomization" + kustomizationFile = "kustomization.yaml" + kustomizationVersion = "kustomize.config.k8s.io/v1beta1" + reconcileInterval = "3m" +) + +type Release struct { + APIVersion string `yaml:"apiVersion"` + Kind string `yaml:"kind"` + Metadata Metadata `yaml:"metadata"` + Spec ReleaseSpec `yaml:"spec"` +} + +type Metadata struct { + Name string `yaml:"name"` + Namespace string `yaml:"namespace,omitempty"` +} + +type ReleaseSpec struct { + Interval string `yaml:"interval"` + ReleaseName string `yaml:"releaseName"` + TargetNamespace string `yaml:"targetNamespace"` + Install InstallSpec `yaml:"install"` + Chart Chart `yaml:"chart"` + Values map[string]any `yaml:"values,omitempty"` +} + +type InstallSpec struct { + CreateNamespace bool `yaml:"createNamespace"` +} + +type Chart struct { + Spec ChartSpec `yaml:"spec"` +} + +type ChartSpec struct { + Chart string `yaml:"chart"` + Version string `yaml:"version"` + SourceRef SourceRef `yaml:"sourceRef"` +} + +type SourceRef struct { + Kind string `yaml:"kind"` + Name string `yaml:"name"` +} + +type kustomization struct { + APIVersion string `yaml:"apiVersion"` + Kind string `yaml:"kind"` + Resources []string `yaml:"resources"` +} + +func Discover(rootDir, env string) ([]Release, error) { + pattern := filepath.Join(rootDir, "*", "*", env+".yaml") + files, err := filepath.Glob(pattern) + if err != nil { + return nil, fmt.Errorf("glob app values: %w", err) + } + + releases := make([]Release, 0, len(files)) + for _, path := range files { + release, err := loadRelease(rootDir, path) + if err != nil { + return nil, err + } + releases = append(releases, release) + } + + return releases, nil +} + +func WriteBundle(outputDir string, releases []Release) error { + if err := os.MkdirAll(outputDir, 0o755); err != nil { + return fmt.Errorf("create bundle dir: %w", err) + } + + resources := make([]string, 0, len(releases)) + for _, release := range releases { + filename := release.Metadata.Name + ".yaml" + path := filepath.Join(outputDir, filename) + + data, err := yaml.Marshal(release) + if err != nil { + return fmt.Errorf("marshal %s: %w", release.Metadata.Name, err) + } + + if err := os.WriteFile(path, data, 0o644); err != nil { + return fmt.Errorf("write %s: %w", path, err) + } + + resources = append(resources, filename) + } + + data, err := yaml.Marshal(kustomization{ + APIVersion: kustomizationVersion, + Kind: kustomizationKind, + Resources: resources, + }) + if err != nil { + return fmt.Errorf("marshal kustomization: %w", err) + } + + path := filepath.Join(outputDir, kustomizationFile) + if err := os.WriteFile(path, data, 0o644); err != nil { + return fmt.Errorf("write %s: %w", path, err) + } + + return nil +} + +func loadRelease(rootDir, path string) (Release, error) { + relPath, err := filepath.Rel(rootDir, path) + if err != nil { + return Release{}, fmt.Errorf("resolve relative path for %s: %w", path, err) + } + + parts := splitPath(relPath) + if len(parts) != 3 { + return Release{}, fmt.Errorf("expected apps/$NAMESPACE/$APP/$ENV.yaml, got %s", relPath) + } + + valuesData, err := os.ReadFile(path) + if err != nil { + return Release{}, fmt.Errorf("read %s: %w", path, err) + } + + values := map[string]any{} + if len(valuesData) > 0 { + if err := yaml.Unmarshal(valuesData, &values); err != nil { + return Release{}, fmt.Errorf("decode %s: %w", path, err) + } + } + + namespace := parts[0] + app := parts[1] + + return Release{ + APIVersion: helmReleaseVersion, + Kind: helmReleaseKind, + Metadata: Metadata{ + Name: namespace + "-" + app, + Namespace: fluxNamespace, + }, + Spec: ReleaseSpec{ + Interval: reconcileInterval, + ReleaseName: app, + TargetNamespace: namespace, + Install: InstallSpec{ + CreateNamespace: true, + }, + Chart: Chart{ + Spec: ChartSpec{ + Chart: chartName, + Version: chartVersion, + SourceRef: SourceRef{ + Kind: helmRepositoryKind, + Name: helmRepositoryName, + }, + }, + }, + Values: values, + }, + }, nil +} + +func splitPath(path string) []string { + return strings.Split(filepath.ToSlash(path), "/") +} diff --git a/toolbox/internal/apps/bundle_test.go b/toolbox/internal/apps/bundle_test.go new file mode 100644 index 0000000..acf12a6 --- /dev/null +++ b/toolbox/internal/apps/bundle_test.go @@ -0,0 +1,156 @@ +package apps + +import ( + "os" + "path/filepath" + "slices" + "testing" + + "gopkg.in/yaml.v3" +) + +func TestDiscoverFiltersByEnvironment(t *testing.T) { + rootDir := t.TempDir() + + writeFile(t, filepath.Join(rootDir, "khuedoan", "blog", "staging.yaml"), "controllers:\n main:\n replicas: 2\n") + writeFile(t, filepath.Join(rootDir, "finance", "actualbudget", "staging.yaml"), "service:\n main:\n controller: main\n") + writeFile(t, filepath.Join(rootDir, "test", "example", "production.yaml"), "ignored: true\n") + + releases, err := Discover(rootDir, "staging") + if err != nil { + t.Fatalf("discover staging apps: %v", err) + } + + if len(releases) != 2 { + t.Fatalf("expected 2 releases, got %d", len(releases)) + } + + gotNames := []string{releases[0].Metadata.Name, releases[1].Metadata.Name} + wantNames := []string{"finance-actualbudget", "khuedoan-blog"} + if !slices.Equal(gotNames, wantNames) { + t.Fatalf("expected release names %v, got %v", wantNames, gotNames) + } + + if releases[0].Spec.TargetNamespace != "finance" { + t.Fatalf("expected finance namespace, got %q", releases[0].Spec.TargetNamespace) + } + + controllers, ok := releases[1].Spec.Values["controllers"].(map[string]any) + if !ok { + t.Fatalf("expected controllers map, got %T", releases[1].Spec.Values["controllers"]) + } + + main, ok := controllers["main"].(map[string]any) + if !ok { + t.Fatalf("expected main controller map, got %T", controllers["main"]) + } + + if main["replicas"] != 2 { + t.Fatalf("expected replicas 2, got %#v", main["replicas"]) + } +} + +func TestWriteBundleCreatesKustomizationAndResources(t *testing.T) { + outputDir := t.TempDir() + releases := []Release{ + { + APIVersion: helmReleaseVersion, + Kind: helmReleaseKind, + Metadata: Metadata{ + Name: "khuedoan-blog", + Namespace: fluxNamespace, + }, + Spec: ReleaseSpec{ + Interval: reconcileInterval, + ReleaseName: "blog", + TargetNamespace: "khuedoan", + Install: InstallSpec{ + CreateNamespace: true, + }, + Chart: Chart{ + Spec: ChartSpec{ + Chart: chartName, + Version: chartVersion, + SourceRef: SourceRef{ + Kind: helmRepositoryKind, + Name: helmRepositoryName, + }, + }, + }, + Values: map[string]any{ + "service": map[string]any{ + "main": map[string]any{ + "controller": "main", + }, + }, + }, + }, + }, + } + + if err := WriteBundle(outputDir, releases); err != nil { + t.Fatalf("write bundle: %v", err) + } + + kustomizationData, err := os.ReadFile(filepath.Join(outputDir, kustomizationFile)) + if err != nil { + t.Fatalf("read kustomization: %v", err) + } + + var manifest kustomization + if err := yaml.Unmarshal(kustomizationData, &manifest); err != nil { + t.Fatalf("decode kustomization: %v", err) + } + + if !slices.Equal(manifest.Resources, []string{"khuedoan-blog.yaml"}) { + t.Fatalf("expected resources [khuedoan-blog.yaml], got %v", manifest.Resources) + } + + resourceData, err := os.ReadFile(filepath.Join(outputDir, "khuedoan-blog.yaml")) + if err != nil { + t.Fatalf("read release manifest: %v", err) + } + + var release Release + if err := yaml.Unmarshal(resourceData, &release); err != nil { + t.Fatalf("decode release manifest: %v", err) + } + + if release.Spec.ReleaseName != "blog" { + t.Fatalf("expected release name blog, got %q", release.Spec.ReleaseName) + } +} + +func TestWriteBundleSupportsNoApps(t *testing.T) { + outputDir := t.TempDir() + + if err := WriteBundle(outputDir, nil); err != nil { + t.Fatalf("write empty bundle: %v", err) + } + + kustomizationData, err := os.ReadFile(filepath.Join(outputDir, kustomizationFile)) + if err != nil { + t.Fatalf("read kustomization: %v", err) + } + + var manifest kustomization + if err := yaml.Unmarshal(kustomizationData, &manifest); err != nil { + t.Fatalf("decode kustomization: %v", err) + } + + if len(manifest.Resources) != 0 { + t.Fatalf("expected no resources, got %v", manifest.Resources) + } +} + +func writeFile(t *testing.T, path, content string) { + t.Helper() + + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("create dir for %s: %v", path, err) + } + + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } +}