diff --git a/toolbox/cmd/root.go b/toolbox/cmd/root.go index 1649101..bbbee18 100644 --- a/toolbox/cmd/root.go +++ b/toolbox/cmd/root.go @@ -28,6 +28,7 @@ func init() { rootCmd.AddCommand(gitopsCmd) rootCmd.AddCommand(secretsCmd) + rootCmd.AddCommand(vendorCmd) } var rootCmd = &cobra.Command{ diff --git a/toolbox/cmd/vendor.go b/toolbox/cmd/vendor.go new file mode 100644 index 0000000..bfd2b96 --- /dev/null +++ b/toolbox/cmd/vendor.go @@ -0,0 +1,82 @@ +package cmd + +import ( + "context" + "fmt" + "os" + "os/exec" + + "github.com/spf13/cobra" + + "github.com/khuedoan/cloudlab/toolbox/internal/cluster" + "github.com/khuedoan/cloudlab/toolbox/internal/vendors" +) + +var vendorCmd = &cobra.Command{ + Use: "vendor", + Args: cobra.NoArgs, + Short: "Vendor charts and images from settings.yaml into the in-cluster registry", + PreRunE: func(_ *cobra.Command, _ []string) error { + if err := validateClusterFlags(); err != nil { + return err + } + if settingsFile == "" { + return fmt.Errorf("--settings is required") + } + for _, name := range []string{"helm", "oras"} { + if _, err := exec.LookPath(name); err != nil { + return fmt.Errorf("find %s CLI: %w", name, err) + } + } + return nil + }, + RunE: runSync, +} + +func init() { + vendorCmd.Flags().StringVar(&settingsFile, "settings", "", "Path to settings YAML file") +} + +func runSync(cmd *cobra.Command, _ []string) error { + entries, err := vendors.LoadVendors(settingsFile) + if err != nil { + return err + } + + connectCtx, cancel := context.WithTimeout(cmd.Context(), connectTimeout) + defer cancel() + + hostAddr, err := cluster.LoadHost(hostsFile, host) + if err != nil { + return fmt.Errorf("load host: %w", err) + } + + conn, err := cluster.Connect(cluster.SSHConfig{ + Host: hostAddr, + User: sshUser, + KeyPath: sshKey, + KnownHostsPath: sshKnownHosts, + Timeout: connectTimeout, + }) + if err != nil { + return fmt.Errorf("connect to cluster: %w", err) + } + defer conn.Close() + + tunnel, err := conn.Forward(connectCtx, cluster.ServiceConfig{ + Namespace: registryNamespace, + Name: registryService, + Port: registryPort, + }) + if err != nil { + return fmt.Errorf("forward registry: %w", err) + } + + workdir, err := os.MkdirTemp("", "toolbox-vendor-*") + if err != nil { + return fmt.Errorf("create temp dir: %w", err) + } + defer os.RemoveAll(workdir) + + return vendors.Sync(cmd.Context(), workdir, tunnel.LocalAddr, entries) +} diff --git a/toolbox/internal/vendors/config.go b/toolbox/internal/vendors/config.go new file mode 100644 index 0000000..42bac59 --- /dev/null +++ b/toolbox/internal/vendors/config.go @@ -0,0 +1,113 @@ +package vendors + +import ( + "fmt" + "os" + "sort" + "strings" + + "gopkg.in/yaml.v3" +) + +type Config struct { + Items map[string]Vendor `yaml:"vendors"` +} + +type Vendor struct { + Kind string `yaml:"kind"` + RepoURL string `yaml:"repo_url,omitempty"` + Ref string `yaml:"ref,omitempty"` + Chart string `yaml:"chart,omitempty"` + Versions []string `yaml:"versions"` + Source string `yaml:"source,omitempty"` +} + +type VendorEntry struct { + Name string + Vendor +} + +func LoadConfig(configPath string) (*Config, error) { + data, err := os.ReadFile(configPath) + if err != nil { + return nil, fmt.Errorf("read file: %w", err) + } + + var config Config + if err := yaml.Unmarshal(data, &config); err != nil { + return nil, fmt.Errorf("parse YAML: %w", err) + } + + return &config, nil +} + +func ParseAndValidate(config *Config) ([]VendorEntry, error) { + names := make([]string, 0, len(config.Items)) + for name := range config.Items { + names = append(names, name) + } + sort.Strings(names) + + entries := make([]VendorEntry, 0, len(names)) + + for _, name := range names { + vendor := config.Items[name] + + if err := validateDestination(name); err != nil { + return nil, err + } + + vendor.Kind = strings.ToLower(vendor.Kind) + + if len(vendor.Versions) == 0 { + return nil, fmt.Errorf("vendors.%s: versions is required", name) + } + + for _, version := range vendor.Versions { + if version == "" { + return nil, fmt.Errorf("vendors.%s: versions cannot be empty", name) + } + } + + switch vendor.Kind { + case "chart": + if vendor.Ref != "" { + if vendor.RepoURL != "" || vendor.Chart != "" { + return nil, fmt.Errorf("vendors.%s: use either ref or repo_url/chart", name) + } + } else { + if vendor.RepoURL == "" || vendor.Chart == "" { + return nil, fmt.Errorf("vendors.%s: repo_url and chart are both required", name) + } + } + + case "image": + if vendor.Source == "" { + return nil, fmt.Errorf("vendors.%s: source is required", name) + } + + default: + if vendor.Kind == "" { + return nil, fmt.Errorf("vendors.%s: kind is required (chart|image)", name) + } + return nil, fmt.Errorf("vendors.%s: invalid kind %q", name, vendor.Kind) + } + + entries = append(entries, VendorEntry{Name: name, Vendor: vendor}) + } + + return entries, nil +} + +func validateDestination(destination string) error { + if destination == "" { + return fmt.Errorf("vendors: destination key is required") + } + if strings.Contains(destination, "://") { + return fmt.Errorf("vendors.%s: destination must be relative to the internal registry", destination) + } + if strings.HasPrefix(destination, "/") { + return fmt.Errorf("vendors.%s: destination must not start with /", destination) + } + return nil +} diff --git a/toolbox/internal/vendors/config_test.go b/toolbox/internal/vendors/config_test.go new file mode 100644 index 0000000..b96cc53 --- /dev/null +++ b/toolbox/internal/vendors/config_test.go @@ -0,0 +1,44 @@ +package vendors + +import ( + "strings" + "testing" +) + +func TestParseAndValidate(t *testing.T) { + cases := []struct { + name string + config *Config + wantErr string + }{ + {"missing chart versions", &Config{Items: map[string]Vendor{ + "vendor/charts/dex": {Kind: "chart", Chart: "dex", RepoURL: "https://charts.dexidp.io"}, + }}, "versions is required"}, + {"missing image kind", &Config{Items: map[string]Vendor{ + "vendor/charts/dex": {Versions: []string{"0.23.0"}, Chart: "dex", RepoURL: "https://charts.dexidp.io"}, + }}, "kind is required"}, + {"image with source and no versions", &Config{Items: map[string]Vendor{ + "vendor/images/dex": {Kind: "image", Source: "ghcr.io/dexidp/dex"}, + }}, "versions is required"}, + {"valid image versions", &Config{Items: map[string]Vendor{ + "vendor/images/dex": {Kind: "image", Source: "ghcr.io/dexidp/dex", Versions: []string{"v2.43.1"}}, + }}, ""}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := ParseAndValidate(tc.config) + if tc.wantErr == "" { + if err != nil { + t.Fatalf("unexpected validation error: %v", err) + } + return + } + if err == nil { + t.Fatal("expected validation error, got nil") + } else if !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("expected validation error %q, got %v", tc.wantErr, err) + } + }) + } +} diff --git a/toolbox/internal/vendors/sync.go b/toolbox/internal/vendors/sync.go new file mode 100644 index 0000000..ae73be5 --- /dev/null +++ b/toolbox/internal/vendors/sync.go @@ -0,0 +1,109 @@ +package vendors + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/charmbracelet/log" +) + +func LoadVendors(configPath string) ([]VendorEntry, error) { + config, err := LoadConfig(configPath) + if err != nil { + return nil, fmt.Errorf("load settings file: %w", err) + } + + entries, err := ParseAndValidate(config) + if err != nil { + return nil, fmt.Errorf("validate settings: %w", err) + } + + return entries, nil +} + +func Sync(ctx context.Context, workdir, registryAddr string, entries []VendorEntry) error { + for _, item := range entries { + switch item.Kind { + case "chart": + if err := syncChart(ctx, workdir, registryAddr, item); err != nil { + return err + } + case "image": + if err := syncImage(ctx, registryAddr, item); err != nil { + return err + } + } + } + return nil +} + +func syncChart(ctx context.Context, workdir, registryAddr string, chart VendorEntry) error { + chartDir := filepath.Join(workdir, chart.Name) + if err := os.MkdirAll(chartDir, 0o755); err != nil { + return fmt.Errorf("create chart temp dir: %w", err) + } + + pullRef := chart.Chart + if chart.Ref != "" { + pullRef = chart.Ref + } + for _, version := range chart.Versions { + log.Infof("vendoring chart %s@%s", chart.Name, version) + + pullArgs := []string{"pull", pullRef, "--version", version, "--destination", chartDir} + if chart.RepoURL != "" { + pullArgs = append(pullArgs, "--repo", chart.RepoURL) + } + if err := runCommand(ctx, "helm", pullArgs...); err != nil { + return fmt.Errorf("pull chart %s@%s: %w", chart.Name, version, err) + } + + archivePath := filepath.Join(chartDir, filepath.Base(pullRef)+"-"+version+".tgz") + pushTarget := fmt.Sprintf("oci://%s/%s", registryAddr, chart.Name) + if err := runCommand(ctx, "helm", "push", archivePath, pushTarget, "--plain-http"); err != nil { + return fmt.Errorf("push chart %s@%s: %w", chart.Name, version, err) + } + } + + return nil +} + +func syncImage(ctx context.Context, registryAddr string, image VendorEntry) error { + for _, version := range image.Versions { + log.Infof("vendoring image %s:%s", image.Name, version) + + source := image.Source + target := image.Name + if strings.HasPrefix(version, "@") { + source += version + target += version + } else { + source += ":" + version + target += ":" + version + } + destination := fmt.Sprintf("%s/%s", registryAddr, target) + copyArgs := []string{"cp", source, destination, "--to-plain-http"} + + if err := runCommand(ctx, "oras", copyArgs...); err != nil { + return fmt.Errorf("copy image %s@%s: %w", image.Name, version, err) + } + } + + return nil +} + +func runCommand(ctx context.Context, name string, args ...string) error { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + if err := cmd.Run(); err != nil { + return fmt.Errorf("%s %v: %w", name, args, err) + } + + return nil +}