diff --git a/docs/content/en-us/guide/configuration/controller.md b/docs/content/en-us/guide/configuration/controller.md index b66e17d..2ce7c26 100644 --- a/docs/content/en-us/guide/configuration/controller.md +++ b/docs/content/en-us/guide/configuration/controller.md @@ -77,6 +77,20 @@ controller: auto_apply: false backup_before_update: false digest_pin: true + forge_auth: + github: + - url: https://github.com + token_file: /run/secrets/github-token + - url: https://github.example.com + api_url: https://github.example.com/api/v3 + token_file: /run/secrets/github-enterprise-token + gitlab: + - url: https://gitlab.com + token_file: /run/secrets/gitlab-token + forgejo: + - url: https://git.example.com + api_url: https://git.example.com/api/v1 + token_file: /run/secrets/forgejo-token semver: default_allow: - patch @@ -89,6 +103,10 @@ controller: | `auto_apply` | No | Controller-level default for automatically applying detected image updates, default `false` | | `backup_before_update` | No | Controller-level default for backing up before image updates, default `false` | | `digest_pin` | No | Controller-level default for pinning tags to `tag@sha256:digest`, default `true` | +| `forge_auth.` | No | Optional single auth object or list of auth objects; public releases can be queried without a token | +| `forge_auth..url` | No | Web URL used to match `discovery.repo_url`, for example `https://github.com` | +| `forge_auth..token` / `token_file` | No | Token used by controller-side release discovery | +| `forge_auth..api_url` | No | API base URL for self-hosted instances; GitHub and GitLab default to public APIs, Forgejo can infer Codeberg from `repo_url` | | `semver.default_allow` | No | Default allowed semver bump types (`patch`, `minor`, `major`); defaults to `patch, minor` when unset | Per-image and per-service overrides take precedence over controller-level defaults. See [Service Definition](../service-definition) for the `update` section. diff --git a/docs/content/en-us/guide/service-definition.md b/docs/content/en-us/guide/service-definition.md index bb33c39..f036301 100644 --- a/docs/content/en-us/guide/service-definition.md +++ b/docs/content/en-us/guide/service-definition.md @@ -240,6 +240,9 @@ update: digest_pin: true discovery_sources: app_release: + auto: true + repo_url: https://github.com/example/api + app_registry: sources: - type: probe - type: registry @@ -305,7 +308,9 @@ Each entry under `images` is keyed by a logical name and supports: | `digest` | Compare the remote digest of the current tag with the local digest; no `filter` is used. | | `probe` | Probe registry manifests for generated semver candidates. | | `registry` | List image tags from the OCI/Docker registry with pagination. | -| `github` / `gitlab` / `forgejo` | Reserved for forge release API discovery. | +| `github` / `gitlab` / `forgejo` | Discover release tags via forge release APIs. Forge API calls run on the controller and are injected into image check tasks as candidates. | + +With `auto: true`, `repo_url` can be used to enable forge release discovery. `github.com` maps to GitHub, `gitlab.com` maps to GitLab, and `codeberg.org` maps to Forgejo. **Filter** types: diff --git a/docs/content/zh-hans/guide/configuration/controller.md b/docs/content/zh-hans/guide/configuration/controller.md index a72be8c..275aac7 100644 --- a/docs/content/zh-hans/guide/configuration/controller.md +++ b/docs/content/zh-hans/guide/configuration/controller.md @@ -77,6 +77,20 @@ controller: auto_apply: false backup_before_update: false digest_pin: true + forge_auth: + github: + - url: https://github.com + token_file: /run/secrets/github-token + - url: https://github.example.com + api_url: https://github.example.com/api/v3 + token_file: /run/secrets/github-enterprise-token + gitlab: + - url: https://gitlab.com + token_file: /run/secrets/gitlab-token + forgejo: + - url: https://git.example.com + api_url: https://git.example.com/api/v1 + token_file: /run/secrets/forgejo-token semver: default_allow: - patch @@ -89,6 +103,10 @@ controller: | `auto_apply` | 否 | 控制器级默认值，是否自动应用检测到的镜像更新，默认 `false` | | `backup_before_update` | 否 | 控制器级默认值，是否在镜像更新前执行备份，默认 `false` | | `digest_pin` | 否 | 控制器级默认值，是否将标签固定到 `tag@sha256:digest`，默认 `true` | +| `forge_auth.` | 否 | 可选的单个 auth 对象或 auth 对象列表；公开 release 可以匿名查询 | +| `forge_auth..url` | 否 | 用于匹配 `discovery.repo_url` 的网页 URL，例如 `https://github.com` | +| `forge_auth..token` / `token_file` | 否 | controller 侧 release discovery 使用的 token | +| `forge_auth..api_url` | 否 | 自托管实例的 API base URL；GitHub/GitLab 默认使用公开 API，Codeberg 可从 `repo_url` 自动推导 | | `semver.default_allow` | 否 | 默认允许的 semver 升级类型（`patch`、`minor`、`major`）；未设置时默认允许 `patch, minor` | 各镜像和各服务的覆盖值优先于控制器级默认值。请参见[服务定义](../service-definition)中的 `update` 段落。 diff --git a/docs/content/zh-hans/guide/service-definition.md b/docs/content/zh-hans/guide/service-definition.md index 2dbbbcd..bbb1284 100644 --- a/docs/content/zh-hans/guide/service-definition.md +++ b/docs/content/zh-hans/guide/service-definition.md @@ -240,6 +240,9 @@ update: digest_pin: true discovery_sources: app_release: + auto: true + repo_url: https://github.com/example/api + app_registry: sources: - type: probe - type: registry @@ -305,7 +308,9 @@ update: | `digest` | 比较当前 tag 的远程 digest 与本地 digest；不使用 `filter`。 | | `probe` | 对生成的 semver 候选执行 registry manifest 探测。 | | `registry` | 从 OCI/Docker registry 分页拉取镜像 tag。 | -| `github` / `gitlab` / `forgejo` | 预留给 forge release API 发现。 | +| `github` / `gitlab` / `forgejo` | 通过 forge release API 发现 release tag。Forge API 调用在 controller 上执行，并作为候选 tag 注入镜像检查任务。 | + +使用 `auto: true` 时可以通过 `repo_url` 启用 forge release discovery。`github.com` 会映射到 GitHub，`gitlab.com` 会映射到 GitLab，`codeberg.org` 会映射到 Forgejo。 **filter** 类型： diff --git a/internal/app/agent/discovery.go b/internal/app/agent/discovery.go index e8d74f5..117a95d 100644 --- a/internal/app/agent/discovery.go +++ b/internal/app/agent/discovery.go @@ -3,22 +3,24 @@ package agent import ( "context" "fmt" + "strconv" "strings" "forgejo.alexma.top/alexma233/composia/internal/core/repo" ) func resolveImageUpdateDiscovery(discovery repo.ImageUpdateDiscovery, sources map[string]repo.ImageUpdateDiscovery) repo.ImageUpdateDiscovery { - if discovery.Ref == "" { - return discovery - } - if source, ok := sources[discovery.Ref]; ok { - return source - } - return discovery + return repo.ResolveImageUpdateDiscovery(discovery, sources) } -func discoverImageUpdateTags(ctx context.Context, imageRef, currentTag string, discovery repo.ImageUpdateDiscovery, filter *repo.ImageUpdateFilter) ([]string, error) { +func discoverImageUpdateTags(ctx context.Context, imageRef, currentTag string, discovery repo.ImageUpdateDiscovery, filter *repo.ImageUpdateFilter, injectedCandidates []string, injectedSourceCandidates map[string][]string) ([]string, error) { + if len(injectedCandidates) > 0 { + base, err := discoverImageUpdateTags(ctx, imageRef, currentTag, discovery, filter, nil, injectedSourceCandidates) + if err != nil { + return nil, err + } + return mergeImageUpdateTags(injectedCandidates, base), nil + } if discovery.Auto != nil && *discovery.Auto { return discoverMergedImageUpdateTags(ctx, imageRef, currentTag, []repo.ImageUpdateDiscoverySource{{Type: "probe"}, {Type: "registry"}}, filter) } @@ -26,7 +28,13 @@ func discoverImageUpdateTags(ctx context.Context, imageRef, currentTag string, d if discovery.Combine == "merge" { return discoverMergedImageUpdateTags(ctx, imageRef, currentTag, discovery.Sources, filter) } - for _, source := range discovery.Sources { + for index, source := range discovery.Sources { + if isAgentExternalDiscoverySource(source.Type) { + if tags := mergeImageUpdateTags(injectedSourceCandidates[strconv.Itoa(index)]); len(tags) > 0 { + return tags, nil + } + continue + } tags, err := discoverImageUpdateTagsFromSource(ctx, imageRef, currentTag, source, filter) if err != nil { return nil, err @@ -38,20 +46,32 @@ func discoverImageUpdateTags(ctx context.Context, imageRef, currentTag string, d return nil, nil } if discovery.Type != "" { - return discoverImageUpdateTagsFromSource(ctx, imageRef, currentTag, repo.ImageUpdateDiscoverySource{Type: discovery.Type, Repo: discovery.Repo, Project: discovery.Project}, filter) + if isAgentExternalDiscoverySource(discovery.Type) { + return mergeImageUpdateTags(injectedSourceCandidates["direct"]), nil + } + return discoverImageUpdateTagsFromSource(ctx, imageRef, currentTag, repo.ImageUpdateDiscoverySource{Type: discovery.Type, Repo: discovery.Repo, RepoURL: discovery.RepoURL, Project: discovery.Project}, filter) } return discoverImageUpdateTagsFromSource(ctx, imageRef, currentTag, repo.ImageUpdateDiscoverySource{Type: "registry"}, filter) } -func discoverMergedImageUpdateTags(ctx context.Context, imageRef, currentTag string, sources []repo.ImageUpdateDiscoverySource, filter *repo.ImageUpdateFilter) ([]string, error) { +func isAgentExternalDiscoverySource(sourceType string) bool { + switch sourceType { + case "github", "gitlab", "forgejo": + return true + default: + return false + } +} + +func mergeImageUpdateTags(groups ...[]string) []string { seen := make(map[string]struct{}) merged := make([]string, 0) - for _, source := range sources { - tags, err := discoverImageUpdateTagsFromSource(ctx, imageRef, currentTag, source, filter) - if err != nil { - return nil, err - } - for _, tag := range tags { + for _, group := range groups { + for _, tag := range group { + tag = strings.TrimSpace(tag) + if tag == "" { + continue + } if _, ok := seen[tag]; ok { continue } @@ -59,7 +79,19 @@ func discoverMergedImageUpdateTags(ctx context.Context, imageRef, currentTag str merged = append(merged, tag) } } - return merged, nil + return merged +} + +func discoverMergedImageUpdateTags(ctx context.Context, imageRef, currentTag string, sources []repo.ImageUpdateDiscoverySource, filter *repo.ImageUpdateFilter) ([]string, error) { + groups := make([][]string, 0, len(sources)) + for _, source := range sources { + tags, err := discoverImageUpdateTagsFromSource(ctx, imageRef, currentTag, source, filter) + if err != nil { + return nil, err + } + groups = append(groups, tags) + } + return mergeImageUpdateTags(groups...), nil } func discoverImageUpdateTagsFromSource(ctx context.Context, imageRef, currentTag string, source repo.ImageUpdateDiscoverySource, filter *repo.ImageUpdateFilter) ([]string, error) { @@ -76,7 +108,7 @@ func discoverImageUpdateTagsFromSource(ctx context.Context, imageRef, currentTag case "registry", "": return listRegistryTags(ctx, imageRef) case "github", "gitlab", "forgejo": - return nil, fmt.Errorf("%s release discovery is not implemented yet", source.Type) + return nil, nil default: return nil, fmt.Errorf("unsupported image update discovery source %q", source.Type) } diff --git a/internal/app/agent/image_update.go b/internal/app/agent/image_update.go index 92ecda6..e087fb5 100644 --- a/internal/app/agent/image_update.go +++ b/internal/app/agent/image_update.go @@ -111,7 +111,7 @@ func reportServiceImageUpdateChecks(ctx context.Context, client agentv1connect.A if image.Filter != nil && image.Filter.Type == "semver" && len(image.Filter.Allow) == 0 && len(params.SemverAllow) > 0 { image.Filter.Allow = append([]string(nil), params.SemverAllow...) } - checks = append(checks, collectServiceImageUpdateCheck(ctx, serviceDir, imageName, image, serviceMeta.Update.DiscoverySources)) + checks = append(checks, collectServiceImageUpdateCheck(ctx, serviceDir, imageName, image, serviceMeta.Update.DiscoverySources, params.ForgeCandidates[imageName], params.ForgeCandidateSources[imageName])) } if len(checks) == 0 { return uploadTaskLog(ctx, logUploader, "no selected image update checks found\n") @@ -128,7 +128,7 @@ func reportServiceImageUpdateChecks(ctx context.Context, client agentv1connect.A return uploadTaskLog(ctx, logUploader, fmt.Sprintf("reported %d service image update check(s)\n", len(checks))) } -func collectServiceImageUpdateCheck(ctx context.Context, serviceDir, imageName string, image repo.ImageUpdateConfig, discoverySources map[string]repo.ImageUpdateDiscovery) *agentv1.ServiceImageUpdateCheck { +func collectServiceImageUpdateCheck(ctx context.Context, serviceDir, imageName string, image repo.ImageUpdateConfig, discoverySources map[string]repo.ImageUpdateDiscovery, injectedCandidates []string, injectedSourceCandidates map[string][]string) *agentv1.ServiceImageUpdateCheck { policyType := "digest" if image.Filter != nil { policyType = image.Filter.Type @@ -165,7 +165,7 @@ func collectServiceImageUpdateCheck(ctx context.Context, serviceDir, imageName s return check } - tags, err := discoverImageUpdateTags(ctx, image.Image, currentTag, discovery, image.Filter) + tags, err := discoverImageUpdateTags(ctx, image.Image, currentTag, discovery, image.Filter, injectedCandidates, injectedSourceCandidates) if err != nil { check.CheckStatus = store.ImageCheckStatusError check.ErrorSummary = err.Error() diff --git a/internal/app/agent/run.go b/internal/app/agent/run.go index 86303cc..188c64c 100644 --- a/internal/app/agent/run.go +++ b/internal/app/agent/run.go @@ -1300,11 +1300,13 @@ func decodeTaskParams(paramsJSON string) (controllerTaskParams, error) { } type controllerTaskParams struct { - ServiceDirs []string `json:"service_dirs,omitempty"` - ImageNames []string `json:"image_names,omitempty"` - SemverAllow []string `json:"semver_allow,omitempty"` - FullRebuild bool `json:"full_rebuild,omitempty"` - ComposeRecreateMode string `json:"compose_recreate_mode,omitempty"` + ServiceDirs []string `json:"service_dirs,omitempty"` + ImageNames []string `json:"image_names,omitempty"` + SemverAllow []string `json:"semver_allow,omitempty"` + ForgeCandidates map[string][]string `json:"forge_candidates,omitempty"` + ForgeCandidateSources map[string]map[string][]string `json:"forge_candidate_sources,omitempty"` + FullRebuild bool `json:"full_rebuild,omitempty"` + ComposeRecreateMode string `json:"compose_recreate_mode,omitempty"` } func syncServiceCaddyFile(ctx context.Context, cfg *config.AgentConfig, serviceDir, serviceRoot string, uploadLog func(string) error) error { diff --git a/internal/app/agent/run_test.go b/internal/app/agent/run_test.go index b5c118b..e9e811d 100644 --- a/internal/app/agent/run_test.go +++ b/internal/app/agent/run_test.go @@ -211,3 +211,18 @@ func TestNextRegistryPageURLResolvesRelativeLink(t *testing.T) { t.Fatalf("nextRegistryPageURL() = %q", got) } } + +func TestMergeImageUpdateTagsDeduplicatesInjectedCandidatesFirst(t *testing.T) { + t.Parallel() + + got := mergeImageUpdateTags([]string{"v1.2.0", "v1.3.0"}, []string{"v1.3.0", "v1.4.0"}) + want := []string{"v1.2.0", "v1.3.0", "v1.4.0"} + if len(got) != len(want) { + t.Fatalf("mergeImageUpdateTags() = %+v, want %+v", got, want) + } + for index := range want { + if got[index] != want[index] { + t.Fatalf("mergeImageUpdateTags() = %+v, want %+v", got, want) + } + } +} diff --git a/internal/app/controller/forge_releases.go b/internal/app/controller/forge_releases.go new file mode 100644 index 0000000..f38dac3 --- /dev/null +++ b/internal/app/controller/forge_releases.go @@ -0,0 +1,296 @@ +package controller + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "forgejo.alexma.top/alexma233/composia/internal/core/config" + "forgejo.alexma.top/alexma233/composia/internal/core/repo" +) + +func collectForgeImageCandidates(ctx context.Context, cfg *config.ControllerConfig, service repo.Service, imageNames []string) (map[string][]string, map[string]map[string][]string, error) { + if service.Meta.Update == nil || len(service.Meta.Update.Images) == 0 { + return nil, nil, nil + } + if len(imageNames) == 0 { + imageNames = make([]string, 0, len(service.Meta.Update.Images)) + for imageName := range service.Meta.Update.Images { + imageNames = append(imageNames, imageName) + } + } + mergedResult := make(map[string][]string) + sourceResult := make(map[string]map[string][]string) + for _, imageName := range imageNames { + image, ok := service.Meta.Update.Images[imageName] + if !ok { + continue + } + discovery := repo.ResolveImageUpdateDiscovery(image.Discovery, service.Meta.Update.DiscoverySources) + merged, bySource, err := collectForgeCandidatesForDiscovery(ctx, cfg, discovery) + if err != nil { + return nil, nil, fmt.Errorf("collect forge candidates for image %q: %w", imageName, err) + } + if len(merged) > 0 { + mergedResult[imageName] = merged + } + if len(bySource) > 0 { + sourceResult[imageName] = bySource + } + } + if len(mergedResult) == 0 { + mergedResult = nil + } + if len(sourceResult) == 0 { + sourceResult = nil + } + return mergedResult, sourceResult, nil +} + +func collectForgeCandidatesForDiscovery(ctx context.Context, cfg *config.ControllerConfig, discovery repo.ImageUpdateDiscovery) ([]string, map[string][]string, error) { + if discovery.Auto != nil && *discovery.Auto { + if strings.TrimSpace(discovery.RepoURL) == "" { + return nil, nil, nil + } + source, ok, err := forgeSourceFromRepoURL(discovery.RepoURL) + if err != nil || !ok { + return nil, nil, err + } + candidates, err := fetchForgeReleaseTags(ctx, cfg, source) + return candidates, nil, err + } + if len(discovery.Sources) > 0 { + if discovery.Combine == "merge" { + candidates, err := mergeForgeCandidateGroups(ctx, cfg, discovery.Sources) + return candidates, nil, err + } + bySource := make(map[string][]string) + for index, source := range discovery.Sources { + if !isForgeDiscoverySource(source.Type) { + continue + } + candidates, err := fetchForgeReleaseTags(ctx, cfg, source) + if err != nil { + return nil, nil, err + } + if len(candidates) > 0 { + bySource[strconv.Itoa(index)] = candidates + } + } + return nil, bySource, nil + } + if isForgeDiscoverySource(discovery.Type) { + candidates, err := fetchForgeReleaseTags(ctx, cfg, repo.ImageUpdateDiscoverySource{Type: discovery.Type, Repo: discovery.Repo, RepoURL: discovery.RepoURL, Project: discovery.Project}) + if len(candidates) == 0 || err != nil { + return nil, nil, err + } + return nil, map[string][]string{"direct": candidates}, nil + } + return nil, nil, nil +} + +func forgeSourceFromRepoURL(rawURL string) (repo.ImageUpdateDiscoverySource, bool, error) { + repoURL, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil { + return repo.ImageUpdateDiscoverySource{}, false, fmt.Errorf("parse repo_url %q: %w", rawURL, err) + } + host := strings.ToLower(repoURL.Hostname()) + parts := make([]string, 0) + for _, part := range strings.Split(strings.Trim(repoURL.Path, "/"), "/") { + if part != "" { + parts = append(parts, part) + } + } + if len(parts) < 2 { + return repo.ImageUpdateDiscoverySource{}, false, fmt.Errorf("repo_url %q must include owner/repo path", rawURL) + } + repoPath := strings.Join(parts[:2], "/") + fullProject := strings.Join(parts, "/") + switch host { + case "github.com": + return repo.ImageUpdateDiscoverySource{Type: "github", Repo: repoPath, RepoURL: rawURL}, true, nil + case "gitlab.com": + return repo.ImageUpdateDiscoverySource{Type: "gitlab", Project: fullProject, RepoURL: rawURL}, true, nil + case "codeberg.org": + return repo.ImageUpdateDiscoverySource{Type: "forgejo", Repo: repoPath, RepoURL: rawURL, APIURL: "https://codeberg.org/api/v1"}, true, nil + default: + return repo.ImageUpdateDiscoverySource{}, false, nil + } +} + +func mergeForgeCandidateGroups(ctx context.Context, cfg *config.ControllerConfig, sources []repo.ImageUpdateDiscoverySource) ([]string, error) { + seen := make(map[string]struct{}) + merged := make([]string, 0) + for _, source := range sources { + if !isForgeDiscoverySource(source.Type) { + continue + } + candidates, err := fetchForgeReleaseTags(ctx, cfg, source) + if err != nil { + return nil, err + } + for _, candidate := range candidates { + candidate = strings.TrimSpace(candidate) + if candidate == "" { + continue + } + if _, ok := seen[candidate]; ok { + continue + } + seen[candidate] = struct{}{} + merged = append(merged, candidate) + } + } + return merged, nil +} + +func isForgeDiscoverySource(sourceType string) bool { + switch sourceType { + case "github", "gitlab", "forgejo": + return true + default: + return false + } +} + +func fetchForgeReleaseTags(ctx context.Context, cfg *config.ControllerConfig, source repo.ImageUpdateDiscoverySource) ([]string, error) { + client := &http.Client{Timeout: 30 * time.Second} + endpoint, headers, err := forgeReleaseRequest(source, cfg) + if err != nil { + return nil, err + } + request, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, err + } + for key, value := range headers { + request.Header.Set(key, value) + } + response, err := client.Do(request) + if err != nil { + return nil, fmt.Errorf("request %s releases: %w", source.Type, err) + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode < 200 || response.StatusCode >= 300 { + return nil, fmt.Errorf("request %s releases returned %s", source.Type, response.Status) + } + var releases []struct { + TagName string `json:"tag_name"` + } + if err := json.NewDecoder(response.Body).Decode(&releases); err != nil { + return nil, fmt.Errorf("decode %s releases: %w", source.Type, err) + } + tags := make([]string, 0, len(releases)) + for _, release := range releases { + if tag := strings.TrimSpace(release.TagName); tag != "" { + tags = append(tags, tag) + } + } + return tags, nil +} + +func forgeReleaseRequest(source repo.ImageUpdateDiscoverySource, cfg *config.ControllerConfig) (string, map[string]string, error) { + headers := map[string]string{"Accept": "application/json"} + updates := (*config.ControllerUpdatesConfig)(nil) + if cfg != nil { + updates = cfg.Updates + } + switch source.Type { + case "github": + repoName := strings.TrimSpace(source.Repo) + if repoName == "" { + return "", nil, fmt.Errorf("github release discovery requires repo") + } + auth := forgeAuth(updates, "github", source.RepoURL) + apiURL := strings.TrimRight(auth.APIURL, "/") + if apiURL == "" { + apiURL = "https://api.github.com" + } + if auth.Token != "" { + headers["Authorization"] = "Bearer " + auth.Token + } + return apiURL + "/repos/" + strings.Trim(repoName, "/") + "/releases?per_page=100", headers, nil + case "gitlab": + project := strings.TrimSpace(source.Project) + if project == "" { + return "", nil, fmt.Errorf("gitlab release discovery requires project") + } + auth := forgeAuth(updates, "gitlab", source.RepoURL) + apiURL := strings.TrimRight(auth.APIURL, "/") + if apiURL == "" { + apiURL = "https://gitlab.com/api/v4" + } + if auth.Token != "" { + headers["PRIVATE-TOKEN"] = auth.Token + } + return apiURL + "/projects/" + url.PathEscape(project) + "/releases?per_page=100", headers, nil + case "forgejo": + repoName := strings.TrimSpace(source.Repo) + if repoName == "" { + return "", nil, fmt.Errorf("forgejo release discovery requires repo") + } + auth := forgeAuth(updates, "forgejo", source.RepoURL) + apiURL := strings.TrimRight(source.APIURL, "/") + if apiURL == "" { + apiURL = strings.TrimRight(auth.APIURL, "/") + } + if apiURL == "" { + return "", nil, fmt.Errorf("controller.updates.forge_auth.forgejo.api_url is required") + } + if auth.Token != "" { + headers["Authorization"] = "token " + auth.Token + } + return apiURL + "/repos/" + strings.Trim(repoName, "/") + "/releases?limit=100", headers, nil + default: + return "", nil, fmt.Errorf("unsupported forge release discovery %q", source.Type) + } +} + +func forgeAuth(updates *config.ControllerUpdatesConfig, sourceType, repoURL string) config.ForgeAuthConfig { + if updates == nil || updates.ForgeAuth == nil { + return config.ForgeAuthConfig{} + } + var auths config.ForgeAuthConfigs + switch sourceType { + case "github": + auths = updates.ForgeAuth.GitHub + case "gitlab": + auths = updates.ForgeAuth.GitLab + case "forgejo": + auths = updates.ForgeAuth.Forgejo + } + if len(auths) == 0 { + return config.ForgeAuthConfig{} + } + if repoURL != "" { + if repoHost := normalizedURLHost(repoURL); repoHost != "" { + for _, auth := range auths { + if auth.URL != "" && normalizedURLHost(auth.URL) == repoHost { + return auth + } + if auth.APIURL != "" && normalizedURLHost(auth.APIURL) == repoHost { + return auth + } + } + } + } + for _, auth := range auths { + if auth.URL == "" && auth.APIURL == "" { + return auth + } + } + return config.ForgeAuthConfig{} +} + +func normalizedURLHost(rawURL string) string { + parsed, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil { + return "" + } + return strings.ToLower(parsed.Hostname()) +} diff --git a/internal/app/controller/forge_releases_test.go b/internal/app/controller/forge_releases_test.go new file mode 100644 index 0000000..db74ee1 --- /dev/null +++ b/internal/app/controller/forge_releases_test.go @@ -0,0 +1,98 @@ +package controller + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "forgejo.alexma.top/alexma233/composia/internal/core/config" + "forgejo.alexma.top/alexma233/composia/internal/core/repo" +) + +func TestCollectForgeImageCandidatesUsesGitHubAutoRepoURL(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.URL.Path != "/repos/example/api/releases" { + t.Fatalf("unexpected path %q", request.URL.Path) + } + if request.URL.Query().Get("per_page") != "100" { + t.Fatalf("unexpected query %q", request.URL.RawQuery) + } + if request.Header.Get("Authorization") != "Bearer test-token" { + t.Fatalf("unexpected auth header %q", request.Header.Get("Authorization")) + } + _, _ = writer.Write([]byte(`[{"tag_name":"v1.2.3"},{"tag_name":"v1.3.0"}]`)) + })) + defer server.Close() + + auto := true + service := repo.Service{Meta: repo.ServiceMeta{Update: &repo.UpdateConfig{Images: map[string]repo.ImageUpdateConfig{ + "api": { + Image: "ghcr.io/example/api", + Current: repo.ImageUpdateCurrent{Env: &repo.ImageUpdateCurrentEnv{File: ".env", Key: "API_VERSION"}}, + Discovery: repo.ImageUpdateDiscovery{Auto: &auto, RepoURL: "https://github.com/example/api"}, + Filter: &repo.ImageUpdateFilter{Type: "semver"}, + }, + }}}} + cfg := &config.ControllerConfig{Updates: &config.ControllerUpdatesConfig{ForgeAuth: &config.ControllerUpdatesForgeAuth{GitHub: config.ForgeAuthConfigs{{URL: "https://github.com", APIURL: server.URL, Token: "test-token"}}}}} + + candidates, sourceCandidates, err := collectForgeImageCandidates(context.Background(), cfg, service, []string{"api"}) + if err != nil { + t.Fatalf("collectForgeImageCandidates() error = %v", err) + } + got := candidates["api"] + if len(got) != 2 || got[0] != "v1.2.3" || got[1] != "v1.3.0" { + t.Fatalf("unexpected candidates %+v", candidates) + } + if len(sourceCandidates) != 0 { + t.Fatalf("unexpected source candidates %+v", sourceCandidates) + } +} + +func TestCollectForgeImageCandidatesSkipsAutoWithoutRepoHint(t *testing.T) { + t.Parallel() + + auto := true + service := repo.Service{Meta: repo.ServiceMeta{Update: &repo.UpdateConfig{Images: map[string]repo.ImageUpdateConfig{ + "api": { + Image: "ghcr.io/example/api", + Current: repo.ImageUpdateCurrent{Env: &repo.ImageUpdateCurrentEnv{File: ".env", Key: "API_VERSION"}}, + Discovery: repo.ImageUpdateDiscovery{Auto: &auto}, + Filter: &repo.ImageUpdateFilter{Type: "semver"}, + }, + }}}} + + candidates, sourceCandidates, err := collectForgeImageCandidates(context.Background(), &config.ControllerConfig{}, service, []string{"api"}) + if err != nil { + t.Fatalf("collectForgeImageCandidates() error = %v", err) + } + if len(candidates) != 0 || len(sourceCandidates) != 0 { + t.Fatalf("expected no candidates, got %+v %+v", candidates, sourceCandidates) + } +} + +func TestForgeSourceFromRepoURLDetectsGitLab(t *testing.T) { + t.Parallel() + + source, ok, err := forgeSourceFromRepoURL("https://gitlab.com/group/subgroup/project") + if err != nil { + t.Fatalf("forgeSourceFromRepoURL() error = %v", err) + } + if !ok || source.Type != "gitlab" || source.Project != "group/subgroup/project" { + t.Fatalf("unexpected source %+v ok=%v", source, ok) + } +} + +func TestForgeSourceFromRepoURLDetectsCodebergAsForgejo(t *testing.T) { + t.Parallel() + + source, ok, err := forgeSourceFromRepoURL("https://codeberg.org/owner/repo") + if err != nil { + t.Fatalf("forgeSourceFromRepoURL() error = %v", err) + } + if !ok || source.Type != "forgejo" || source.Repo != "owner/repo" || source.APIURL != "https://codeberg.org/api/v1" { + t.Fatalf("unexpected source %+v ok=%v", source, ok) + } +} diff --git a/internal/app/controller/run.go b/internal/app/controller/run.go index 8fae36d..8144fa9 100644 --- a/internal/app/controller/run.go +++ b/internal/app/controller/run.go @@ -1362,16 +1362,18 @@ type serviceInstanceServer struct { } type serviceTaskParams struct { - ServiceDir string `json:"service_dir"` - ServiceDirs []string `json:"service_dirs,omitempty"` - DataNames []string `json:"data_names,omitempty"` - ImageNames []string `json:"image_names,omitempty"` - SemverAllow []string `json:"semver_allow,omitempty"` - FullRebuild bool `json:"full_rebuild,omitempty"` - SourceNodeID string `json:"source_node_id,omitempty"` - TargetNodeID string `json:"target_node_id,omitempty"` - RestoreItems []restoreTaskItem `json:"restore_items,omitempty"` - ComposeRecreateMode string `json:"compose_recreate_mode,omitempty"` + ServiceDir string `json:"service_dir"` + ServiceDirs []string `json:"service_dirs,omitempty"` + DataNames []string `json:"data_names,omitempty"` + ImageNames []string `json:"image_names,omitempty"` + SemverAllow []string `json:"semver_allow,omitempty"` + ForgeCandidates map[string][]string `json:"forge_candidates,omitempty"` + ForgeCandidateSources map[string]map[string][]string `json:"forge_candidate_sources,omitempty"` + FullRebuild bool `json:"full_rebuild,omitempty"` + SourceNodeID string `json:"source_node_id,omitempty"` + TargetNodeID string `json:"target_node_id,omitempty"` + RestoreItems []restoreTaskItem `json:"restore_items,omitempty"` + ComposeRecreateMode string `json:"compose_recreate_mode,omitempty"` } type restoreTaskItem struct { @@ -2020,12 +2022,14 @@ func (server *serviceCommandServer) RunServiceAction(ctx context.Context, req *c } type serviceTaskCreateOptions struct { - AttemptOfTaskID string - Source task.Source - CreatedAt *time.Time - ComposeRecreateMode string - ImageNames []string - SemverAllow []string + AttemptOfTaskID string + Source task.Source + CreatedAt *time.Time + ComposeRecreateMode string + ImageNames []string + SemverAllow []string + ForgeCandidates map[string][]string + ForgeCandidateSources map[string]map[string][]string } func composeRecreateModeParam(taskType task.Type, mode controllerv1.ComposeRecreateMode) string { @@ -2499,6 +2503,14 @@ func (server *serviceCommandServer) createServiceTaskWithOptions(ctx context.Con if len(targetNodeIDs) == 0 { return task.Record{}, connect.NewError(connect.CodeFailedPrecondition, fmt.Errorf("service %q does not have any target nodes", serviceName)) } + if taskType == task.TypeImageCheck && options.ForgeCandidates == nil { + forgeCandidates, forgeCandidateSources, err := collectForgeImageCandidates(ctx, server.cfg, service, options.ImageNames) + if err != nil { + return task.Record{}, connect.NewError(connect.CodeFailedPrecondition, err) + } + options.ForgeCandidates = forgeCandidates + options.ForgeCandidateSources = forgeCandidateSources + } for _, nodeID := range targetNodeIDs { if err := validateTaskTargetNode(ctx, server.db, server.cfg, nodeID, taskType); err != nil { return task.Record{}, err @@ -2514,7 +2526,7 @@ func (server *serviceCommandServer) createServiceTaskWithOptions(ctx context.Con if err != nil { return task.Record{}, connect.NewError(connect.CodeInternal, fmt.Errorf("resolve service directory: %w", err)) } - paramsJSON, err := json.Marshal(serviceTaskParams{ServiceDir: serviceDir, DataNames: dataNames, ImageNames: options.ImageNames, SemverAllow: options.SemverAllow, ComposeRecreateMode: options.ComposeRecreateMode}) + paramsJSON, err := json.Marshal(serviceTaskParams{ServiceDir: serviceDir, DataNames: dataNames, ImageNames: options.ImageNames, SemverAllow: options.SemverAllow, ForgeCandidates: options.ForgeCandidates, ForgeCandidateSources: options.ForgeCandidateSources, ComposeRecreateMode: options.ComposeRecreateMode}) if err != nil { return task.Record{}, connect.NewError(connect.CodeInternal, fmt.Errorf("encode task params: %w", err)) } diff --git a/internal/app/controller/scheduler.go b/internal/app/controller/scheduler.go index 0a637a4..fdb3346 100644 --- a/internal/app/controller/scheduler.go +++ b/internal/app/controller/scheduler.go @@ -84,7 +84,11 @@ func scheduleServiceImageChecks(ctx context.Context, db *store.DB, serviceServer return err } semverAllow := effectiveControllerSemverAllow(serviceServer.cfg) - paramsJSONBytes, err := json.Marshal(serviceTaskParams{ServiceDir: serviceDir, ImageNames: dueImageNames, SemverAllow: semverAllow}) + forgeCandidates, forgeCandidateSources, err := collectForgeImageCandidates(ctx, serviceServer.cfg, service, dueImageNames) + if err != nil { + return err + } + paramsJSONBytes, err := json.Marshal(serviceTaskParams{ServiceDir: serviceDir, ImageNames: dueImageNames, SemverAllow: semverAllow, ForgeCandidates: forgeCandidates, ForgeCandidateSources: forgeCandidateSources}) if err != nil { return err } @@ -98,7 +102,7 @@ func scheduleServiceImageChecks(ctx context.Context, db *store.DB, serviceServer if exists { continue } - if _, err := serviceServer.createServiceTaskWithOptions(ctx, service.Name, []string{nodeID}, task.TypeImageCheck, nil, serviceTaskCreateOptions{Source: task.SourceSchedule, CreatedAt: &createdAt, ImageNames: dueImageNames, SemverAllow: semverAllow}); err != nil { + if _, err := serviceServer.createServiceTaskWithOptions(ctx, service.Name, []string{nodeID}, task.TypeImageCheck, nil, serviceTaskCreateOptions{Source: task.SourceSchedule, CreatedAt: &createdAt, ImageNames: dueImageNames, SemverAllow: semverAllow, ForgeCandidates: forgeCandidates, ForgeCandidateSources: forgeCandidateSources}); err != nil { log.Printf("scheduler skipped image check for service=%s node=%s: %v", service.Name, nodeID, err) } } diff --git a/internal/core/config/config.go b/internal/core/config/config.go index 3a9afec..097d2c2 100644 --- a/internal/core/config/config.go +++ b/internal/core/config/config.go @@ -51,17 +51,37 @@ type ControllerUpdatesSemverConfig struct { } type ControllerUpdatesForgeAuth struct { - GitHub *ForgeAuthConfig `yaml:"github"` - GitLab *ForgeAuthConfig `yaml:"gitlab"` - Forgejo *ForgeAuthConfig `yaml:"forgejo"` + GitHub ForgeAuthConfigs `yaml:"github"` + GitLab ForgeAuthConfigs `yaml:"gitlab"` + Forgejo ForgeAuthConfigs `yaml:"forgejo"` } type ForgeAuthConfig struct { + URL string `yaml:"url"` Token string `yaml:"token"` TokenFile string `yaml:"token_file"` APIURL string `yaml:"api_url"` } +type ForgeAuthConfigs []ForgeAuthConfig + +func (configs *ForgeAuthConfigs) UnmarshalYAML(value *yaml.Node) error { + if value.Kind == yaml.SequenceNode { + var items []ForgeAuthConfig + if err := value.Decode(&items); err != nil { + return err + } + *configs = items + return nil + } + var item ForgeAuthConfig + if err := value.Decode(&item); err != nil { + return err + } + *configs = []ForgeAuthConfig{item} + return nil +} + type ControllerGitConfig struct { RemoteURL string `yaml:"remote_url"` Branch string `yaml:"branch"` diff --git a/internal/core/config/config_test.go b/internal/core/config/config_test.go index bdaf23c..fb32fd2 100644 --- a/internal/core/config/config_test.go +++ b/internal/core/config/config_test.go @@ -202,6 +202,46 @@ controller: } } +func TestLoadControllerParsesMultipleForgeAuthEntries(t *testing.T) { + t.Parallel() + + configPath := filepath.Join(t.TempDir(), "config.yaml") + content := strings.TrimSpace(` +controller: + listen_addr: ":8080" + repo_dir: "/srv/composia/repo" + state_dir: "/srv/composia/state-controller" + log_dir: "/srv/composia/logs" + nodes: + - id: "main" + token: "main-token" + updates: + forge_auth: + github: + - url: https://github.com + token: github-token + - url: https://github.example.com + api_url: https://github.example.com/api/v3 + token: enterprise-token +`) + "\n" + + if err := os.WriteFile(configPath, []byte(content), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + controller, err := LoadController(configPath) + if err != nil { + t.Fatalf("load controller: %v", err) + } + entries := controller.Updates.ForgeAuth.GitHub + if len(entries) != 2 { + t.Fatalf("expected 2 github auth entries, got %+v", entries) + } + if entries[1].APIURL != "https://github.example.com/api/v3" || entries[1].Token != "enterprise-token" { + t.Fatalf("unexpected second auth entry %+v", entries[1]) + } +} + func TestLoadControllerRejectsInvalidUpdatesDefaults(t *testing.T) { t.Parallel() diff --git a/internal/core/config/secret_value.go b/internal/core/config/secret_value.go index feb9f9a..be618d7 100644 --- a/internal/core/config/secret_value.go +++ b/internal/core/config/secret_value.go @@ -55,13 +55,13 @@ func resolveControllerInlineOrFileConfig(controller *ControllerConfig) error { controller.DNS.Cloudflare.APIToken = resolved } if controller.Updates != nil && controller.Updates.ForgeAuth != nil { - if err := resolveForgeAuthConfig(controller.Updates.ForgeAuth.GitHub, "controller.updates.forge_auth.github.token"); err != nil { + if err := resolveForgeAuthConfigs(controller.Updates.ForgeAuth.GitHub, "controller.updates.forge_auth.github"); err != nil { return err } - if err := resolveForgeAuthConfig(controller.Updates.ForgeAuth.GitLab, "controller.updates.forge_auth.gitlab.token"); err != nil { + if err := resolveForgeAuthConfigs(controller.Updates.ForgeAuth.GitLab, "controller.updates.forge_auth.gitlab"); err != nil { return err } - if err := resolveForgeAuthConfig(controller.Updates.ForgeAuth.Forgejo, "controller.updates.forge_auth.forgejo.token"); err != nil { + if err := resolveForgeAuthConfigs(controller.Updates.ForgeAuth.Forgejo, "controller.updates.forge_auth.forgejo"); err != nil { return err } } @@ -84,15 +84,14 @@ func resolveControllerInlineOrFileConfig(controller *ControllerConfig) error { return nil } -func resolveForgeAuthConfig(auth *ForgeAuthConfig, fieldPath string) error { - if auth == nil { - return nil - } - resolved, err := resolveInlineOrFileValue(auth.Token, auth.TokenFile, fieldPath, false) - if err != nil { - return err +func resolveForgeAuthConfigs(auths ForgeAuthConfigs, fieldPath string) error { + for index := range auths { + resolved, err := resolveInlineOrFileValue(auths[index].Token, auths[index].TokenFile, fmt.Sprintf("%s[%d].token", fieldPath, index), false) + if err != nil { + return err + } + auths[index].Token = resolved } - auth.Token = resolved return nil } diff --git a/internal/core/repo/services.go b/internal/core/repo/services.go index 9282426..5014e11 100644 --- a/internal/core/repo/services.go +++ b/internal/core/repo/services.go @@ -144,6 +144,7 @@ type ImageUpdateDiscovery struct { Auto *bool `yaml:"auto"` Type string `yaml:"type"` Repo string `yaml:"repo"` + RepoURL string `yaml:"repo_url"` Project string `yaml:"project"` Sources []ImageUpdateDiscoverySource `yaml:"sources"` Combine string `yaml:"combine"` @@ -152,7 +153,9 @@ type ImageUpdateDiscovery struct { type ImageUpdateDiscoverySource struct { Type string `yaml:"type"` Repo string `yaml:"repo"` + RepoURL string `yaml:"repo_url"` Project string `yaml:"project"` + APIURL string `yaml:"api_url"` } type ImageUpdateFilter struct { @@ -876,7 +879,7 @@ func validateImageUpdateDiscovery(fieldPrefix string, discovery ImageUpdateDisco return nil } if strings.TrimSpace(discovery.Type) != "" { - return validateImageUpdateDiscoverySource(fieldPrefix, ImageUpdateDiscoverySource{Type: discovery.Type, Repo: discovery.Repo, Project: discovery.Project}, filter) + return validateImageUpdateDiscoverySource(fieldPrefix, ImageUpdateDiscoverySource{Type: discovery.Type, Repo: discovery.Repo, RepoURL: discovery.RepoURL, Project: discovery.Project}, filter) } return nil } @@ -952,12 +955,17 @@ func isDigestDiscovery(discovery ImageUpdateDiscovery) bool { } func IsDigestImageDiscovery(discovery ImageUpdateDiscovery, sources map[string]ImageUpdateDiscovery) bool { - if discovery.Ref != "" { - if source, ok := sources[discovery.Ref]; ok { - return isDigestDiscovery(source) - } + return isDigestDiscovery(ResolveImageUpdateDiscovery(discovery, sources)) +} + +func ResolveImageUpdateDiscovery(discovery ImageUpdateDiscovery, sources map[string]ImageUpdateDiscovery) ImageUpdateDiscovery { + if discovery.Ref == "" { + return discovery + } + if source, ok := sources[discovery.Ref]; ok { + return source } - return isDigestDiscovery(discovery) + return discovery } func ImageUpdateCurrentFile(current ImageUpdateCurrent) string {