// Package arch resolves Arch Linux pkgbases to their upstream repositories. // // Data sources (all read through fetch.Client): // // - pacman's repository databases (core, extra, multilib) list every pkgbase with its // version, binaries and upstream url (syncdb.go); // - the packaging "state" repository pins each pkgbase to its released packaging commit // (state.go); // - the packaging repository on gitlab.archlinux.org (archlinux/packaging/packages/) // holds the .SRCINFO, or for older commits only the PKGBUILD, at that commit: source= // names git repositories and release tarballs, with their sha256. package arch import ( "context" "fmt" "net/url" "strings" "time" "tangled.org/odd.computer/isomorph/internal/distro" "tangled.org/odd.computer/isomorph/internal/fetch" "tangled.org/odd.computer/isomorph/internal/model" "tangled.org/odd.computer/isomorph/internal/repourl" ) // DefaultGitLabBase hosts the packaging repositories. const DefaultGitLabBase = "https://gitlab.archlinux.org" // packagesGroup is the GitLab group holding one project per pkgbase. const packagesGroup = "archlinux/packaging/packages" // Evidence sources and weights (prior trust in the field). const ( srcSourceVCS = "SRCINFO:source:git" srcSource = "SRCINFO:source" srcURL = "SRCINFO:url" weightSourceVCS = 0.9 weightSource = 0.5 weightURL = 0.5 ) // Resolver reads Arch's packaging data. type Resolver struct { c *fetch.Client // GitLabBase hosts the packaging repositories (DefaultGitLabBase by default). GitLabBase string // Mirror serves the repository databases (DefaultMirror when empty). Mirror string } // New returns a Resolver using c, with the default endpoints. func New(c *fetch.Client) *Resolver { return &Resolver{c: c, GitLabBase: DefaultGitLabBase} } // projectID is the URL-encoded project path used in /api/v4/projects//... calls. func projectID(path string) string { return url.PathEscape(packagesGroup + "/" + path) } // Candidates returns the upstream candidates a .SRCINFO supports: every remote source (VCS // sources weigh most) and the url field. One observation per (kind, key) from the source array, // so that a series of patches from one host does not inflate the score. The result is not yet // ranked. func Candidates(si *SRCINFO) []distro.Candidate { var cands []distro.Candidate seen := map[string]bool{} for _, s := range si.Sources { if s.Local || isSignatureFile(s.URL) { continue } c, err := repourl.Parse(s.Remote()) if err != nil { continue } k := string(c.Kind) + ":" + c.Key if seen[k] { continue } seen[k] = true source, weight := srcSource, weightSource if s.VCS != "" { source, weight = srcSourceVCS, weightSourceVCS if s.VCS != "git" { source = "SRCINFO:source:" + s.VCS } } cands = distro.AddCandidate(cands, s.Remote(), model.Evidence{Source: source, Value: s.Raw, Weight: weight}) } if si.URL != "" { cands = distro.AddCandidate(cands, si.URL, model.Evidence{Source: srcURL, Value: si.URL, Weight: weightURL}) } return cands } // SourceSHA256 returns the sha256 of the pkgbase's remote, non-VCS sources (release tarballs), // pairing the architecture-independent source array with sha256sums by position. func SourceSHA256(si *SRCINFO) []string { sums := si.Fields["sha256sums"] var out []string i := 0 for _, s := range si.Sources { if s.Arch != "" { continue } if i < len(sums) && !s.Local && s.VCS == "" && !isSignatureFile(s.URL) && sums[i] != "SKIP" { out = append(out, sums[i]) } i++ } return out } // srcinfoTTL keeps packaging files fetched at a commit for good: commits are immutable. const srcinfoTTL = 10 * 365 * 24 * time.Hour // SRCINFOAt returns the .SRCINFO of pkgbase at a packaging commit (or what its PKGBUILD says, // where the commit carries no .SRCINFO) from whichever cached response exists, else through the // API. func (r *Resolver) SRCINFOAt(ctx context.Context, pkgbase, commit string) (*SRCINFO, error) { for _, web := range []bool{false, true} { if r.c.Cached(r.fileURL(pkgbase, commit, ".SRCINFO", web), fetch.WithTTL(srcinfoTTL)) || r.c.Cached(r.fileURL(pkgbase, commit, "PKGBUILD", web), fetch.WithTTL(srcinfoTTL)) { return r.FetchSRCINFO(ctx, pkgbase, commit, web) } } return r.FetchSRCINFO(ctx, pkgbase, commit, false) } // FetchSRCINFO fetches through the API (web=false) or the web raw route (web=true); on // gitlab.archlinux.org the two count against separate anonymous rate-limit buckets. A commit // without .SRCINFO (older packaging repos) is read from its PKGBUILD instead. func (r *Resolver) FetchSRCINFO(ctx context.Context, pkgbase, commit string, web bool) (*SRCINFO, error) { parse := ParseSRCINFO raw, err := r.c.Get(ctx, r.fileURL(pkgbase, commit, ".SRCINFO", web), fetch.WithTTL(srcinfoTTL)) if fetch.IsNotFound(err) { parse = ParsePKGBUILD raw, err = r.c.Get(ctx, r.fileURL(pkgbase, commit, "PKGBUILD", web), fetch.WithTTL(srcinfoTTL)) } if err != nil { return nil, fmt.Errorf("arch: %s@%s: %w", pkgbase, commit, err) } si, err := parse(raw) if err == nil && si.PkgBase == "" { // The web route answers a missing project with the sign-in page. err = fmt.Errorf("arch: %s@%s: response is not a .SRCINFO or PKGBUILD", pkgbase, commit) } return si, err } // HasSRCINFO reports whether the packaging metadata of pkgbase at commit is in the local cache // (through either route), so bulk runs can use what earlier crawls fetched without touching // GitLab. func (r *Resolver) HasSRCINFO(pkgbase, commit string) bool { for _, web := range []bool{false, true} { for _, f := range []string{".SRCINFO", "PKGBUILD"} { if r.c.Cached(r.fileURL(pkgbase, commit, f, web), fetch.WithTTL(srcinfoTTL)) { return true } } } return false } // fileURL is the API (web=false) or web raw (web=true) URL of a packaging file at a commit. func (r *Resolver) fileURL(pkgbase, commit, file string, web bool) string { if web { return r.GitLabBase + "/" + packagesGroup + "/" + ProjectPath(pkgbase) + "/-/raw/" + url.PathEscape(commit) + "/" + file } return r.GitLabBase + "/api/v4/projects/" + projectID(ProjectPath(pkgbase)) + "/repository/files/" + url.PathEscape(file) + "/raw?ref=" + url.QueryEscape(commit) } func isSignatureFile(u string) bool { l := strings.ToLower(u) for _, ext := range []string{".sig", ".asc", ".sign", ".gpg", ".minisig"} { if strings.HasSuffix(l, ext) { return true } } return false }