Something went wrong. Try again.
Maps Linux distribution source packages to their upstream repositories, and through them to each other
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185// Package arch resolves Arch Linux pkgbases to their upstream repositories.//// Data sources (all read through fetch.Client)://// - pacman's repository databases (core, extra, multilib) list every pkgbase with its// version, binaries and upstream url (syncdb.go);// - the packaging "state" repository pins each pkgbase to its released packaging commit// (state.go);// - the packaging repository on gitlab.archlinux.org (archlinux/packaging/packages/<path>)// holds the .SRCINFO, or for older commits only the PKGBUILD, at that commit: source=// names git repositories and release tarballs, with their sha256.package arch
import ( "context" "fmt" "net/url" "strings" "time"
"tangled.org/odd.computer/isomorph/internal/distro" "tangled.org/odd.computer/isomorph/internal/fetch" "tangled.org/odd.computer/isomorph/internal/model" "tangled.org/odd.computer/isomorph/internal/repourl")
// DefaultGitLabBase hosts the packaging repositories.const DefaultGitLabBase = "https://gitlab.archlinux.org"
// packagesGroup is the GitLab group holding one project per pkgbase.const packagesGroup = "archlinux/packaging/packages"
// Evidence sources and weights (prior trust in the field).const ( srcSourceVCS = "SRCINFO:source:git" srcSource = "SRCINFO:source" srcURL = "SRCINFO:url"
weightSourceVCS = 0.9 weightSource = 0.5 weightURL = 0.5)
// Resolver reads Arch's packaging data.type Resolver struct { c *fetch.Client
// GitLabBase hosts the packaging repositories (DefaultGitLabBase by default). GitLabBase string // Mirror serves the repository databases (DefaultMirror when empty). Mirror string}
// New returns a Resolver using c, with the default endpoints.func New(c *fetch.Client) *Resolver { return &Resolver{c: c, GitLabBase: DefaultGitLabBase}}
// projectID is the URL-encoded project path used in /api/v4/projects/<id>/... calls.func projectID(path string) string { return url.PathEscape(packagesGroup + "/" + path)}
// Candidates returns the upstream candidates a .SRCINFO supports: every remote source (VCS// sources weigh most) and the url field. One observation per (kind, key) from the source array,// so that a series of patches from one host does not inflate the score. The result is not yet// ranked.func Candidates(si *SRCINFO) []distro.Candidate { var cands []distro.Candidate seen := map[string]bool{} for _, s := range si.Sources { if s.Local || isSignatureFile(s.URL) { continue } c, err := repourl.Parse(s.Remote()) if err != nil { continue } k := string(c.Kind) + ":" + c.Key if seen[k] { continue } seen[k] = true source, weight := srcSource, weightSource if s.VCS != "" { source, weight = srcSourceVCS, weightSourceVCS if s.VCS != "git" { source = "SRCINFO:source:" + s.VCS } } cands = distro.AddCandidate(cands, s.Remote(), model.Evidence{Source: source, Value: s.Raw, Weight: weight}) } if si.URL != "" { cands = distro.AddCandidate(cands, si.URL, model.Evidence{Source: srcURL, Value: si.URL, Weight: weightURL}) } return cands}
// SourceSHA256 returns the sha256 of the pkgbase's remote, non-VCS sources (release tarballs),// pairing the architecture-independent source array with sha256sums by position.func SourceSHA256(si *SRCINFO) []string { sums := si.Fields["sha256sums"] var out []string i := 0 for _, s := range si.Sources { if s.Arch != "" { continue } if i < len(sums) && !s.Local && s.VCS == "" && !isSignatureFile(s.URL) && sums[i] != "SKIP" { out = append(out, sums[i]) } i++ } return out}
// srcinfoTTL keeps packaging files fetched at a commit for good: commits are immutable.const srcinfoTTL = 10 * 365 * 24 * time.Hour
// SRCINFOAt returns the .SRCINFO of pkgbase at a packaging commit (or what its PKGBUILD says,// where the commit carries no .SRCINFO) from whichever cached response exists, else through the// API.func (r *Resolver) SRCINFOAt(ctx context.Context, pkgbase, commit string) (*SRCINFO, error) { for _, web := range []bool{false, true} { if r.c.Cached(r.fileURL(pkgbase, commit, ".SRCINFO", web), fetch.WithTTL(srcinfoTTL)) || r.c.Cached(r.fileURL(pkgbase, commit, "PKGBUILD", web), fetch.WithTTL(srcinfoTTL)) { return r.FetchSRCINFO(ctx, pkgbase, commit, web) } } return r.FetchSRCINFO(ctx, pkgbase, commit, false)}
// FetchSRCINFO fetches through the API (web=false) or the web raw route (web=true); on// gitlab.archlinux.org the two count against separate anonymous rate-limit buckets. A commit// without .SRCINFO (older packaging repos) is read from its PKGBUILD instead.func (r *Resolver) FetchSRCINFO(ctx context.Context, pkgbase, commit string, web bool) (*SRCINFO, error) { parse := ParseSRCINFO raw, err := r.c.Get(ctx, r.fileURL(pkgbase, commit, ".SRCINFO", web), fetch.WithTTL(srcinfoTTL)) if fetch.IsNotFound(err) { parse = ParsePKGBUILD raw, err = r.c.Get(ctx, r.fileURL(pkgbase, commit, "PKGBUILD", web), fetch.WithTTL(srcinfoTTL)) } if err != nil { return nil, fmt.Errorf("arch: %s@%s: %w", pkgbase, commit, err) } si, err := parse(raw) if err == nil && si.PkgBase == "" { // The web route answers a missing project with the sign-in page. err = fmt.Errorf("arch: %s@%s: response is not a .SRCINFO or PKGBUILD", pkgbase, commit) } return si, err}
// HasSRCINFO reports whether the packaging metadata of pkgbase at commit is in the local cache// (through either route), so bulk runs can use what earlier crawls fetched without touching// GitLab.func (r *Resolver) HasSRCINFO(pkgbase, commit string) bool { for _, web := range []bool{false, true} { for _, f := range []string{".SRCINFO", "PKGBUILD"} { if r.c.Cached(r.fileURL(pkgbase, commit, f, web), fetch.WithTTL(srcinfoTTL)) { return true } } } return false}
// fileURL is the API (web=false) or web raw (web=true) URL of a packaging file at a commit.func (r *Resolver) fileURL(pkgbase, commit, file string, web bool) string { if web { return r.GitLabBase + "/" + packagesGroup + "/" + ProjectPath(pkgbase) + "/-/raw/" + url.PathEscape(commit) + "/" + file } return r.GitLabBase + "/api/v4/projects/" + projectID(ProjectPath(pkgbase)) + "/repository/files/" + url.PathEscape(file) + "/raw?ref=" + url.QueryEscape(commit)}
func isSignatureFile(u string) bool { l := strings.ToLower(u) for _, ext := range []string{".sig", ".asc", ".sign", ".gpg", ".minisig"} { if strings.HasSuffix(l, ext) { return true } } return false}