Something went wrong. Try again.
Maps Linux distribution source packages to their upstream repositories, and through them to each other
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517package bulk
import ( "context" "database/sql" "regexp" "sort" "strings" "sync" "time"
"tangled.org/odd.computer/isomorph/internal/distro" "tangled.org/odd.computer/isomorph/internal/fetch" "tangled.org/odd.computer/isomorph/internal/model" "tangled.org/odd.computer/isomorph/internal/repourl")
// Aliases map an upstream key onto the key that names the same repository canonically: a// renamed or transferred GitHub repository onto its current name, and a known mirror onto the// project's main repository. They are applied by rescoring each package from its evidence, so// two fields naming the same repository under different keys reinforce each other instead of// competing.
// Alias sources.const ( AliasGitHubRename = "github-rename" AliasMirrorRule = "mirror-rule" AliasKDEProjects = "kde-projects" // AliasGitHubGone maps a GitHub repository that no longer exists to "": its evidence // drops out, and the package falls back to its other evidence or is left unresolved. AliasGitHubGone = "github-gone")
// Alias records one mapping and how it was learned.type Alias struct { From, To, Source string CheckedAt time.Time}
// MirrorAliases applies the documented mirror rules to keys:// - gitbox.apache.org/[repos/asf/]X -> github.com/apache/X (the GitHub mirror, which the GitHub// API can follow further);// - github.com/gnome/X -> gitlab.gnome.org/gnome/X;// - github.com/golang/X -> go.googlesource.com/X;// - github.com/openstack/X -> opendev.org/openstack/X.func MirrorAliases(keys []string) []Alias { var out []Alias for _, k := range keys { to := "" switch { case strings.HasPrefix(k, "gitbox.apache.org/"): // repourl keys gitbox repositories as gitbox.apache.org/<name>; older keys kept // the repos/asf/ prefix. to = "github.com/apache/" + strings.TrimPrefix(strings.TrimPrefix(k, "gitbox.apache.org/"), "repos/asf/") case strings.HasPrefix(k, "github.com/gnome/"): to = "gitlab.gnome.org/gnome/" + strings.TrimPrefix(k, "github.com/gnome/") case strings.HasPrefix(k, "github.com/golang/"): to = "go.googlesource.com/" + strings.TrimPrefix(k, "github.com/golang/") case strings.HasPrefix(k, "github.com/openstack/"): to = "opendev.org/openstack/" + strings.TrimPrefix(k, "github.com/openstack/") } if to != "" && to != k { out = append(out, Alias{From: k, To: to, Source: AliasMirrorRule}) } } return out}
// KDEProjectsURL lists every KDE repository path (accessibility/kmousetool, ...).const KDEProjectsURL = "https://projects.kde.org/api/v1/projects"
// KDEAliases gives KDE keys without a group their repository path from KDE's own project list:// download.kde.org tarballs and apps.kde.org pages name a project (invent.kde.org/kmousetool),// and KDE's GitHub mirror drops the group (github.com/kde/kmousetool); both map to// invent.kde.org/accessibility/kmousetool when exactly one KDE repository has that name.func KDEAliases(ctx context.Context, c *fetch.Client, keys []string) ([]Alias, error) { var paths []string if err := c.GetJSON(ctx, KDEProjectsURL, &paths, fetch.WithTTL(7*24*time.Hour)); err != nil { return nil, err } return kdeAliases(paths, keys), nil}
func kdeAliases(paths, keys []string) []Alias { byName := map[string][]string{} for _, p := range paths { p = strings.ToLower(p) byName[p[strings.LastIndexByte(p, '/')+1:]] = append(byName[p[strings.LastIndexByte(p, '/')+1:]], p) } var out []Alias for _, k := range keys { name := "" switch { case strings.HasPrefix(k, "github.com/kde/"): name = strings.TrimPrefix(k, "github.com/kde/") case strings.HasPrefix(k, "invent.kde.org/") && strings.Count(k, "/") == 1: name = strings.TrimPrefix(k, "invent.kde.org/") } if c := byName[name]; name != "" && len(c) == 1 { out = append(out, Alias{From: k, To: "invent.kde.org/" + c[0], Source: AliasKDEProjects}) } } return out}
// Resolve follows an alias chain to its end (bounded, in case of cycles).func Resolve(alias map[string]string, k string) string { for i := 0; i < 5; i++ { to, ok := alias[k] if !ok || to == k { return k } k = to } return k}
// Rescore recomputes a package's upstream from its evidence after mapping every evidence key// through alias: evidence for keys that alias together combines as 1-prod(1-w).func Rescore(p *Package, alias map[string]string) { type cand struct { key string miss float64 // prod(1-w) basis string } byKey := map[string]*cand{} var order []string for i := range p.Evidence { e := &p.Evidence[i] if e.Key == "" { continue } if _, name, ok := registryKey(e.Key); ok && !SameProject(p.Name, name) { continue // a dependency fetched from a registry (vendored crates), not this project } e.Key = Resolve(alias, e.Key) if !isRepoKey(e.Key) { continue // a registry entry that no alias resolved } c := byKey[e.Key] if c == nil { c = &cand{key: e.Key, miss: 1} byKey[e.Key] = c order = append(order, e.Key) } c.miss *= 1 - e.Weight if b := BasisOf(e.Source); c.basis == "" || basisRank[b] < basisRank[c.basis] { c.basis = b } } best, bestScore := "", 0.0 for _, k := range order { if byKey[k].basis == BasisSecondary { continue // corroborates, never decides } if siteOnly(p.Evidence, k) && !namedLike(p.Name, k) { continue // a project site standing for another package's repository (flagship.go) } if s := (1 - byKey[k].miss) * distro.PackagingPenalty(k); s > bestScore { best, bestScore = k, s } } if best == "" { // Nothing left: every key was aliased away (a repository that no longer exists) or is // not a repository. p.UpstreamKey, p.Repo, p.Status, p.Basis = "", "", StatusUnresolved, "" return } if byKey[best].basis == BasisHomepage && onStaleHomeHost(best) { // See staleHomeHosts: the best release or VCS source wins over such a homepage. alt, altScore := "", 0.0 for _, k := range order { if b := byKey[k].basis; k != best && (b == BasisReleaseURL || b == BasisVCSSource) { if s := (1 - byKey[k].miss) * distro.PackagingPenalty(k); s > altScore { alt, altScore = k, s } } } if alt != "" { best = alt } } if best != p.UpstreamKey { p.UpstreamKey, p.Repo = best, "https://"+best if c, err := repourl.Parse("https://" + best); err == nil { p.Repo = c.URL } } p.Status = StatusResolved p.Basis = byKey[best].basis}
// ApplyAliases rescores every package whose evidence names an aliased key.func ApplyAliases(pkgs []Package, alias map[string]string) int { n := 0 for i := range pkgs { for _, e := range pkgs[i].Evidence { if _, ok := alias[e.Key]; ok { before := pkgs[i].UpstreamKey Rescore(&pkgs[i], alias) if pkgs[i].UpstreamKey != before { n++ } break } } } return n}
// LoadAliases reads the aliases table.func LoadAliases(ctx context.Context, db *sql.DB) (map[string]string, error) { rows, err := db.QueryContext(ctx, `select from_key, to_key from aliases`) if err != nil { return nil, err } defer rows.Close() out := map[string]string{} for rows.Next() { var f, t string if err := rows.Scan(&f, &t); err != nil { return nil, err } out[f] = t } return out, rows.Err()}
// SaveAliases upserts aliases.func SaveAliases(ctx context.Context, db *sql.DB, as []Alias) error { return saveAliases(ctx, db, "", as)}
// ReplaceAliases replaces every stored alias of source with as: rule-based sources are// recomputed in full on each run, so a rule that no longer applies must not linger.func ReplaceAliases(ctx context.Context, db *sql.DB, source string, as []Alias) error { return saveAliases(ctx, db, source, as)}
func saveAliases(ctx context.Context, db *sql.DB, replace string, as []Alias) error { tx, err := db.BeginTx(ctx, nil) if err != nil { return err } defer tx.Rollback() if replace != "" { if _, err := tx.ExecContext(ctx, `delete from aliases where source = ?`, replace); err != nil { return err } } for _, a := range as { if _, err := tx.ExecContext(ctx, `insert or replace into aliases values (?,?,?,?)`, a.From, a.To, a.Source, a.CheckedAt.UTC().Format(time.RFC3339)); err != nil { return err } } return tx.Commit()}
// LoadSlice reads one (distro, suite) slice back with its evidence, binaries and hashes, one// query per table.func LoadSlice(ctx context.Context, db *sql.DB, d model.Distro, suite string) ([]Package, error) { rows, err := db.QueryContext(ctx, `select distro, suite, package, version, coalesce(osv_ecosystem,''), coalesce(purl,''), coalesce(homepage,''), coalesce(upstream_key,''), coalesce(repo,''), status, coalesce(basis,'') from packages where distro = ? and suite = ? order by package`, d, suite) if err != nil { return nil, err } var out []Package idx := map[string]int{} for rows.Next() { var p Package if err := rows.Scan(&p.Distro, &p.Suite, &p.Name, &p.Version, &p.OSVEcosystem, &p.PURL, &p.Homepage, &p.UpstreamKey, &p.Repo, &p.Status, &p.Basis); err != nil { rows.Close() return nil, err } idx[p.Name] = len(out) out = append(out, p) } rows.Close() if err := rows.Err(); err != nil { return nil, err } scan := func(q string, add func(r *sql.Rows) error) error { r, err := db.QueryContext(ctx, q, d, suite) if err != nil { return err } defer r.Close() for r.Next() { if err := add(r); err != nil { return err } } return r.Err() } if err := scan(`select package, source, value, coalesce(key,''), weight from evidence where distro = ? and suite = ? order by rowid`, func(r *sql.Rows) error { var name string var e model.Evidence if err := r.Scan(&name, &e.Source, &e.Value, &e.Key, &e.Weight); err != nil { return err } if i, ok := idx[name]; ok { out[i].Evidence = append(out[i].Evidence, e) } return nil }); err != nil { return nil, err } for _, t := range []struct { q string dst func(p *Package) *[]string }{ {`select package, binary from binaries where distro = ? and suite = ? order by rowid`, func(p *Package) *[]string { return &p.Binaries }}, {`select package, sha256 from source_sha256 where distro = ? and suite = ? order by rowid`, func(p *Package) *[]string { return &p.SourceSHA256 }}, } { if err := scan(t.q, func(r *sql.Rows) error { var name, v string if err := r.Scan(&name, &v); err != nil { return err } if i, ok := idx[name]; ok { dst := t.dst(&out[i]) *dst = append(*dst, v) } return nil }); err != nil { return nil, err } } return out, nil}
// Slices lists the (distro, suite) slices in the table.func Slices(ctx context.Context, db *sql.DB) ([][2]string, error) { rows, err := db.QueryContext(ctx, `select distro, suite from sources order by distro, suite`) if err != nil { return nil, err } defer rows.Close() var out [][2]string for rows.Next() { var d, s string if err := rows.Scan(&d, &s); err != nil { return nil, err } out = append(out, [2]string{d, s}) } return out, rows.Err()}
// ConflictingKeys returns the keys worth canonicalising: keys of packages whose evidence names// more than one repository on the same host (a rename leaves the old and new name side by// side), plus the keys of cross-distro disagreements, restricted to hosts (sorted, unique).func ConflictingKeys(ctx context.Context, db *sql.DB, hostPrefix string, disagreements [][2]string) ([]string, error) { rows, err := db.QueryContext(ctx, `select distinct e.key from evidence e join ( select distro, suite, package from evidence where key like ? || '%' group by distro, suite, package having count(distinct key) > 1) c on e.distro = c.distro and e.suite = c.suite and e.package = c.package where e.key like ? || '%'`, hostPrefix, hostPrefix) if err != nil { return nil, err } set := map[string]bool{} for rows.Next() { var k string if err := rows.Scan(&k); err != nil { rows.Close() return nil, err } set[k] = true } rows.Close() for _, d := range disagreements { for _, k := range d { if strings.HasPrefix(k, hostPrefix) { set[k] = true } } } out := make([]string, 0, len(set)) for k := range set { out = append(out, k) } sort.Strings(out) return out, nil}
// AliasGitLabProject marks aliases from a numeric GitLab project id to the project's path.const AliasGitLabProject = "gitlab-project-id"
// gitlabProjectKeyRe matches the keys repourl gives GitLab URLs that name a project by number.var gitlabProjectKeyRe = regexp.MustCompile(`^([^/]+)/-/project/(\d+)$`)
// GitLabProjectAliases resolves "<host>/-/project/<id>" keys through the host's API// (/api/v4/projects/<id> answers anonymously for public projects). base maps a host onto the// scheme and host to query (tests point it at an httptest server).func GitLabProjectAliases(ctx context.Context, c *fetch.Client, keys []string, base func(host string) string) ([]Alias, int) { var out []Alias failed := 0 for _, k := range keys { m := gitlabProjectKeyRe.FindStringSubmatch(k) if m == nil { continue } var resp struct { Path string `json:"path_with_namespace"` } if err := c.GetJSON(ctx, base(m[1])+"/api/v4/projects/"+m[2], &resp, fetch.WithTTL(30*24*time.Hour)); err != nil || resp.Path == "" { failed++ continue } cn, err := repourl.Parse("https://" + m[1] + "/" + resp.Path) if err != nil || cn.Kind != repourl.KindRepo { failed++ continue } out = append(out, Alias{From: k, To: cn.Key, Source: AliasGitLabProject}) } return out, failed}
var repoKeyCache sync.Map
// isRepoKey reports whether key names a repository (not a registry entry or other hint).func isRepoKey(key string) bool { if v, ok := repoKeyCache.Load(key); ok { return v.(bool) } c, err := repourl.Parse("https://" + key) ok := err == nil && c.Kind == repourl.KindRepo repoKeyCache.Store(key, ok) return ok}
// AliasRegistry marks aliases from a registry entry (pypi.org/<name>, crates.io/<name>) to the// repository its metadata names.const AliasRegistry = "registry-metadata"
// RegistryAliases resolves registry keys through the registry's metadata: PyPI with// oss-rebuild's link selection (pypi.go) and crates.io's repository field. base// maps a registry host onto the scheme and host to query (tests point it at an httptest// server).func RegistryAliases(ctx context.Context, c *fetch.Client, keys []string, base func(host string) string) ([]Alias, int) { var out []Alias failed := 0 for _, k := range keys { host, name, ok := strings.Cut(k, "/") if !ok { continue } var urls []string switch host { case "pypi.org": var resp struct { Info pypiInfo `json:"info"` } if err := c.GetJSON(ctx, base(host)+"/pypi/"+name+"/json", &resp, fetch.WithTTL(30*24*time.Hour)); err != nil { failed++ continue } if r := pypiRepo(name, resp.Info); r != "" { if !strings.Contains(r, "://") { r = "https://" + r } urls = []string{r} } case "crates.io": var resp struct { Crate struct { Repository string `json:"repository"` Homepage string `json:"homepage"` } `json:"crate"` } if err := c.GetJSON(ctx, base(host)+"/api/v1/crates/"+name, &resp, fetch.WithTTL(30*24*time.Hour)); err != nil { failed++ continue } urls = []string{resp.Crate.Repository, resp.Crate.Homepage} default: continue } for _, u := range urls { if cn, err := repourl.Parse(u); err == nil && cn.Kind == repourl.KindRepo { out = append(out, Alias{From: k, To: cn.Key, Source: AliasRegistry}) break } } } return out, failed}
// registryKey splits a registry key (pypi.org/<name>, crates.io/<name>).func registryKey(k string) (registry, name string, ok bool) { registry, name, ok = strings.Cut(k, "/") if !ok || (registry != "pypi.org" && registry != "crates.io") { return "", "", false } return registry, name, true}
// SameProject reports whether a registry project name is the distro package itself rather than// one of its dependencies: equal after lower-casing, unifying - _ . and dropping the ecosystem// prefixes distros add (python-, python3-, py3-, rust-, ...).func SameProject(pkg, registryName string) bool { norm := func(s string) string { s = strings.NewReplacer("_", "-", ".", "-").Replace(strings.ToLower(s)) for _, p := range []string{"python3-", "python-", "py3-", "py-", "rust-", "golang-"} { s = strings.TrimPrefix(s, p) } return s } return norm(pkg) == norm(registryName)}