From 675e7b5bb4b61e361b670803ca12bc0f9c05ba3a Mon Sep 17 00:00:00 2001 From: Matthew Suozzo Date: Wed, 30 Sep 2026 11:00:49 -0400 Subject: [PATCH] bulk: the association table, canonicalisation and comparison isomorph bulk debian|arch|opensuse|alpine resolves every source package of a distribution suite into a SQLite table: package, version, OSV ecosystem, purl, binaries, homepage, upstream key and clone URL, the basis of the mapping (metadata, vcs-source, release-url, derived, bug-tracker, homepage) and every piece of evidence behind it. The Arch and openSUSE commands crawl per-package packaging files at a published-limit-respecting pace and read them back from the cache on later runs. bulk canonicalize maps keys that name the same repository onto one: GitHub renames and transfers (and repositories that no longer exist), GitLab numeric project ids, PyPI and crates.io metadata for the package's own registry entry, KDE's project list for keys without a group, and mirror rules. Packages are rescored from their stored evidence, so fields that name one repository under different keys reinforce each other. bulk compare prints, for any two distributions or all pairs, a confusion matrix of their mappings over same-named packages and over identical source tarballs. bulk debian also resolves every package with a DEP-12 Repository again without it and reports agreement per basis. docs/EVALUATION.md reports coverage, agreement with DEP-12 and between distributions, and a labelled error analysis (eval/) with the fixes it led to. --- cmd/isomorph/bulk.go | 622 ++++++++++++++++++++++++++++++++++ cmd/isomorph/main.go | 107 ++++++ docs/EVALUATION.md | 222 ++++++++++++ eval/errors-2026-09-28.tsv | 99 ++++++ eval/sample.py | 68 ++++ go.mod | 10 + go.sum | 52 ++- internal/bulk/alias.go | 510 ++++++++++++++++++++++++++++ internal/bulk/alias_test.go | 188 ++++++++++ internal/bulk/alpine.go | 60 ++++ internal/bulk/arch.go | 130 +++++++ internal/bulk/arch_test.go | 158 +++++++++ internal/bulk/bulk.go | 116 +++++++ internal/bulk/bulk_test.go | 161 +++++++++ internal/bulk/compare.go | 73 ++++ internal/bulk/compare_test.go | 51 +++ internal/bulk/debian.go | 152 +++++++++ internal/bulk/github.go | 61 ++++ internal/bulk/github_test.go | 63 ++++ internal/bulk/opensuse.go | 58 ++++ internal/bulk/pypi.go | 128 +++++++ internal/bulk/pypi_test.go | 23 ++ internal/bulk/sqlite.go | 208 ++++++++++++ 23 files changed, 3318 insertions(+), 2 deletions(-) create mode 100644 cmd/isomorph/bulk.go create mode 100644 cmd/isomorph/main.go create mode 100644 docs/EVALUATION.md create mode 100644 eval/errors-2026-09-28.tsv create mode 100644 eval/sample.py create mode 100644 internal/bulk/alias.go create mode 100644 internal/bulk/alias_test.go create mode 100644 internal/bulk/alpine.go create mode 100644 internal/bulk/arch.go create mode 100644 internal/bulk/arch_test.go create mode 100644 internal/bulk/bulk.go create mode 100644 internal/bulk/bulk_test.go create mode 100644 internal/bulk/compare.go create mode 100644 internal/bulk/compare_test.go create mode 100644 internal/bulk/debian.go create mode 100644 internal/bulk/github.go create mode 100644 internal/bulk/github_test.go create mode 100644 internal/bulk/opensuse.go create mode 100644 internal/bulk/pypi.go create mode 100644 internal/bulk/pypi_test.go create mode 100644 internal/bulk/sqlite.go diff --git a/cmd/isomorph/bulk.go b/cmd/isomorph/bulk.go new file mode 100644 index 0000000..cd993f0 --- /dev/null +++ b/cmd/isomorph/bulk.go @@ -0,0 +1,622 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "flag" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "text/tabwriter" + "time" + + "tangled.org/odd.computer/isomorph/internal/bulk" + "tangled.org/odd.computer/isomorph/internal/distro/alpine" + "tangled.org/odd.computer/isomorph/internal/distro/arch" + "tangled.org/odd.computer/isomorph/internal/distro/debian" + "tangled.org/odd.computer/isomorph/internal/distro/opensuse" + "tangled.org/odd.computer/isomorph/internal/model" +) + +func init() { + register(command{name: "bulk", usage: "debian|arch|opensuse|alpine|canonicalize|compare [flags]: resolve whole distributions into the association table, canonicalise its keys, or compare distributions", run: runBulk}) +} + +func defaultDB(env *Env) string { return filepath.Join(env.DataDir, "packages.sqlite") } + +func runBulk(ctx context.Context, env *Env, args []string) error { + if len(args) == 0 { + return errors.New("usage: bulk debian|arch|opensuse|alpine|compare|canonicalize [flags]") + } + switch args[0] { + case "debian": + return bulkDebian(ctx, env, args[1:]) + case "arch": + return bulkArch(ctx, env, args[1:]) + case "opensuse": + return bulkOpenSUSE(ctx, env, args[1:]) + case "alpine": + return bulkAlpine(ctx, env, args[1:]) + case "compare": + return bulkCompare(ctx, env, args[1:]) + case "canonicalize": + return bulkCanonicalize(ctx, env, args[1:]) + } + return fmt.Errorf("bulk: unknown source %q (want debian, arch, opensuse, alpine, compare or canonicalize)", args[0]) +} + +// writeSlices replaces each (distro, suite) slice of pkgs in the database and prints coverage +// per basis. +func writeSlices(ctx context.Context, env *Env, dbPath, origin string, fetched time.Time, pkgs []bulk.Package) error { + if err := os.MkdirAll(filepath.Dir(dbPath), 0o755); err != nil { + return err + } + db, err := bulk.Open(ctx, dbPath) + if err != nil { + return err + } + defer db.Close() + alias, err := bulk.LoadAliases(ctx, db) + if err != nil { + return err + } + if n := bulk.ApplyAliases(pkgs, alias); n > 0 { + env.Logf("aliases changed the upstream of %d packages", n) + } + slices := map[[2]string][]bulk.Package{} + var order [][2]string + for _, p := range pkgs { + k := [2]string{string(p.Distro), p.Suite} + if _, ok := slices[k]; !ok { + order = append(order, k) + } + slices[k] = append(slices[k], p) + } + for _, k := range order { + src := bulk.Source{Distro: model.Distro(k[0]), Suite: k[1], Origin: origin, FetchedAt: fetched, Packages: len(slices[k])} + if err := bulk.Replace(ctx, db, src, slices[k]); err != nil { + return err + } + } + resolved, byBasis := 0, map[string]int{} + for _, p := range pkgs { + if p.Status == bulk.StatusResolved { + resolved++ + byBasis[p.Basis]++ + } + } + fmt.Fprintf(env.Stdout, "%d packages, %d resolved to a repository (%.1f%%); by basis: %v\n", len(pkgs), resolved, pct(resolved, len(pkgs)), byBasis) + fmt.Fprintf(env.Stdout, "wrote %s\n", dbPath) + return nil +} + +func pct(a, b int) float64 { return 100 * float64(a) / float64(max(1, b)) } + +func bulkDebian(ctx context.Context, env *Env, args []string) error { + fs := flag.NewFlagSet("bulk debian", flag.ContinueOnError) + suite := fs.String("suite", "sid", "Debian suite (UDD release name)") + component := fs.String("component", "main", "archive component") + dbPath := fs.String("db", defaultDB(env), "SQLite database to update") + heldOutPath := fs.String("heldout", "", "write every held-out disagreement (package, repository key, fallback key) as TSV") + if err := fs.Parse(args); err != nil { + return err + } + start := time.Now() + rows, err := debian.QueryUDD(ctx, debian.UDDDSN(), *suite, *component) + if err != nil { + return err + } + env.Logf("udd: %d source packages in %s/%s (%s)", len(rows), *suite, *component, time.Since(start).Round(time.Second)) + fmt.Fprintf(env.Stdout, "debian/%s: ", *suite) + if err := writeSlices(ctx, env, *dbPath, "udd-mirror.debian.net", start, bulk.FromUDD(rows, *suite)); err != nil { + return err + } + h := bulk.DebianHeldOut(rows) + fmt.Fprintf(env.Stdout, "held out DEP-12 Repository: %d packages have one; the other fields resolve %d (%.1f%%), agreeing on %d (%.1f%% of those)\n", + h.WithRepository, h.FallbackResolved, pct(h.FallbackResolved, h.WithRepository), h.Agree, pct(h.Agree, h.FallbackResolved)) + for b, n := range h.ByBasis { + fmt.Fprintf(env.Stdout, " fallback basis %-12s resolved %5d, agree %5d (%.1f%%)\n", b, n[0], n[1], pct(n[1], n[0])) + } + if *heldOutPath != "" { + var b strings.Builder + for _, d := range h.Disagree { + b.WriteString(strings.Join(d[:], "\t") + "\n") + } + if err := os.WriteFile(*heldOutPath, []byte(b.String()), 0o644); err != nil { + return err + } + } + return nil +} + +func bulkArch(ctx context.Context, env *Env, args []string) error { + fs := flag.NewFlagSet("bulk arch", flag.ContinueOnError) + dbPath := fs.String("db", defaultDB(env), "SQLite database to update") + crawl := fs.Int("crawl", 0, "fetch up to N .SRCINFO files not yet cached (gitlab.archlinux.org rate-limits anonymous API use; set ARCH_GITLAB_TOKEN or ~/.config/isomorph/gitlab.archlinux.org.token)") + if err := fs.Parse(args); err != nil { + return err + } + start := time.Now() + r := arch.New(env.Client) + var dbs []arch.SyncPackage + for _, repo := range arch.SyncRepos { + p, err := r.SyncDB(ctx, repo) + if err != nil { + return err + } + dbs = append(dbs, p...) + } + state, err := r.State(ctx, nil) + if err != nil { + return err + } + env.Logf("arch: %d binary packages in %v, %d pkgbases pinned in the state repository", len(dbs), arch.SyncRepos, len(state)) + pkgs, st := bulk.FromArch(ctx, r, dbs, state, *crawl, func(n int) { + if n%200 == 0 { + env.Logf("arch: fetched %d .SRCINFO (%s)", n, time.Since(start).Round(time.Second)) + } + }) + if ctx.Err() != nil { + return ctx.Err() + } + fmt.Fprintf(env.Stdout, "arch: %d pkgbases, %d resolved from .SRCINFO (%d fetched now, %d errors), the rest from the repository databases' url\n", + st.PkgBases, st.FromSRCINFO, st.Fetched, st.FetchErrors) + if st.Stopped != nil { + fmt.Fprintf(env.Stdout, "arch: crawl stopped early: %v\n", st.Stopped) + } + for name, e := range st.Errors { + env.Logf("arch: %s: %s", name, e) + } + fmt.Fprint(env.Stdout, "arch: ") + return writeSlices(ctx, env, *dbPath, "pacman repository databases + gitlab.archlinux.org .SRCINFO", start, pkgs) +} + +func bulkOpenSUSE(ctx context.Context, env *Env, args []string) error { + fs := flag.NewFlagSet("bulk opensuse", flag.ContinueOnError) + dbPath := fs.String("db", defaultDB(env), "SQLite database to update") + crawl := fs.Int("crawl", 0, "fetch the OBS spec and _service of up to N unresolved packages not yet cached (api.opensuse.org, one request per second)") + crawlResolved := fs.Bool("crawl-resolved", false, "with -crawl, also fetch packages the metadata already resolves, after the unresolved ones") + if err := fs.Parse(args); err != nil { + return err + } + start := time.Now() + srcs, err := opensuse.Primary(ctx, env.Client, "", "src-oss") + if err != nil { + return err + } + bins, err := opensuse.Primary(ctx, env.Client, "", "oss") + if err != nil { + return err + } + r := opensuse.New(env.Client) + giturls, err := r.GitURLs(ctx) + if err != nil { + env.Logf("opensuse: giturls unavailable, continuing without: %v", err) + } + env.Logf("opensuse: %d source and %d binary packages, %d giturls entries", len(srcs), len(bins), len(giturls)) + files, err := openSUSEFiles(ctx, env, r, srcs, bins, giturls, *crawl, *crawlResolved) + if err != nil { + return err + } + fmt.Fprint(env.Stdout, "opensuse/tumbleweed: ") + return writeSlices(ctx, env, *dbPath, "download.opensuse.org tumbleweed repodata + rb.zq1.de giturls.good + api.opensuse.org spec/_service", start, bulk.FromOpenSUSE(srcs, bins, giturls, files)) +} + +// openSUSEFiles reads every source package's cached OBS spec and _service, then fetches those +// of up to crawl more packages: unresolved ones first, and resolved ones too if asked. +func openSUSEFiles(ctx context.Context, env *Env, r *opensuse.Resolver, srcs, bins []opensuse.RPM, giturls map[string][]string, crawl int, crawlResolved bool) (map[string]*opensuse.OBSFiles, error) { + list, err := r.ListPackages(ctx) + if err != nil { + return nil, err + } + obsPkgs := map[string]bool{} + for _, p := range list { + obsPkgs[p] = true + } + files := map[string]*opensuse.OBSFiles{} + var missing []string + errs := 0 + seen := map[string]bool{} + for _, s := range srcs { + if (s.Arch != "src" && s.Arch != "nosrc") || seen[s.Name] { + continue + } + seen[s.Name] = true + f, err := r.OBSFiles(ctx, s.Name, opensuse.OBSPackageCandidates(s.Name, obsPkgs), false) + switch { + case errors.Is(err, opensuse.ErrNotCached): + missing = append(missing, s.Name) + case err != nil: + errs++ + env.Debugf("opensuse: %s: %v", s.Name, err) + default: + files[s.Name] = f + } + } + if crawl > 0 && len(missing) > 0 { + // Unresolved first: resolve the missing packages from what is already known. + resolved := map[string]bool{} + for _, p := range bulk.FromOpenSUSE(srcs, bins, giturls, files) { + resolved[p.Name] = p.UpstreamKey != "" + } + var queue []string + for _, n := range missing { + if !resolved[n] { + queue = append(queue, n) + } + } + if crawlResolved { + for _, n := range missing { + if resolved[n] { + queue = append(queue, n) + } + } + } + if len(queue) > crawl { + queue = queue[:crawl] + } + env.Logf("opensuse: fetching OBS files of %d packages (%d not cached)", len(queue), len(missing)) + fetched, failed := 0, 0 + t0 := time.Now() + for _, n := range queue { + f, err := r.OBSFiles(ctx, n, opensuse.OBSPackageCandidates(n, obsPkgs), true) + if err != nil { + failed++ + env.Logf("opensuse: %s: %v", n, err) + if strings.Contains(err.Error(), "rate limited") || strings.Contains(err.Error(), "429") || ctx.Err() != nil { + env.Logf("opensuse: stopping the crawl") + break + } + continue + } + files[n] = f + fetched++ + if fetched%200 == 0 { + env.Logf("opensuse: fetched OBS files of %d packages (%s)", fetched, time.Since(t0).Round(time.Second)) + } + } + env.Logf("opensuse: fetched %d, %d failed", fetched, failed) + } + withSpec, withService, notCached := 0, 0, 0 + for _, f := range files { + if f != nil { + withSpec++ + if len(f.Service) > 0 { + withService++ + } + } + } + for _, n := range missing { + if _, ok := files[n]; !ok { + notCached++ + } + } + env.Logf("opensuse: OBS spec for %d packages (%d with _service), %d without a spec found, %d not cached, %d errors", + withSpec, withService, len(files)-withSpec, notCached, errs) + return files, nil +} + +func bulkAlpine(ctx context.Context, env *Env, args []string) error { + fs := flag.NewFlagSet("bulk alpine", flag.ContinueOnError) + dbPath := fs.String("db", defaultDB(env), "SQLite database to update") + archive := fs.String("aports", alpine.DefaultAportsArchive, "aports archive (tar.gz) to read APKBUILDs from") + if err := fs.Parse(args); err != nil { + return err + } + start := time.Now() + sections := []string{"main", "community"} + r := alpine.New(env.Client) + index := map[string][]alpine.IndexEntry{} + for _, s := range sections { + e, err := r.Index(ctx, s) + if err != nil { + return err + } + index[s] = e + } + apkbuilds := map[string]*alpine.APKBUILD{} + failed := 0 + if err := alpine.StreamAPKBUILDs(ctx, env.Client, *archive, sections, func(section, dir string, ab *alpine.APKBUILD, err error) { + if err != nil { + failed++ + return + } + apkbuilds[section+"/"+dir] = ab + }); err != nil { + return err + } + env.Logf("alpine: %d APKBUILDs (%d unparseable) in %s", len(apkbuilds), failed, time.Since(start).Round(time.Second)) + fmt.Fprint(env.Stdout, "alpine/edge: ") + return writeSlices(ctx, env, *dbPath, "dl-cdn.alpinelinux.org edge APKINDEX + github.com/alpinelinux/aports master", start, bulk.FromAlpine(index, apkbuilds)) +} + +func bulkCompare(ctx context.Context, env *Env, args []string) error { + fs := flag.NewFlagSet("bulk compare", flag.ContinueOnError) + dbPath := fs.String("db", defaultDB(env), "SQLite database") + a := fs.String("a", "debian", "first distro") + b := fs.String("b", "arch", "second distro") + all := fs.Bool("all", false, "compare every pair of distros in the database, then summarise the name join as a distro-by-distro grid") + samples := fs.Int("samples", -1, "disagreements to print per join (default 15 for one pair, 0 with -all)") + if err := fs.Parse(args); err != nil { + return err + } + if *samples < 0 { + *samples = 15 + if *all { + *samples = 0 + } + } + db, err := bulk.Open(ctx, *dbPath) + if err != nil { + return err + } + defer db.Close() + pairs := [][2]string{{*a, *b}} + var distros []string + if *all { + if distros, err = distroNames(ctx, db); err != nil { + return err + } + if pairs, err = distroPairs(ctx, db); err != nil { + return err + } + } + byName := map[[2]string]bulk.Agreement{} + for _, p := range pairs { + ags, err := bulk.Compare(ctx, db, p[0], p[1]) + if err != nil { + return err + } + for _, ag := range ags { + printConfusion(env, p[0], p[1], ag, *samples) + if ag.Join == "name" { + byName[p] = ag + byName[[2]string{p[1], p[0]}] = ag + } + } + } + if *all { + fmt.Fprintln(env.Stdout, "agreement where both resolve, joined by name (of pairs both resolved):") + tw := tabwriter.NewWriter(env.Stdout, 0, 0, 3, ' ', 0) + fmt.Fprintf(tw, "\t%s\n", strings.Join(distros, "\t")) + for _, r := range distros { + fmt.Fprint(tw, r) + for _, c := range distros { + if ag, ok := byName[[2]string{r, c}]; ok { + fmt.Fprintf(tw, "\t%.1f%% of %d", pct(ag.Agree, ag.BothResolved), ag.BothResolved) + } else { + fmt.Fprint(tw, "\t-") + } + } + fmt.Fprintln(tw) + } + tw.Flush() + } + return nil +} + +// distroNames lists the distributions in db, in name order. +func distroNames(ctx context.Context, db *sql.DB) ([]string, error) { + slices, err := bulk.Slices(ctx, db) + if err != nil { + return nil, err + } + var distros []string + for _, s := range slices { + if len(distros) == 0 || distros[len(distros)-1] != s[0] { + distros = append(distros, s[0]) + } + } + return distros, nil +} + +// distroPairs lists every unordered pair of distributions in db, in name order. +func distroPairs(ctx context.Context, db *sql.DB) ([][2]string, error) { + distros, err := distroNames(ctx, db) + if err != nil { + return nil, err + } + var pairs [][2]string + for i := range distros { + for _, d := range distros[i+1:] { + pairs = append(pairs, [2]string{distros[i], d}) + } + } + return pairs, nil +} + +// printConfusion prints one join as a confusion matrix of a's outcome (rows) against b's +// (columns), then up to samples of the disagreements. +func printConfusion(env *Env, a, b string, ag bulk.Agreement, samples int) { + fmt.Fprintf(env.Stdout, "%s vs %s joined by %s: %d pairs\n", a, b, ag.Join, ag.Pairs) + if ag.Pairs == 0 { + fmt.Fprintln(env.Stdout) + return + } + tw := tabwriter.NewWriter(env.Stdout, 0, 0, 3, ' ', 0) + fmt.Fprintf(tw, "\t%s resolved\t%s unresolved\n", b, b) + fmt.Fprintf(tw, "%s resolved\t%d agree, %d differ\t%d\n", a, ag.Agree, len(ag.Disagree), ag.OnlyA) + fmt.Fprintf(tw, "%s unresolved\t%d\t%d\n", a, ag.OnlyB, ag.Neither) + tw.Flush() + fmt.Fprintf(env.Stdout, "%.1f%% agree where both resolve\n", pct(ag.Agree, ag.BothResolved)) + for i, d := range ag.Disagree { + if i == samples { + if samples > 0 { + fmt.Fprintf(env.Stdout, " ... %d more\n", len(ag.Disagree)-samples) + } + break + } + fmt.Fprintf(env.Stdout, " %s=%s %s=%s\n", d[0], d[2], d[1], d[3]) + } + fmt.Fprintln(env.Stdout) +} + +func bulkCanonicalize(ctx context.Context, env *Env, args []string) error { + fs := flag.NewFlagSet("bulk canonicalize", flag.ContinueOnError) + dbPath := fs.String("db", defaultDB(env), "SQLite database") + if err := fs.Parse(args); err != nil { + return err + } + db, err := bulk.Open(ctx, *dbPath) + if err != nil { + return err + } + defer db.Close() + // The keys worth a lookup are those in conflict: within a package's own evidence, and + // between distributions' mappings of same-named packages (every pair). The lookups only + // ask GitHub about a key; no distribution's mapping is copied to another. + pairs, err := distroPairs(ctx, db) + if err != nil { + return err + } + var disagreements [][2]string + for _, p := range pairs { + ags, err := bulk.Compare(ctx, db, p[0], p[1]) + if err != nil { + return err + } + for _, ag := range ags { + for _, d := range ag.Disagree { + disagreements = append(disagreements, [2]string{d[2], d[3]}) + } + } + } + keys, err := bulk.ConflictingKeys(ctx, db, "github.com/", disagreements) + if err != nil { + return err + } + env.Logf("canonicalize: %d GitHub keys in conflicts across %d distribution pairs", len(keys), len(pairs)) + now := time.Now() + var aliases []bulk.Alias + failed, gone := 0, 0 + for i, k := range keys { + to, isGone, err := bulk.LookupGitHub(ctx, env.Client, k) + if err != nil { + failed++ + env.Debugf("canonicalize %s: %v", k, err) + continue + } + switch { + case isGone: + gone++ + aliases = append(aliases, bulk.Alias{From: k, To: "", Source: bulk.AliasGitHubGone, CheckedAt: now}) + case to != k: + aliases = append(aliases, bulk.Alias{From: k, To: to, Source: bulk.AliasGitHubRename, CheckedAt: now}) + } + if (i+1)%200 == 0 { + env.Logf("canonicalize: %d/%d", i+1, len(keys)) + } + } + var projectKeys []string + pk, err := db.QueryContext(ctx, `select distinct key from evidence where key like '%/-/project/%'`) + if err != nil { + return err + } + for pk.Next() { + var k string + pk.Scan(&k) + projectKeys = append(projectKeys, k) + } + pk.Close() + projects, projectFailed := bulk.GitLabProjectAliases(ctx, env.Client, projectKeys, func(host string) string { return "https://" + host }) + // Only registry entries that are the package itself: a Rust package lists every vendored + // crate as a source, and those name dependencies, not the upstream. + var registryKeys []string + seenReg := map[string]bool{} + rk, err := db.QueryContext(ctx, `select distinct package, key from evidence where key like 'pypi.org/%' or key like 'crates.io/%'`) + if err != nil { + return err + } + for rk.Next() { + var pkg, k string + rk.Scan(&pkg, &k) + if _, name, _ := strings.Cut(k, "/"); bulk.SameProject(pkg, name) && !seenReg[k] { + seenReg[k] = true + registryKeys = append(registryKeys, k) + } + } + rk.Close() + sort.Strings(registryKeys) + registries, registryFailed := bulk.RegistryAliases(ctx, env.Client, registryKeys, func(host string) string { return "https://" + host }) + for i := range registries { + registries[i].CheckedAt = now + } + aliases = append(aliases, registries...) + for i := range projects { + projects[i].CheckedAt = now + } + aliases = append(aliases, projects...) + // Rule-based aliases are recomputed in full. Stored evidence keeps its keys after aliasing, + // so the keys a rule mapped before are re-checked from the aliases table as well; otherwise + // the next rebuild from raw data would find those aliases gone. + var all []string + rows, err := db.QueryContext(ctx, `select key from evidence where key != '' union select from_key from aliases where source in (?, ?)`, + bulk.AliasMirrorRule, bulk.AliasKDEProjects) + if err != nil { + return err + } + for rows.Next() { + var k string + rows.Scan(&k) + all = append(all, k) + } + rows.Close() + mirrors := bulk.MirrorAliases(all) + for i := range mirrors { + mirrors[i].CheckedAt = now + } + kde, kdeErr := bulk.KDEAliases(ctx, env.Client, all) + if kdeErr != nil { + env.Logf("canonicalize: KDE project list unavailable, keeping the stored KDE aliases: %v", kdeErr) + } + for i := range kde { + kde[i].CheckedAt = now + } + if err := bulk.SaveAliases(ctx, db, aliases); err != nil { + return err + } + if err := bulk.ReplaceAliases(ctx, db, bulk.AliasMirrorRule, mirrors); err != nil { + return err + } + if kdeErr == nil { + if err := bulk.ReplaceAliases(ctx, db, bulk.AliasKDEProjects, kde); err != nil { + return err + } + } + alias, err := bulk.LoadAliases(ctx, db) + if err != nil { + return err + } + slices, err := bulk.Slices(ctx, db) + if err != nil { + return err + } + changed := 0 + for _, sl := range slices { + pkgs, err := bulk.LoadSlice(ctx, db, model.Distro(sl[0]), sl[1]) + if err != nil { + return err + } + n := bulk.ApplyAliases(pkgs, alias) + if n == 0 { + continue + } + changed += n + var src bulk.Source + var fetched string + if err := db.QueryRowContext(ctx, `select origin, fetched_at from sources where distro = ? and suite = ?`, sl[0], sl[1]).Scan(&src.Origin, &fetched); err != nil { + return err + } + src.Distro, src.Suite, src.Packages = model.Distro(sl[0]), sl[1], len(pkgs) + src.FetchedAt, _ = time.Parse(time.RFC3339, fetched) + if err := bulk.Replace(ctx, db, src, pkgs); err != nil { + return err + } + } + fmt.Fprintf(env.Stdout, "canonicalize: %d GitHub keys checked (%d failed), %d renamed, %d gone; %d GitLab project ids resolved (%d failed); %d registry entries resolved of %d (%d failed); %d mirror aliases; %d KDE aliases; %d package mappings changed\n", + len(keys), failed, len(aliases)-len(projects)-len(registries)-gone, gone, len(projects), projectFailed, len(registries), len(registryKeys), registryFailed, len(mirrors), len(kde), changed) + return nil +} diff --git a/cmd/isomorph/main.go b/cmd/isomorph/main.go new file mode 100644 index 0000000..60e58e9 --- /dev/null +++ b/cmd/isomorph/main.go @@ -0,0 +1,107 @@ +// Command isomorph maps distribution source packages to their upstream repositories and, +// through them, to their counterparts in other distributions. +// +// isomorph [global flags] [args] +// +// Global flags: -data DIR (database and exports, default ./data), -cache DIR (HTTP cache), +// -offline, -v. +package main + +import ( + "context" + "flag" + "fmt" + "io" + "os" + "os/signal" + "sort" + "syscall" + + "tangled.org/odd.computer/isomorph/internal/fetch" +) + +// Env is shared by all commands. +type Env struct { + // DataDir holds the database (packages.sqlite) and exports. + DataDir string + Client *fetch.Client + Verbose bool + Stdout io.Writer +} + +// Logf prints a progress line to stderr. +func (e *Env) Logf(format string, args ...any) { + fmt.Fprintf(os.Stderr, format+"\n", args...) +} + +// Debugf prints only with -v. +func (e *Env) Debugf(format string, args ...any) { + if e.Verbose { + e.Logf(format, args...) + } +} + +type command struct { + name string + usage string + run func(ctx context.Context, env *Env, args []string) error +} + +var commands = map[string]command{} + +func register(c command) { commands[c.name] = c } + +func usage() { + fmt.Fprintf(os.Stderr, "usage: isomorph [-data DIR] [-cache DIR] [-offline] [-v] [args]\n\ncommands:\n") + var names []string + for n := range commands { + names = append(names, n) + } + sort.Strings(names) + for _, n := range names { + fmt.Fprintf(os.Stderr, " %-8s %s\n", n, commands[n].usage) + } +} + +func main() { + fs := flag.NewFlagSet("isomorph", flag.ExitOnError) + data := fs.String("data", envOr("ISOMORPH_DATA", "data"), "directory for the database and exports (default $ISOMORPH_DATA or ./data)") + cache := fs.String("cache", "", "HTTP cache directory (default $ISOMORPH_CACHE or the user cache dir)") + offline := fs.Bool("offline", false, "never hit the network; fail on cache misses") + verbose := fs.Bool("v", false, "verbose logging") + fs.Usage = usage + fs.Parse(os.Args[1:]) + if fs.NArg() == 0 { + usage() + os.Exit(2) + } + cmd, ok := commands[fs.Arg(0)] + if !ok { + fmt.Fprintf(os.Stderr, "unknown command %q\n\n", fs.Arg(0)) + usage() + os.Exit(2) + } + client := fetch.New(*cache) + if *offline { + client.Offline = true + } + env := &Env{DataDir: *data, Client: client, Verbose: *verbose, Stdout: os.Stdout} + client.Logf = env.Debugf + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + if err := cmd.run(ctx, env, fs.Args()[1:]); err != nil { + fatal(err) + } +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, "isomorph:", err) + os.Exit(1) +} + +func envOr(k, def string) string { + if v := os.Getenv(k); v != "" { + return v + } + return def +} diff --git a/docs/EVALUATION.md b/docs/EVALUATION.md new file mode 100644 index 0000000..43d91af --- /dev/null +++ b/docs/EVALUATION.md @@ -0,0 +1,222 @@ +# Evaluation + +How well does each distribution's mapping name the right upstream repository, and how +often do distributions agree? This document measures three things: + +1. **Coverage:** the share of source packages mapped to a repository. +2. **Agreement with a ground truth:** Debian's DEP-12 `Repository` field is the one field + whose purpose is to name the upstream repository. Its packages are resolved again without + it, and the result is compared with it. +3. **Agreement between distributions:** same-named packages, and packages built from the + identical release tarball, should map to the same repository. + +The numbers below come from one run on 2026-09-28 over Debian sid/main, Arch (core, extra, +multilib), openSUSE Tumbleweed and Alpine edge (main, community). "How to reproduce" at the +end has the commands. + +Every mapping comes from its own distribution's data: the packaging, and the upstream +projects' own services (GitHub, KDE's project list, PyPI, crates.io). Other distributions +are used only to measure agreement, and to choose which keys are worth a GitHub lookup. + +## Coverage + +| distribution | source packages | mapped to a repository | +|---|---:|---:| +| Arch | 12,358 | 85.3% | +| Alpine edge | 9,494 | 78.3% | +| Debian sid | 41,417 | 77.9% | +| openSUSE Tumbleweed | 17,211 | 75.0% | + +Every mapping records its **basis**, the strongest kind of evidence behind it: + +| basis | Debian | Arch | openSUSE | Alpine | +|---|---:|---:|---:|---:| +| metadata (DEP-12 `Repository`) | 11,404 | | | | +| vcs-source (a git source, `_service` scm, `_repo_url`) | | 4,367 | 564 | 453 | +| release-url (watch URL, release tarball) | 15,984 | 5,014 | 1,192 | 6,693 | +| derived (rb.zq1.de giturls) | | | 6,399 | | +| bug-tracker (DEP-12 `Bug-Database`) | 68 | | | | +| homepage | 4,790 | 1,157 | 4,757 | 285 | + +The basis stands in for a confidence score. Measured agreement per basis, below, says how +far each kind of evidence can be trusted. + +- **Arch is a moving target.** About 1,100 pkgbases were released after the `.SRCINFO` + crawl, so their new packaging commits were not cached for this run and those rows use the + repository database's url. + +## Agreement with Debian's DEP-12 `Repository` + +10,872 Debian packages have a DEP-12 `Repository` that names a repository. Each is resolved +again without `Repository` and `Repository-Browse`: + +- The remaining fields (watch URL, Bug-Database, Homepage) resolve **10,283 (94.6%)**. +- They agree with `Repository` on **9,950 (96.8%)**. + +| basis of the fallback | resolved | agree | +|---|---:|---:| +| release-url | 6,547 | 96.7% | +| bug-tracker | 3,624 | 97.3% | +| homepage | 112 | 81.2% | + +- **Why homepage agreement is lower:** most of its misses are GitHub renames (this check does + not apply aliases), umbrella project pages (gtk2-perl's SourceForge project), and the GNU + Octave packages repository. +- **Limits:** DEP-12 is itself sometimes stale or wrong, so 96.8% is a lower bound on the + fallbacks' precision. Packages with DEP-12 metadata are also better maintained than the + average. + +The same field scores the other distributions, for their same-named packages: + +| distribution | basis | packages | agree with DEP-12 | +|---|---|---:|---:| +| Arch | release-url | 427 | 94.4% | +| Arch | vcs-source | 415 | 93.3% | +| Arch | homepage | 16 | 87.5% | +| Alpine | release-url | 272 | 92.6% | +| Alpine | homepage | 7 | 57.1% | +| openSUSE | derived | 556 | 95.3% | +| openSUSE | homepage | 227 | 88.5% | +| openSUSE | release-url | 70 | 87.1% | +| openSUSE | vcs-source | 29 | 93.1% | + +Most remaining misses are projects that moved (soundtouch to Codeberg, pmbootstrap to +postmarketOS's GitLab), where the other distribution is newer than Debian's field, or +different software with the same name. + +## Agreement between distributions + +`isomorph bulk compare -all` joins every pair of distributions twice, independently of +the mappings: +- by package name; +- for Debian and Arch, by identical source tarball: the sha256 of Debian's `.orig` tarball + and of Arch's release source. + +Each join is shown as a confusion matrix of one distribution's outcome against the other's. + +**By package name.** Each row flattens one matrix. + +| a vs b | both, agree | both, differ | only a | only b | neither | agree where both | +|---|---:|---:|---:|---:|---:|---:| +| Arch vs Debian | 4,584 | 304 | 611 | 156 | 574 | 93.8% | +| Alpine vs Arch | 2,857 | 192 | 54 | 281 | 681 | 93.7% | +| Alpine vs Debian | 2,694 | 194 | 236 | 133 | 478 | 93.3% | +| Arch vs openSUSE | 3,882 | 356 | 355 | 89 | 321 | 91.6% | +| Alpine vs openSUSE | 2,318 | 231 | 138 | 123 | 378 | 90.9% | +| Debian vs openSUSE | 3,898 | 401 | 376 | 367 | 694 | 90.7% | + +- **The name join understates agreement.** It pairs different software that shares a name + (fff, hyperlink, forks of mg), so "differ" overstates real disagreement. +- **Arch fills in for Debian.** 611 shared names resolve in Arch but not in Debian, against + 156 the other way. + +**By identical source tarball (Debian vs Arch):** 1,162 pairs; 668 resolved on both sides, +of which 644 (96.4%) agree. This join needs no names. It stays small because Debian repacks +tarballs and Arch builds many packages from git. + +### Before the error-analysis fixes + +The morning run on 2026-09-28 came before the fixes below and before the openSUSE spec crawl. + +| | before | after | +|---|---:|---:| +| Arch coverage | 83.8% | 85.3% | +| Alpine coverage | 75.4% | 78.3% | +| Debian coverage | 76.7% | 77.9% | +| openSUSE coverage | 65.3% | 75.0% | +| Arch vs Debian | 93.0% of 4,662 | 93.8% of 4,888 | +| Alpine vs Arch | 92.1% of 2,809 | 93.7% of 3,049 | +| Alpine vs Debian | 92.7% of 2,631 | 93.3% of 2,888 | +| Arch vs openSUSE | 89.2% of 3,547 | 91.6% of 4,238 | +| Alpine vs openSUSE | 89.6% of 1,972 | 90.9% of 2,549 | +| Debian vs openSUSE | 90.2% of 3,608 | 90.7% of 4,299 | + +## Error analysis + +**Sample.** It was drawn from the morning run with seed 20260928, and stratified: +- 8 pairs from each of the six "both, differ" cells, 48 in total; +- 50 packages that are unresolved while a same-named package elsewhere resolves: 20 from + openSUSE and 10 from each other distribution. + +**Method.** Each item was labelled from its evidence in all four distributions, and some were +checked against the GitHub API. Shares are weighted by stratum size; with about 50 items per +sample they carry roughly ±10 points. +- The sampler is `eval/sample.py`. +- The labels are in `eval/errors-2026-09-28.tsv`. + +**Why two resolved packages differed** (1,684 pairs): + +| cause | sampled | share | examples | +|---|---:|---:|---| +| stale or dead distribution data: the row names only an old home | 12 | 25% | SourceForge downloads for GitHub projects, notabug to Codeberg, repositories that 404 | +| wrong repository in openSUSE's giturls, or an umbrella homepage | 7 | 18% | awesome-releases, the parent hfst, dead github.com/freedesktop mirrors | +| selection: the right repository was in the row but lost | 9 | 17% | a SourceForge or Launchpad homepage beat a GitHub release URL | +| different software sharing a name | 8 | 15% | hyperlink, fff, forks of mg, each distribution's ca-certificates | +| same project, both keys valid | 7 | 15% | GitHub mirrors, renames not yet canonicalised | +| URL mapping | 5 | 10% | `*.wiki.kernel.org` to Linux, `download.samba.org/pub/rsync` to Samba | + +**Why a package was unresolved while a counterpart resolved** (3,456 packages): + +| cause | sampled | share | +|---|---:|---:| +| only a project website, no repository or forge URL (openSUSE: 16 of 20) | 27 | 55% | +| a host rule would map the homepage (docs.xfce.org, gnu.org/software, netfilter.org, apps.kde.org) | 11 | 23% | +| the evidence named the repository but the key was incomplete (KDE tarballs name no group; an unevaluated `case` in APKBUILDs) | 8 | 12% | +| no URL at all, a Debian-native package, or a registry entry naming no repository | 4 | 10% | + +**What was fixed.** Each fix was counted over all packages, not only the sample. + +1. **KDE.** + - Keys without a group take it from KDE's project list; 489 packages were unresolved this + way. + - The APKBUILD evaluator takes the matching branch of a `case` over a known variable + (47 Alpine Plasma packages), and reads `_repo_url`. +2. **Host rules.** The findvcs-style rules the openSUSE resolver used (GNU and nongnu to + Savannah, cgit to GitLab, Apache, GitHub Pages) now apply to every distribution. About 435 + rows had a homepage one of them covers. New rules cover docs.xfce.org, netfilter.org and + freedesktop.org/software/pulseaudio. +3. **SourceForge and Launchpad homepages.** A homepage on these hosts no longer beats a + release or VCS source naming another repository (130 packages). + - Against DEP-12, where such a homepage and the watch URL disagreed, the watch URL was + right 22 times out of 23. + - GitHub homepages split 16 to 17, so the rule is limited to these two hosts. +4. **URL mapping:** + - kernel.org project subdomains (66 packages) and samba.org's other projects (23) no longer + hint at Linux or Samba; + - GitLab API URLs that name a project by path now name it. +5. **Canonicalisation** checks conflicts across every pair of distributions. It aliases + repositories that no longer exist (GitHub answers 404) to nothing, so the package falls back + to its next candidate. +6. **openSUSE's gap was missing data.** Its repository metadata carries only the spec + `Url:`. The OBS spec and `_service` of the ~5,900 unresolved packages are now read, one + request a second; coverage went from 65.3% to 75.0%. +7. **Secondary spec sources** (`Source1` and later: configuration files, completions, test + data) corroborate a candidate but never decide one. Of the 43 packages they alone + resolved, about a third named another project. + +**Measured and not changed.** When openSUSE's giturls and spec `Url:` disagree (214 +packages), neither is reliably right. On the 22 with a DEP-12 `Repository`, giturls matches +in 12 and `Url:` in 9, so their precedence stays. + +**What remains:** +- **Stale or wrong distribution data** was about 43% of the disagreements before the fixes. + Manual overrides or cross-distribution consensus could address it; neither is built. Each + distribution's mapping stays its own. +- **Packages whose only URL is a project website.** +- **Arch and Alpine source order.** Neither records which source is primary, so a + dependency's git source can still outrank the main tarball (telegram-desktop). + +## How to reproduce + +```sh +isomorph bulk debian # one UDD query +isomorph bulk arch -crawl 20000 # paced .SRCINFO crawl; later runs read the cache +isomorph bulk opensuse -crawl 7000 # paced OBS spec/_service crawl +isomorph bulk alpine # one streamed aports download +GITHUB_TOKEN=... isomorph bulk canonicalize +isomorph bulk compare -all +python3 eval/sample.py differ # the error-analysis sample +``` + +- **Held-out check:** `bulk debian` prints it. `-heldout FILE` writes each disagreement. +- **`GITHUB_TOKEN`:** anonymous GitHub API use is capped at 60 requests an hour. diff --git a/eval/errors-2026-09-28.tsv b/eval/errors-2026-09-28.tsv new file mode 100644 index 0000000..d108f14 --- /dev/null +++ b/eval/errors-2026-09-28.tsv @@ -0,0 +1,99 @@ +cell a b package key_a key_b category wrong_side note +differ alpine arch b4 git.kernel.org/pub/scm/utils/b4/b4 github.com/mricon/b4 same-project arch uses the maintainer's GitHub mirror of the kernel.org repository +differ alpine arch hyphen sourceforge.net/hunspell github.com/hunspell/hyphen stale-data alpine APKBUILD downloads from the old SourceForge hunspell project; development is on GitHub +differ alpine arch ngrep sourceforge.net/ngrep github.com/jpr5/ngrep selection alpine SourceForge url outranks the GitHub source tarball in the same APKBUILD +differ alpine arch rsync git.samba.org/samba github.com/rsyncproject/rsync url-mapping alpine download.samba.org/pub/rsync mapped to the samba repository +differ alpine arch ruby-builder github.com/jimweirich/builder github.com/rails/builder stale-data alpine jimweirich/builder is the original, unmaintained repository; rails/builder continues it +differ alpine arch tcpreplay sourceforge.net/tcpreplay github.com/appneta/tcpreplay stale-data alpine release tarball from SourceForge; development on GitHub +differ alpine arch uvicorn github.com/kludex/uvicorn github.com/encode/uvicorn same-project encode/uvicorn was transferred to Kludex/uvicorn; the rename lookup only covered Debian vs Arch conflicts +differ alpine arch xcb-util-errors gitlab.freedesktop.org/xorg/lib/libxcb-errors cgit.freedesktop.org/xcb/util-errors stale-data arch cgit.freedesktop.org url predates the move to gitlab.freedesktop.org/xorg/lib/libxcb-errors +differ alpine debian edk2 github.com/tianocore/tianocore.github.io github.com/tianocore/edk2 selection alpine wiki page of tianocore.github.io outranks the edk2 source tarball +differ alpine debian guile-sqlite3 codeberg.org/guile-sqlite3/guile-sqlite3 notabug.org/guile-sqlite3/guile-sqlite3 stale-data debian notabug.org homepage; the project moved to Codeberg +differ alpine debian hyperlink github.com/untitaker/hyperlink github.com/python-hyper/hyperlink different-software Rust link checker vs Python URL library +differ alpine debian libraw1394 sourceforge.net/libraw1394 git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux url-mapping debian ieee1394.wiki.kernel.org mapped to torvalds/linux +differ alpine debian libutempter github.com/altlinux/libutempter git.altlinux.org/libutempter same-project git.altlinux.org and its GitHub mirror +differ alpine debian metalog github.com/hvisage/metalog sourceforge.net/metalog selection debian SourceForge homepage outranks the GitHub watch URL +differ alpine debian scons sourceforge.net/scons github.com/scons/scons selection alpine SourceForge download listed before the GitHub archive in the same APKBUILD +differ alpine debian w3m git.sr.ht/~rkta/w3m salsa.debian.org/debian/w3m different-software Debian tracks its own w3m fork on salsa; others package the sourcehut continuation +differ alpine opensuse awesome github.com/awesomewm/awesome github.com/awesomewm/awesome-releases source-data opensuse giturls names awesome-releases, a release-artifact repository +differ alpine opensuse ca-certificates gitlab.alpinelinux.org/alpine/ca-certificates github.com/opensuse/ca-certificates different-software each distribution ships its own ca-certificates +differ alpine opensuse iwd git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux git.kernel.org/pub/scm/network/wireless/iwd url-mapping alpine iwd.wiki.kernel.org mapped to torvalds/linux +differ alpine opensuse libwmf sourceforge.net/wvware github.com/caolanm/libwmf selection alpine SourceForge url outranks the GitHub source tarball +differ alpine opensuse mg github.com/hboetes/mg github.com/troglobit/mg different-software two portable forks of OpenBSD mg (hboetes, troglobit) +differ alpine opensuse ocaml-xml-light github.com/ncannasse/xml-light github.com/gasche/xml-light different-software gasche/xml-light is a GitHub fork of ncannasse/xml-light +differ alpine opensuse tree-sitter-xml github.com/observeroftime/tree-sitter-xml github.com/tree-sitter-grammars/tree-sitter-xml same-project ObserverOfTime/tree-sitter-xml was transferred to tree-sitter-grammars +differ alpine opensuse vde2 github.com/virtualsquare/vde-2 sourceforge.net/vde stale-data opensuse SourceForge homepage only; development on GitHub (virtualsquare/vde-2) +differ arch debian biber github.com/plk/biber sourceforge.net/biblatex-biber selection debian SourceForge homepage outranks the GitHub watch URL +differ arch debian cloud-utils github.com/canonical/cloud-utils launchpad.net/cloud-utils selection debian Launchpad homepage outranks the GitHub watch URL +differ arch debian evolution-ews gitlab.gnome.org/gnome/evolution-ews gitlab.gnome.org/gnome/evolution selection debian parent project's wiki (evolution) outranks the evolution-ews watch URL +differ arch debian haskell-xdg-desktop-entry github.com/taffybar/taffybar github.com/taffybar/xdg-desktop-entry stale-data debian old standalone repository; now a subdirectory of the taffybar monorepo +differ arch debian liborcus gitlab.com/api/v4/projects/orcus/orcus/packages/generic/source/0.21.0/liborcus-0.21.0.tar.xz gitlab.com/orcus/orcus url-mapping arch GitLab generic package-registry URL taken as the key +differ arch debian ruby-net-dns github.com/ruby-dns/net-dns gitlab.com/gitlab-org/gitlab-net-dns different-software Debian packages GitLab's fork (gitlab-net-dns) +differ arch debian smartmontools sourceforge.net/smartmontools github.com/smartmontools/smartmontools stale-data arch release tarball from SourceForge; development on GitHub +differ arch debian telegram-desktop github.com/tdlib/td github.com/telegramdesktop/tdesktop selection arch a git source for the tdlib dependency outranks the tdesktop release tarball +differ arch opensuse helmfile github.com/helmfile/helmfile github.com/roboll/helmfile source-data opensuse giturls names the abandoned roboll/helmfile; spec Url: is right but ranks lower +differ arch opensuse libdazzle gitlab.gnome.org/archive/libdazzle gitlab.gnome.org/gnome/libdazzle same-project GNOME/libdazzle moved to the Archive group (GitLab rename, not canonicalised) +differ arch opensuse libqmi gitlab.freedesktop.org/mobile-broadband/libqmi github.com/freedesktop/libqmi source-data opensuse giturls names github.com/freedesktop/libqmi, which does not exist +differ arch opensuse mdds gitlab.com/api/v4/projects/mdds/mdds/packages/generic/source/3.2.1/mdds-3.2.1.tar.xz gitlab.com/mdds/mdds url-mapping arch GitLab generic package-registry URL taken as the key +differ arch opensuse python-parsedatetime github.com/bear/parsedatetime github.com/regebro/parsedatetime stale-data opensuse spec Url: and giturls name regebro/parsedatetime, which does not exist +differ arch opensuse schroedinger launchpad.net/schroedinger sourceforge.net/schrodinger stale-data opensuse old SourceForge FAQ page; releases on Launchpad +differ arch opensuse startup-notification gitlab.freedesktop.org/xdg/startup-notification github.com/freedesktop/startup-notification source-data opensuse giturls names github.com/freedesktop/startup-notification, which does not exist +differ arch opensuse switchboard-plug-applications github.com/elementary/switchboard-plug-applications github.com/elementary/settings-applications same-project renamed to elementary/settings-applications +differ debian opensuse fff github.com/meekrosoft/fff github.com/dmtrkovalenko/fff different-software C fake-function framework vs a file finder +differ debian opensuse hfst-ospell github.com/hfst/hfst-ospell github.com/hfst/hfst source-data opensuse giturls names the parent hfst repository +differ debian opensuse ibus-chewing codeberg.org/chewing/ibus-chewing github.com/chewing/ibus-chewing stale-data opensuse GitHub repository says 'Migrated to Codeberg' +differ debian opensuse jline sourceforge.net/jline github.com/jline/jline2 different-software Debian's jline is jline 1 (SourceForge); openSUSE's is jline2 +differ debian opensuse kernelshark git.kernel.org/pub/scm/utils/trace-cmd/kernel-shark git.kernel.org/pub/scm/linux/kernel/git/rostedt/trace-cmd source-data opensuse giturls names trace-cmd, the repository kernelshark left; spec Url: is right +differ debian opensuse libaec github.com/deutsches-klimarechenzentrum/libaec gitlab.dkrz.de/k202009/libaec same-project gitlab.dkrz.de and the DKRZ GitHub organisation +differ debian opensuse libcdio-paranoia github.com/libcdio/libcdio-paranoia git.savannah.gnu.org/libcdio source-data opensuse spec Url: is the umbrella libcdio Savannah project +differ debian opensuse luajit github.com/openresty/luajit2 github.com/openresty/luajit stale-data opensuse spec Url: names openresty/luajit, which does not exist (openresty/luajit2) +unresolved alpine botan3 website-only alpine botan.randombit.net +unresolved alpine flatpak-kcm key-incomplete alpine source uses $_rel set in a case block; _repo_url names invent.kde.org/plasma/flatpak-kcm +unresolved alpine kmousetool key-incomplete alpine download.kde.org tarball gives invent.kde.org/kmousetool without its group +unresolved alpine kphotoalbum key-incomplete alpine download.kde.org tarball gives invent.kde.org/kphotoalbum without its group +unresolved alpine libopensmtpd website-only alpine imperialat.at page; arch has the git source +unresolved alpine msmtp website-only alpine marlam.de; arch has the git source +unresolved alpine nss website-only alpine developer.mozilla.org page +unresolved alpine ristretto host-rule alpine docs.xfce.org/apps/ristretto -> gitlab.xfce.org/apps/ristretto +unresolved alpine ulogd website-only alpine netfilter.org/projects/ulogd; the repository is ulogd2 +unresolved alpine xfce4-places-plugin host-rule alpine docs.xfce.org/panel-plugins/... -> gitlab.xfce.org +unresolved arch batctl website-only arch open-mesh.org +unresolved arch blinken key-incomplete arch KDE key without group +unresolved arch kate key-incomplete arch KDE key without group +unresolved arch kspaceduel key-incomplete arch KDE key without group +unresolved arch libnetfilter_cttimeout host-rule arch netfilter.org/projects/X -> git.netfilter.org/X +unresolved arch python-sgmllib3k registry-only arch PyPI entry names no repository +unresolved arch suil website-only arch drobilla.net +unresolved arch unifdef website-only arch dotat.at +unresolved arch xbindkeys host-rule arch nongnu.org rule exists in the openSUSE resolver only +unresolved arch zanshin key-incomplete arch KDE key without group +unresolved debian bridge-utils no-url debian no Homepage or watch URL +unresolved debian dillo website-only debian dillo-browser.org +unresolved debian gnuchess host-rule debian gnu.org/software rule exists in the openSUSE resolver only +unresolved debian jetring distro-native debian Debian-native package; its upstream is salsa +unresolved debian kdenlive key-incomplete debian KDE key without group +unresolved debian libeot no-url debian no Homepage or watch URL +unresolved debian libx86 website-only debian codon.org.uk page +unresolved debian mate-common website-only debian mate-desktop.org +unresolved debian paprefs host-rule debian freedesktop.org/software/pulseaudio/paprefs -> gitlab.freedesktop.org/pulseaudio/paprefs +unresolved debian xfce4-fsguard-plugin host-rule debian docs.xfce.org -> gitlab.xfce.org +unresolved opensuse a52dec host-rule opensuse Url: is a Gitea repository on an unknown host (git.adelielinux.org) +unresolved opensuse apcupsd website-only opensuse apcupsd.com +unresolved opensuse cardpeek website-only opensuse pannetrat.com +unresolved opensuse fooyin website-only opensuse fooyin.org +unresolved opensuse iotop website-only opensuse guichaz.free.fr +unresolved opensuse kasts host-rule opensuse apps.kde.org/kasts -> KDE project lookup +unresolved opensuse kf6-kplotting website-only opensuse www.kde.org +unresolved opensuse kf6-syndication website-only opensuse www.kde.org +unresolved opensuse lapackpp website-only opensuse icl.utk.edu/slate +unresolved opensuse libcap website-only opensuse sites.google.com +unresolved opensuse libgedit-gtksourceview website-only opensuse gedit-technology.net +unresolved opensuse mosh website-only opensuse mosh.org +unresolved opensuse multitail website-only opensuse vanheusden.com +unresolved opensuse poezio website-only opensuse poez.io +unresolved opensuse python-scikit-image website-only opensuse scikit-image.org +unresolved opensuse sratom website-only opensuse drobilla.net +unresolved opensuse unknown-horizons website-only opensuse unknown-horizons.org +unresolved opensuse xf86-video-tdfx website-only opensuse xorg.freedesktop.org +unresolved opensuse xfce4-clipman-plugin host-rule opensuse docs.xfce.org -> gitlab.xfce.org +unresolved opensuse xfce4-panel host-rule opensuse docs.xfce.org -> gitlab.xfce.org diff --git a/eval/sample.py b/eval/sample.py new file mode 100644 index 0000000..5eab535 --- /dev/null +++ b/eval/sample.py @@ -0,0 +1,68 @@ +"""Stratified error-analysis sample over the name-join confusion matrices. + +differ: pairs (a, b, name) where both resolve to different keys, 8 per distro pair. +unres: (distro, name) unresolved while a same-named package elsewhere resolves, + 20 for openSUSE and 10 for each other distro. +Prints stratum sizes and, per item, every distro's row and evidence for that name. +The labels for the 2026-09-28 sample are in errors-2026-09-28.tsv. + +Usage: python3 eval/sample.py differ|unres [packages.sqlite] +""" +import itertools, random, sqlite3, sys + +DB = sys.argv[2] if len(sys.argv) > 2 else "data/packages.sqlite" +SEED = 20260928 +DISTROS = ["alpine", "arch", "debian", "opensuse"] +db = sqlite3.connect(DB) + +def rows(name): + out = {} + for d, suite, key, basis, home in db.execute( + "select distro, suite, coalesce(upstream_key,''), coalesce(basis,''), coalesce(homepage,'') " + "from packages where package=? order by distro, suite", (name,)): + ev = db.execute( + "select source, value, coalesce(key,'') from evidence where distro=? and suite=? and package=? " + "order by weight desc", (d, suite, name)).fetchall() + out.setdefault(d, (suite, key, basis, home, ev)) + return out + +def dump(tag, header, name): + print(f"[{tag}] {header} name={name}") + for d, (suite, key, basis, home, ev) in rows(name).items(): + print(f" {d:8} {key or '-'} ({basis or 'unresolved'}) home={home}") + seen, n = set(), 0 + for src, val, k in ev: + if val in seen: + continue + seen.add(val) + n += 1 + if n > 6: + print(f" ... {len(set(v for _, v, _ in ev)) - 6} more") + break + print(f" {src}: {val[:110]}" + (f" -> {k}" if k else "")) + print() + +rng = random.Random(SEED) +resolved = {} +for d, name, key in db.execute("select distro, package, coalesce(upstream_key,'') from packages"): + if key or name not in resolved.get(d, {}): + resolved.setdefault(d, {})[name] = key + +mode = sys.argv[1] +if mode == "differ": + for a, b in itertools.combinations(DISTROS, 2): + pop = sorted(n for n in resolved[a].keys() & resolved[b].keys() + if resolved[a][n] and resolved[b][n] and resolved[a][n] != resolved[b][n]) + pick = rng.sample(pop, min(8, len(pop))) + print(f"### stratum {a} vs {b}: {len(pop)} differing names, sampled {len(pick)}\n") + for i, n in enumerate(sorted(pick)): + dump(f"D-{a[:2]}{b[:2]}-{i}", f"{a} vs {b}", n) +elif mode == "unres": + want = {"opensuse": 20, "debian": 10, "arch": 10, "alpine": 10} + for d in DISTROS: + pop = sorted(n for n, k in resolved[d].items() if not k and any( + resolved[o].get(n) for o in DISTROS if o != d)) + pick = rng.sample(pop, want[d]) + print(f"### stratum {d} unresolved, a counterpart resolved: {len(pop)} names, sampled {len(pick)}\n") + for i, n in enumerate(sorted(pick)): + dump(f"U-{d[:2]}-{i}", f"{d} unresolved", n) diff --git a/go.mod b/go.mod index 55627d4..1237b82 100644 --- a/go.mod +++ b/go.mod @@ -5,10 +5,20 @@ go 1.27.1 require ( github.com/jackc/pgx/v5 v5.11.0 github.com/klauspost/compress v1.20.1 + modernc.org/sqlite v1.59.0 ) require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.29.0 // indirect + modernc.org/libc v1.75.7 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.12.1 // indirect ) diff --git a/go.sum b/go.sum index e8680e9..8726946 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,14 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= @@ -11,18 +19,58 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ= github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8= +modernc.org/cc/v4 v4.29.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.35.0 h1:F+TUsmw09QxLzmi3aeYYGxjAXarmZaKgj3mKQHNaA8w= +modernc.org/ccgo/v4 v4.35.0/go.mod h1:qrVGs9S3Sr2Ztcg9ve+kTAYMp5a3YvWjo+SoN06kJ5I= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc= +modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.75.7 h1:o3DTP9/0p9pKmY2WCKQaySW6wIiZhNM7wc2lUoyhfew= +modernc.org/libc v1.75.7/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= +modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.59.0 h1:X1es1GpqBlS/5T+vbM4HLUdaa8OtQx468DF2vrx+38A= +modernc.org/sqlite v1.59.0/go.mod h1:+paeT2A3iPRHkQDwG7oA6Tk0zQd5woMEI8q7orfry8k= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/internal/bulk/alias.go b/internal/bulk/alias.go new file mode 100644 index 0000000..364cd77 --- /dev/null +++ b/internal/bulk/alias.go @@ -0,0 +1,510 @@ +package bulk + +import ( + "context" + "database/sql" + "regexp" + "sort" + "strings" + "sync" + "time" + + "tangled.org/odd.computer/isomorph/internal/distro" + "tangled.org/odd.computer/isomorph/internal/fetch" + "tangled.org/odd.computer/isomorph/internal/model" + "tangled.org/odd.computer/isomorph/internal/repourl" +) + +// Aliases map an upstream key onto the key that names the same repository canonically: a +// renamed or transferred GitHub repository onto its current name, and a known mirror onto the +// project's main repository. They are applied by rescoring each package from its evidence, so +// two fields naming the same repository under different keys reinforce each other instead of +// competing. + +// Alias sources. +const ( + AliasGitHubRename = "github-rename" + AliasMirrorRule = "mirror-rule" + AliasKDEProjects = "kde-projects" + // AliasGitHubGone maps a GitHub repository that no longer exists to "": its evidence + // drops out, and the package falls back to its other evidence or is left unresolved. + AliasGitHubGone = "github-gone" +) + +// Alias records one mapping and how it was learned. +type Alias struct { + From, To, Source string + CheckedAt time.Time +} + +// MirrorAliases applies the documented mirror rules to keys: +// - gitbox.apache.org/[repos/asf/]X -> github.com/apache/X (the GitHub mirror, which the GitHub +// API can follow further); +// - github.com/gnome/X -> gitlab.gnome.org/gnome/X; +// - github.com/golang/X -> go.googlesource.com/X. +func MirrorAliases(keys []string) []Alias { + var out []Alias + for _, k := range keys { + to := "" + switch { + case strings.HasPrefix(k, "gitbox.apache.org/"): + // repourl keys gitbox repositories as gitbox.apache.org/; older keys kept + // the repos/asf/ prefix. + to = "github.com/apache/" + strings.TrimPrefix(strings.TrimPrefix(k, "gitbox.apache.org/"), "repos/asf/") + case strings.HasPrefix(k, "github.com/gnome/"): + to = "gitlab.gnome.org/gnome/" + strings.TrimPrefix(k, "github.com/gnome/") + case strings.HasPrefix(k, "github.com/golang/"): + to = "go.googlesource.com/" + strings.TrimPrefix(k, "github.com/golang/") + } + if to != "" && to != k { + out = append(out, Alias{From: k, To: to, Source: AliasMirrorRule}) + } + } + return out +} + +// KDEProjectsURL lists every KDE repository path (accessibility/kmousetool, ...). +const KDEProjectsURL = "https://projects.kde.org/api/v1/projects" + +// KDEAliases gives KDE keys without a group their repository path from KDE's own project list: +// download.kde.org tarballs and apps.kde.org pages name a project (invent.kde.org/kmousetool), +// and KDE's GitHub mirror drops the group (github.com/kde/kmousetool); both map to +// invent.kde.org/accessibility/kmousetool when exactly one KDE repository has that name. +func KDEAliases(ctx context.Context, c *fetch.Client, keys []string) ([]Alias, error) { + var paths []string + if err := c.GetJSON(ctx, KDEProjectsURL, &paths, fetch.WithTTL(7*24*time.Hour)); err != nil { + return nil, err + } + return kdeAliases(paths, keys), nil +} + +func kdeAliases(paths, keys []string) []Alias { + byName := map[string][]string{} + for _, p := range paths { + p = strings.ToLower(p) + byName[p[strings.LastIndexByte(p, '/')+1:]] = append(byName[p[strings.LastIndexByte(p, '/')+1:]], p) + } + var out []Alias + for _, k := range keys { + name := "" + switch { + case strings.HasPrefix(k, "github.com/kde/"): + name = strings.TrimPrefix(k, "github.com/kde/") + case strings.HasPrefix(k, "invent.kde.org/") && strings.Count(k, "/") == 1: + name = strings.TrimPrefix(k, "invent.kde.org/") + } + if c := byName[name]; name != "" && len(c) == 1 { + out = append(out, Alias{From: k, To: "invent.kde.org/" + c[0], Source: AliasKDEProjects}) + } + } + return out +} + +// Resolve follows an alias chain to its end (bounded, in case of cycles). +func Resolve(alias map[string]string, k string) string { + for i := 0; i < 5; i++ { + to, ok := alias[k] + if !ok || to == k { + return k + } + k = to + } + return k +} + +// Rescore recomputes a package's upstream from its evidence after mapping every evidence key +// through alias: evidence for keys that alias together combines as 1-prod(1-w). +func Rescore(p *Package, alias map[string]string) { + type cand struct { + key string + miss float64 // prod(1-w) + basis string + } + byKey := map[string]*cand{} + var order []string + for i := range p.Evidence { + e := &p.Evidence[i] + if e.Key == "" { + continue + } + if _, name, ok := registryKey(e.Key); ok && !SameProject(p.Name, name) { + continue // a dependency fetched from a registry (vendored crates), not this project + } + e.Key = Resolve(alias, e.Key) + if !isRepoKey(e.Key) { + continue // a registry entry that no alias resolved + } + c := byKey[e.Key] + if c == nil { + c = &cand{key: e.Key, miss: 1} + byKey[e.Key] = c + order = append(order, e.Key) + } + c.miss *= 1 - e.Weight + if b := BasisOf(e.Source); c.basis == "" || basisRank[b] < basisRank[c.basis] { + c.basis = b + } + } + best, bestScore := "", 0.0 + for _, k := range order { + if byKey[k].basis == BasisSecondary { + continue // corroborates, never decides + } + if s := (1 - byKey[k].miss) * distro.PackagingPenalty(k); s > bestScore { + best, bestScore = k, s + } + } + if best == "" { + // Nothing left: every key was aliased away (a repository that no longer exists) or is + // not a repository. + p.UpstreamKey, p.Repo, p.Status, p.Basis = "", "", StatusUnresolved, "" + return + } + if byKey[best].basis == BasisHomepage && onStaleHomeHost(best) { + // See staleHomeHosts: the best release or VCS source wins over such a homepage. + alt, altScore := "", 0.0 + for _, k := range order { + if b := byKey[k].basis; k != best && (b == BasisReleaseURL || b == BasisVCSSource) { + if s := (1 - byKey[k].miss) * distro.PackagingPenalty(k); s > altScore { + alt, altScore = k, s + } + } + } + if alt != "" { + best = alt + } + } + if best != p.UpstreamKey { + p.UpstreamKey, p.Repo = best, "https://"+best + if c, err := repourl.Parse("https://" + best); err == nil { + p.Repo = c.URL + } + } + p.Status = StatusResolved + p.Basis = byKey[best].basis +} + +// ApplyAliases rescores every package whose evidence names an aliased key. +func ApplyAliases(pkgs []Package, alias map[string]string) int { + n := 0 + for i := range pkgs { + for _, e := range pkgs[i].Evidence { + if _, ok := alias[e.Key]; ok { + before := pkgs[i].UpstreamKey + Rescore(&pkgs[i], alias) + if pkgs[i].UpstreamKey != before { + n++ + } + break + } + } + } + return n +} + +// LoadAliases reads the aliases table. +func LoadAliases(ctx context.Context, db *sql.DB) (map[string]string, error) { + rows, err := db.QueryContext(ctx, `select from_key, to_key from aliases`) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]string{} + for rows.Next() { + var f, t string + if err := rows.Scan(&f, &t); err != nil { + return nil, err + } + out[f] = t + } + return out, rows.Err() +} + +// SaveAliases upserts aliases. +func SaveAliases(ctx context.Context, db *sql.DB, as []Alias) error { + return saveAliases(ctx, db, "", as) +} + +// ReplaceAliases replaces every stored alias of source with as: rule-based sources are +// recomputed in full on each run, so a rule that no longer applies must not linger. +func ReplaceAliases(ctx context.Context, db *sql.DB, source string, as []Alias) error { + return saveAliases(ctx, db, source, as) +} + +func saveAliases(ctx context.Context, db *sql.DB, replace string, as []Alias) error { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + if replace != "" { + if _, err := tx.ExecContext(ctx, `delete from aliases where source = ?`, replace); err != nil { + return err + } + } + for _, a := range as { + if _, err := tx.ExecContext(ctx, `insert or replace into aliases values (?,?,?,?)`, + a.From, a.To, a.Source, a.CheckedAt.UTC().Format(time.RFC3339)); err != nil { + return err + } + } + return tx.Commit() +} + +// LoadSlice reads one (distro, suite) slice back with its evidence, binaries and hashes, one +// query per table. +func LoadSlice(ctx context.Context, db *sql.DB, d model.Distro, suite string) ([]Package, error) { + rows, err := db.QueryContext(ctx, `select distro, suite, package, version, coalesce(osv_ecosystem,''), coalesce(purl,''), + coalesce(homepage,''), coalesce(upstream_key,''), coalesce(repo,''), status, coalesce(basis,'') + from packages where distro = ? and suite = ? order by package`, d, suite) + if err != nil { + return nil, err + } + var out []Package + idx := map[string]int{} + for rows.Next() { + var p Package + if err := rows.Scan(&p.Distro, &p.Suite, &p.Name, &p.Version, &p.OSVEcosystem, &p.PURL, &p.Homepage, + &p.UpstreamKey, &p.Repo, &p.Status, &p.Basis); err != nil { + rows.Close() + return nil, err + } + idx[p.Name] = len(out) + out = append(out, p) + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, err + } + scan := func(q string, add func(r *sql.Rows) error) error { + r, err := db.QueryContext(ctx, q, d, suite) + if err != nil { + return err + } + defer r.Close() + for r.Next() { + if err := add(r); err != nil { + return err + } + } + return r.Err() + } + if err := scan(`select package, source, value, coalesce(key,''), weight from evidence where distro = ? and suite = ? order by rowid`, + func(r *sql.Rows) error { + var name string + var e model.Evidence + if err := r.Scan(&name, &e.Source, &e.Value, &e.Key, &e.Weight); err != nil { + return err + } + if i, ok := idx[name]; ok { + out[i].Evidence = append(out[i].Evidence, e) + } + return nil + }); err != nil { + return nil, err + } + for _, t := range []struct { + q string + dst func(p *Package) *[]string + }{ + {`select package, binary from binaries where distro = ? and suite = ? order by rowid`, func(p *Package) *[]string { return &p.Binaries }}, + {`select package, sha256 from source_sha256 where distro = ? and suite = ? order by rowid`, func(p *Package) *[]string { return &p.SourceSHA256 }}, + } { + if err := scan(t.q, func(r *sql.Rows) error { + var name, v string + if err := r.Scan(&name, &v); err != nil { + return err + } + if i, ok := idx[name]; ok { + dst := t.dst(&out[i]) + *dst = append(*dst, v) + } + return nil + }); err != nil { + return nil, err + } + } + return out, nil +} + +// Slices lists the (distro, suite) slices in the table. +func Slices(ctx context.Context, db *sql.DB) ([][2]string, error) { + rows, err := db.QueryContext(ctx, `select distro, suite from sources order by distro, suite`) + if err != nil { + return nil, err + } + defer rows.Close() + var out [][2]string + for rows.Next() { + var d, s string + if err := rows.Scan(&d, &s); err != nil { + return nil, err + } + out = append(out, [2]string{d, s}) + } + return out, rows.Err() +} + +// ConflictingKeys returns the keys worth canonicalising: keys of packages whose evidence names +// more than one repository on the same host (a rename leaves the old and new name side by +// side), plus the keys of cross-distro disagreements, restricted to hosts (sorted, unique). +func ConflictingKeys(ctx context.Context, db *sql.DB, hostPrefix string, disagreements [][2]string) ([]string, error) { + rows, err := db.QueryContext(ctx, `select distinct e.key from evidence e join ( + select distro, suite, package from evidence where key like ? || '%' + group by distro, suite, package having count(distinct key) > 1) c + on e.distro = c.distro and e.suite = c.suite and e.package = c.package + where e.key like ? || '%'`, hostPrefix, hostPrefix) + if err != nil { + return nil, err + } + set := map[string]bool{} + for rows.Next() { + var k string + if err := rows.Scan(&k); err != nil { + rows.Close() + return nil, err + } + set[k] = true + } + rows.Close() + for _, d := range disagreements { + for _, k := range d { + if strings.HasPrefix(k, hostPrefix) { + set[k] = true + } + } + } + out := make([]string, 0, len(set)) + for k := range set { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +// AliasGitLabProject marks aliases from a numeric GitLab project id to the project's path. +const AliasGitLabProject = "gitlab-project-id" + +// gitlabProjectKeyRe matches the keys repourl gives GitLab URLs that name a project by number. +var gitlabProjectKeyRe = regexp.MustCompile(`^([^/]+)/-/project/(\d+)$`) + +// GitLabProjectAliases resolves "/-/project/" keys through the host's API +// (/api/v4/projects/ answers anonymously for public projects). base maps a host onto the +// scheme and host to query (tests point it at an httptest server). +func GitLabProjectAliases(ctx context.Context, c *fetch.Client, keys []string, base func(host string) string) ([]Alias, int) { + var out []Alias + failed := 0 + for _, k := range keys { + m := gitlabProjectKeyRe.FindStringSubmatch(k) + if m == nil { + continue + } + var resp struct { + Path string `json:"path_with_namespace"` + } + if err := c.GetJSON(ctx, base(m[1])+"/api/v4/projects/"+m[2], &resp, fetch.WithTTL(30*24*time.Hour)); err != nil || resp.Path == "" { + failed++ + continue + } + cn, err := repourl.Parse("https://" + m[1] + "/" + resp.Path) + if err != nil || cn.Kind != repourl.KindRepo { + failed++ + continue + } + out = append(out, Alias{From: k, To: cn.Key, Source: AliasGitLabProject}) + } + return out, failed +} + +var repoKeyCache sync.Map + +// isRepoKey reports whether key names a repository (not a registry entry or other hint). +func isRepoKey(key string) bool { + if v, ok := repoKeyCache.Load(key); ok { + return v.(bool) + } + c, err := repourl.Parse("https://" + key) + ok := err == nil && c.Kind == repourl.KindRepo + repoKeyCache.Store(key, ok) + return ok +} + +// AliasRegistry marks aliases from a registry entry (pypi.org/, crates.io/) to the +// repository its metadata names. +const AliasRegistry = "registry-metadata" + +// RegistryAliases resolves registry keys through the registry's metadata: PyPI with +// oss-rebuild's link selection (pypi.go) and crates.io's repository field. base +// maps a registry host onto the scheme and host to query (tests point it at an httptest +// server). +func RegistryAliases(ctx context.Context, c *fetch.Client, keys []string, base func(host string) string) ([]Alias, int) { + var out []Alias + failed := 0 + for _, k := range keys { + host, name, ok := strings.Cut(k, "/") + if !ok { + continue + } + var urls []string + switch host { + case "pypi.org": + var resp struct { + Info pypiInfo `json:"info"` + } + if err := c.GetJSON(ctx, base(host)+"/pypi/"+name+"/json", &resp, fetch.WithTTL(30*24*time.Hour)); err != nil { + failed++ + continue + } + if r := pypiRepo(resp.Info); r != "" { + if !strings.Contains(r, "://") { + r = "https://" + r + } + urls = []string{r} + } + case "crates.io": + var resp struct { + Crate struct { + Repository string `json:"repository"` + Homepage string `json:"homepage"` + } `json:"crate"` + } + if err := c.GetJSON(ctx, base(host)+"/api/v1/crates/"+name, &resp, fetch.WithTTL(30*24*time.Hour)); err != nil { + failed++ + continue + } + urls = []string{resp.Crate.Repository, resp.Crate.Homepage} + default: + continue + } + for _, u := range urls { + if cn, err := repourl.Parse(u); err == nil && cn.Kind == repourl.KindRepo { + out = append(out, Alias{From: k, To: cn.Key, Source: AliasRegistry}) + break + } + } + } + return out, failed +} + +// registryKey splits a registry key (pypi.org/, crates.io/). +func registryKey(k string) (registry, name string, ok bool) { + registry, name, ok = strings.Cut(k, "/") + if !ok || (registry != "pypi.org" && registry != "crates.io") { + return "", "", false + } + return registry, name, true +} + +// SameProject reports whether a registry project name is the distro package itself rather than +// one of its dependencies: equal after lower-casing, unifying - _ . and dropping the ecosystem +// prefixes distros add (python-, python3-, py3-, rust-, ...). +func SameProject(pkg, registryName string) bool { + norm := func(s string) string { + s = strings.NewReplacer("_", "-", ".", "-").Replace(strings.ToLower(s)) + for _, p := range []string{"python3-", "python-", "py3-", "py-", "rust-", "golang-"} { + s = strings.TrimPrefix(s, p) + } + return s + } + return norm(pkg) == norm(registryName) +} diff --git a/internal/bulk/alias_test.go b/internal/bulk/alias_test.go new file mode 100644 index 0000000..b8bd02e --- /dev/null +++ b/internal/bulk/alias_test.go @@ -0,0 +1,188 @@ +package bulk + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "tangled.org/odd.computer/isomorph/internal/distro/opensuse" + "tangled.org/odd.computer/isomorph/internal/fetch" + "tangled.org/odd.computer/isomorph/internal/model" +) + +func TestRescoreMergesAliases(t *testing.T) { + // DEP-12 names the pre-transfer repository, the homepage and watch URL the new one; without + // the alias the stale key wins on weight, with it all three agree. + p := Package{ + UpstreamKey: "github.com/spacetelescope/asdf-standard", Status: StatusResolved, Basis: BasisMetadata, + Evidence: []model.Evidence{ + {Source: "debian/upstream/metadata:Repository", Key: "github.com/spacetelescope/asdf-standard", Weight: 0.95}, + {Source: "debian/watch:upstream_url", Key: "github.com/asdf-format/asdf-standard", Weight: 0.36}, + {Source: "debian/control:Homepage", Key: "github.com/asdf-format/asdf-standard", Weight: 0.4}, + }, + } + alias := map[string]string{"github.com/spacetelescope/asdf-standard": "github.com/asdf-format/asdf-standard"} + pkgs := []Package{p} + if n := ApplyAliases(pkgs, alias); n != 1 { + t.Fatalf("changed %d", n) + } + got := pkgs[0] + if got.UpstreamKey != "github.com/asdf-format/asdf-standard" || got.Repo != "https://github.com/asdf-format/asdf-standard" || got.Basis != BasisMetadata { + t.Fatalf("rescored: %+v", got) + } +} + +func TestMirrorAliases(t *testing.T) { + got := MirrorAliases([]string{ + "gitbox.apache.org/repos/asf/activemq", "gitbox.apache.org/zookeeper", "github.com/gnome/glib", "github.com/golang/text", + "github.com/kde/heaptrack", "github.com/other/x", "invent.kde.org/heaptrack", + }) + want := map[string]string{ + "gitbox.apache.org/repos/asf/activemq": "github.com/apache/activemq", + "gitbox.apache.org/zookeeper": "github.com/apache/zookeeper", + "github.com/gnome/glib": "gitlab.gnome.org/gnome/glib", + "github.com/golang/text": "go.googlesource.com/text", + } + if len(got) != len(want) { + t.Fatalf("got %+v", got) + } + for _, a := range got { + if want[a.From] != a.To || a.Source != AliasMirrorRule { + t.Errorf("alias %+v", a) + } + } + kde := kdeAliases([]string{"sdk/heaptrack", "a/dup", "b/dup", "accessibility/KMouseTool"}, []string{ + "github.com/kde/heaptrack", "github.com/kde/dup", "github.com/kde/unknown", + "invent.kde.org/kmousetool", "invent.kde.org/sdk/heaptrack", + }) + wantKDE := map[string]string{ + "github.com/kde/heaptrack": "invent.kde.org/sdk/heaptrack", + "invent.kde.org/kmousetool": "invent.kde.org/accessibility/kmousetool", + } + if len(kde) != len(wantKDE) { + t.Fatalf("kde aliases %+v", kde) + } + for _, a := range kde { + if wantKDE[a.From] != a.To || a.Source != AliasKDEProjects { + t.Errorf("kde alias %+v", a) + } + } + if Resolve(map[string]string{"a": "b", "b": "c", "c": "c"}, "a") != "c" { + t.Error("alias chains must be followed") + } +} + +func TestGitLabProjectAliases(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v4/projects/20183" { + fmt.Fprint(w, `{"id":20183,"path_with_namespace":"World/amberol"}`) + return + } + http.NotFound(w, r) + })) + defer srv.Close() + c := fetch.New(t.TempDir()) + c.MinInterval, c.Retries = 0, 0 + got, failed := GitLabProjectAliases(context.Background(), c, + []string{"gitlab.gnome.org/-/project/20183", "gitlab.gnome.org/-/project/404", "github.com/x/y"}, + func(string) string { return srv.URL }) + if failed != 1 || len(got) != 1 || got[0].From != "gitlab.gnome.org/-/project/20183" || got[0].To != "gitlab.gnome.org/world/amberol" || got[0].Source != AliasGitLabProject { + t.Fatalf("aliases %+v failed %d", got, failed) + } +} + +func TestRegistryAliases(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/pypi/acme-tiny/json": + fmt.Fprint(w, `{"info":{"home_page":"","project_urls":{"Documentation":"https://docs.example.org","Source":"https://github.com/diafygi/acme-tiny"}}}`) + case "/pypi/docs-only/json": + fmt.Fprint(w, `{"info":{"home_page":"https://docs-only.example.org","project_urls":{}}}`) + case "/api/v1/crates/ripgrep": + fmt.Fprint(w, `{"crate":{"repository":"https://github.com/BurntSushi/ripgrep","homepage":""}}`) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + c := fetch.New(t.TempDir()) + c.MinInterval, c.Retries = 0, 0 + got, failed := RegistryAliases(context.Background(), c, + []string{"pypi.org/acme-tiny", "pypi.org/docs-only", "pypi.org/gone", "crates.io/ripgrep", "github.com/x/y"}, + func(string) string { return srv.URL }) + want := map[string]string{"pypi.org/acme-tiny": "github.com/diafygi/acme-tiny", "crates.io/ripgrep": "github.com/burntsushi/ripgrep"} + if failed != 1 || len(got) != len(want) { + t.Fatalf("got %+v failed %d", got, failed) + } + for _, a := range got { + if want[a.From] != a.To || a.Source != AliasRegistry { + t.Errorf("alias %+v", a) + } + } +} + +func TestRegistryHintIsNotARepository(t *testing.T) { + // A PyPI sdist outweighs the homepage, but a registry entry is not a repository: the + // homepage's repository is chosen, and rescoring without an alias keeps it. + pkgs := FromOpenSUSE([]opensuse.RPM{{Name: "acme-tiny", Arch: "src", Version: "5-1", URL: "https://github.com/diafygi/acme-tiny"}}, nil, nil, nil) + p := pkgs[0] + p.Evidence = append(p.Evidence, model.Evidence{Source: "APKBUILD:source", Key: "pypi.org/acme-tiny", Weight: 0.9}) + Rescore(&p, nil) + if p.UpstreamKey != "github.com/diafygi/acme-tiny" { + t.Fatalf("registry key chosen: %+v", p) + } + Rescore(&p, map[string]string{"pypi.org/acme-tiny": "github.com/diafygi/acme-tiny"}) + if p.UpstreamKey != "github.com/diafygi/acme-tiny" || p.Basis != BasisReleaseURL { + t.Fatalf("aliased registry evidence should support the repository: %+v", p) + } +} + +// TestRescoreGone: a repository aliased away as gone drops out; the package falls back to its +// next candidate, or is left unresolved when there is none. +func TestRescoreGone(t *testing.T) { + ev := func(src, key string, w float64) model.Evidence { + return model.Evidence{Source: src, Value: "https://" + key, Key: key, Weight: w} + } + alias := map[string]string{"github.com/freedesktop/libqmi": ""} + p := Package{Name: "libqmi", UpstreamKey: "github.com/freedesktop/libqmi", Status: StatusResolved, Evidence: []model.Evidence{ + ev("zq1:giturls", "github.com/freedesktop/libqmi", 0.7), + ev("spec:URL", "gitlab.freedesktop.org/mobile-broadband/libqmi", 0.5), + }} + Rescore(&p, alias) + if p.UpstreamKey != "gitlab.freedesktop.org/mobile-broadband/libqmi" || p.Basis != BasisHomepage { + t.Errorf("fallback: %+v", p) + } + q := Package{Name: "x", UpstreamKey: "github.com/freedesktop/libqmi", Status: StatusResolved, Evidence: []model.Evidence{ + ev("zq1:giturls", "github.com/freedesktop/libqmi", 0.7), + }} + Rescore(&q, alias) + if q.UpstreamKey != "" || q.Status != StatusUnresolved { + t.Errorf("no fallback: %+v", q) + } +} + +// TestSecondarySource: a secondary spec source (a config file from another repository) +// corroborates but never decides, even when it outscores the primary evidence. +func TestSecondarySource(t *testing.T) { + ev := func(src, key string, w float64) model.Evidence { + return model.Evidence{Source: src, Value: "https://" + key, Key: key, Weight: w} + } + p := Package{Name: "caddy", Evidence: []model.Evidence{ + ev("spec:Source2", "github.com/caddyserver/dist", 0.7), + ev("spec:URL", "github.com/caddyserver/caddy", 0.5), + }} + Rescore(&p, nil) + if p.UpstreamKey != "github.com/caddyserver/caddy" || p.Basis != BasisHomepage { + t.Errorf("caddy: %+v", p) + } + q := Package{Name: "cmocka", Evidence: []model.Evidence{ev("spec:Source4", "github.com/jothepro/doxygen-awesome-css", 0.7)}} + Rescore(&q, nil) + if q.UpstreamKey != "" || q.Status != StatusUnresolved { + t.Errorf("secondary only: %+v", q) + } + if BasisOf("spec:Source") != BasisReleaseURL || BasisOf("spec:Source0") != BasisReleaseURL || BasisOf("spec:Source12") != BasisSecondary { + t.Error("spec source bases") + } +} diff --git a/internal/bulk/alpine.go b/internal/bulk/alpine.go new file mode 100644 index 0000000..b6e2c81 --- /dev/null +++ b/internal/bulk/alpine.go @@ -0,0 +1,60 @@ +package bulk + +import ( + "sort" + + "tangled.org/odd.computer/isomorph/internal/distro" + "tangled.org/odd.computer/isomorph/internal/distro/alpine" + "tangled.org/odd.computer/isomorph/internal/model" +) + +// FromAlpine builds one row per origin (source package) of the edge APKINDEX of each section. +// The APKINDEX supplies the built version, url and binary names; the origin's APKBUILD (keyed +// "
/") supplies the source URLs. OSV defines Alpine ecosystems per release +// (Alpine:v3.22), not for edge, so OSVEcosystem stays empty for these rows. +func FromAlpine(index map[string][]alpine.IndexEntry, apkbuilds map[string]*alpine.APKBUILD) []Package { + var out []Package + for _, section := range sortedKeys(index) { + type origin struct { + version, url string + bins []string + } + origins := map[string]*origin{} + for _, e := range index[section] { + name := e.Origin + if name == "" { + name = e.Name + } + o := origins[name] + if o == nil { + o = &origin{version: e.Version, url: e.URL} + origins[name] = o + } + o.bins = append(o.bins, e.Name) + } + for _, name := range sortedKeys(origins) { + o := origins[name] + p := Package{Distro: model.Alpine, Suite: "edge-" + section, Name: name, Version: o.version, + Binaries: dedupe(o.bins), Homepage: o.url, Status: StatusUnresolved} + if ab := apkbuilds[section+"/"+name]; ab != nil { + if ab.URL != "" { + p.Homepage = ab.URL + } + fill(&p, distro.RankCandidates(alpine.Candidates(ab))) + } else if o.url != "" { + fill(&p, distro.RankCandidates(distro.AddCandidate(nil, o.url, model.Evidence{Source: "APKINDEX:url", Value: o.url, Weight: 0.5}))) + } + out = append(out, p) + } + } + return out +} + +func sortedKeys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/internal/bulk/arch.go b/internal/bulk/arch.go new file mode 100644 index 0000000..37d0ff9 --- /dev/null +++ b/internal/bulk/arch.go @@ -0,0 +1,130 @@ +package bulk + +import ( + "context" + "sort" + "strings" + "time" + + "tangled.org/odd.computer/isomorph/internal/distro" + "tangled.org/odd.computer/isomorph/internal/distro/arch" + "tangled.org/odd.computer/isomorph/internal/model" +) + +// ArchSource provides .SRCINFO files at released packaging commits (arch.Resolver does). +type ArchSource interface { + SRCINFOAt(ctx context.Context, pkgbase, commit string) (*arch.SRCINFO, error) + FetchSRCINFO(ctx context.Context, pkgbase, commit string, web bool) (*arch.SRCINFO, error) + HasSRCINFO(pkgbase, commit string) bool +} + +// CrawlInterval paces .SRCINFO fetches. gitlab.archlinux.org allows anonymous clients about 600 +// requests per ten minutes on each of the API and web routes; alternating the two at this +// interval stays under both even when a third of the packages need a second request (the +// PKGBUILD fallback). +var CrawlInterval = 700 * time.Millisecond + +// ArchStats describes how an Arch slice was built. +type ArchStats struct { + PkgBases int + // FromSRCINFO counts rows resolved from .SRCINFO (cached or fetched this run); the rest use + // the repository databases' url field only. + FromSRCINFO int + // Fetched and FetchErrors count .SRCINFO requests made this run. + Fetched, FetchErrors int + // Stopped is set when the crawl hit a rate limit and stopped early. + Stopped error + // Errors samples fetch errors by pkgbase (at most 20). + Errors map[string]string +} + +// FromArch builds one row per pkgbase of the repository databases. Each row starts from the +// databases (version, binary names, url). A pkgbase whose .SRCINFO at its released commit (from +// the state repository) is cached, or can be fetched within the crawl budget, is resolved from +// .SRCINFO instead: its source array names git repositories and release tarballs (with +// sha256), which the url field does not. +func FromArch(ctx context.Context, src ArchSource, dbs []arch.SyncPackage, state []arch.StateEntry, crawl int, progress func(fetched int)) ([]Package, ArchStats) { + type base struct { + repo, name, version, url string + bins []string + } + bases := map[[2]string]*base{} + var order [][2]string + for _, p := range dbs { + k := [2]string{p.Repo, p.Base} + b := bases[k] + if b == nil { + b = &base{repo: p.Repo, name: p.Base, version: p.Version} + bases[k] = b + order = append(order, k) + } + if b.url == "" { + b.url = p.URL + } + b.bins = append(b.bins, p.Name) + } + sort.Slice(order, func(i, j int) bool { + if order[i][0] != order[j][0] { + return order[i][0] < order[j][0] + } + return order[i][1] < order[j][1] + }) + commits := map[[2]string]string{} + for _, e := range state { + commits[[2]string{e.Repo, e.PkgBase}] = e.Commit + } + var st ArchStats + var last time.Time + out := make([]Package, 0, len(order)) + for _, k := range order { + b := bases[k] + sort.Strings(b.bins) + p := Package{Distro: model.Arch, Suite: b.repo, Name: b.name, Version: b.version, Binaries: b.bins, Homepage: b.url, Status: StatusUnresolved} + var si *arch.SRCINFO + if c := commits[k]; c != "" && ctx.Err() == nil { + switch { + case src.HasSRCINFO(b.name, c): + si, _ = src.SRCINFOAt(ctx, b.name, c) + case st.Fetched < crawl && st.Stopped == nil: + if wait := CrawlInterval - time.Since(last); wait > 0 { + select { + case <-ctx.Done(): + case <-time.After(wait): + } + } + last = time.Now() + st.Fetched++ + var err error + if si, err = src.FetchSRCINFO(ctx, b.name, c, st.Fetched%2 == 0); err != nil { + si = nil + st.FetchErrors++ + if st.Errors == nil { + st.Errors = map[string]string{} + } + if len(st.Errors) < 20 { + st.Errors[b.name] = err.Error() + } + if strings.Contains(err.Error(), "rate limited") { + st.Stopped = err + } + } + if progress != nil { + progress(st.Fetched) + } + } + } + if si != nil { + st.FromSRCINFO++ + if si.URL != "" { + p.Homepage = si.URL + } + p.SourceSHA256 = arch.SourceSHA256(si) + fill(&p, distro.RankCandidates(arch.Candidates(si))) + } else if b.url != "" { + fill(&p, distro.RankCandidates(distro.AddCandidate(nil, b.url, model.Evidence{Source: "pacman-db:url", Value: b.url, Weight: 0.5}))) + } + out = append(out, p) + } + st.PkgBases = len(out) + return out, st +} diff --git a/internal/bulk/arch_test.go b/internal/bulk/arch_test.go new file mode 100644 index 0000000..00837ca --- /dev/null +++ b/internal/bulk/arch_test.go @@ -0,0 +1,158 @@ +package bulk + +import ( + "context" + "errors" + "path/filepath" + "testing" + "time" + + "tangled.org/odd.computer/isomorph/internal/distro/alpine" + "tangled.org/odd.computer/isomorph/internal/distro/arch" + "tangled.org/odd.computer/isomorph/internal/distro/opensuse" + "tangled.org/odd.computer/isomorph/internal/model" +) + +type fakeSRCINFO struct { + files map[string]string + cached map[string]bool +} + +func (f fakeSRCINFO) SRCINFOAt(_ context.Context, pkgbase, commit string) (*arch.SRCINFO, error) { + body, ok := f.files[pkgbase+"@"+commit] + if !ok { + return nil, errors.New("404") + } + return arch.ParseSRCINFO([]byte(body)) +} + +func (f fakeSRCINFO) FetchSRCINFO(ctx context.Context, pkgbase, commit string, _ bool) (*arch.SRCINFO, error) { + return f.SRCINFOAt(ctx, pkgbase, commit) +} + +func (f fakeSRCINFO) HasSRCINFO(pkgbase, commit string) bool { return f.cached[pkgbase+"@"+commit] } + +func TestFromArchAndCompare(t *testing.T) { + defer func(d time.Duration) { CrawlInterval = d }(CrawlInterval) + CrawlInterval = 0 + f := fakeSRCINFO{files: map[string]string{ + "json-c@aaaa": "pkgbase = json-c\n\tpkgver = 0.18\n\tpkgrel = 1\n\turl = https://github.com/json-c/json-c/wiki\n" + + "\tsource = git+https://github.com/json-c/json-c#tag=json-c-0.18-20240915\n\tsha256sums = SKIP\n\npkgname = json-c\n", + "gzip@bbbb": "pkgbase = gzip\n\tpkgver = 1.13\n\tpkgrel = 1\n\turl = https://www.gnu.org/software/gzip/\n" + + "\tsource = https://ftp.gnu.org/pub/gnu/gzip/gzip-1.13.tar.xz\n\tsha256sums = 7777\n\npkgname = gzip\n", + }, cached: map[string]bool{"json-c@aaaa": true}} + dbs := []arch.SyncPackage{ + {Repo: "extra", Name: "json-c", Base: "json-c", Version: "0.18-1", URL: "https://github.com/json-c/json-c/wiki"}, + {Repo: "core", Name: "gzip", Base: "gzip", Version: "1.13-1", URL: "https://www.gnu.org/software/gzip/"}, + {Repo: "extra", Name: "gone-bin", Base: "gone", Version: "1-1", URL: "https://github.com/gone/gone"}, + {Repo: "extra", Name: "gone-docs", Base: "gone", Version: "1-1"}, + } + state := []arch.StateEntry{ + {Repo: "extra", PkgBase: "json-c", Version: "0.18-1", Commit: "aaaa"}, + {Repo: "core", PkgBase: "gzip", Version: "1.13-1", Commit: "bbbb"}, + {Repo: "extra", PkgBase: "gone", Version: "1-1", Commit: "cccc"}, + } + // Cache only: json-c from .SRCINFO, the others from the databases' url. + pkgs, st := FromArch(context.Background(), f, dbs, state, 0, nil) + if st.PkgBases != 3 || st.FromSRCINFO != 1 || st.Fetched != 0 { + t.Fatalf("cache-only stats: %+v", st) + } + by := map[string]Package{} + for _, p := range pkgs { + by[p.Name] = p + } + if p := by["json-c"]; p.Basis != BasisVCSSource || p.UpstreamKey != "github.com/json-c/json-c" { + t.Errorf("json-c from cached .SRCINFO: %+v", p) + } + if p := by["gone"]; p.UpstreamKey != "github.com/gone/gone" || p.Basis != BasisHomepage || len(p.Binaries) != 2 { + t.Errorf("gone from the database url: %+v", p) + } + // With a crawl budget the uncached ones are fetched; a failing fetch keeps the baseline. + pkgs, st = FromArch(context.Background(), f, dbs, state, 10, nil) + if st.FromSRCINFO != 2 || st.Fetched != 2 || st.FetchErrors != 1 { + t.Fatalf("crawl stats: %+v", st) + } + by = map[string]Package{} + for _, p := range pkgs { + by[p.Name] = p + } + if p := by["gzip"]; p.UpstreamKey != "git.savannah.gnu.org/gzip" || p.Basis != BasisReleaseURL || len(p.SourceSHA256) != 1 { + t.Errorf("gzip: %+v", p) + } + if p := by["gone"]; p.UpstreamKey != "github.com/gone/gone" { + t.Errorf("gone after a failed fetch: %+v", p) + } + pkgs = []Package{by["json-c"], by["gzip"], by["gone"]} + + ctx := context.Background() + db, err := Open(ctx, filepath.Join(t.TempDir(), "p.sqlite")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + deb := FromUDD(udd(), "sid") + deb[1].SourceSHA256 = []string{"7777"} // gzip: the same release tarball as Arch + Replace(ctx, db, Source{Distro: model.Debian, Suite: "sid", Origin: "t", FetchedAt: time.Now()}, deb) + Replace(ctx, db, Source{Distro: model.Arch, Suite: "extra", Origin: "t", FetchedAt: time.Now()}, []Package{pkgs[0], pkgs[2]}) + Replace(ctx, db, Source{Distro: model.Arch, Suite: "core", Origin: "t", FetchedAt: time.Now()}, []Package{pkgs[1]}) + _ = st + ags, err := Compare(ctx, db, "debian", "arch") + if err != nil { + t.Fatal(err) + } + if sha := ags[0]; sha.Join != "sha256" || sha.Pairs != 1 || sha.Agree != 1 { + t.Errorf("sha256 join: %+v", sha) + } + if name := ags[1]; name.Join != "name" || name.Pairs != 2 || name.BothResolved != 2 || name.Agree != 2 { + t.Errorf("name join: %+v", name) + } +} + +func TestFromOpenSUSE(t *testing.T) { + srcs := []opensuse.RPM{ + {Name: "json-c", Arch: "src", Version: "0.18-1.2", URL: "https://github.com/json-c/json-c/wiki"}, + {Name: "at", Arch: "src", Version: "1:3.2-4"}, + } + bins := []opensuse.RPM{ + {Name: "libjson-c5", Arch: "x86_64", SourceRPM: "json-c-0.18-1.2.src.rpm"}, + {Name: "libjson-c-devel", Arch: "x86_64", SourceRPM: "json-c-0.18-1.2.src.rpm"}, + {Name: "libjson-c5", Arch: "i586", SourceRPM: "json-c-0.18-1.2.src.rpm"}, + } + pkgs := FromOpenSUSE(srcs, bins, map[string][]string{"at": {"https://salsa.debian.org/debian/at.git"}}, nil) + if len(pkgs) != 2 { + t.Fatalf("%d rows", len(pkgs)) + } + at, j := pkgs[0], pkgs[1] + if j.Name != "json-c" || j.UpstreamKey != "github.com/json-c/json-c" || j.Basis != BasisHomepage || j.OSVEcosystem != "openSUSE:Tumbleweed" || + len(j.Binaries) != 2 || j.PURL != "pkg:rpm/opensuse/json-c?arch=src" { + t.Errorf("json-c: %+v", j) + } + if at.UpstreamKey != "salsa.debian.org/debian/at" || at.Basis != BasisDerived { + t.Errorf("at via giturls: %+v", at) + } +} + +func TestFromAlpine(t *testing.T) { + index := map[string][]alpine.IndexEntry{ + "community": { + {Name: "sysstat", Version: "12.7.1-r0", Origin: "sysstat", URL: "https://github.com/sysstat/sysstat"}, + {Name: "sysstat-doc", Version: "12.7.1-r0", Origin: "sysstat", URL: "https://github.com/sysstat/sysstat"}, + }, + "main": {{Name: "libfoo", Version: "1-r0", Origin: "foo", URL: "https://gitlab.com/foo/foo"}}, + } + ab, err := alpine.ParseAPKBUILD([]byte("pkgname=sysstat\npkgver=12.7.1\npkgrel=0\nurl=\"https://sebastien.godard.pagesperso-orange.fr/\"\nsource=\"$pkgname-$pkgver.tar.gz::https://github.com/sysstat/sysstat/archive/v$pkgver.tar.gz\"\n")) + if err != nil { + t.Fatal(err) + } + pkgs := FromAlpine(index, map[string]*alpine.APKBUILD{"community/sysstat": ab}) + if len(pkgs) != 2 { + t.Fatalf("%d rows", len(pkgs)) + } + s, f := pkgs[0], pkgs[1] + if s.Name != "sysstat" || s.Suite != "edge-community" || s.UpstreamKey != "github.com/sysstat/sysstat" || s.Basis != BasisReleaseURL || len(s.Binaries) != 2 { + t.Errorf("sysstat: %+v", s) + } + if f.Name != "foo" || f.UpstreamKey != "gitlab.com/foo/foo" || f.Basis != BasisHomepage { + t.Errorf("foo from the index url: %+v", f) + } +} diff --git a/internal/bulk/bulk.go b/internal/bulk/bulk.go new file mode 100644 index 0000000..f70411c --- /dev/null +++ b/internal/bulk/bulk.go @@ -0,0 +1,116 @@ +// Package bulk holds the association table: every source package of a distribution suite, +// resolved to an upstream repository, with the evidence behind each mapping. It is built from +// whole-archive sources (UDD for Debian) rather than one package at a time, and published as +// JSONL (canonical, diffable) and SQLite (for queries; Python reads it with the standard +// library). +package bulk + +import ( + "regexp" + "strings" + "time" + + "tangled.org/odd.computer/isomorph/internal/model" +) + +// Status values of Package.Status. +const ( + // StatusResolved: the best candidate is a repository. + StatusResolved = "resolved" + // StatusUnresolved: no field named or hinted at a repository. + StatusUnresolved = "unresolved" +) + +// Basis values: the kind of evidence behind a mapping, strongest first. +const ( + // BasisMetadata: a field whose purpose is to name the upstream repository (DEP-12 + // Repository, Repository-Browse). + BasisMetadata = "metadata" + // BasisVCSSource: the package builds from the repository (an Arch git source). + BasisVCSSource = "vcs-source" + // BasisReleaseURL: a release tarball or debian/watch URL on a forge, or a tarball on a + // host whose repository is known (ftp.gnu.org -> Savannah). + BasisReleaseURL = "release-url" + // BasisDerived: a third-party dataset derived from the packaging (rb.zq1.de giturls, itself + // computed from spec Url:/Source: fields). + BasisDerived = "derived" + // BasisBugTracker: a bug tracker URL on a forge (DEP-12 Bug-Database). + BasisBugTracker = "bug-tracker" + // BasisHomepage: a homepage on a forge or a well-known project site. + BasisHomepage = "homepage" + // BasisSecondary is a secondary spec source (Source1 and up: configuration files, shell + // completions, test dependencies). It corroborates a candidate but never decides one: of + // the openSUSE packages it alone resolved, about a third named another project's + // repository (docs/EVALUATION.md). + BasisSecondary = "secondary-source" +) + +// basisRank orders the bases, strongest first. +var basisRank = map[string]int{BasisMetadata: 0, BasisVCSSource: 1, BasisReleaseURL: 2, BasisDerived: 3, BasisBugTracker: 4, BasisHomepage: 5, BasisSecondary: 6} + +var secondarySpecSourceRe = regexp.MustCompile(`^spec:Source[1-9][0-9]*$`) + +// BasisOf maps an evidence source name onto its basis. +func BasisOf(source string) string { + switch { + case source == "debian/upstream/metadata:Repository", source == "debian/upstream/metadata:Repository-Browse": + return BasisMetadata + case strings.HasPrefix(source, "SRCINFO:source:"), strings.HasPrefix(source, "APKBUILD:source:"), source == "APKBUILD:_repo_url", source == "spec:Git-Clone": + return BasisVCSSource + case strings.HasPrefix(source, "_service:"): + // obs_scm, tar_scm and the like fetch a repository; download_url and download_files + // fetch release files. + if strings.HasPrefix(source, "_service:download") { + return BasisReleaseURL + } + return BasisVCSSource + case secondarySpecSourceRe.MatchString(source): + return BasisSecondary + case source == "SRCINFO:source", source == "APKBUILD:source", source == "debian/watch:upstream_url", strings.HasPrefix(source, "spec:Source"): + return BasisReleaseURL + case source == "debian/upstream/metadata:Bug-Database": + return BasisBugTracker + case source == "zq1:giturls": + return BasisDerived + } + return BasisHomepage +} + +// Package is one source package (Debian source, openSUSE spec, Arch pkgbase, Alpine origin) +// in one suite, with its resolved upstream. +type Package struct { + Distro model.Distro `json:"distro"` + Suite string `json:"suite"` + Name string `json:"package"` + // Version is the distro version, as the distro writes it. + Version string `json:"version"` + // OSVEcosystem is the OSV ecosystem string ("Debian:sid"); empty where OSV defines none. + OSVEcosystem string `json:"osv_ecosystem,omitempty"` + // PURL identifies the source package (advisory: purl's distro qualifiers are not + // standardised, so consumers should key on (OSVEcosystem, Name)). + PURL string `json:"purl,omitempty"` + Binaries []string `json:"binaries,omitempty"` + Homepage string `json:"homepage,omitempty"` + // UpstreamKey is the repourl key of the best candidate; Repo is its clone URL, ready for an + // OSV GIT range. + UpstreamKey string `json:"upstream_key,omitempty"` + Repo string `json:"repo,omitempty"` + Status string `json:"status"` + // Basis is the kind of evidence that decided the mapping (Basis* constants). The evaluation + // reports measured agreement per basis; there is deliberately no numeric confidence. + Basis string `json:"basis,omitempty"` + // Evidence lists every observation considered, highest weight first. + Evidence []model.Evidence `json:"evidence,omitempty"` + // SourceSHA256 are the sha256 of the upstream source archives the package builds from + // (Debian .orig tarballs); equal hashes across distros mean the same upstream release. + SourceSHA256 []string `json:"source_sha256,omitempty"` +} + +// Source describes where one (distro, suite) slice of the table came from. +type Source struct { + Distro model.Distro `json:"distro"` + Suite string `json:"suite"` + Origin string `json:"origin"` + FetchedAt time.Time `json:"fetched_at"` + Packages int `json:"packages"` +} diff --git a/internal/bulk/bulk_test.go b/internal/bulk/bulk_test.go new file mode 100644 index 0000000..caba262 --- /dev/null +++ b/internal/bulk/bulk_test.go @@ -0,0 +1,161 @@ +package bulk + +import ( + "context" + "path/filepath" + "testing" + "time" + + "tangled.org/odd.computer/isomorph/internal/distro/debian" + "tangled.org/odd.computer/isomorph/internal/model" +) + +func udd() []debian.UDDRow { + return []debian.UDDRow{ + { // DEP-12 Repository agrees with the homepage. + Source: "json-c", Version: "0.18+ds-1", Homepage: "https://github.com/json-c/json-c/wiki", + Binaries: []string{"libjson-c5", "libjson-c-dev"}, + Metadata: map[string]string{"Repository": "https://github.com/json-c/json-c.git"}, + WatchURL: "https://github.com/json-c/json-c/archive/refs/tags/json-c-0.18-20240915.tar.gz", + }, + { // Only a tarball watch URL on a known host. + Source: "gzip", Version: "1.13-1", Homepage: "https://www.gnu.org/software/gzip/", + WatchURL: "https://ftp.gnu.org/gnu/gzip/gzip-1.13.tar.xz", + }, + { // Nothing points at a repository; the packaging repo on salsa must not win. + Source: "debian-native-tool", Version: "3", VcsURL: "https://salsa.debian.org/debian/debian-native-tool.git", + }, + { // Repository disagrees with the homepage's forge (a mirror). + Source: "mirrored", Version: "1-1", Homepage: "https://github.com/someone/mirrored", + Metadata: map[string]string{"Repository": "https://gitlab.com/upstream/mirrored.git"}, + }, + } +} + +func TestFromUDD(t *testing.T) { + pkgs := FromUDD(udd(), "sid") + by := map[string]Package{} + for _, p := range pkgs { + by[p.Name] = p + } + j := by["json-c"] + if j.UpstreamKey != "github.com/json-c/json-c" || j.Status != StatusResolved || j.Basis != BasisMetadata || + j.OSVEcosystem != "Debian:sid" || j.PURL != "pkg:deb/debian/json-c?arch=source" || len(j.Binaries) != 2 { + t.Errorf("json-c: %+v", j) + } + if g := by["gzip"]; g.UpstreamKey != "git.savannah.gnu.org/gzip" || g.Status != StatusResolved || g.Basis != BasisReleaseURL { + t.Errorf("gzip via the GNU tarball hint: %+v", g) + } + if n := by["debian-native-tool"]; n.Status != StatusUnresolved || n.UpstreamKey != "" { + t.Errorf("packaging repo taken as upstream: %+v", n) + } + if m := by["mirrored"]; m.UpstreamKey != "gitlab.com/upstream/mirrored" { + t.Errorf("Repository must outweigh the homepage: %+v", m) + } + h := DebianHeldOut(udd()) + if h.WithRepository != 2 || h.FallbackResolved != 2 || h.Agree != 1 || len(h.Disagree) != 1 || h.Disagree[0][0] != "mirrored" || + h.ByBasis[BasisReleaseURL] != [2]int{1, 1} || h.ByBasis[BasisHomepage] != [2]int{1, 0} { + t.Errorf("held out: %+v", h) + } +} + +func TestSQLiteRoundTrip(t *testing.T) { + ctx := context.Background() + db, err := Open(ctx, filepath.Join(t.TempDir(), "p.sqlite")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + deb := FromUDD(udd(), "sid") + if err := Replace(ctx, db, Source{Distro: model.Debian, Suite: "sid", Origin: "test", FetchedAt: time.Now()}, deb); err != nil { + t.Fatal(err) + } + arch := []Package{{Distro: model.Arch, Suite: "extra", Name: "json-c", Version: "0.18-1", UpstreamKey: "github.com/json-c/json-c", Status: StatusResolved, Basis: BasisVCSSource}} + if err := Replace(ctx, db, Source{Distro: model.Arch, Suite: "extra", Origin: "test", FetchedAt: time.Now()}, arch); err != nil { + t.Fatal(err) + } + got, err := Lookup(ctx, db, "debian/sid:json-c") + if err != nil || len(got) != 1 || got[0].UpstreamKey != "github.com/json-c/json-c" || len(got[0].Evidence) == 0 { + t.Fatalf("lookup by package: %v %+v", err, got) + } + both, err := Lookup(ctx, db, "github.com/json-c/json-c") + if err != nil || len(both) != 2 { + t.Fatalf("lookup by upstream: %v %+v", err, both) + } + var n int + if err := db.QueryRowContext(ctx, `select count(*) from counterparts where package = 'json-c'`).Scan(&n); err != nil || n != 2 { + t.Fatalf("counterparts view: %v %d", err, n) + } + var bin string + if err := db.QueryRowContext(ctx, `select package from binaries where binary = 'libjson-c5'`).Scan(&bin); err != nil || bin != "json-c" { + t.Fatalf("binaries: %v %q", err, bin) + } + back, err := LoadSlice(ctx, db, model.Debian, "sid") + if err != nil || len(back) != len(deb) { + t.Fatalf("LoadSlice: %v %d", err, len(back)) + } + for i := range back { + w := deb[i] + for j := range deb { + if deb[j].Name == back[i].Name { + w = deb[j] + } + } + if back[i].UpstreamKey != w.UpstreamKey || len(back[i].Evidence) != len(w.Evidence) || len(back[i].Binaries) != len(w.Binaries) || back[i].Basis != w.Basis { + t.Errorf("LoadSlice %s: got %+v want %+v", w.Name, back[i], w) + } + } + // Replacing a slice drops its previous rows. + if err := Replace(ctx, db, Source{Distro: model.Debian, Suite: "sid", Origin: "test", FetchedAt: time.Now()}, deb[:1]); err != nil { + t.Fatal(err) + } + if err := db.QueryRowContext(ctx, `select count(*) from packages where distro = 'debian'`).Scan(&n); err != nil || n != 1 { + t.Fatalf("replace: %v %d", err, n) + } +} + +func TestOpenRefusesOtherSchemaVersions(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "p.sqlite") + db, err := Open(ctx, path) + if err != nil { + t.Fatal(err) + } + db.ExecContext(ctx, "pragma user_version = 1") + db.Close() + if _, err := Open(ctx, path); err == nil { + t.Fatal("opened a database with another schema version") + } +} + +// TestStaleHomeHost: a SourceForge or Launchpad homepage outscores a GitHub release URL (the +// URL counts as a hint at 0.6 * 0.6), but loses to it; a GitHub homepage keeps winning. +func TestStaleHomeHost(t *testing.T) { + pkgs := FromUDD([]debian.UDDRow{ + {Source: "biber", Version: "2.22-1", Homepage: "http://biblatex-biber.sourceforge.net/", + WatchURL: "https://api.github.com/repos/plk/biber/tarball/refs/tags/v2.22"}, + {Source: "cloud-utils", Version: "0.34-1", Homepage: "https://launchpad.net/cloud-utils", + WatchURL: "https://github.com/canonical/cloud-utils/archive/refs/tags/0.34.tar.gz"}, + {Source: "vde2", Version: "2.3.3-1", Homepage: "http://vde.sourceforge.net"}, + {Source: "gh", Version: "1-1", Homepage: "https://github.com/a/gh", + WatchURL: "https://github.com/b/gh/archive/refs/tags/1.tar.gz"}, + }, "sid") + want := map[string][2]string{ + "biber": {"github.com/plk/biber", BasisReleaseURL}, + "cloud-utils": {"github.com/canonical/cloud-utils", BasisReleaseURL}, + "vde2": {"sourceforge.net/vde", BasisHomepage}, + "gh": {"github.com/a/gh", BasisHomepage}, + } + for _, p := range pkgs { + if w := want[p.Name]; p.UpstreamKey != w[0] || p.Basis != w[1] { + t.Errorf("%s: %s (%s), want %v", p.Name, p.UpstreamKey, p.Basis, w) + } + // Rescoring from the stored evidence keeps the choice. + q := p + q.UpstreamKey = "" + Rescore(&q, nil) + if q.UpstreamKey != p.UpstreamKey { + t.Errorf("%s: rescored to %s", p.Name, q.UpstreamKey) + } + } +} diff --git a/internal/bulk/compare.go b/internal/bulk/compare.go new file mode 100644 index 0000000..4b158e2 --- /dev/null +++ b/internal/bulk/compare.go @@ -0,0 +1,73 @@ +package bulk + +import ( + "context" + "database/sql" +) + +// Agreement compares two distributions' mappings over pairs of packages joined independently +// of the mapping: by an identical upstream source archive (sha256) or by an identical name. +// Where both packages resolve, the pair should name the same upstream key. The counts form a +// confusion matrix of a's outcome against b's: both resolve (agreeing or not), only one side +// resolves, or neither does. +type Agreement struct { + // Join is "sha256" or "name". + Join string + // Pairs counts joined package pairs; BothResolved those where both sides resolve; Agree + // those naming the same key. + Pairs, BothResolved, Agree int + // OnlyA and OnlyB count pairs where only a's or only b's package resolves (the other + // side's gap); Neither those where neither does. + OnlyA, OnlyB, Neither int + // Disagree lists every pair where both resolve to different keys + // (a package, b package, a key, b key). + Disagree [][4]string +} + +// Compare measures agreement between distros a and b in db. +func Compare(ctx context.Context, db *sql.DB, a, b string) ([]Agreement, error) { + joins := []struct{ name, from string }{ + {"sha256", `from source_sha256 x join source_sha256 y on x.sha256 = y.sha256 + join packages pa on pa.distro = x.distro and pa.suite = x.suite and pa.package = x.package + join packages pb on pb.distro = y.distro and pb.suite = y.suite and pb.package = y.package + where x.distro = ? and y.distro = ?`}, + {"name", `from packages pa join packages pb on pa.package = pb.package where pa.distro = ? and pb.distro = ?`}, + } + var out []Agreement + for _, j := range joins { + rows, err := db.QueryContext(ctx, `select distinct pa.package, pb.package, coalesce(pa.upstream_key,''), coalesce(pb.upstream_key,'') `+j.from, a, b) + if err != nil { + return nil, err + } + ag := Agreement{Join: j.name} + for rows.Next() { + var pa, pb, ka, kb string + if err := rows.Scan(&pa, &pb, &ka, &kb); err != nil { + rows.Close() + return nil, err + } + ag.Pairs++ + switch { + case ka != "" && kb != "": + ag.BothResolved++ + if ka == kb { + ag.Agree++ + } else { + ag.Disagree = append(ag.Disagree, [4]string{pa, pb, ka, kb}) + } + case ka != "": + ag.OnlyA++ + case kb != "": + ag.OnlyB++ + default: + ag.Neither++ + } + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, err + } + out = append(out, ag) + } + return out, nil +} diff --git a/internal/bulk/compare_test.go b/internal/bulk/compare_test.go new file mode 100644 index 0000000..99343b9 --- /dev/null +++ b/internal/bulk/compare_test.go @@ -0,0 +1,51 @@ +package bulk + +import ( + "context" + "path/filepath" + "testing" + "time" + + "tangled.org/odd.computer/isomorph/internal/model" +) + +func TestCompareMatrix(t *testing.T) { + ctx := context.Background() + db, err := Open(ctx, filepath.Join(t.TempDir(), "p.sqlite")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + pkg := func(d model.Distro, name, key string) Package { + return Package{Distro: d, Suite: "s", Name: name, UpstreamKey: key} + } + a := []Package{ + pkg(model.Debian, "same", "github.com/x/same"), + pkg(model.Debian, "differ", "github.com/x/one"), + pkg(model.Debian, "only-a", "github.com/x/only-a"), + pkg(model.Debian, "only-b", ""), + pkg(model.Debian, "neither", ""), + pkg(model.Debian, "unpaired", "github.com/x/unpaired"), + } + b := []Package{ + pkg(model.Arch, "same", "github.com/x/same"), + pkg(model.Arch, "differ", "github.com/x/other"), + pkg(model.Arch, "only-a", ""), + pkg(model.Arch, "only-b", "github.com/x/only-b"), + pkg(model.Arch, "neither", ""), + } + Replace(ctx, db, Source{Distro: model.Debian, Suite: "s", Origin: "t", FetchedAt: time.Now()}, a) + Replace(ctx, db, Source{Distro: model.Arch, Suite: "s", Origin: "t", FetchedAt: time.Now()}, b) + ags, err := Compare(ctx, db, "debian", "arch") + if err != nil { + t.Fatal(err) + } + got := ags[1] + if got.Join != "name" || got.Pairs != 5 || got.BothResolved != 2 || got.Agree != 1 || + len(got.Disagree) != 1 || got.OnlyA != 1 || got.OnlyB != 1 || got.Neither != 1 { + t.Errorf("name join: %+v", got) + } + if d := got.Disagree[0]; d != [4]string{"differ", "differ", "github.com/x/one", "github.com/x/other"} { + t.Errorf("disagreement: %v", d) + } +} diff --git a/internal/bulk/debian.go b/internal/bulk/debian.go new file mode 100644 index 0000000..9f85a54 --- /dev/null +++ b/internal/bulk/debian.go @@ -0,0 +1,152 @@ +package bulk + +import ( + "sort" + "strings" + + "tangled.org/odd.computer/isomorph/internal/distro" + "tangled.org/odd.computer/isomorph/internal/distro/debian" + "tangled.org/odd.computer/isomorph/internal/model" + "tangled.org/odd.computer/isomorph/internal/repourl" +) + +// FromUDD turns UDD rows of one suite into association rows. +func FromUDD(rows []debian.UDDRow, suite string) []Package { + out := make([]Package, 0, len(rows)) + for _, r := range rows { + p := Package{ + Distro: model.Debian, + Suite: suite, + Name: r.Source, + Version: r.Version, + OSVEcosystem: "Debian:" + suite, + PURL: "pkg:deb/debian/" + r.Source + "?arch=source", + Binaries: r.Binaries, + Homepage: r.Homepage, + Status: StatusUnresolved, + SourceSHA256: r.OrigSHA256, + } + cands := debian.ResolveRow(r, nil) + fill(&p, cands) + out = append(out, p) + } + return out +} + +// fill sets the upstream fields from ranked candidates. +func fill(p *Package, cands []distro.Candidate) { + for _, c := range cands { + p.Evidence = append(p.Evidence, c.Evidence...) + } + sort.SliceStable(p.Evidence, func(i, j int) bool { return p.Evidence[i].Weight > p.Evidence[j].Weight }) + if best := choose(cands); best != nil { + p.UpstreamKey, p.Repo, p.Status = best.Key, best.URL, StatusResolved + p.Basis = basis(*best) + } +} + +// choose picks the upstream among ranked candidates: the first repository, unless it rests on +// a homepage on a staleHomeHosts host and a release or VCS source names another repository. +func choose(cands []distro.Candidate) *distro.Candidate { + var best *distro.Candidate + for i := range cands { + c := &cands[i] + if c.Kind != repourl.KindRepo || basis(*c) == BasisSecondary { + continue + } + if best == nil { + best = c + if basis(*c) != BasisHomepage || !onStaleHomeHost(c.Key) { + break + } + } else if b := basis(*c); b == BasisReleaseURL || b == BasisVCSSource { + return c // see staleHomeHosts + } + } + return best +} + +// staleHomeHosts are hosts whose project pages outlive a project's move elsewhere. A homepage +// there loses to a release or VCS source of the same package that names another repository: +// on Debian, where the Homepage is on one of these hosts and the watch URL names a different +// repository, DEP-12 Repository sides with the watch URL 22 times out of 23 (GitHub homepages, +// by contrast, split 16 to 17, so the rule stays narrow; docs/EVALUATION.md). +var staleHomeHosts = []string{"sourceforge.net/", "launchpad.net/"} + +func onStaleHomeHost(key string) bool { + for _, h := range staleHomeHosts { + if strings.HasPrefix(key, h) { + return true + } + } + return false +} + +// basis is the strongest kind of evidence supporting a candidate, by the fixed precedence of +// basisRank (not by the ranking weights, which scale hints down). +func basis(c distro.Candidate) string { + best := "" + for _, e := range c.Evidence { + if b := BasisOf(e.Source); best == "" || basisRank[b] < basisRank[best] { + best = b + } + } + return best +} + +// HeldOut measures Debian's other fields against DEP-12 Repository. Among packages whose +// Repository resolves to a repository, it resolves again without Repository and +// Repository-Browse and counts how often the remaining fields (watch URL, Homepage, +// Bug-Database) resolve at all, and to the same key. +type HeldOut struct { + WithRepository int + // FallbackResolved counts packages the other fields still resolve; Agree counts those that + // resolve to the Repository key. + FallbackResolved int + Agree int + // ByBasis counts, per basis of the fallback's decision, [resolved, agreeing]. + ByBasis map[string][2]int + // Disagree lists (package, repository key, fallback key, fallback basis) for inspection. + Disagree [][4]string +} + +// DebianHeldOut runs the held-out comparison. +func DebianHeldOut(rows []debian.UDDRow) HeldOut { + skip := map[string]bool{ + "debian/upstream/metadata:Repository": true, + "debian/upstream/metadata:Repository-Browse": true, + } + h := HeldOut{ByBasis: map[string][2]int{}} + for _, r := range rows { + full := choose(debian.ResolveRow(r, nil)) + if full == nil || !fromRepository(*full) { + continue + } + h.WithRepository++ + held := choose(debian.ResolveRow(r, skip)) + if held == nil { + continue + } + h.FallbackResolved++ + hb := basis(*held) + b := h.ByBasis[hb] + b[0]++ + if held.Key == full.Key { + h.Agree++ + b[1]++ + } else { + h.Disagree = append(h.Disagree, [4]string{r.Source, full.Key, held.Key, hb}) + } + h.ByBasis[hb] = b + } + return h +} + +func fromRepository(c distro.Candidate) bool { + for _, e := range c.Evidence { + if e.Source == "debian/upstream/metadata:Repository" { + return true + } + } + return false +} diff --git a/internal/bulk/github.go b/internal/bulk/github.go new file mode 100644 index 0000000..50fe2a5 --- /dev/null +++ b/internal/bulk/github.go @@ -0,0 +1,61 @@ +package bulk + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "tangled.org/odd.computer/isomorph/internal/fetch" + "tangled.org/odd.computer/isomorph/internal/repourl" +) + +const ( + defaultGitHubAPI = "https://api.github.com" + githubAccept = "application/vnd.github+json" + // githubTTL is how long a repository lookup is cached. Renames are rare and GitHub keeps + // redirecting the old name for as long as it is unclaimed, so a stale answer is harmless. + githubTTL = 30 * 24 * time.Hour +) + +// githubAPI overrides the GitHub REST base URL; tests point it at an httptest server. Empty +// means defaultGitHubAPI. +var githubAPI string + +// LookupGitHub resolves a github.com key through the repositories API, which follows renames +// and ownership transfers: the key derived from the response's full_name is the current one. +// gone is true (with key returned unchanged) when the API answers 404, the repository no longer +// exists. Rate limits, offline mode and network failures are returned as errors. Keys on other +// hosts are returned as is. +func LookupGitHub(ctx context.Context, client *fetch.Client, key string) (canonical string, gone bool, err error) { + if !strings.HasPrefix(key, "github.com/") { + return key, false, nil + } + c, err := repourl.Parse("https://" + key) + if err != nil || c.Host != "github.com" || c.Kind != repourl.KindRepo { + return key, false, nil + } + if client == nil { + return key, false, errors.New("bulk: rename lookup needs a fetch.Client") + } + api := githubAPI + if api == "" { + api = defaultGitHubAPI + } + var resp struct { + FullName string `json:"full_name"` + } + u := api + "/repos/" + c.Path + err = client.GetJSON(ctx, u, &resp, fetch.WithHeader("Accept", githubAccept), fetch.WithTTL(githubTTL)) + if fetch.IsNotFound(err) { + return key, true, nil + } + if err != nil { + return key, false, fmt.Errorf("bulk: %s: rename lookup: %w", key, err) + } + if resp.FullName == "" { + return key, false, fmt.Errorf("bulk: %s: rename lookup: response has no full_name", key) + } + return "github.com/" + strings.ToLower(resp.FullName), false, nil +} diff --git a/internal/bulk/github_test.go b/internal/bulk/github_test.go new file mode 100644 index 0000000..3994b33 --- /dev/null +++ b/internal/bulk/github_test.go @@ -0,0 +1,63 @@ +package bulk + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "tangled.org/odd.computer/isomorph/internal/fetch" +) + +func TestLookupGitHub(t *testing.T) { + mux := http.NewServeMux() + // GitHub answers a renamed repository with a 301 to its id-based URL; the client follows. + mux.HandleFunc("/repos/bagder/curl", func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "/repositories/1", http.StatusMovedPermanently) + }) + mux.HandleFunc("/repositories/1", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{"id":1,"full_name":"Curl/Curl"}`)) + }) + mux.HandleFunc("/repos/curl/curl", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{"id":1,"full_name":"curl/curl"}`)) + }) + mux.HandleFunc("/repos/gone/repo", func(w http.ResponseWriter, r *http.Request) { + http.Error(w, `{"message":"Not Found"}`, http.StatusNotFound) + }) + mux.HandleFunc("/repos/limited/repo", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-RateLimit-Remaining", "0") + w.Header().Set("X-RateLimit-Reset", "1700000000") + http.Error(w, `{"message":"API rate limit exceeded"}`, http.StatusForbidden) + }) + srv := httptest.NewServer(mux) + defer srv.Close() + client := fetch.New(t.TempDir()) + client.HTTP = srv.Client() + client.MinInterval = 0 + client.Retries = 0 + client.Offline = false + client.Tokens = map[string]string{} + old := githubAPI + githubAPI = srv.URL + defer func() { githubAPI = old }() + ctx := context.Background() + + for _, c := range []struct { + key, want string + gone bool + }{ + {"github.com/bagder/curl", "github.com/curl/curl", false}, + {"github.com/curl/curl", "github.com/curl/curl", false}, + {"github.com/gone/repo", "github.com/gone/repo", true}, + {"gitlab.com/gnutls/gnutls", "gitlab.com/gnutls/gnutls", false}, // other hosts pass through + {"github.com/curl", "github.com/curl", false}, // not a repository + } { + got, gone, err := LookupGitHub(ctx, client, c.key) + if err != nil || got != c.want || gone != c.gone { + t.Errorf("%s: %q gone=%v err=%v, want %q gone=%v", c.key, got, gone, err, c.want, c.gone) + } + } + if got, _, err := LookupGitHub(ctx, client, "github.com/limited/repo"); err == nil { + t.Errorf("rate limit: expected an error, got %q", got) + } +} diff --git a/internal/bulk/opensuse.go b/internal/bulk/opensuse.go new file mode 100644 index 0000000..0724513 --- /dev/null +++ b/internal/bulk/opensuse.go @@ -0,0 +1,58 @@ +package bulk + +import ( + "sort" + + "tangled.org/odd.computer/isomorph/internal/distro/opensuse" + "tangled.org/odd.computer/isomorph/internal/model" +) + +// FromOpenSUSE builds one row per source package of Tumbleweed's src-oss repository. bins (the +// oss repository) supplies binary names through each binary's source RPM; giturls (package -> +// git URLs, rb.zq1.de) corroborates the spec's Url:. +func FromOpenSUSE(srcs, bins []opensuse.RPM, giturls map[string][]string, files map[string]*opensuse.OBSFiles) []Package { + binaries := map[string][]string{} + for _, b := range bins { + if b.Arch == "src" || b.Arch == "nosrc" || b.SourceRPM == "" { + continue + } + if n := opensuse.SourceName(b.SourceRPM); n != "" { + binaries[n] = append(binaries[n], b.Name) + } + } + seen := map[string]bool{} + var out []Package + for _, s := range srcs { + if (s.Arch != "src" && s.Arch != "nosrc") || seen[s.Name] { + continue + } + seen[s.Name] = true + bs := dedupe(binaries[s.Name]) + p := Package{ + Distro: model.OpenSUSE, + Suite: "tumbleweed", + Name: s.Name, + Version: s.Version, + OSVEcosystem: "openSUSE:Tumbleweed", + PURL: "pkg:rpm/opensuse/" + s.Name + "?arch=src", + Binaries: bs, + Homepage: s.URL, + Status: StatusUnresolved, + } + fill(&p, opensuse.Candidates(s.URL, giturls[s.Name], files[s.Name])) + out = append(out, p) + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out +} + +func dedupe(s []string) []string { + sort.Strings(s) + var out []string + for i, v := range s { + if i == 0 || v != s[i-1] { + out = append(out, v) + } + } + return out +} diff --git a/internal/bulk/pypi.go b/internal/bulk/pypi.go new file mode 100644 index 0000000..38d33fe --- /dev/null +++ b/internal/bulk/pypi.go @@ -0,0 +1,128 @@ +package bulk + +import ( + "regexp" + "slices" + "strings" +) + +// The PyPI repository selection below is adapted from oss-rebuild's pypi.InferRepo and +// uri.FindCommonRepo (github.com/google/oss-rebuild, Apache-2.0), where it was validated on PyPI +// rebuilds. PyPI metadata names repositories inconsistently (a "Source" link, a GitHub home +// page, only a mention in the description), so links are tried in order of how reliably they +// name the source: +// +// source links: +// - the home page, on a known repo host +// - a link named like a source link (commonRepoLinks), on a known repo host +// - a link named like a source link, on any host +// other mentions: +// - the first known repo host cited in the description +// - a link under any other name, on a known repo host, sponsor pages excluded +// +// Unlike oss-rebuild, only the project's current metadata is read (no per-release metadata), +// and the result goes through repourl like every other candidate. + +// pypiInfo is the part of PyPI's /pypi//json "info" object the selection reads. +type pypiInfo struct { + HomePage string `json:"home_page"` + ProjectURLs map[string]string `json:"project_urls"` + Description string `json:"description"` +} + +// commonRepoLinks are project_urls names (lower-cased, spaces removed) that usually name the +// source repository. +var commonRepoLinks = []string{"source", "sourcecode", "repository", "project", "github", "code"} + +var ( + pypiGitHubRE = regexp.MustCompile(`(?i)\bgithub(?:\.com[:/]|:)([\w-]+/[\w.-]+)`) + pypiGitLabRE = regexp.MustCompile(`(?i)\bgitlab(?:\.com[:/]|:)([\w-]+(?:/[\w.-]+)+)`) + pypiBitbucketRE = regexp.MustCompile(`(?i)\bbitbucket(?:\.org[:/]|:)([\w-]+/[\w.-]+)`) + pypiCodebergRE = regexp.MustCompile(`(?i)\bcodeberg(?:\.org[:/]|:)([\w-]+/[\w.-]+)`) + pypiCommonRepos = []*regexp.Regexp{pypiGitHubRE, pypiGitLabRE, pypiBitbucketRE, pypiCodebergRE} + // gitlabProjectRoutes directly follow a GitLab project path in older links without /-/. + gitlabProjectRoutes = []string{"badges", "blob", "commits", "container_registry", "issues", "packages", + "pipelines", "raw", "releases", "tags", "tree", "uploads", "wikis"} +) + +// pypiRepo returns the repository URL a PyPI project's metadata names, or "". +func pypiRepo(info pypiInfo) string { + if r := repoFromSourceLinks(info); r != "" { + return r + } + return repoFromOtherLinks(info) +} + +func repoFromSourceLinks(info pypiInfo) string { + byName := make(map[string]string, len(info.ProjectURLs)) + for name, u := range info.ProjectURLs { + byName[strings.ReplaceAll(strings.ToLower(name), " ", "")] = u + } + var sources []string + for _, name := range commonRepoLinks { + if u, ok := byName[name]; ok { + sources = append(sources, u) + } + } + for _, u := range append([]string{info.HomePage, byName["homepage"]}, sources...) { + if r := findCommonRepo(u); r != "" { + return r + } + } + if len(sources) != 0 { + return sources[0] + } + return "" +} + +func repoFromOtherLinks(info pypiInfo) string { + if r := findCommonRepo(info.Description); r != "" && !strings.Contains(r, "sponsors") { + return r + } + names := make([]string, 0, len(info.ProjectURLs)) + for name := range info.ProjectURLs { + names = append(names, name) + } + slices.Sort(names) // map order is random; make the choice deterministic + for _, name := range names { + u := info.ProjectURLs[name] + if strings.Contains(u, "sponsors") { + continue + } + if r := findCommonRepo(u); r != "" { + return r + } + } + return "" +} + +// findCommonRepo returns the first known repository host path in text ("github.com/o/r"), or "". +func findCommonRepo(text string) string { + for _, re := range pypiCommonRepos { + if r := re.FindString(text); r != "" { + if re == pypiGitLabRE { + r = trimGitLabRoute(r) + } + return r + } + } + return "" +} + +// trimGitLabRoute drops a project route from a matched GitLab path. +func trimGitLabRoute(repo string) string { + if root, _, found := strings.Cut(repo, "/-/"); found { + return root + } + start := strings.IndexAny(repo, ":/") + if start < 0 { + return repo + } + segs := strings.Split(repo[start+1:], "/") + for i := 2; i < len(segs); i++ { + if slices.Contains(gitlabProjectRoutes, segs[i]) { + return repo[:start+1] + strings.Join(segs[:i], "/") + } + } + return repo +} diff --git a/internal/bulk/pypi_test.go b/internal/bulk/pypi_test.go new file mode 100644 index 0000000..20808e1 --- /dev/null +++ b/internal/bulk/pypi_test.go @@ -0,0 +1,23 @@ +package bulk + +import "testing" + +func TestPyPIRepo(t *testing.T) { + for _, tc := range []struct { + name string + info pypiInfo + want string + }{ + {"homepage on a repo host wins", pypiInfo{HomePage: "https://github.com/o/home", ProjectURLs: map[string]string{"Source": "https://gitlab.com/o/src"}}, "github.com/o/home"}, + {"source link, spaces and case folded", pypiInfo{ProjectURLs: map[string]string{"Source Code": "https://gitlab.com/g/sub/p/-/tree/main", "Documentation": "https://docs.example.org"}}, "gitlab.com/g/sub/p"}, + {"legacy gitlab route trimmed", pypiInfo{ProjectURLs: map[string]string{"Repository": "https://gitlab.com/g/p/tree/main"}}, "gitlab.com/g/p"}, + {"source link on any host", pypiInfo{ProjectURLs: map[string]string{"Source": "https://git.example.org/p.git"}}, "https://git.example.org/p.git"}, + {"description mention", pypiInfo{HomePage: "https://p.readthedocs.io", Description: "Clone https://github.com/o/desc to hack."}, "github.com/o/desc"}, + {"sponsors skipped", pypiInfo{ProjectURLs: map[string]string{"Funding": "https://github.com/sponsors/o", "Tracker": "https://github.com/o/real/issues"}}, "github.com/o/real"}, + {"nothing", pypiInfo{HomePage: "https://example.org"}, ""}, + } { + if got := pypiRepo(tc.info); got != tc.want { + t.Errorf("%s: pypiRepo = %q, want %q", tc.name, got, tc.want) + } + } +} diff --git a/internal/bulk/sqlite.go b/internal/bulk/sqlite.go new file mode 100644 index 0000000..ffb66f0 --- /dev/null +++ b/internal/bulk/sqlite.go @@ -0,0 +1,208 @@ +package bulk + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "strings" + + _ "modernc.org/sqlite" // pure-Go driver, registered as "sqlite" + + "tangled.org/odd.computer/isomorph/internal/model" +) + +// schema is the published SQLite layout. Rows for one (distro, suite) are replaced as a unit, +// so each slice can be refreshed independently. +const schema = ` +create table if not exists sources ( + distro text not null, suite text not null, origin text not null, fetched_at text not null, + packages integer not null, primary key (distro, suite)); +create table if not exists packages ( + distro text not null, suite text not null, package text not null, version text not null, + osv_ecosystem text, purl text, homepage text, upstream_key text, repo text, + status text not null, basis text, + primary key (distro, suite, package)); +create index if not exists packages_upstream on packages (upstream_key); +create table if not exists evidence ( + distro text not null, suite text not null, package text not null, + source text not null, value text not null, key text, weight real not null); +create index if not exists evidence_pkg on evidence (distro, suite, package); +create table if not exists binaries ( + distro text not null, suite text not null, binary text not null, package text not null); +create index if not exists binaries_name on binaries (binary); +create table if not exists source_sha256 ( + distro text not null, suite text not null, package text not null, sha256 text not null); +create index if not exists source_sha256_hash on source_sha256 (sha256); +create table if not exists aliases ( + from_key text primary key, to_key text not null, source text not null, checked_at text not null); +create view if not exists counterparts as + select a.distro, a.suite, a.package, a.upstream_key, + b.distro as other_distro, b.suite as other_suite, b.package as other_package + from packages a join packages b + on a.upstream_key = b.upstream_key + and not (a.distro = b.distro and a.suite = b.suite and a.package = b.package) + where a.upstream_key is not null and a.upstream_key != ''; +` + +// schemaVersion is stored in SQLite's user_version; bump it whenever schema changes shape. +const schemaVersion = 2 + +// Open opens (creating if needed) the SQLite database at path. A database written with another +// schema version is refused rather than silently mixed: the file is derived, so rebuild it. +func Open(ctx context.Context, path string) (*sql.DB, error) { + db, err := sql.Open("sqlite", path) + if err != nil { + return nil, err + } + var version, tables int + if err := db.QueryRowContext(ctx, `pragma user_version`).Scan(&version); err != nil { + db.Close() + return nil, err + } + if err := db.QueryRowContext(ctx, `select count(*) from sqlite_master where type = 'table'`).Scan(&tables); err != nil { + db.Close() + return nil, err + } + if tables > 0 && version != schemaVersion { + db.Close() + return nil, fmt.Errorf("bulk: %s has schema version %d, want %d; delete it and rebuild", path, version, schemaVersion) + } + if _, err := db.ExecContext(ctx, fmt.Sprintf("pragma user_version = %d", schemaVersion)); err != nil { + db.Close() + return nil, err + } + if _, err := db.ExecContext(ctx, schema); err != nil { + db.Close() + return nil, fmt.Errorf("bulk: schema: %w", err) + } + return db, nil +} + +// Replace writes pkgs as the complete (src.Distro, src.Suite) slice, removing what was there. +func Replace(ctx context.Context, db *sql.DB, src Source, pkgs []Package) error { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + for _, t := range []string{"packages", "evidence", "binaries", "source_sha256"} { + if _, err := tx.ExecContext(ctx, "delete from "+t+" where distro = ? and suite = ?", src.Distro, src.Suite); err != nil { + return err + } + } + ins := func(q string) (*sql.Stmt, error) { return tx.PrepareContext(ctx, q) } + pkgStmt, err := ins(`insert into packages values (?,?,?,?,?,?,?,?,?,?,?)`) + if err != nil { + return err + } + evStmt, err := ins(`insert into evidence values (?,?,?,?,?,?,?)`) + if err != nil { + return err + } + binStmt, err := ins(`insert into binaries values (?,?,?,?)`) + if err != nil { + return err + } + shaStmt, err := ins(`insert into source_sha256 values (?,?,?,?)`) + if err != nil { + return err + } + for _, p := range pkgs { + if _, err := pkgStmt.ExecContext(ctx, p.Distro, p.Suite, p.Name, p.Version, p.OSVEcosystem, p.PURL, p.Homepage, + p.UpstreamKey, p.Repo, p.Status, p.Basis); err != nil { + return fmt.Errorf("bulk: %s/%s: %w", p.Distro, p.Name, err) + } + for _, e := range p.Evidence { + if _, err := evStmt.ExecContext(ctx, p.Distro, p.Suite, p.Name, e.Source, e.Value, e.Key, e.Weight); err != nil { + return err + } + } + for _, b := range p.Binaries { + if _, err := binStmt.ExecContext(ctx, p.Distro, p.Suite, b, p.Name); err != nil { + return err + } + } + for _, h := range p.SourceSHA256 { + if _, err := shaStmt.ExecContext(ctx, p.Distro, p.Suite, p.Name, h); err != nil { + return err + } + } + } + if _, err := tx.ExecContext(ctx, `insert or replace into sources values (?,?,?,?,?)`, + src.Distro, src.Suite, src.Origin, src.FetchedAt.UTC().Format("2006-01-02T15:04:05Z"), len(pkgs)); err != nil { + return err + } + return tx.Commit() +} + +// Lookup returns the packages matching ref: "distro:package", "distro/suite:package", or an +// upstream key (every package mapped to it). +func Lookup(ctx context.Context, db *sql.DB, ref string) ([]Package, error) { + q := `select distro, suite, package, version, coalesce(osv_ecosystem,''), coalesce(purl,''), coalesce(homepage,''), + coalesce(upstream_key,''), coalesce(repo,''), status, coalesce(basis,'') from packages where ` + var args []any + if d, name, ok := strings.Cut(ref, ":"); ok && !strings.Contains(d, ".") { + distro, suite, _ := strings.Cut(d, "/") + q += "distro = ? and package = ?" + args = append(args, distro, name) + if suite != "" { + q += " and suite = ?" + args = append(args, suite) + } + } else { + q += "upstream_key = ?" + args = append(args, ref) + } + rows, err := db.QueryContext(ctx, q+" order by distro, suite, package", args...) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Package + for rows.Next() { + var p Package + if err := rows.Scan(&p.Distro, &p.Suite, &p.Name, &p.Version, &p.OSVEcosystem, &p.PURL, &p.Homepage, + &p.UpstreamKey, &p.Repo, &p.Status, &p.Basis); err != nil { + return nil, err + } + out = append(out, p) + } + if err := rows.Err(); err != nil { + return nil, err + } + for i := range out { + if err := loadEvidence(ctx, db, &out[i]); err != nil { + return nil, err + } + } + return out, nil +} + +func loadEvidence(ctx context.Context, db *sql.DB, p *Package) error { + rows, err := db.QueryContext(ctx, `select source, value, coalesce(key,''), weight from evidence + where distro = ? and suite = ? and package = ? order by weight desc`, p.Distro, p.Suite, p.Name) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var e model.Evidence + if err := rows.Scan(&e.Source, &e.Value, &e.Key, &e.Weight); err != nil { + return err + } + p.Evidence = append(p.Evidence, e) + } + return rows.Err() +} + +// WriteJSONL writes one JSON object per package. +func WriteJSONL(w interface{ Write([]byte) (int, error) }, pkgs []Package) error { + enc := json.NewEncoder(w) + for _, p := range pkgs { + if err := enc.Encode(p); err != nil { + return err + } + } + return nil +} -- 2.51.2