diff --git a/cmd/isomorph/bulk.go b/cmd/isomorph/bulk.go new file mode 100644 index 0000000..cd993f0 --- /dev/null +++ b/cmd/isomorph/bulk.go @@ -0,0 +1,622 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "flag" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "text/tabwriter" + "time" + + "tangled.org/odd.computer/isomorph/internal/bulk" + "tangled.org/odd.computer/isomorph/internal/distro/alpine" + "tangled.org/odd.computer/isomorph/internal/distro/arch" + "tangled.org/odd.computer/isomorph/internal/distro/debian" + "tangled.org/odd.computer/isomorph/internal/distro/opensuse" + "tangled.org/odd.computer/isomorph/internal/model" +) + +func init() { + register(command{name: "bulk", usage: "debian|arch|opensuse|alpine|canonicalize|compare [flags]: resolve whole distributions into the association table, canonicalise its keys, or compare distributions", run: runBulk}) +} + +func defaultDB(env *Env) string { return filepath.Join(env.DataDir, "packages.sqlite") } + +func runBulk(ctx context.Context, env *Env, args []string) error { + if len(args) == 0 { + return errors.New("usage: bulk debian|arch|opensuse|alpine|compare|canonicalize [flags]") + } + switch args[0] { + case "debian": + return bulkDebian(ctx, env, args[1:]) + case "arch": + return bulkArch(ctx, env, args[1:]) + case "opensuse": + return bulkOpenSUSE(ctx, env, args[1:]) + case "alpine": + return bulkAlpine(ctx, env, args[1:]) + case "compare": + return bulkCompare(ctx, env, args[1:]) + case "canonicalize": + return bulkCanonicalize(ctx, env, args[1:]) + } + return fmt.Errorf("bulk: unknown source %q (want debian, arch, opensuse, alpine, compare or canonicalize)", args[0]) +} + +// writeSlices replaces each (distro, suite) slice of pkgs in the database and prints coverage +// per basis. +func writeSlices(ctx context.Context, env *Env, dbPath, origin string, fetched time.Time, pkgs []bulk.Package) error { + if err := os.MkdirAll(filepath.Dir(dbPath), 0o755); err != nil { + return err + } + db, err := bulk.Open(ctx, dbPath) + if err != nil { + return err + } + defer db.Close() + alias, err := bulk.LoadAliases(ctx, db) + if err != nil { + return err + } + if n := bulk.ApplyAliases(pkgs, alias); n > 0 { + env.Logf("aliases changed the upstream of %d packages", n) + } + slices := map[[2]string][]bulk.Package{} + var order [][2]string + for _, p := range pkgs { + k := [2]string{string(p.Distro), p.Suite} + if _, ok := slices[k]; !ok { + order = append(order, k) + } + slices[k] = append(slices[k], p) + } + for _, k := range order { + src := bulk.Source{Distro: model.Distro(k[0]), Suite: k[1], Origin: origin, FetchedAt: fetched, Packages: len(slices[k])} + if err := bulk.Replace(ctx, db, src, slices[k]); err != nil { + return err + } + } + resolved, byBasis := 0, map[string]int{} + for _, p := range pkgs { + if p.Status == bulk.StatusResolved { + resolved++ + byBasis[p.Basis]++ + } + } + fmt.Fprintf(env.Stdout, "%d packages, %d resolved to a repository (%.1f%%); by basis: %v\n", len(pkgs), resolved, pct(resolved, len(pkgs)), byBasis) + fmt.Fprintf(env.Stdout, "wrote %s\n", dbPath) + return nil +} + +func pct(a, b int) float64 { return 100 * float64(a) / float64(max(1, b)) } + +func bulkDebian(ctx context.Context, env *Env, args []string) error { + fs := flag.NewFlagSet("bulk debian", flag.ContinueOnError) + suite := fs.String("suite", "sid", "Debian suite (UDD release name)") + component := fs.String("component", "main", "archive component") + dbPath := fs.String("db", defaultDB(env), "SQLite database to update") + heldOutPath := fs.String("heldout", "", "write every held-out disagreement (package, repository key, fallback key) as TSV") + if err := fs.Parse(args); err != nil { + return err + } + start := time.Now() + rows, err := debian.QueryUDD(ctx, debian.UDDDSN(), *suite, *component) + if err != nil { + return err + } + env.Logf("udd: %d source packages in %s/%s (%s)", len(rows), *suite, *component, time.Since(start).Round(time.Second)) + fmt.Fprintf(env.Stdout, "debian/%s: ", *suite) + if err := writeSlices(ctx, env, *dbPath, "udd-mirror.debian.net", start, bulk.FromUDD(rows, *suite)); err != nil { + return err + } + h := bulk.DebianHeldOut(rows) + fmt.Fprintf(env.Stdout, "held out DEP-12 Repository: %d packages have one; the other fields resolve %d (%.1f%%), agreeing on %d (%.1f%% of those)\n", + h.WithRepository, h.FallbackResolved, pct(h.FallbackResolved, h.WithRepository), h.Agree, pct(h.Agree, h.FallbackResolved)) + for b, n := range h.ByBasis { + fmt.Fprintf(env.Stdout, " fallback basis %-12s resolved %5d, agree %5d (%.1f%%)\n", b, n[0], n[1], pct(n[1], n[0])) + } + if *heldOutPath != "" { + var b strings.Builder + for _, d := range h.Disagree { + b.WriteString(strings.Join(d[:], "\t") + "\n") + } + if err := os.WriteFile(*heldOutPath, []byte(b.String()), 0o644); err != nil { + return err + } + } + return nil +} + +func bulkArch(ctx context.Context, env *Env, args []string) error { + fs := flag.NewFlagSet("bulk arch", flag.ContinueOnError) + dbPath := fs.String("db", defaultDB(env), "SQLite database to update") + crawl := fs.Int("crawl", 0, "fetch up to N .SRCINFO files not yet cached (gitlab.archlinux.org rate-limits anonymous API use; set ARCH_GITLAB_TOKEN or ~/.config/isomorph/gitlab.archlinux.org.token)") + if err := fs.Parse(args); err != nil { + return err + } + start := time.Now() + r := arch.New(env.Client) + var dbs []arch.SyncPackage + for _, repo := range arch.SyncRepos { + p, err := r.SyncDB(ctx, repo) + if err != nil { + return err + } + dbs = append(dbs, p...) + } + state, err := r.State(ctx, nil) + if err != nil { + return err + } + env.Logf("arch: %d binary packages in %v, %d pkgbases pinned in the state repository", len(dbs), arch.SyncRepos, len(state)) + pkgs, st := bulk.FromArch(ctx, r, dbs, state, *crawl, func(n int) { + if n%200 == 0 { + env.Logf("arch: fetched %d .SRCINFO (%s)", n, time.Since(start).Round(time.Second)) + } + }) + if ctx.Err() != nil { + return ctx.Err() + } + fmt.Fprintf(env.Stdout, "arch: %d pkgbases, %d resolved from .SRCINFO (%d fetched now, %d errors), the rest from the repository databases' url\n", + st.PkgBases, st.FromSRCINFO, st.Fetched, st.FetchErrors) + if st.Stopped != nil { + fmt.Fprintf(env.Stdout, "arch: crawl stopped early: %v\n", st.Stopped) + } + for name, e := range st.Errors { + env.Logf("arch: %s: %s", name, e) + } + fmt.Fprint(env.Stdout, "arch: ") + return writeSlices(ctx, env, *dbPath, "pacman repository databases + gitlab.archlinux.org .SRCINFO", start, pkgs) +} + +func bulkOpenSUSE(ctx context.Context, env *Env, args []string) error { + fs := flag.NewFlagSet("bulk opensuse", flag.ContinueOnError) + dbPath := fs.String("db", defaultDB(env), "SQLite database to update") + crawl := fs.Int("crawl", 0, "fetch the OBS spec and _service of up to N unresolved packages not yet cached (api.opensuse.org, one request per second)") + crawlResolved := fs.Bool("crawl-resolved", false, "with -crawl, also fetch packages the metadata already resolves, after the unresolved ones") + if err := fs.Parse(args); err != nil { + return err + } + start := time.Now() + srcs, err := opensuse.Primary(ctx, env.Client, "", "src-oss") + if err != nil { + return err + } + bins, err := opensuse.Primary(ctx, env.Client, "", "oss") + if err != nil { + return err + } + r := opensuse.New(env.Client) + giturls, err := r.GitURLs(ctx) + if err != nil { + env.Logf("opensuse: giturls unavailable, continuing without: %v", err) + } + env.Logf("opensuse: %d source and %d binary packages, %d giturls entries", len(srcs), len(bins), len(giturls)) + files, err := openSUSEFiles(ctx, env, r, srcs, bins, giturls, *crawl, *crawlResolved) + if err != nil { + return err + } + fmt.Fprint(env.Stdout, "opensuse/tumbleweed: ") + return writeSlices(ctx, env, *dbPath, "download.opensuse.org tumbleweed repodata + rb.zq1.de giturls.good + api.opensuse.org spec/_service", start, bulk.FromOpenSUSE(srcs, bins, giturls, files)) +} + +// openSUSEFiles reads every source package's cached OBS spec and _service, then fetches those +// of up to crawl more packages: unresolved ones first, and resolved ones too if asked. +func openSUSEFiles(ctx context.Context, env *Env, r *opensuse.Resolver, srcs, bins []opensuse.RPM, giturls map[string][]string, crawl int, crawlResolved bool) (map[string]*opensuse.OBSFiles, error) { + list, err := r.ListPackages(ctx) + if err != nil { + return nil, err + } + obsPkgs := map[string]bool{} + for _, p := range list { + obsPkgs[p] = true + } + files := map[string]*opensuse.OBSFiles{} + var missing []string + errs := 0 + seen := map[string]bool{} + for _, s := range srcs { + if (s.Arch != "src" && s.Arch != "nosrc") || seen[s.Name] { + continue + } + seen[s.Name] = true + f, err := r.OBSFiles(ctx, s.Name, opensuse.OBSPackageCandidates(s.Name, obsPkgs), false) + switch { + case errors.Is(err, opensuse.ErrNotCached): + missing = append(missing, s.Name) + case err != nil: + errs++ + env.Debugf("opensuse: %s: %v", s.Name, err) + default: + files[s.Name] = f + } + } + if crawl > 0 && len(missing) > 0 { + // Unresolved first: resolve the missing packages from what is already known. + resolved := map[string]bool{} + for _, p := range bulk.FromOpenSUSE(srcs, bins, giturls, files) { + resolved[p.Name] = p.UpstreamKey != "" + } + var queue []string + for _, n := range missing { + if !resolved[n] { + queue = append(queue, n) + } + } + if crawlResolved { + for _, n := range missing { + if resolved[n] { + queue = append(queue, n) + } + } + } + if len(queue) > crawl { + queue = queue[:crawl] + } + env.Logf("opensuse: fetching OBS files of %d packages (%d not cached)", len(queue), len(missing)) + fetched, failed := 0, 0 + t0 := time.Now() + for _, n := range queue { + f, err := r.OBSFiles(ctx, n, opensuse.OBSPackageCandidates(n, obsPkgs), true) + if err != nil { + failed++ + env.Logf("opensuse: %s: %v", n, err) + if strings.Contains(err.Error(), "rate limited") || strings.Contains(err.Error(), "429") || ctx.Err() != nil { + env.Logf("opensuse: stopping the crawl") + break + } + continue + } + files[n] = f + fetched++ + if fetched%200 == 0 { + env.Logf("opensuse: fetched OBS files of %d packages (%s)", fetched, time.Since(t0).Round(time.Second)) + } + } + env.Logf("opensuse: fetched %d, %d failed", fetched, failed) + } + withSpec, withService, notCached := 0, 0, 0 + for _, f := range files { + if f != nil { + withSpec++ + if len(f.Service) > 0 { + withService++ + } + } + } + for _, n := range missing { + if _, ok := files[n]; !ok { + notCached++ + } + } + env.Logf("opensuse: OBS spec for %d packages (%d with _service), %d without a spec found, %d not cached, %d errors", + withSpec, withService, len(files)-withSpec, notCached, errs) + return files, nil +} + +func bulkAlpine(ctx context.Context, env *Env, args []string) error { + fs := flag.NewFlagSet("bulk alpine", flag.ContinueOnError) + dbPath := fs.String("db", defaultDB(env), "SQLite database to update") + archive := fs.String("aports", alpine.DefaultAportsArchive, "aports archive (tar.gz) to read APKBUILDs from") + if err := fs.Parse(args); err != nil { + return err + } + start := time.Now() + sections := []string{"main", "community"} + r := alpine.New(env.Client) + index := map[string][]alpine.IndexEntry{} + for _, s := range sections { + e, err := r.Index(ctx, s) + if err != nil { + return err + } + index[s] = e + } + apkbuilds := map[string]*alpine.APKBUILD{} + failed := 0 + if err := alpine.StreamAPKBUILDs(ctx, env.Client, *archive, sections, func(section, dir string, ab *alpine.APKBUILD, err error) { + if err != nil { + failed++ + return + } + apkbuilds[section+"/"+dir] = ab + }); err != nil { + return err + } + env.Logf("alpine: %d APKBUILDs (%d unparseable) in %s", len(apkbuilds), failed, time.Since(start).Round(time.Second)) + fmt.Fprint(env.Stdout, "alpine/edge: ") + return writeSlices(ctx, env, *dbPath, "dl-cdn.alpinelinux.org edge APKINDEX + github.com/alpinelinux/aports master", start, bulk.FromAlpine(index, apkbuilds)) +} + +func bulkCompare(ctx context.Context, env *Env, args []string) error { + fs := flag.NewFlagSet("bulk compare", flag.ContinueOnError) + dbPath := fs.String("db", defaultDB(env), "SQLite database") + a := fs.String("a", "debian", "first distro") + b := fs.String("b", "arch", "second distro") + all := fs.Bool("all", false, "compare every pair of distros in the database, then summarise the name join as a distro-by-distro grid") + samples := fs.Int("samples", -1, "disagreements to print per join (default 15 for one pair, 0 with -all)") + if err := fs.Parse(args); err != nil { + return err + } + if *samples < 0 { + *samples = 15 + if *all { + *samples = 0 + } + } + db, err := bulk.Open(ctx, *dbPath) + if err != nil { + return err + } + defer db.Close() + pairs := [][2]string{{*a, *b}} + var distros []string + if *all { + if distros, err = distroNames(ctx, db); err != nil { + return err + } + if pairs, err = distroPairs(ctx, db); err != nil { + return err + } + } + byName := map[[2]string]bulk.Agreement{} + for _, p := range pairs { + ags, err := bulk.Compare(ctx, db, p[0], p[1]) + if err != nil { + return err + } + for _, ag := range ags { + printConfusion(env, p[0], p[1], ag, *samples) + if ag.Join == "name" { + byName[p] = ag + byName[[2]string{p[1], p[0]}] = ag + } + } + } + if *all { + fmt.Fprintln(env.Stdout, "agreement where both resolve, joined by name (of pairs both resolved):") + tw := tabwriter.NewWriter(env.Stdout, 0, 0, 3, ' ', 0) + fmt.Fprintf(tw, "\t%s\n", strings.Join(distros, "\t")) + for _, r := range distros { + fmt.Fprint(tw, r) + for _, c := range distros { + if ag, ok := byName[[2]string{r, c}]; ok { + fmt.Fprintf(tw, "\t%.1f%% of %d", pct(ag.Agree, ag.BothResolved), ag.BothResolved) + } else { + fmt.Fprint(tw, "\t-") + } + } + fmt.Fprintln(tw) + } + tw.Flush() + } + return nil +} + +// distroNames lists the distributions in db, in name order. +func distroNames(ctx context.Context, db *sql.DB) ([]string, error) { + slices, err := bulk.Slices(ctx, db) + if err != nil { + return nil, err + } + var distros []string + for _, s := range slices { + if len(distros) == 0 || distros[len(distros)-1] != s[0] { + distros = append(distros, s[0]) + } + } + return distros, nil +} + +// distroPairs lists every unordered pair of distributions in db, in name order. +func distroPairs(ctx context.Context, db *sql.DB) ([][2]string, error) { + distros, err := distroNames(ctx, db) + if err != nil { + return nil, err + } + var pairs [][2]string + for i := range distros { + for _, d := range distros[i+1:] { + pairs = append(pairs, [2]string{distros[i], d}) + } + } + return pairs, nil +} + +// printConfusion prints one join as a confusion matrix of a's outcome (rows) against b's +// (columns), then up to samples of the disagreements. +func printConfusion(env *Env, a, b string, ag bulk.Agreement, samples int) { + fmt.Fprintf(env.Stdout, "%s vs %s joined by %s: %d pairs\n", a, b, ag.Join, ag.Pairs) + if ag.Pairs == 0 { + fmt.Fprintln(env.Stdout) + return + } + tw := tabwriter.NewWriter(env.Stdout, 0, 0, 3, ' ', 0) + fmt.Fprintf(tw, "\t%s resolved\t%s unresolved\n", b, b) + fmt.Fprintf(tw, "%s resolved\t%d agree, %d differ\t%d\n", a, ag.Agree, len(ag.Disagree), ag.OnlyA) + fmt.Fprintf(tw, "%s unresolved\t%d\t%d\n", a, ag.OnlyB, ag.Neither) + tw.Flush() + fmt.Fprintf(env.Stdout, "%.1f%% agree where both resolve\n", pct(ag.Agree, ag.BothResolved)) + for i, d := range ag.Disagree { + if i == samples { + if samples > 0 { + fmt.Fprintf(env.Stdout, " ... %d more\n", len(ag.Disagree)-samples) + } + break + } + fmt.Fprintf(env.Stdout, " %s=%s %s=%s\n", d[0], d[2], d[1], d[3]) + } + fmt.Fprintln(env.Stdout) +} + +func bulkCanonicalize(ctx context.Context, env *Env, args []string) error { + fs := flag.NewFlagSet("bulk canonicalize", flag.ContinueOnError) + dbPath := fs.String("db", defaultDB(env), "SQLite database") + if err := fs.Parse(args); err != nil { + return err + } + db, err := bulk.Open(ctx, *dbPath) + if err != nil { + return err + } + defer db.Close() + // The keys worth a lookup are those in conflict: within a package's own evidence, and + // between distributions' mappings of same-named packages (every pair). The lookups only + // ask GitHub about a key; no distribution's mapping is copied to another. + pairs, err := distroPairs(ctx, db) + if err != nil { + return err + } + var disagreements [][2]string + for _, p := range pairs { + ags, err := bulk.Compare(ctx, db, p[0], p[1]) + if err != nil { + return err + } + for _, ag := range ags { + for _, d := range ag.Disagree { + disagreements = append(disagreements, [2]string{d[2], d[3]}) + } + } + } + keys, err := bulk.ConflictingKeys(ctx, db, "github.com/", disagreements) + if err != nil { + return err + } + env.Logf("canonicalize: %d GitHub keys in conflicts across %d distribution pairs", len(keys), len(pairs)) + now := time.Now() + var aliases []bulk.Alias + failed, gone := 0, 0 + for i, k := range keys { + to, isGone, err := bulk.LookupGitHub(ctx, env.Client, k) + if err != nil { + failed++ + env.Debugf("canonicalize %s: %v", k, err) + continue + } + switch { + case isGone: + gone++ + aliases = append(aliases, bulk.Alias{From: k, To: "", Source: bulk.AliasGitHubGone, CheckedAt: now}) + case to != k: + aliases = append(aliases, bulk.Alias{From: k, To: to, Source: bulk.AliasGitHubRename, CheckedAt: now}) + } + if (i+1)%200 == 0 { + env.Logf("canonicalize: %d/%d", i+1, len(keys)) + } + } + var projectKeys []string + pk, err := db.QueryContext(ctx, `select distinct key from evidence where key like '%/-/project/%'`) + if err != nil { + return err + } + for pk.Next() { + var k string + pk.Scan(&k) + projectKeys = append(projectKeys, k) + } + pk.Close() + projects, projectFailed := bulk.GitLabProjectAliases(ctx, env.Client, projectKeys, func(host string) string { return "https://" + host }) + // Only registry entries that are the package itself: a Rust package lists every vendored + // crate as a source, and those name dependencies, not the upstream. + var registryKeys []string + seenReg := map[string]bool{} + rk, err := db.QueryContext(ctx, `select distinct package, key from evidence where key like 'pypi.org/%' or key like 'crates.io/%'`) + if err != nil { + return err + } + for rk.Next() { + var pkg, k string + rk.Scan(&pkg, &k) + if _, name, _ := strings.Cut(k, "/"); bulk.SameProject(pkg, name) && !seenReg[k] { + seenReg[k] = true + registryKeys = append(registryKeys, k) + } + } + rk.Close() + sort.Strings(registryKeys) + registries, registryFailed := bulk.RegistryAliases(ctx, env.Client, registryKeys, func(host string) string { return "https://" + host }) + for i := range registries { + registries[i].CheckedAt = now + } + aliases = append(aliases, registries...) + for i := range projects { + projects[i].CheckedAt = now + } + aliases = append(aliases, projects...) + // Rule-based aliases are recomputed in full. Stored evidence keeps its keys after aliasing, + // so the keys a rule mapped before are re-checked from the aliases table as well; otherwise + // the next rebuild from raw data would find those aliases gone. + var all []string + rows, err := db.QueryContext(ctx, `select key from evidence where key != '' union select from_key from aliases where source in (?, ?)`, + bulk.AliasMirrorRule, bulk.AliasKDEProjects) + if err != nil { + return err + } + for rows.Next() { + var k string + rows.Scan(&k) + all = append(all, k) + } + rows.Close() + mirrors := bulk.MirrorAliases(all) + for i := range mirrors { + mirrors[i].CheckedAt = now + } + kde, kdeErr := bulk.KDEAliases(ctx, env.Client, all) + if kdeErr != nil { + env.Logf("canonicalize: KDE project list unavailable, keeping the stored KDE aliases: %v", kdeErr) + } + for i := range kde { + kde[i].CheckedAt = now + } + if err := bulk.SaveAliases(ctx, db, aliases); err != nil { + return err + } + if err := bulk.ReplaceAliases(ctx, db, bulk.AliasMirrorRule, mirrors); err != nil { + return err + } + if kdeErr == nil { + if err := bulk.ReplaceAliases(ctx, db, bulk.AliasKDEProjects, kde); err != nil { + return err + } + } + alias, err := bulk.LoadAliases(ctx, db) + if err != nil { + return err + } + slices, err := bulk.Slices(ctx, db) + if err != nil { + return err + } + changed := 0 + for _, sl := range slices { + pkgs, err := bulk.LoadSlice(ctx, db, model.Distro(sl[0]), sl[1]) + if err != nil { + return err + } + n := bulk.ApplyAliases(pkgs, alias) + if n == 0 { + continue + } + changed += n + var src bulk.Source + var fetched string + if err := db.QueryRowContext(ctx, `select origin, fetched_at from sources where distro = ? and suite = ?`, sl[0], sl[1]).Scan(&src.Origin, &fetched); err != nil { + return err + } + src.Distro, src.Suite, src.Packages = model.Distro(sl[0]), sl[1], len(pkgs) + src.FetchedAt, _ = time.Parse(time.RFC3339, fetched) + if err := bulk.Replace(ctx, db, src, pkgs); err != nil { + return err + } + } + fmt.Fprintf(env.Stdout, "canonicalize: %d GitHub keys checked (%d failed), %d renamed, %d gone; %d GitLab project ids resolved (%d failed); %d registry entries resolved of %d (%d failed); %d mirror aliases; %d KDE aliases; %d package mappings changed\n", + len(keys), failed, len(aliases)-len(projects)-len(registries)-gone, gone, len(projects), projectFailed, len(registries), len(registryKeys), registryFailed, len(mirrors), len(kde), changed) + return nil +} diff --git a/cmd/isomorph/main.go b/cmd/isomorph/main.go new file mode 100644 index 0000000..60e58e9 --- /dev/null +++ b/cmd/isomorph/main.go @@ -0,0 +1,107 @@ +// Command isomorph maps distribution source packages to their upstream repositories and, +// through them, to their counterparts in other distributions. +// +// isomorph [global flags] [args] +// +// Global flags: -data DIR (database and exports, default ./data), -cache DIR (HTTP cache), +// -offline, -v. +package main + +import ( + "context" + "flag" + "fmt" + "io" + "os" + "os/signal" + "sort" + "syscall" + + "tangled.org/odd.computer/isomorph/internal/fetch" +) + +// Env is shared by all commands. +type Env struct { + // DataDir holds the database (packages.sqlite) and exports. + DataDir string + Client *fetch.Client + Verbose bool + Stdout io.Writer +} + +// Logf prints a progress line to stderr. +func (e *Env) Logf(format string, args ...any) { + fmt.Fprintf(os.Stderr, format+"\n", args...) +} + +// Debugf prints only with -v. +func (e *Env) Debugf(format string, args ...any) { + if e.Verbose { + e.Logf(format, args...) + } +} + +type command struct { + name string + usage string + run func(ctx context.Context, env *Env, args []string) error +} + +var commands = map[string]command{} + +func register(c command) { commands[c.name] = c } + +func usage() { + fmt.Fprintf(os.Stderr, "usage: isomorph [-data DIR] [-cache DIR] [-offline] [-v] [args]\n\ncommands:\n") + var names []string + for n := range commands { + names = append(names, n) + } + sort.Strings(names) + for _, n := range names { + fmt.Fprintf(os.Stderr, " %-8s %s\n", n, commands[n].usage) + } +} + +func main() { + fs := flag.NewFlagSet("isomorph", flag.ExitOnError) + data := fs.String("data", envOr("ISOMORPH_DATA", "data"), "directory for the database and exports (default $ISOMORPH_DATA or ./data)") + cache := fs.String("cache", "", "HTTP cache directory (default $ISOMORPH_CACHE or the user cache dir)") + offline := fs.Bool("offline", false, "never hit the network; fail on cache misses") + verbose := fs.Bool("v", false, "verbose logging") + fs.Usage = usage + fs.Parse(os.Args[1:]) + if fs.NArg() == 0 { + usage() + os.Exit(2) + } + cmd, ok := commands[fs.Arg(0)] + if !ok { + fmt.Fprintf(os.Stderr, "unknown command %q\n\n", fs.Arg(0)) + usage() + os.Exit(2) + } + client := fetch.New(*cache) + if *offline { + client.Offline = true + } + env := &Env{DataDir: *data, Client: client, Verbose: *verbose, Stdout: os.Stdout} + client.Logf = env.Debugf + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + if err := cmd.run(ctx, env, fs.Args()[1:]); err != nil { + fatal(err) + } +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, "isomorph:", err) + os.Exit(1) +} + +func envOr(k, def string) string { + if v := os.Getenv(k); v != "" { + return v + } + return def +} diff --git a/docs/EVALUATION.md b/docs/EVALUATION.md new file mode 100644 index 0000000..43d91af --- /dev/null +++ b/docs/EVALUATION.md @@ -0,0 +1,222 @@ +# Evaluation + +How well does each distribution's mapping name the right upstream repository, and how +often do distributions agree? This document measures three things: + +1. **Coverage:** the share of source packages mapped to a repository. +2. **Agreement with a ground truth:** Debian's DEP-12 `Repository` field is the one field + whose purpose is to name the upstream repository. Its packages are resolved again without + it, and the result is compared with it. +3. **Agreement between distributions:** same-named packages, and packages built from the + identical release tarball, should map to the same repository. + +The numbers below come from one run on 2026-09-28 over Debian sid/main, Arch (core, extra, +multilib), openSUSE Tumbleweed and Alpine edge (main, community). "How to reproduce" at the +end has the commands. + +Every mapping comes from its own distribution's data: the packaging, and the upstream +projects' own services (GitHub, KDE's project list, PyPI, crates.io). Other distributions +are used only to measure agreement, and to choose which keys are worth a GitHub lookup. + +## Coverage + +| distribution | source packages | mapped to a repository | +|---|---:|---:| +| Arch | 12,358 | 85.3% | +| Alpine edge | 9,494 | 78.3% | +| Debian sid | 41,417 | 77.9% | +| openSUSE Tumbleweed | 17,211 | 75.0% | + +Every mapping records its **basis**, the strongest kind of evidence behind it: + +| basis | Debian | Arch | openSUSE | Alpine | +|---|---:|---:|---:|---:| +| metadata (DEP-12 `Repository`) | 11,404 | | | | +| vcs-source (a git source, `_service` scm, `_repo_url`) | | 4,367 | 564 | 453 | +| release-url (watch URL, release tarball) | 15,984 | 5,014 | 1,192 | 6,693 | +| derived (rb.zq1.de giturls) | | | 6,399 | | +| bug-tracker (DEP-12 `Bug-Database`) | 68 | | | | +| homepage | 4,790 | 1,157 | 4,757 | 285 | + +The basis stands in for a confidence score. Measured agreement per basis, below, says how +far each kind of evidence can be trusted. + +- **Arch is a moving target.** About 1,100 pkgbases were released after the `.SRCINFO` + crawl, so their new packaging commits were not cached for this run and those rows use the + repository database's url. + +## Agreement with Debian's DEP-12 `Repository` + +10,872 Debian packages have a DEP-12 `Repository` that names a repository. Each is resolved +again without `Repository` and `Repository-Browse`: + +- The remaining fields (watch URL, Bug-Database, Homepage) resolve **10,283 (94.6%)**. +- They agree with `Repository` on **9,950 (96.8%)**. + +| basis of the fallback | resolved | agree | +|---|---:|---:| +| release-url | 6,547 | 96.7% | +| bug-tracker | 3,624 | 97.3% | +| homepage | 112 | 81.2% | + +- **Why homepage agreement is lower:** most of its misses are GitHub renames (this check does + not apply aliases), umbrella project pages (gtk2-perl's SourceForge project), and the GNU + Octave packages repository. +- **Limits:** DEP-12 is itself sometimes stale or wrong, so 96.8% is a lower bound on the + fallbacks' precision. Packages with DEP-12 metadata are also better maintained than the + average. + +The same field scores the other distributions, for their same-named packages: + +| distribution | basis | packages | agree with DEP-12 | +|---|---|---:|---:| +| Arch | release-url | 427 | 94.4% | +| Arch | vcs-source | 415 | 93.3% | +| Arch | homepage | 16 | 87.5% | +| Alpine | release-url | 272 | 92.6% | +| Alpine | homepage | 7 | 57.1% | +| openSUSE | derived | 556 | 95.3% | +| openSUSE | homepage | 227 | 88.5% | +| openSUSE | release-url | 70 | 87.1% | +| openSUSE | vcs-source | 29 | 93.1% | + +Most remaining misses are projects that moved (soundtouch to Codeberg, pmbootstrap to +postmarketOS's GitLab), where the other distribution is newer than Debian's field, or +different software with the same name. + +## Agreement between distributions + +`isomorph bulk compare -all` joins every pair of distributions twice, independently of +the mappings: +- by package name; +- for Debian and Arch, by identical source tarball: the sha256 of Debian's `.orig` tarball + and of Arch's release source. + +Each join is shown as a confusion matrix of one distribution's outcome against the other's. + +**By package name.** Each row flattens one matrix. + +| a vs b | both, agree | both, differ | only a | only b | neither | agree where both | +|---|---:|---:|---:|---:|---:|---:| +| Arch vs Debian | 4,584 | 304 | 611 | 156 | 574 | 93.8% | +| Alpine vs Arch | 2,857 | 192 | 54 | 281 | 681 | 93.7% | +| Alpine vs Debian | 2,694 | 194 | 236 | 133 | 478 | 93.3% | +| Arch vs openSUSE | 3,882 | 356 | 355 | 89 | 321 | 91.6% | +| Alpine vs openSUSE | 2,318 | 231 | 138 | 123 | 378 | 90.9% | +| Debian vs openSUSE | 3,898 | 401 | 376 | 367 | 694 | 90.7% | + +- **The name join understates agreement.** It pairs different software that shares a name + (fff, hyperlink, forks of mg), so "differ" overstates real disagreement. +- **Arch fills in for Debian.** 611 shared names resolve in Arch but not in Debian, against + 156 the other way. + +**By identical source tarball (Debian vs Arch):** 1,162 pairs; 668 resolved on both sides, +of which 644 (96.4%) agree. This join needs no names. It stays small because Debian repacks +tarballs and Arch builds many packages from git. + +### Before the error-analysis fixes + +The morning run on 2026-09-28 came before the fixes below and before the openSUSE spec crawl. + +| | before | after | +|---|---:|---:| +| Arch coverage | 83.8% | 85.3% | +| Alpine coverage | 75.4% | 78.3% | +| Debian coverage | 76.7% | 77.9% | +| openSUSE coverage | 65.3% | 75.0% | +| Arch vs Debian | 93.0% of 4,662 | 93.8% of 4,888 | +| Alpine vs Arch | 92.1% of 2,809 | 93.7% of 3,049 | +| Alpine vs Debian | 92.7% of 2,631 | 93.3% of 2,888 | +| Arch vs openSUSE | 89.2% of 3,547 | 91.6% of 4,238 | +| Alpine vs openSUSE | 89.6% of 1,972 | 90.9% of 2,549 | +| Debian vs openSUSE | 90.2% of 3,608 | 90.7% of 4,299 | + +## Error analysis + +**Sample.** It was drawn from the morning run with seed 20260928, and stratified: +- 8 pairs from each of the six "both, differ" cells, 48 in total; +- 50 packages that are unresolved while a same-named package elsewhere resolves: 20 from + openSUSE and 10 from each other distribution. + +**Method.** Each item was labelled from its evidence in all four distributions, and some were +checked against the GitHub API. Shares are weighted by stratum size; with about 50 items per +sample they carry roughly ±10 points. +- The sampler is `eval/sample.py`. +- The labels are in `eval/errors-2026-09-28.tsv`. + +**Why two resolved packages differed** (1,684 pairs): + +| cause | sampled | share | examples | +|---|---:|---:|---| +| stale or dead distribution data: the row names only an old home | 12 | 25% | SourceForge downloads for GitHub projects, notabug to Codeberg, repositories that 404 | +| wrong repository in openSUSE's giturls, or an umbrella homepage | 7 | 18% | awesome-releases, the parent hfst, dead github.com/freedesktop mirrors | +| selection: the right repository was in the row but lost | 9 | 17% | a SourceForge or Launchpad homepage beat a GitHub release URL | +| different software sharing a name | 8 | 15% | hyperlink, fff, forks of mg, each distribution's ca-certificates | +| same project, both keys valid | 7 | 15% | GitHub mirrors, renames not yet canonicalised | +| URL mapping | 5 | 10% | `*.wiki.kernel.org` to Linux, `download.samba.org/pub/rsync` to Samba | + +**Why a package was unresolved while a counterpart resolved** (3,456 packages): + +| cause | sampled | share | +|---|---:|---:| +| only a project website, no repository or forge URL (openSUSE: 16 of 20) | 27 | 55% | +| a host rule would map the homepage (docs.xfce.org, gnu.org/software, netfilter.org, apps.kde.org) | 11 | 23% | +| the evidence named the repository but the key was incomplete (KDE tarballs name no group; an unevaluated `case` in APKBUILDs) | 8 | 12% | +| no URL at all, a Debian-native package, or a registry entry naming no repository | 4 | 10% | + +**What was fixed.** Each fix was counted over all packages, not only the sample. + +1. **KDE.** + - Keys without a group take it from KDE's project list; 489 packages were unresolved this + way. + - The APKBUILD evaluator takes the matching branch of a `case` over a known variable + (47 Alpine Plasma packages), and reads `_repo_url`. +2. **Host rules.** The findvcs-style rules the openSUSE resolver used (GNU and nongnu to + Savannah, cgit to GitLab, Apache, GitHub Pages) now apply to every distribution. About 435 + rows had a homepage one of them covers. New rules cover docs.xfce.org, netfilter.org and + freedesktop.org/software/pulseaudio. +3. **SourceForge and Launchpad homepages.** A homepage on these hosts no longer beats a + release or VCS source naming another repository (130 packages). + - Against DEP-12, where such a homepage and the watch URL disagreed, the watch URL was + right 22 times out of 23. + - GitHub homepages split 16 to 17, so the rule is limited to these two hosts. +4. **URL mapping:** + - kernel.org project subdomains (66 packages) and samba.org's other projects (23) no longer + hint at Linux or Samba; + - GitLab API URLs that name a project by path now name it. +5. **Canonicalisation** checks conflicts across every pair of distributions. It aliases + repositories that no longer exist (GitHub answers 404) to nothing, so the package falls back + to its next candidate. +6. **openSUSE's gap was missing data.** Its repository metadata carries only the spec + `Url:`. The OBS spec and `_service` of the ~5,900 unresolved packages are now read, one + request a second; coverage went from 65.3% to 75.0%. +7. **Secondary spec sources** (`Source1` and later: configuration files, completions, test + data) corroborate a candidate but never decide one. Of the 43 packages they alone + resolved, about a third named another project. + +**Measured and not changed.** When openSUSE's giturls and spec `Url:` disagree (214 +packages), neither is reliably right. On the 22 with a DEP-12 `Repository`, giturls matches +in 12 and `Url:` in 9, so their precedence stays. + +**What remains:** +- **Stale or wrong distribution data** was about 43% of the disagreements before the fixes. + Manual overrides or cross-distribution consensus could address it; neither is built. Each + distribution's mapping stays its own. +- **Packages whose only URL is a project website.** +- **Arch and Alpine source order.** Neither records which source is primary, so a + dependency's git source can still outrank the main tarball (telegram-desktop). + +## How to reproduce + +```sh +isomorph bulk debian # one UDD query +isomorph bulk arch -crawl 20000 # paced .SRCINFO crawl; later runs read the cache +isomorph bulk opensuse -crawl 7000 # paced OBS spec/_service crawl +isomorph bulk alpine # one streamed aports download +GITHUB_TOKEN=... isomorph bulk canonicalize +isomorph bulk compare -all +python3 eval/sample.py differ # the error-analysis sample +``` + +- **Held-out check:** `bulk debian` prints it. `-heldout FILE` writes each disagreement. +- **`GITHUB_TOKEN`:** anonymous GitHub API use is capped at 60 requests an hour. diff --git a/eval/errors-2026-09-28.tsv b/eval/errors-2026-09-28.tsv new file mode 100644 index 0000000..d108f14 --- /dev/null +++ b/eval/errors-2026-09-28.tsv @@ -0,0 +1,99 @@ +cell a b package key_a key_b category wrong_side note +differ alpine arch b4 git.kernel.org/pub/scm/utils/b4/b4 github.com/mricon/b4 same-project arch uses the maintainer's GitHub mirror of the kernel.org repository +differ alpine arch hyphen sourceforge.net/hunspell github.com/hunspell/hyphen stale-data alpine APKBUILD downloads from the old SourceForge hunspell project; development is on GitHub +differ alpine arch ngrep sourceforge.net/ngrep github.com/jpr5/ngrep selection alpine SourceForge url outranks the GitHub source tarball in the same APKBUILD +differ alpine arch rsync git.samba.org/samba github.com/rsyncproject/rsync url-mapping alpine download.samba.org/pub/rsync mapped to the samba repository +differ alpine arch ruby-builder github.com/jimweirich/builder github.com/rails/builder stale-data alpine jimweirich/builder is the original, unmaintained repository; rails/builder continues it +differ alpine arch tcpreplay sourceforge.net/tcpreplay github.com/appneta/tcpreplay stale-data alpine release tarball from SourceForge; development on GitHub +differ alpine arch uvicorn github.com/kludex/uvicorn github.com/encode/uvicorn same-project encode/uvicorn was transferred to Kludex/uvicorn; the rename lookup only covered Debian vs Arch conflicts +differ alpine arch xcb-util-errors gitlab.freedesktop.org/xorg/lib/libxcb-errors cgit.freedesktop.org/xcb/util-errors stale-data arch cgit.freedesktop.org url predates the move to gitlab.freedesktop.org/xorg/lib/libxcb-errors +differ alpine debian edk2 github.com/tianocore/tianocore.github.io github.com/tianocore/edk2 selection alpine wiki page of tianocore.github.io outranks the edk2 source tarball +differ alpine debian guile-sqlite3 codeberg.org/guile-sqlite3/guile-sqlite3 notabug.org/guile-sqlite3/guile-sqlite3 stale-data debian notabug.org homepage; the project moved to Codeberg +differ alpine debian hyperlink github.com/untitaker/hyperlink github.com/python-hyper/hyperlink different-software Rust link checker vs Python URL library +differ alpine debian libraw1394 sourceforge.net/libraw1394 git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux url-mapping debian ieee1394.wiki.kernel.org mapped to torvalds/linux +differ alpine debian libutempter github.com/altlinux/libutempter git.altlinux.org/libutempter same-project git.altlinux.org and its GitHub mirror +differ alpine debian metalog github.com/hvisage/metalog sourceforge.net/metalog selection debian SourceForge homepage outranks the GitHub watch URL +differ alpine debian scons sourceforge.net/scons github.com/scons/scons selection alpine SourceForge download listed before the GitHub archive in the same APKBUILD +differ alpine debian w3m git.sr.ht/~rkta/w3m salsa.debian.org/debian/w3m different-software Debian tracks its own w3m fork on salsa; others package the sourcehut continuation +differ alpine opensuse awesome github.com/awesomewm/awesome github.com/awesomewm/awesome-releases source-data opensuse giturls names awesome-releases, a release-artifact repository +differ alpine opensuse ca-certificates gitlab.alpinelinux.org/alpine/ca-certificates github.com/opensuse/ca-certificates different-software each distribution ships its own ca-certificates +differ alpine opensuse iwd git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux git.kernel.org/pub/scm/network/wireless/iwd url-mapping alpine iwd.wiki.kernel.org mapped to torvalds/linux +differ alpine opensuse libwmf sourceforge.net/wvware github.com/caolanm/libwmf selection alpine SourceForge url outranks the GitHub source tarball +differ alpine opensuse mg github.com/hboetes/mg github.com/troglobit/mg different-software two portable forks of OpenBSD mg (hboetes, troglobit) +differ alpine opensuse ocaml-xml-light github.com/ncannasse/xml-light github.com/gasche/xml-light different-software gasche/xml-light is a GitHub fork of ncannasse/xml-light +differ alpine opensuse tree-sitter-xml github.com/observeroftime/tree-sitter-xml github.com/tree-sitter-grammars/tree-sitter-xml same-project ObserverOfTime/tree-sitter-xml was transferred to tree-sitter-grammars +differ alpine opensuse vde2 github.com/virtualsquare/vde-2 sourceforge.net/vde stale-data opensuse SourceForge homepage only; development on GitHub (virtualsquare/vde-2) +differ arch debian biber github.com/plk/biber sourceforge.net/biblatex-biber selection debian SourceForge homepage outranks the GitHub watch URL +differ arch debian cloud-utils github.com/canonical/cloud-utils launchpad.net/cloud-utils selection debian Launchpad homepage outranks the GitHub watch URL +differ arch debian evolution-ews gitlab.gnome.org/gnome/evolution-ews gitlab.gnome.org/gnome/evolution selection debian parent project's wiki (evolution) outranks the evolution-ews watch URL +differ arch debian haskell-xdg-desktop-entry github.com/taffybar/taffybar github.com/taffybar/xdg-desktop-entry stale-data debian old standalone repository; now a subdirectory of the taffybar monorepo +differ arch debian liborcus gitlab.com/api/v4/projects/orcus/orcus/packages/generic/source/0.21.0/liborcus-0.21.0.tar.xz gitlab.com/orcus/orcus url-mapping arch GitLab generic package-registry URL taken as the key +differ arch debian ruby-net-dns github.com/ruby-dns/net-dns gitlab.com/gitlab-org/gitlab-net-dns different-software Debian packages GitLab's fork (gitlab-net-dns) +differ arch debian smartmontools sourceforge.net/smartmontools github.com/smartmontools/smartmontools stale-data arch release tarball from SourceForge; development on GitHub +differ arch debian telegram-desktop github.com/tdlib/td github.com/telegramdesktop/tdesktop selection arch a git source for the tdlib dependency outranks the tdesktop release tarball +differ arch opensuse helmfile github.com/helmfile/helmfile github.com/roboll/helmfile source-data opensuse giturls names the abandoned roboll/helmfile; spec Url: is right but ranks lower +differ arch opensuse libdazzle gitlab.gnome.org/archive/libdazzle gitlab.gnome.org/gnome/libdazzle same-project GNOME/libdazzle moved to the Archive group (GitLab rename, not canonicalised) +differ arch opensuse libqmi gitlab.freedesktop.org/mobile-broadband/libqmi github.com/freedesktop/libqmi source-data opensuse giturls names github.com/freedesktop/libqmi, which does not exist +differ arch opensuse mdds gitlab.com/api/v4/projects/mdds/mdds/packages/generic/source/3.2.1/mdds-3.2.1.tar.xz gitlab.com/mdds/mdds url-mapping arch GitLab generic package-registry URL taken as the key +differ arch opensuse python-parsedatetime github.com/bear/parsedatetime github.com/regebro/parsedatetime stale-data opensuse spec Url: and giturls name regebro/parsedatetime, which does not exist +differ arch opensuse schroedinger launchpad.net/schroedinger sourceforge.net/schrodinger stale-data opensuse old SourceForge FAQ page; releases on Launchpad +differ arch opensuse startup-notification gitlab.freedesktop.org/xdg/startup-notification github.com/freedesktop/startup-notification source-data opensuse giturls names github.com/freedesktop/startup-notification, which does not exist +differ arch opensuse switchboard-plug-applications github.com/elementary/switchboard-plug-applications github.com/elementary/settings-applications same-project renamed to elementary/settings-applications +differ debian opensuse fff github.com/meekrosoft/fff github.com/dmtrkovalenko/fff different-software C fake-function framework vs a file finder +differ debian opensuse hfst-ospell github.com/hfst/hfst-ospell github.com/hfst/hfst source-data opensuse giturls names the parent hfst repository +differ debian opensuse ibus-chewing codeberg.org/chewing/ibus-chewing github.com/chewing/ibus-chewing stale-data opensuse GitHub repository says 'Migrated to Codeberg' +differ debian opensuse jline sourceforge.net/jline github.com/jline/jline2 different-software Debian's jline is jline 1 (SourceForge); openSUSE's is jline2 +differ debian opensuse kernelshark git.kernel.org/pub/scm/utils/trace-cmd/kernel-shark git.kernel.org/pub/scm/linux/kernel/git/rostedt/trace-cmd source-data opensuse giturls names trace-cmd, the repository kernelshark left; spec Url: is right +differ debian opensuse libaec github.com/deutsches-klimarechenzentrum/libaec gitlab.dkrz.de/k202009/libaec same-project gitlab.dkrz.de and the DKRZ GitHub organisation +differ debian opensuse libcdio-paranoia github.com/libcdio/libcdio-paranoia git.savannah.gnu.org/libcdio source-data opensuse spec Url: is the umbrella libcdio Savannah project +differ debian opensuse luajit github.com/openresty/luajit2 github.com/openresty/luajit stale-data opensuse spec Url: names openresty/luajit, which does not exist (openresty/luajit2) +unresolved alpine botan3 website-only alpine botan.randombit.net +unresolved alpine flatpak-kcm key-incomplete alpine source uses $_rel set in a case block; _repo_url names invent.kde.org/plasma/flatpak-kcm +unresolved alpine kmousetool key-incomplete alpine download.kde.org tarball gives invent.kde.org/kmousetool without its group +unresolved alpine kphotoalbum key-incomplete alpine download.kde.org tarball gives invent.kde.org/kphotoalbum without its group +unresolved alpine libopensmtpd website-only alpine imperialat.at page; arch has the git source +unresolved alpine msmtp website-only alpine marlam.de; arch has the git source +unresolved alpine nss website-only alpine developer.mozilla.org page +unresolved alpine ristretto host-rule alpine docs.xfce.org/apps/ristretto -> gitlab.xfce.org/apps/ristretto +unresolved alpine ulogd website-only alpine netfilter.org/projects/ulogd; the repository is ulogd2 +unresolved alpine xfce4-places-plugin host-rule alpine docs.xfce.org/panel-plugins/... -> gitlab.xfce.org +unresolved arch batctl website-only arch open-mesh.org +unresolved arch blinken key-incomplete arch KDE key without group +unresolved arch kate key-incomplete arch KDE key without group +unresolved arch kspaceduel key-incomplete arch KDE key without group +unresolved arch libnetfilter_cttimeout host-rule arch netfilter.org/projects/X -> git.netfilter.org/X +unresolved arch python-sgmllib3k registry-only arch PyPI entry names no repository +unresolved arch suil website-only arch drobilla.net +unresolved arch unifdef website-only arch dotat.at +unresolved arch xbindkeys host-rule arch nongnu.org rule exists in the openSUSE resolver only +unresolved arch zanshin key-incomplete arch KDE key without group +unresolved debian bridge-utils no-url debian no Homepage or watch URL +unresolved debian dillo website-only debian dillo-browser.org +unresolved debian gnuchess host-rule debian gnu.org/software rule exists in the openSUSE resolver only +unresolved debian jetring distro-native debian Debian-native package; its upstream is salsa +unresolved debian kdenlive key-incomplete debian KDE key without group +unresolved debian libeot no-url debian no Homepage or watch URL +unresolved debian libx86 website-only debian codon.org.uk page +unresolved debian mate-common website-only debian mate-desktop.org +unresolved debian paprefs host-rule debian freedesktop.org/software/pulseaudio/paprefs -> gitlab.freedesktop.org/pulseaudio/paprefs +unresolved debian xfce4-fsguard-plugin host-rule debian docs.xfce.org -> gitlab.xfce.org +unresolved opensuse a52dec host-rule opensuse Url: is a Gitea repository on an unknown host (git.adelielinux.org) +unresolved opensuse apcupsd website-only opensuse apcupsd.com +unresolved opensuse cardpeek website-only opensuse pannetrat.com +unresolved opensuse fooyin website-only opensuse fooyin.org +unresolved opensuse iotop website-only opensuse guichaz.free.fr +unresolved opensuse kasts host-rule opensuse apps.kde.org/kasts -> KDE project lookup +unresolved opensuse kf6-kplotting website-only opensuse www.kde.org +unresolved opensuse kf6-syndication website-only opensuse www.kde.org +unresolved opensuse lapackpp website-only opensuse icl.utk.edu/slate +unresolved opensuse libcap website-only opensuse sites.google.com +unresolved opensuse libgedit-gtksourceview website-only opensuse gedit-technology.net +unresolved opensuse mosh website-only opensuse mosh.org +unresolved opensuse multitail website-only opensuse vanheusden.com +unresolved opensuse poezio website-only opensuse poez.io +unresolved opensuse python-scikit-image website-only opensuse scikit-image.org +unresolved opensuse sratom website-only opensuse drobilla.net +unresolved opensuse unknown-horizons website-only opensuse unknown-horizons.org +unresolved opensuse xf86-video-tdfx website-only opensuse xorg.freedesktop.org +unresolved opensuse xfce4-clipman-plugin host-rule opensuse docs.xfce.org -> gitlab.xfce.org +unresolved opensuse xfce4-panel host-rule opensuse docs.xfce.org -> gitlab.xfce.org diff --git a/eval/sample.py b/eval/sample.py new file mode 100644 index 0000000..5eab535 --- /dev/null +++ b/eval/sample.py @@ -0,0 +1,68 @@ +"""Stratified error-analysis sample over the name-join confusion matrices. + +differ: pairs (a, b, name) where both resolve to different keys, 8 per distro pair. +unres: (distro, name) unresolved while a same-named package elsewhere resolves, + 20 for openSUSE and 10 for each other distro. +Prints stratum sizes and, per item, every distro's row and evidence for that name. +The labels for the 2026-09-28 sample are in errors-2026-09-28.tsv. + +Usage: python3 eval/sample.py differ|unres [packages.sqlite] +""" +import itertools, random, sqlite3, sys + +DB = sys.argv[2] if len(sys.argv) > 2 else "data/packages.sqlite" +SEED = 20260928 +DISTROS = ["alpine", "arch", "debian", "opensuse"] +db = sqlite3.connect(DB) + +def rows(name): + out = {} + for d, suite, key, basis, home in db.execute( + "select distro, suite, coalesce(upstream_key,''), coalesce(basis,''), coalesce(homepage,'') " + "from packages where package=? order by distro, suite", (name,)): + ev = db.execute( + "select source, value, coalesce(key,'') from evidence where distro=? and suite=? and package=? " + "order by weight desc", (d, suite, name)).fetchall() + out.setdefault(d, (suite, key, basis, home, ev)) + return out + +def dump(tag, header, name): + print(f"[{tag}] {header} name={name}") + for d, (suite, key, basis, home, ev) in rows(name).items(): + print(f" {d:8} {key or '-'} ({basis or 'unresolved'}) home={home}") + seen, n = set(), 0 + for src, val, k in ev: + if val in seen: + continue + seen.add(val) + n += 1 + if n > 6: + print(f" ... {len(set(v for _, v, _ in ev)) - 6} more") + break + print(f" {src}: {val[:110]}" + (f" -> {k}" if k else "")) + print() + +rng = random.Random(SEED) +resolved = {} +for d, name, key in db.execute("select distro, package, coalesce(upstream_key,'') from packages"): + if key or name not in resolved.get(d, {}): + resolved.setdefault(d, {})[name] = key + +mode = sys.argv[1] +if mode == "differ": + for a, b in itertools.combinations(DISTROS, 2): + pop = sorted(n for n in resolved[a].keys() & resolved[b].keys() + if resolved[a][n] and resolved[b][n] and resolved[a][n] != resolved[b][n]) + pick = rng.sample(pop, min(8, len(pop))) + print(f"### stratum {a} vs {b}: {len(pop)} differing names, sampled {len(pick)}\n") + for i, n in enumerate(sorted(pick)): + dump(f"D-{a[:2]}{b[:2]}-{i}", f"{a} vs {b}", n) +elif mode == "unres": + want = {"opensuse": 20, "debian": 10, "arch": 10, "alpine": 10} + for d in DISTROS: + pop = sorted(n for n, k in resolved[d].items() if not k and any( + resolved[o].get(n) for o in DISTROS if o != d)) + pick = rng.sample(pop, want[d]) + print(f"### stratum {d} unresolved, a counterpart resolved: {len(pop)} names, sampled {len(pick)}\n") + for i, n in enumerate(sorted(pick)): + dump(f"U-{d[:2]}-{i}", f"{d} unresolved", n) diff --git a/go.mod b/go.mod index 55627d4..1237b82 100644 --- a/go.mod +++ b/go.mod @@ -5,10 +5,20 @@ go 1.27.1 require ( github.com/jackc/pgx/v5 v5.11.0 github.com/klauspost/compress v1.20.1 + modernc.org/sqlite v1.59.0 ) require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.29.0 // indirect + modernc.org/libc v1.75.7 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.12.1 // indirect ) diff --git a/go.sum b/go.sum index e8680e9..8726946 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,14 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= @@ -11,18 +19,58 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ= github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8= +modernc.org/cc/v4 v4.29.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.35.0 h1:F+TUsmw09QxLzmi3aeYYGxjAXarmZaKgj3mKQHNaA8w= +modernc.org/ccgo/v4 v4.35.0/go.mod h1:qrVGs9S3Sr2Ztcg9ve+kTAYMp5a3YvWjo+SoN06kJ5I= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc= +modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.75.7 h1:o3DTP9/0p9pKmY2WCKQaySW6wIiZhNM7wc2lUoyhfew= +modernc.org/libc v1.75.7/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= +modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.59.0 h1:X1es1GpqBlS/5T+vbM4HLUdaa8OtQx468DF2vrx+38A= +modernc.org/sqlite v1.59.0/go.mod h1:+paeT2A3iPRHkQDwG7oA6Tk0zQd5woMEI8q7orfry8k= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/internal/bulk/alias.go b/internal/bulk/alias.go new file mode 100644 index 0000000..364cd77 --- /dev/null +++ b/internal/bulk/alias.go @@ -0,0 +1,510 @@ +package bulk + +import ( + "context" + "database/sql" + "regexp" + "sort" + "strings" + "sync" + "time" + + "tangled.org/odd.computer/isomorph/internal/distro" + "tangled.org/odd.computer/isomorph/internal/fetch" + "tangled.org/odd.computer/isomorph/internal/model" + "tangled.org/odd.computer/isomorph/internal/repourl" +) + +// Aliases map an upstream key onto the key that names the same repository canonically: a +// renamed or transferred GitHub repository onto its current name, and a known mirror onto the +// project's main repository. They are applied by rescoring each package from its evidence, so +// two fields naming the same repository under different keys reinforce each other instead of +// competing. + +// Alias sources. +const ( + AliasGitHubRename = "github-rename" + AliasMirrorRule = "mirror-rule" + AliasKDEProjects = "kde-projects" + // AliasGitHubGone maps a GitHub repository that no longer exists to "": its evidence + // drops out, and the package falls back to its other evidence or is left unresolved. + AliasGitHubGone = "github-gone" +) + +// Alias records one mapping and how it was learned. +type Alias struct { + From, To, Source string + CheckedAt time.Time +} + +// MirrorAliases applies the documented mirror rules to keys: +// - gitbox.apache.org/[repos/asf/]X -> github.com/apache/X (the GitHub mirror, which the GitHub +// API can follow further); +// - github.com/gnome/X -> gitlab.gnome.org/gnome/X; +// - github.com/golang/X -> go.googlesource.com/X. +func MirrorAliases(keys []string) []Alias { + var out []Alias + for _, k := range keys { + to := "" + switch { + case strings.HasPrefix(k, "gitbox.apache.org/"): + // repourl keys gitbox repositories as gitbox.apache.org/; older keys kept + // the repos/asf/ prefix. + to = "github.com/apache/" + strings.TrimPrefix(strings.TrimPrefix(k, "gitbox.apache.org/"), "repos/asf/") + case strings.HasPrefix(k, "github.com/gnome/"): + to = "gitlab.gnome.org/gnome/" + strings.TrimPrefix(k, "github.com/gnome/") + case strings.HasPrefix(k, "github.com/golang/"): + to = "go.googlesource.com/" + strings.TrimPrefix(k, "github.com/golang/") + } + if to != "" && to != k { + out = append(out, Alias{From: k, To: to, Source: AliasMirrorRule}) + } + } + return out +} + +// KDEProjectsURL lists every KDE repository path (accessibility/kmousetool, ...). +const KDEProjectsURL = "https://projects.kde.org/api/v1/projects" + +// KDEAliases gives KDE keys without a group their repository path from KDE's own project list: +// download.kde.org tarballs and apps.kde.org pages name a project (invent.kde.org/kmousetool), +// and KDE's GitHub mirror drops the group (github.com/kde/kmousetool); both map to +// invent.kde.org/accessibility/kmousetool when exactly one KDE repository has that name. +func KDEAliases(ctx context.Context, c *fetch.Client, keys []string) ([]Alias, error) { + var paths []string + if err := c.GetJSON(ctx, KDEProjectsURL, &paths, fetch.WithTTL(7*24*time.Hour)); err != nil { + return nil, err + } + return kdeAliases(paths, keys), nil +} + +func kdeAliases(paths, keys []string) []Alias { + byName := map[string][]string{} + for _, p := range paths { + p = strings.ToLower(p) + byName[p[strings.LastIndexByte(p, '/')+1:]] = append(byName[p[strings.LastIndexByte(p, '/')+1:]], p) + } + var out []Alias + for _, k := range keys { + name := "" + switch { + case strings.HasPrefix(k, "github.com/kde/"): + name = strings.TrimPrefix(k, "github.com/kde/") + case strings.HasPrefix(k, "invent.kde.org/") && strings.Count(k, "/") == 1: + name = strings.TrimPrefix(k, "invent.kde.org/") + } + if c := byName[name]; name != "" && len(c) == 1 { + out = append(out, Alias{From: k, To: "invent.kde.org/" + c[0], Source: AliasKDEProjects}) + } + } + return out +} + +// Resolve follows an alias chain to its end (bounded, in case of cycles). +func Resolve(alias map[string]string, k string) string { + for i := 0; i < 5; i++ { + to, ok := alias[k] + if !ok || to == k { + return k + } + k = to + } + return k +} + +// Rescore recomputes a package's upstream from its evidence after mapping every evidence key +// through alias: evidence for keys that alias together combines as 1-prod(1-w). +func Rescore(p *Package, alias map[string]string) { + type cand struct { + key string + miss float64 // prod(1-w) + basis string + } + byKey := map[string]*cand{} + var order []string + for i := range p.Evidence { + e := &p.Evidence[i] + if e.Key == "" { + continue + } + if _, name, ok := registryKey(e.Key); ok && !SameProject(p.Name, name) { + continue // a dependency fetched from a registry (vendored crates), not this project + } + e.Key = Resolve(alias, e.Key) + if !isRepoKey(e.Key) { + continue // a registry entry that no alias resolved + } + c := byKey[e.Key] + if c == nil { + c = &cand{key: e.Key, miss: 1} + byKey[e.Key] = c + order = append(order, e.Key) + } + c.miss *= 1 - e.Weight + if b := BasisOf(e.Source); c.basis == "" || basisRank[b] < basisRank[c.basis] { + c.basis = b + } + } + best, bestScore := "", 0.0 + for _, k := range order { + if byKey[k].basis == BasisSecondary { + continue // corroborates, never decides + } + if s := (1 - byKey[k].miss) * distro.PackagingPenalty(k); s > bestScore { + best, bestScore = k, s + } + } + if best == "" { + // Nothing left: every key was aliased away (a repository that no longer exists) or is + // not a repository. + p.UpstreamKey, p.Repo, p.Status, p.Basis = "", "", StatusUnresolved, "" + return + } + if byKey[best].basis == BasisHomepage && onStaleHomeHost(best) { + // See staleHomeHosts: the best release or VCS source wins over such a homepage. + alt, altScore := "", 0.0 + for _, k := range order { + if b := byKey[k].basis; k != best && (b == BasisReleaseURL || b == BasisVCSSource) { + if s := (1 - byKey[k].miss) * distro.PackagingPenalty(k); s > altScore { + alt, altScore = k, s + } + } + } + if alt != "" { + best = alt + } + } + if best != p.UpstreamKey { + p.UpstreamKey, p.Repo = best, "https://"+best + if c, err := repourl.Parse("https://" + best); err == nil { + p.Repo = c.URL + } + } + p.Status = StatusResolved + p.Basis = byKey[best].basis +} + +// ApplyAliases rescores every package whose evidence names an aliased key. +func ApplyAliases(pkgs []Package, alias map[string]string) int { + n := 0 + for i := range pkgs { + for _, e := range pkgs[i].Evidence { + if _, ok := alias[e.Key]; ok { + before := pkgs[i].UpstreamKey + Rescore(&pkgs[i], alias) + if pkgs[i].UpstreamKey != before { + n++ + } + break + } + } + } + return n +} + +// LoadAliases reads the aliases table. +func LoadAliases(ctx context.Context, db *sql.DB) (map[string]string, error) { + rows, err := db.QueryContext(ctx, `select from_key, to_key from aliases`) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]string{} + for rows.Next() { + var f, t string + if err := rows.Scan(&f, &t); err != nil { + return nil, err + } + out[f] = t + } + return out, rows.Err() +} + +// SaveAliases upserts aliases. +func SaveAliases(ctx context.Context, db *sql.DB, as []Alias) error { + return saveAliases(ctx, db, "", as) +} + +// ReplaceAliases replaces every stored alias of source with as: rule-based sources are +// recomputed in full on each run, so a rule that no longer applies must not linger. +func ReplaceAliases(ctx context.Context, db *sql.DB, source string, as []Alias) error { + return saveAliases(ctx, db, source, as) +} + +func saveAliases(ctx context.Context, db *sql.DB, replace string, as []Alias) error { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + if replace != "" { + if _, err := tx.ExecContext(ctx, `delete from aliases where source = ?`, replace); err != nil { + return err + } + } + for _, a := range as { + if _, err := tx.ExecContext(ctx, `insert or replace into aliases values (?,?,?,?)`, + a.From, a.To, a.Source, a.CheckedAt.UTC().Format(time.RFC3339)); err != nil { + return err + } + } + return tx.Commit() +} + +// LoadSlice reads one (distro, suite) slice back with its evidence, binaries and hashes, one +// query per table. +func LoadSlice(ctx context.Context, db *sql.DB, d model.Distro, suite string) ([]Package, error) { + rows, err := db.QueryContext(ctx, `select distro, suite, package, version, coalesce(osv_ecosystem,''), coalesce(purl,''), + coalesce(homepage,''), coalesce(upstream_key,''), coalesce(repo,''), status, coalesce(basis,'') + from packages where distro = ? and suite = ? order by package`, d, suite) + if err != nil { + return nil, err + } + var out []Package + idx := map[string]int{} + for rows.Next() { + var p Package + if err := rows.Scan(&p.Distro, &p.Suite, &p.Name, &p.Version, &p.OSVEcosystem, &p.PURL, &p.Homepage, + &p.UpstreamKey, &p.Repo, &p.Status, &p.Basis); err != nil { + rows.Close() + return nil, err + } + idx[p.Name] = len(out) + out = append(out, p) + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, err + } + scan := func(q string, add func(r *sql.Rows) error) error { + r, err := db.QueryContext(ctx, q, d, suite) + if err != nil { + return err + } + defer r.Close() + for r.Next() { + if err := add(r); err != nil { + return err + } + } + return r.Err() + } + if err := scan(`select package, source, value, coalesce(key,''), weight from evidence where distro = ? and suite = ? order by rowid`, + func(r *sql.Rows) error { + var name string + var e model.Evidence + if err := r.Scan(&name, &e.Source, &e.Value, &e.Key, &e.Weight); err != nil { + return err + } + if i, ok := idx[name]; ok { + out[i].Evidence = append(out[i].Evidence, e) + } + return nil + }); err != nil { + return nil, err + } + for _, t := range []struct { + q string + dst func(p *Package) *[]string + }{ + {`select package, binary from binaries where distro = ? and suite = ? order by rowid`, func(p *Package) *[]string { return &p.Binaries }}, + {`select package, sha256 from source_sha256 where distro = ? and suite = ? order by rowid`, func(p *Package) *[]string { return &p.SourceSHA256 }}, + } { + if err := scan(t.q, func(r *sql.Rows) error { + var name, v string + if err := r.Scan(&name, &v); err != nil { + return err + } + if i, ok := idx[name]; ok { + dst := t.dst(&out[i]) + *dst = append(*dst, v) + } + return nil + }); err != nil { + return nil, err + } + } + return out, nil +} + +// Slices lists the (distro, suite) slices in the table. +func Slices(ctx context.Context, db *sql.DB) ([][2]string, error) { + rows, err := db.QueryContext(ctx, `select distro, suite from sources order by distro, suite`) + if err != nil { + return nil, err + } + defer rows.Close() + var out [][2]string + for rows.Next() { + var d, s string + if err := rows.Scan(&d, &s); err != nil { + return nil, err + } + out = append(out, [2]string{d, s}) + } + return out, rows.Err() +} + +// ConflictingKeys returns the keys worth canonicalising: keys of packages whose evidence names +// more than one repository on the same host (a rename leaves the old and new name side by +// side), plus the keys of cross-distro disagreements, restricted to hosts (sorted, unique). +func ConflictingKeys(ctx context.Context, db *sql.DB, hostPrefix string, disagreements [][2]string) ([]string, error) { + rows, err := db.QueryContext(ctx, `select distinct e.key from evidence e join ( + select distro, suite, package from evidence where key like ? || '%' + group by distro, suite, package having count(distinct key) > 1) c + on e.distro = c.distro and e.suite = c.suite and e.package = c.package + where e.key like ? || '%'`, hostPrefix, hostPrefix) + if err != nil { + return nil, err + } + set := map[string]bool{} + for rows.Next() { + var k string + if err := rows.Scan(&k); err != nil { + rows.Close() + return nil, err + } + set[k] = true + } + rows.Close() + for _, d := range disagreements { + for _, k := range d { + if strings.HasPrefix(k, hostPrefix) { + set[k] = true + } + } + } + out := make([]string, 0, len(set)) + for k := range set { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +// AliasGitLabProject marks aliases from a numeric GitLab project id to the project's path. +const AliasGitLabProject = "gitlab-project-id" + +// gitlabProjectKeyRe matches the keys repourl gives GitLab URLs that name a project by number. +var gitlabProjectKeyRe = regexp.MustCompile(`^([^/]+)/-/project/(\d+)$`) + +// GitLabProjectAliases resolves "/-/project/" keys through the host's API +// (/api/v4/projects/ answers anonymously for public projects). base maps a host onto the +// scheme and host to query (tests point it at an httptest server). +func GitLabProjectAliases(ctx context.Context, c *fetch.Client, keys []string, base func(host string) string) ([]Alias, int) { + var out []Alias + failed := 0 + for _, k := range keys { + m := gitlabProjectKeyRe.FindStringSubmatch(k) + if m == nil { + continue + } + var resp struct { + Path string `json:"path_with_namespace"` + } + if err := c.GetJSON(ctx, base(m[1])+"/api/v4/projects/"+m[2], &resp, fetch.WithTTL(30*24*time.Hour)); err != nil || resp.Path == "" { + failed++ + continue + } + cn, err := repourl.Parse("https://" + m[1] + "/" + resp.Path) + if err != nil || cn.Kind != repourl.KindRepo { + failed++ + continue + } + out = append(out, Alias{From: k, To: cn.Key, Source: AliasGitLabProject}) + } + return out, failed +} + +var repoKeyCache sync.Map + +// isRepoKey reports whether key names a repository (not a registry entry or other hint). +func isRepoKey(key string) bool { + if v, ok := repoKeyCache.Load(key); ok { + return v.(bool) + } + c, err := repourl.Parse("https://" + key) + ok := err == nil && c.Kind == repourl.KindRepo + repoKeyCache.Store(key, ok) + return ok +} + +// AliasRegistry marks aliases from a registry entry (pypi.org/, crates.io/) to the +// repository its metadata names. +const AliasRegistry = "registry-metadata" + +// RegistryAliases resolves registry keys through the registry's metadata: PyPI with +// oss-rebuild's link selection (pypi.go) and crates.io's repository field. base +// maps a registry host onto the scheme and host to query (tests point it at an httptest +// server). +func RegistryAliases(ctx context.Context, c *fetch.Client, keys []string, base func(host string) string) ([]Alias, int) { + var out []Alias + failed := 0 + for _, k := range keys { + host, name, ok := strings.Cut(k, "/") + if !ok { + continue + } + var urls []string + switch host { + case "pypi.org": + var resp struct { + Info pypiInfo `json:"info"` + } + if err := c.GetJSON(ctx, base(host)+"/pypi/"+name+"/json", &resp, fetch.WithTTL(30*24*time.Hour)); err != nil { + failed++ + continue + } + if r := pypiRepo(resp.Info); r != "" { + if !strings.Contains(r, "://") { + r = "https://" + r + } + urls = []string{r} + } + case "crates.io": + var resp struct { + Crate struct { + Repository string `json:"repository"` + Homepage string `json:"homepage"` + } `json:"crate"` + } + if err := c.GetJSON(ctx, base(host)+"/api/v1/crates/"+name, &resp, fetch.WithTTL(30*24*time.Hour)); err != nil { + failed++ + continue + } + urls = []string{resp.Crate.Repository, resp.Crate.Homepage} + default: + continue + } + for _, u := range urls { + if cn, err := repourl.Parse(u); err == nil && cn.Kind == repourl.KindRepo { + out = append(out, Alias{From: k, To: cn.Key, Source: AliasRegistry}) + break + } + } + } + return out, failed +} + +// registryKey splits a registry key (pypi.org/, crates.io/). +func registryKey(k string) (registry, name string, ok bool) { + registry, name, ok = strings.Cut(k, "/") + if !ok || (registry != "pypi.org" && registry != "crates.io") { + return "", "", false + } + return registry, name, true +} + +// SameProject reports whether a registry project name is the distro package itself rather than +// one of its dependencies: equal after lower-casing, unifying - _ . and dropping the ecosystem +// prefixes distros add (python-, python3-, py3-, rust-, ...). +func SameProject(pkg, registryName string) bool { + norm := func(s string) string { + s = strings.NewReplacer("_", "-", ".", "-").Replace(strings.ToLower(s)) + for _, p := range []string{"python3-", "python-", "py3-", "py-", "rust-", "golang-"} { + s = strings.TrimPrefix(s, p) + } + return s + } + return norm(pkg) == norm(registryName) +} diff --git a/internal/bulk/alias_test.go b/internal/bulk/alias_test.go new file mode 100644 index 0000000..b8bd02e --- /dev/null +++ b/internal/bulk/alias_test.go @@ -0,0 +1,188 @@ +package bulk + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "tangled.org/odd.computer/isomorph/internal/distro/opensuse" + "tangled.org/odd.computer/isomorph/internal/fetch" + "tangled.org/odd.computer/isomorph/internal/model" +) + +func TestRescoreMergesAliases(t *testing.T) { + // DEP-12 names the pre-transfer repository, the homepage and watch URL the new one; without + // the alias the stale key wins on weight, with it all three agree. + p := Package{ + UpstreamKey: "github.com/spacetelescope/asdf-standard", Status: StatusResolved, Basis: BasisMetadata, + Evidence: []model.Evidence{ + {Source: "debian/upstream/metadata:Repository", Key: "github.com/spacetelescope/asdf-standard", Weight: 0.95}, + {Source: "debian/watch:upstream_url", Key: "github.com/asdf-format/asdf-standard", Weight: 0.36}, + {Source: "debian/control:Homepage", Key: "github.com/asdf-format/asdf-standard", Weight: 0.4}, + }, + } + alias := map[string]string{"github.com/spacetelescope/asdf-standard": "github.com/asdf-format/asdf-standard"} + pkgs := []Package{p} + if n := ApplyAliases(pkgs, alias); n != 1 { + t.Fatalf("changed %d", n) + } + got := pkgs[0] + if got.UpstreamKey != "github.com/asdf-format/asdf-standard" || got.Repo != "https://github.com/asdf-format/asdf-standard" || got.Basis != BasisMetadata { + t.Fatalf("rescored: %+v", got) + } +} + +func TestMirrorAliases(t *testing.T) { + got := MirrorAliases([]string{ + "gitbox.apache.org/repos/asf/activemq", "gitbox.apache.org/zookeeper", "github.com/gnome/glib", "github.com/golang/text", + "github.com/kde/heaptrack", "github.com/other/x", "invent.kde.org/heaptrack", + }) + want := map[string]string{ + "gitbox.apache.org/repos/asf/activemq": "github.com/apache/activemq", + "gitbox.apache.org/zookeeper": "github.com/apache/zookeeper", + "github.com/gnome/glib": "gitlab.gnome.org/gnome/glib", + "github.com/golang/text": "go.googlesource.com/text", + } + if len(got) != len(want) { + t.Fatalf("got %+v", got) + } + for _, a := range got { + if want[a.From] != a.To || a.Source != AliasMirrorRule { + t.Errorf("alias %+v", a) + } + } + kde := kdeAliases([]string{"sdk/heaptrack", "a/dup", "b/dup", "accessibility/KMouseTool"}, []string{ + "github.com/kde/heaptrack", "github.com/kde/dup", "github.com/kde/unknown", + "invent.kde.org/kmousetool", "invent.kde.org/sdk/heaptrack", + }) + wantKDE := map[string]string{ + "github.com/kde/heaptrack": "invent.kde.org/sdk/heaptrack", + "invent.kde.org/kmousetool": "invent.kde.org/accessibility/kmousetool", + } + if len(kde) != len(wantKDE) { + t.Fatalf("kde aliases %+v", kde) + } + for _, a := range kde { + if wantKDE[a.From] != a.To || a.Source != AliasKDEProjects { + t.Errorf("kde alias %+v", a) + } + } + if Resolve(map[string]string{"a": "b", "b": "c", "c": "c"}, "a") != "c" { + t.Error("alias chains must be followed") + } +} + +func TestGitLabProjectAliases(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v4/projects/20183" { + fmt.Fprint(w, `{"id":20183,"path_with_namespace":"World/amberol"}`) + return + } + http.NotFound(w, r) + })) + defer srv.Close() + c := fetch.New(t.TempDir()) + c.MinInterval, c.Retries = 0, 0 + got, failed := GitLabProjectAliases(context.Background(), c, + []string{"gitlab.gnome.org/-/project/20183", "gitlab.gnome.org/-/project/404", "github.com/x/y"}, + func(string) string { return srv.URL }) + if failed != 1 || len(got) != 1 || got[0].From != "gitlab.gnome.org/-/project/20183" || got[0].To != "gitlab.gnome.org/world/amberol" || got[0].Source != AliasGitLabProject { + t.Fatalf("aliases %+v failed %d", got, failed) + } +} + +func TestRegistryAliases(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/pypi/acme-tiny/json": + fmt.Fprint(w, `{"info":{"home_page":"","project_urls":{"Documentation":"https://docs.example.org","Source":"https://github.com/diafygi/acme-tiny"}}}`) + case "/pypi/docs-only/json": + fmt.Fprint(w, `{"info":{"home_page":"https://docs-only.example.org","project_urls":{}}}`) + case "/api/v1/crates/ripgrep": + fmt.Fprint(w, `{"crate":{"repository":"https://github.com/BurntSushi/ripgrep","homepage":""}}`) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + c := fetch.New(t.TempDir()) + c.MinInterval, c.Retries = 0, 0 + got, failed := RegistryAliases(context.Background(), c, + []string{"pypi.org/acme-tiny", "pypi.org/docs-only", "pypi.org/gone", "crates.io/ripgrep", "github.com/x/y"}, + func(string) string { return srv.URL }) + want := map[string]string{"pypi.org/acme-tiny": "github.com/diafygi/acme-tiny", "crates.io/ripgrep": "github.com/burntsushi/ripgrep"} + if failed != 1 || len(got) != len(want) { + t.Fatalf("got %+v failed %d", got, failed) + } + for _, a := range got { + if want[a.From] != a.To || a.Source != AliasRegistry { + t.Errorf("alias %+v", a) + } + } +} + +func TestRegistryHintIsNotARepository(t *testing.T) { + // A PyPI sdist outweighs the homepage, but a registry entry is not a repository: the + // homepage's repository is chosen, and rescoring without an alias keeps it. + pkgs := FromOpenSUSE([]opensuse.RPM{{Name: "acme-tiny", Arch: "src", Version: "5-1", URL: "https://github.com/diafygi/acme-tiny"}}, nil, nil, nil) + p := pkgs[0] + p.Evidence = append(p.Evidence, model.Evidence{Source: "APKBUILD:source", Key: "pypi.org/acme-tiny", Weight: 0.9}) + Rescore(&p, nil) + if p.UpstreamKey != "github.com/diafygi/acme-tiny" { + t.Fatalf("registry key chosen: %+v", p) + } + Rescore(&p, map[string]string{"pypi.org/acme-tiny": "github.com/diafygi/acme-tiny"}) + if p.UpstreamKey != "github.com/diafygi/acme-tiny" || p.Basis != BasisReleaseURL { + t.Fatalf("aliased registry evidence should support the repository: %+v", p) + } +} + +// TestRescoreGone: a repository aliased away as gone drops out; the package falls back to its +// next candidate, or is left unresolved when there is none. +func TestRescoreGone(t *testing.T) { + ev := func(src, key string, w float64) model.Evidence { + return model.Evidence{Source: src, Value: "https://" + key, Key: key, Weight: w} + } + alias := map[string]string{"github.com/freedesktop/libqmi": ""} + p := Package{Name: "libqmi", UpstreamKey: "github.com/freedesktop/libqmi", Status: StatusResolved, Evidence: []model.Evidence{ + ev("zq1:giturls", "github.com/freedesktop/libqmi", 0.7), + ev("spec:URL", "gitlab.freedesktop.org/mobile-broadband/libqmi", 0.5), + }} + Rescore(&p, alias) + if p.UpstreamKey != "gitlab.freedesktop.org/mobile-broadband/libqmi" || p.Basis != BasisHomepage { + t.Errorf("fallback: %+v", p) + } + q := Package{Name: "x", UpstreamKey: "github.com/freedesktop/libqmi", Status: StatusResolved, Evidence: []model.Evidence{ + ev("zq1:giturls", "github.com/freedesktop/libqmi", 0.7), + }} + Rescore(&q, alias) + if q.UpstreamKey != "" || q.Status != StatusUnresolved { + t.Errorf("no fallback: %+v", q) + } +} + +// TestSecondarySource: a secondary spec source (a config file from another repository) +// corroborates but never decides, even when it outscores the primary evidence. +func TestSecondarySource(t *testing.T) { + ev := func(src, key string, w float64) model.Evidence { + return model.Evidence{Source: src, Value: "https://" + key, Key: key, Weight: w} + } + p := Package{Name: "caddy", Evidence: []model.Evidence{ + ev("spec:Source2", "github.com/caddyserver/dist", 0.7), + ev("spec:URL", "github.com/caddyserver/caddy", 0.5), + }} + Rescore(&p, nil) + if p.UpstreamKey != "github.com/caddyserver/caddy" || p.Basis != BasisHomepage { + t.Errorf("caddy: %+v", p) + } + q := Package{Name: "cmocka", Evidence: []model.Evidence{ev("spec:Source4", "github.com/jothepro/doxygen-awesome-css", 0.7)}} + Rescore(&q, nil) + if q.UpstreamKey != "" || q.Status != StatusUnresolved { + t.Errorf("secondary only: %+v", q) + } + if BasisOf("spec:Source") != BasisReleaseURL || BasisOf("spec:Source0") != BasisReleaseURL || BasisOf("spec:Source12") != BasisSecondary { + t.Error("spec source bases") + } +} diff --git a/internal/bulk/alpine.go b/internal/bulk/alpine.go new file mode 100644 index 0000000..b6e2c81 --- /dev/null +++ b/internal/bulk/alpine.go @@ -0,0 +1,60 @@ +package bulk + +import ( + "sort" + + "tangled.org/odd.computer/isomorph/internal/distro" + "tangled.org/odd.computer/isomorph/internal/distro/alpine" + "tangled.org/odd.computer/isomorph/internal/model" +) + +// FromAlpine builds one row per origin (source package) of the edge APKINDEX of each section. +// The APKINDEX supplies the built version, url and binary names; the origin's APKBUILD (keyed +// "
/") supplies the source URLs. OSV defines Alpine ecosystems per release +// (Alpine:v3.22), not for edge, so OSVEcosystem stays empty for these rows. +func FromAlpine(index map[string][]alpine.IndexEntry, apkbuilds map[string]*alpine.APKBUILD) []Package { + var out []Package + for _, section := range sortedKeys(index) { + type origin struct { + version, url string + bins []string + } + origins := map[string]*origin{} + for _, e := range index[section] { + name := e.Origin + if name == "" { + name = e.Name + } + o := origins[name] + if o == nil { + o = &origin{version: e.Version, url: e.URL} + origins[name] = o + } + o.bins = append(o.bins, e.Name) + } + for _, name := range sortedKeys(origins) { + o := origins[name] + p := Package{Distro: model.Alpine, Suite: "edge-" + section, Name: name, Version: o.version, + Binaries: dedupe(o.bins), Homepage: o.url, Status: StatusUnresolved} + if ab := apkbuilds[section+"/"+name]; ab != nil { + if ab.URL != "" { + p.Homepage = ab.URL + } + fill(&p, distro.RankCandidates(alpine.Candidates(ab))) + } else if o.url != "" { + fill(&p, distro.RankCandidates(distro.AddCandidate(nil, o.url, model.Evidence{Source: "APKINDEX:url", Value: o.url, Weight: 0.5}))) + } + out = append(out, p) + } + } + return out +} + +func sortedKeys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/internal/bulk/arch.go b/internal/bulk/arch.go new file mode 100644 index 0000000..37d0ff9 --- /dev/null +++ b/internal/bulk/arch.go @@ -0,0 +1,130 @@ +package bulk + +import ( + "context" + "sort" + "strings" + "time" + + "tangled.org/odd.computer/isomorph/internal/distro" + "tangled.org/odd.computer/isomorph/internal/distro/arch" + "tangled.org/odd.computer/isomorph/internal/model" +) + +// ArchSource provides .SRCINFO files at released packaging commits (arch.Resolver does). +type ArchSource interface { + SRCINFOAt(ctx context.Context, pkgbase, commit string) (*arch.SRCINFO, error) + FetchSRCINFO(ctx context.Context, pkgbase, commit string, web bool) (*arch.SRCINFO, error) + HasSRCINFO(pkgbase, commit string) bool +} + +// CrawlInterval paces .SRCINFO fetches. gitlab.archlinux.org allows anonymous clients about 600 +// requests per ten minutes on each of the API and web routes; alternating the two at this +// interval stays under both even when a third of the packages need a second request (the +// PKGBUILD fallback). +var CrawlInterval = 700 * time.Millisecond + +// ArchStats describes how an Arch slice was built. +type ArchStats struct { + PkgBases int + // FromSRCINFO counts rows resolved from .SRCINFO (cached or fetched this run); the rest use + // the repository databases' url field only. + FromSRCINFO int + // Fetched and FetchErrors count .SRCINFO requests made this run. + Fetched, FetchErrors int + // Stopped is set when the crawl hit a rate limit and stopped early. + Stopped error + // Errors samples fetch errors by pkgbase (at most 20). + Errors map[string]string +} + +// FromArch builds one row per pkgbase of the repository databases. Each row starts from the +// databases (version, binary names, url). A pkgbase whose .SRCINFO at its released commit (from +// the state repository) is cached, or can be fetched within the crawl budget, is resolved from +// .SRCINFO instead: its source array names git repositories and release tarballs (with +// sha256), which the url field does not. +func FromArch(ctx context.Context, src ArchSource, dbs []arch.SyncPackage, state []arch.StateEntry, crawl int, progress func(fetched int)) ([]Package, ArchStats) { + type base struct { + repo, name, version, url string + bins []string + } + bases := map[[2]string]*base{} + var order [][2]string + for _, p := range dbs { + k := [2]string{p.Repo, p.Base} + b := bases[k] + if b == nil { + b = &base{repo: p.Repo, name: p.Base, version: p.Version} + bases[k] = b + order = append(order, k) + } + if b.url == "" { + b.url = p.URL + } + b.bins = append(b.bins, p.Name) + } + sort.Slice(order, func(i, j int) bool { + if order[i][0] != order[j][0] { + return order[i][0] < order[j][0] + } + return order[i][1] < order[j][1] + }) + commits := map[[2]string]string{} + for _, e := range state { + commits[[2]string{e.Repo, e.PkgBase}] = e.Commit + } + var st ArchStats + var last time.Time + out := make([]Package, 0, len(order)) + for _, k := range order { + b := bases[k] + sort.Strings(b.bins) + p := Package{Distro: model.Arch, Suite: b.repo, Name: b.name, Version: b.version, Binaries: b.bins, Homepage: b.url, Status: StatusUnresolved} + var si *arch.SRCINFO + if c := commits[k]; c != "" && ctx.Err() == nil { + switch { + case src.HasSRCINFO(b.name, c): + si, _ = src.SRCINFOAt(ctx, b.name, c) + case st.Fetched < crawl && st.Stopped == nil: + if wait := CrawlInterval - time.Since(last); wait > 0 { + select { + case <-ctx.Done(): + case <-time.After(wait): + } + } + last = time.Now() + st.Fetched++ + var err error + if si, err = src.FetchSRCINFO(ctx, b.name, c, st.Fetched%2 == 0); err != nil { + si = nil + st.FetchErrors++ + if st.Errors == nil { + st.Errors = map[string]string{} + } + if len(st.Errors) < 20 { + st.Errors[b.name] = err.Error() + } + if strings.Contains(err.Error(), "rate limited") { + st.Stopped = err + } + } + if progress != nil { + progress(st.Fetched) + } + } + } + if si != nil { + st.FromSRCINFO++ + if si.URL != "" { + p.Homepage = si.URL + } + p.SourceSHA256 = arch.SourceSHA256(si) + fill(&p, distro.RankCandidates(arch.Candidates(si))) + } else if b.url != "" { + fill(&p, distro.RankCandidates(distro.AddCandidate(nil, b.url, model.Evidence{Source: "pacman-db:url", Value: b.url, Weight: 0.5}))) + } + out = append(out, p) + } + st.PkgBases = len(out) + return out, st +} diff --git a/internal/bulk/arch_test.go b/internal/bulk/arch_test.go new file mode 100644 index 0000000..00837ca --- /dev/null +++ b/internal/bulk/arch_test.go @@ -0,0 +1,158 @@ +package bulk + +import ( + "context" + "errors" + "path/filepath" + "testing" + "time" + + "tangled.org/odd.computer/isomorph/internal/distro/alpine" + "tangled.org/odd.computer/isomorph/internal/distro/arch" + "tangled.org/odd.computer/isomorph/internal/distro/opensuse" + "tangled.org/odd.computer/isomorph/internal/model" +) + +type fakeSRCINFO struct { + files map[string]string + cached map[string]bool +} + +func (f fakeSRCINFO) SRCINFOAt(_ context.Context, pkgbase, commit string) (*arch.SRCINFO, error) { + body, ok := f.files[pkgbase+"@"+commit] + if !ok { + return nil, errors.New("404") + } + return arch.ParseSRCINFO([]byte(body)) +} + +func (f fakeSRCINFO) FetchSRCINFO(ctx context.Context, pkgbase, commit string, _ bool) (*arch.SRCINFO, error) { + return f.SRCINFOAt(ctx, pkgbase, commit) +} + +func (f fakeSRCINFO) HasSRCINFO(pkgbase, commit string) bool { return f.cached[pkgbase+"@"+commit] } + +func TestFromArchAndCompare(t *testing.T) { + defer func(d time.Duration) { CrawlInterval = d }(CrawlInterval) + CrawlInterval = 0 + f := fakeSRCINFO{files: map[string]string{ + "json-c@aaaa": "pkgbase = json-c\n\tpkgver = 0.18\n\tpkgrel = 1\n\turl = https://github.com/json-c/json-c/wiki\n" + + "\tsource = git+https://github.com/json-c/json-c#tag=json-c-0.18-20240915\n\tsha256sums = SKIP\n\npkgname = json-c\n", + "gzip@bbbb": "pkgbase = gzip\n\tpkgver = 1.13\n\tpkgrel = 1\n\turl = https://www.gnu.org/software/gzip/\n" + + "\tsource = https://ftp.gnu.org/pub/gnu/gzip/gzip-1.13.tar.xz\n\tsha256sums = 7777\n\npkgname = gzip\n", + }, cached: map[string]bool{"json-c@aaaa": true}} + dbs := []arch.SyncPackage{ + {Repo: "extra", Name: "json-c", Base: "json-c", Version: "0.18-1", URL: "https://github.com/json-c/json-c/wiki"}, + {Repo: "core", Name: "gzip", Base: "gzip", Version: "1.13-1", URL: "https://www.gnu.org/software/gzip/"}, + {Repo: "extra", Name: "gone-bin", Base: "gone", Version: "1-1", URL: "https://github.com/gone/gone"}, + {Repo: "extra", Name: "gone-docs", Base: "gone", Version: "1-1"}, + } + state := []arch.StateEntry{ + {Repo: "extra", PkgBase: "json-c", Version: "0.18-1", Commit: "aaaa"}, + {Repo: "core", PkgBase: "gzip", Version: "1.13-1", Commit: "bbbb"}, + {Repo: "extra", PkgBase: "gone", Version: "1-1", Commit: "cccc"}, + } + // Cache only: json-c from .SRCINFO, the others from the databases' url. + pkgs, st := FromArch(context.Background(), f, dbs, state, 0, nil) + if st.PkgBases != 3 || st.FromSRCINFO != 1 || st.Fetched != 0 { + t.Fatalf("cache-only stats: %+v", st) + } + by := map[string]Package{} + for _, p := range pkgs { + by[p.Name] = p + } + if p := by["json-c"]; p.Basis != BasisVCSSource || p.UpstreamKey != "github.com/json-c/json-c" { + t.Errorf("json-c from cached .SRCINFO: %+v", p) + } + if p := by["gone"]; p.UpstreamKey != "github.com/gone/gone" || p.Basis != BasisHomepage || len(p.Binaries) != 2 { + t.Errorf("gone from the database url: %+v", p) + } + // With a crawl budget the uncached ones are fetched; a failing fetch keeps the baseline. + pkgs, st = FromArch(context.Background(), f, dbs, state, 10, nil) + if st.FromSRCINFO != 2 || st.Fetched != 2 || st.FetchErrors != 1 { + t.Fatalf("crawl stats: %+v", st) + } + by = map[string]Package{} + for _, p := range pkgs { + by[p.Name] = p + } + if p := by["gzip"]; p.UpstreamKey != "git.savannah.gnu.org/gzip" || p.Basis != BasisReleaseURL || len(p.SourceSHA256) != 1 { + t.Errorf("gzip: %+v", p) + } + if p := by["gone"]; p.UpstreamKey != "github.com/gone/gone" { + t.Errorf("gone after a failed fetch: %+v", p) + } + pkgs = []Package{by["json-c"], by["gzip"], by["gone"]} + + ctx := context.Background() + db, err := Open(ctx, filepath.Join(t.TempDir(), "p.sqlite")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + deb := FromUDD(udd(), "sid") + deb[1].SourceSHA256 = []string{"7777"} // gzip: the same release tarball as Arch + Replace(ctx, db, Source{Distro: model.Debian, Suite: "sid", Origin: "t", FetchedAt: time.Now()}, deb) + Replace(ctx, db, Source{Distro: model.Arch, Suite: "extra", Origin: "t", FetchedAt: time.Now()}, []Package{pkgs[0], pkgs[2]}) + Replace(ctx, db, Source{Distro: model.Arch, Suite: "core", Origin: "t", FetchedAt: time.Now()}, []Package{pkgs[1]}) + _ = st + ags, err := Compare(ctx, db, "debian", "arch") + if err != nil { + t.Fatal(err) + } + if sha := ags[0]; sha.Join != "sha256" || sha.Pairs != 1 || sha.Agree != 1 { + t.Errorf("sha256 join: %+v", sha) + } + if name := ags[1]; name.Join != "name" || name.Pairs != 2 || name.BothResolved != 2 || name.Agree != 2 { + t.Errorf("name join: %+v", name) + } +} + +func TestFromOpenSUSE(t *testing.T) { + srcs := []opensuse.RPM{ + {Name: "json-c", Arch: "src", Version: "0.18-1.2", URL: "https://github.com/json-c/json-c/wiki"}, + {Name: "at", Arch: "src", Version: "1:3.2-4"}, + } + bins := []opensuse.RPM{ + {Name: "libjson-c5", Arch: "x86_64", SourceRPM: "json-c-0.18-1.2.src.rpm"}, + {Name: "libjson-c-devel", Arch: "x86_64", SourceRPM: "json-c-0.18-1.2.src.rpm"}, + {Name: "libjson-c5", Arch: "i586", SourceRPM: "json-c-0.18-1.2.src.rpm"}, + } + pkgs := FromOpenSUSE(srcs, bins, map[string][]string{"at": {"https://salsa.debian.org/debian/at.git"}}, nil) + if len(pkgs) != 2 { + t.Fatalf("%d rows", len(pkgs)) + } + at, j := pkgs[0], pkgs[1] + if j.Name != "json-c" || j.UpstreamKey != "github.com/json-c/json-c" || j.Basis != BasisHomepage || j.OSVEcosystem != "openSUSE:Tumbleweed" || + len(j.Binaries) != 2 || j.PURL != "pkg:rpm/opensuse/json-c?arch=src" { + t.Errorf("json-c: %+v", j) + } + if at.UpstreamKey != "salsa.debian.org/debian/at" || at.Basis != BasisDerived { + t.Errorf("at via giturls: %+v", at) + } +} + +func TestFromAlpine(t *testing.T) { + index := map[string][]alpine.IndexEntry{ + "community": { + {Name: "sysstat", Version: "12.7.1-r0", Origin: "sysstat", URL: "https://github.com/sysstat/sysstat"}, + {Name: "sysstat-doc", Version: "12.7.1-r0", Origin: "sysstat", URL: "https://github.com/sysstat/sysstat"}, + }, + "main": {{Name: "libfoo", Version: "1-r0", Origin: "foo", URL: "https://gitlab.com/foo/foo"}}, + } + ab, err := alpine.ParseAPKBUILD([]byte("pkgname=sysstat\npkgver=12.7.1\npkgrel=0\nurl=\"https://sebastien.godard.pagesperso-orange.fr/\"\nsource=\"$pkgname-$pkgver.tar.gz::https://github.com/sysstat/sysstat/archive/v$pkgver.tar.gz\"\n")) + if err != nil { + t.Fatal(err) + } + pkgs := FromAlpine(index, map[string]*alpine.APKBUILD{"community/sysstat": ab}) + if len(pkgs) != 2 { + t.Fatalf("%d rows", len(pkgs)) + } + s, f := pkgs[0], pkgs[1] + if s.Name != "sysstat" || s.Suite != "edge-community" || s.UpstreamKey != "github.com/sysstat/sysstat" || s.Basis != BasisReleaseURL || len(s.Binaries) != 2 { + t.Errorf("sysstat: %+v", s) + } + if f.Name != "foo" || f.UpstreamKey != "gitlab.com/foo/foo" || f.Basis != BasisHomepage { + t.Errorf("foo from the index url: %+v", f) + } +} diff --git a/internal/bulk/bulk.go b/internal/bulk/bulk.go new file mode 100644 index 0000000..f70411c --- /dev/null +++ b/internal/bulk/bulk.go @@ -0,0 +1,116 @@ +// Package bulk holds the association table: every source package of a distribution suite, +// resolved to an upstream repository, with the evidence behind each mapping. It is built from +// whole-archive sources (UDD for Debian) rather than one package at a time, and published as +// JSONL (canonical, diffable) and SQLite (for queries; Python reads it with the standard +// library). +package bulk + +import ( + "regexp" + "strings" + "time" + + "tangled.org/odd.computer/isomorph/internal/model" +) + +// Status values of Package.Status. +const ( + // StatusResolved: the best candidate is a repository. + StatusResolved = "resolved" + // StatusUnresolved: no field named or hinted at a repository. + StatusUnresolved = "unresolved" +) + +// Basis values: the kind of evidence behind a mapping, strongest first. +const ( + // BasisMetadata: a field whose purpose is to name the upstream repository (DEP-12 + // Repository, Repository-Browse). + BasisMetadata = "metadata" + // BasisVCSSource: the package builds from the repository (an Arch git source). + BasisVCSSource = "vcs-source" + // BasisReleaseURL: a release tarball or debian/watch URL on a forge, or a tarball on a + // host whose repository is known (ftp.gnu.org -> Savannah). + BasisReleaseURL = "release-url" + // BasisDerived: a third-party dataset derived from the packaging (rb.zq1.de giturls, itself + // computed from spec Url:/Source: fields). + BasisDerived = "derived" + // BasisBugTracker: a bug tracker URL on a forge (DEP-12 Bug-Database). + BasisBugTracker = "bug-tracker" + // BasisHomepage: a homepage on a forge or a well-known project site. + BasisHomepage = "homepage" + // BasisSecondary is a secondary spec source (Source1 and up: configuration files, shell + // completions, test dependencies). It corroborates a candidate but never decides one: of + // the openSUSE packages it alone resolved, about a third named another project's + // repository (docs/EVALUATION.md). + BasisSecondary = "secondary-source" +) + +// basisRank orders the bases, strongest first. +var basisRank = map[string]int{BasisMetadata: 0, BasisVCSSource: 1, BasisReleaseURL: 2, BasisDerived: 3, BasisBugTracker: 4, BasisHomepage: 5, BasisSecondary: 6} + +var secondarySpecSourceRe = regexp.MustCompile(`^spec:Source[1-9][0-9]*$`) + +// BasisOf maps an evidence source name onto its basis. +func BasisOf(source string) string { + switch { + case source == "debian/upstream/metadata:Repository", source == "debian/upstream/metadata:Repository-Browse": + return BasisMetadata + case strings.HasPrefix(source, "SRCINFO:source:"), strings.HasPrefix(source, "APKBUILD:source:"), source == "APKBUILD:_repo_url", source == "spec:Git-Clone": + return BasisVCSSource + case strings.HasPrefix(source, "_service:"): + // obs_scm, tar_scm and the like fetch a repository; download_url and download_files + // fetch release files. + if strings.HasPrefix(source, "_service:download") { + return BasisReleaseURL + } + return BasisVCSSource + case secondarySpecSourceRe.MatchString(source): + return BasisSecondary + case source == "SRCINFO:source", source == "APKBUILD:source", source == "debian/watch:upstream_url", strings.HasPrefix(source, "spec:Source"): + return BasisReleaseURL + case source == "debian/upstream/metadata:Bug-Database": + return BasisBugTracker + case source == "zq1:giturls": + return BasisDerived + } + return BasisHomepage +} + +// Package is one source package (Debian source, openSUSE spec, Arch pkgbase, Alpine origin) +// in one suite, with its resolved upstream. +type Package struct { + Distro model.Distro `json:"distro"` + Suite string `json:"suite"` + Name string `json:"package"` + // Version is the distro version, as the distro writes it. + Version string `json:"version"` + // OSVEcosystem is the OSV ecosystem string ("Debian:sid"); empty where OSV defines none. + OSVEcosystem string `json:"osv_ecosystem,omitempty"` + // PURL identifies the source package (advisory: purl's distro qualifiers are not + // standardised, so consumers should key on (OSVEcosystem, Name)). + PURL string `json:"purl,omitempty"` + Binaries []string `json:"binaries,omitempty"` + Homepage string `json:"homepage,omitempty"` + // UpstreamKey is the repourl key of the best candidate; Repo is its clone URL, ready for an + // OSV GIT range. + UpstreamKey string `json:"upstream_key,omitempty"` + Repo string `json:"repo,omitempty"` + Status string `json:"status"` + // Basis is the kind of evidence that decided the mapping (Basis* constants). The evaluation + // reports measured agreement per basis; there is deliberately no numeric confidence. + Basis string `json:"basis,omitempty"` + // Evidence lists every observation considered, highest weight first. + Evidence []model.Evidence `json:"evidence,omitempty"` + // SourceSHA256 are the sha256 of the upstream source archives the package builds from + // (Debian .orig tarballs); equal hashes across distros mean the same upstream release. + SourceSHA256 []string `json:"source_sha256,omitempty"` +} + +// Source describes where one (distro, suite) slice of the table came from. +type Source struct { + Distro model.Distro `json:"distro"` + Suite string `json:"suite"` + Origin string `json:"origin"` + FetchedAt time.Time `json:"fetched_at"` + Packages int `json:"packages"` +} diff --git a/internal/bulk/bulk_test.go b/internal/bulk/bulk_test.go new file mode 100644 index 0000000..caba262 --- /dev/null +++ b/internal/bulk/bulk_test.go @@ -0,0 +1,161 @@ +package bulk + +import ( + "context" + "path/filepath" + "testing" + "time" + + "tangled.org/odd.computer/isomorph/internal/distro/debian" + "tangled.org/odd.computer/isomorph/internal/model" +) + +func udd() []debian.UDDRow { + return []debian.UDDRow{ + { // DEP-12 Repository agrees with the homepage. + Source: "json-c", Version: "0.18+ds-1", Homepage: "https://github.com/json-c/json-c/wiki", + Binaries: []string{"libjson-c5", "libjson-c-dev"}, + Metadata: map[string]string{"Repository": "https://github.com/json-c/json-c.git"}, + WatchURL: "https://github.com/json-c/json-c/archive/refs/tags/json-c-0.18-20240915.tar.gz", + }, + { // Only a tarball watch URL on a known host. + Source: "gzip", Version: "1.13-1", Homepage: "https://www.gnu.org/software/gzip/", + WatchURL: "https://ftp.gnu.org/gnu/gzip/gzip-1.13.tar.xz", + }, + { // Nothing points at a repository; the packaging repo on salsa must not win. + Source: "debian-native-tool", Version: "3", VcsURL: "https://salsa.debian.org/debian/debian-native-tool.git", + }, + { // Repository disagrees with the homepage's forge (a mirror). + Source: "mirrored", Version: "1-1", Homepage: "https://github.com/someone/mirrored", + Metadata: map[string]string{"Repository": "https://gitlab.com/upstream/mirrored.git"}, + }, + } +} + +func TestFromUDD(t *testing.T) { + pkgs := FromUDD(udd(), "sid") + by := map[string]Package{} + for _, p := range pkgs { + by[p.Name] = p + } + j := by["json-c"] + if j.UpstreamKey != "github.com/json-c/json-c" || j.Status != StatusResolved || j.Basis != BasisMetadata || + j.OSVEcosystem != "Debian:sid" || j.PURL != "pkg:deb/debian/json-c?arch=source" || len(j.Binaries) != 2 { + t.Errorf("json-c: %+v", j) + } + if g := by["gzip"]; g.UpstreamKey != "git.savannah.gnu.org/gzip" || g.Status != StatusResolved || g.Basis != BasisReleaseURL { + t.Errorf("gzip via the GNU tarball hint: %+v", g) + } + if n := by["debian-native-tool"]; n.Status != StatusUnresolved || n.UpstreamKey != "" { + t.Errorf("packaging repo taken as upstream: %+v", n) + } + if m := by["mirrored"]; m.UpstreamKey != "gitlab.com/upstream/mirrored" { + t.Errorf("Repository must outweigh the homepage: %+v", m) + } + h := DebianHeldOut(udd()) + if h.WithRepository != 2 || h.FallbackResolved != 2 || h.Agree != 1 || len(h.Disagree) != 1 || h.Disagree[0][0] != "mirrored" || + h.ByBasis[BasisReleaseURL] != [2]int{1, 1} || h.ByBasis[BasisHomepage] != [2]int{1, 0} { + t.Errorf("held out: %+v", h) + } +} + +func TestSQLiteRoundTrip(t *testing.T) { + ctx := context.Background() + db, err := Open(ctx, filepath.Join(t.TempDir(), "p.sqlite")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + deb := FromUDD(udd(), "sid") + if err := Replace(ctx, db, Source{Distro: model.Debian, Suite: "sid", Origin: "test", FetchedAt: time.Now()}, deb); err != nil { + t.Fatal(err) + } + arch := []Package{{Distro: model.Arch, Suite: "extra", Name: "json-c", Version: "0.18-1", UpstreamKey: "github.com/json-c/json-c", Status: StatusResolved, Basis: BasisVCSSource}} + if err := Replace(ctx, db, Source{Distro: model.Arch, Suite: "extra", Origin: "test", FetchedAt: time.Now()}, arch); err != nil { + t.Fatal(err) + } + got, err := Lookup(ctx, db, "debian/sid:json-c") + if err != nil || len(got) != 1 || got[0].UpstreamKey != "github.com/json-c/json-c" || len(got[0].Evidence) == 0 { + t.Fatalf("lookup by package: %v %+v", err, got) + } + both, err := Lookup(ctx, db, "github.com/json-c/json-c") + if err != nil || len(both) != 2 { + t.Fatalf("lookup by upstream: %v %+v", err, both) + } + var n int + if err := db.QueryRowContext(ctx, `select count(*) from counterparts where package = 'json-c'`).Scan(&n); err != nil || n != 2 { + t.Fatalf("counterparts view: %v %d", err, n) + } + var bin string + if err := db.QueryRowContext(ctx, `select package from binaries where binary = 'libjson-c5'`).Scan(&bin); err != nil || bin != "json-c" { + t.Fatalf("binaries: %v %q", err, bin) + } + back, err := LoadSlice(ctx, db, model.Debian, "sid") + if err != nil || len(back) != len(deb) { + t.Fatalf("LoadSlice: %v %d", err, len(back)) + } + for i := range back { + w := deb[i] + for j := range deb { + if deb[j].Name == back[i].Name { + w = deb[j] + } + } + if back[i].UpstreamKey != w.UpstreamKey || len(back[i].Evidence) != len(w.Evidence) || len(back[i].Binaries) != len(w.Binaries) || back[i].Basis != w.Basis { + t.Errorf("LoadSlice %s: got %+v want %+v", w.Name, back[i], w) + } + } + // Replacing a slice drops its previous rows. + if err := Replace(ctx, db, Source{Distro: model.Debian, Suite: "sid", Origin: "test", FetchedAt: time.Now()}, deb[:1]); err != nil { + t.Fatal(err) + } + if err := db.QueryRowContext(ctx, `select count(*) from packages where distro = 'debian'`).Scan(&n); err != nil || n != 1 { + t.Fatalf("replace: %v %d", err, n) + } +} + +func TestOpenRefusesOtherSchemaVersions(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "p.sqlite") + db, err := Open(ctx, path) + if err != nil { + t.Fatal(err) + } + db.ExecContext(ctx, "pragma user_version = 1") + db.Close() + if _, err := Open(ctx, path); err == nil { + t.Fatal("opened a database with another schema version") + } +} + +// TestStaleHomeHost: a SourceForge or Launchpad homepage outscores a GitHub release URL (the +// URL counts as a hint at 0.6 * 0.6), but loses to it; a GitHub homepage keeps winning. +func TestStaleHomeHost(t *testing.T) { + pkgs := FromUDD([]debian.UDDRow{ + {Source: "biber", Version: "2.22-1", Homepage: "http://biblatex-biber.sourceforge.net/", + WatchURL: "https://api.github.com/repos/plk/biber/tarball/refs/tags/v2.22"}, + {Source: "cloud-utils", Version: "0.34-1", Homepage: "https://launchpad.net/cloud-utils", + WatchURL: "https://github.com/canonical/cloud-utils/archive/refs/tags/0.34.tar.gz"}, + {Source: "vde2", Version: "2.3.3-1", Homepage: "http://vde.sourceforge.net"}, + {Source: "gh", Version: "1-1", Homepage: "https://github.com/a/gh", + WatchURL: "https://github.com/b/gh/archive/refs/tags/1.tar.gz"}, + }, "sid") + want := map[string][2]string{ + "biber": {"github.com/plk/biber", BasisReleaseURL}, + "cloud-utils": {"github.com/canonical/cloud-utils", BasisReleaseURL}, + "vde2": {"sourceforge.net/vde", BasisHomepage}, + "gh": {"github.com/a/gh", BasisHomepage}, + } + for _, p := range pkgs { + if w := want[p.Name]; p.UpstreamKey != w[0] || p.Basis != w[1] { + t.Errorf("%s: %s (%s), want %v", p.Name, p.UpstreamKey, p.Basis, w) + } + // Rescoring from the stored evidence keeps the choice. + q := p + q.UpstreamKey = "" + Rescore(&q, nil) + if q.UpstreamKey != p.UpstreamKey { + t.Errorf("%s: rescored to %s", p.Name, q.UpstreamKey) + } + } +} diff --git a/internal/bulk/compare.go b/internal/bulk/compare.go new file mode 100644 index 0000000..4b158e2 --- /dev/null +++ b/internal/bulk/compare.go @@ -0,0 +1,73 @@ +package bulk + +import ( + "context" + "database/sql" +) + +// Agreement compares two distributions' mappings over pairs of packages joined independently +// of the mapping: by an identical upstream source archive (sha256) or by an identical name. +// Where both packages resolve, the pair should name the same upstream key. The counts form a +// confusion matrix of a's outcome against b's: both resolve (agreeing or not), only one side +// resolves, or neither does. +type Agreement struct { + // Join is "sha256" or "name". + Join string + // Pairs counts joined package pairs; BothResolved those where both sides resolve; Agree + // those naming the same key. + Pairs, BothResolved, Agree int + // OnlyA and OnlyB count pairs where only a's or only b's package resolves (the other + // side's gap); Neither those where neither does. + OnlyA, OnlyB, Neither int + // Disagree lists every pair where both resolve to different keys + // (a package, b package, a key, b key). + Disagree [][4]string +} + +// Compare measures agreement between distros a and b in db. +func Compare(ctx context.Context, db *sql.DB, a, b string) ([]Agreement, error) { + joins := []struct{ name, from string }{ + {"sha256", `from source_sha256 x join source_sha256 y on x.sha256 = y.sha256 + join packages pa on pa.distro = x.distro and pa.suite = x.suite and pa.package = x.package + join packages pb on pb.distro = y.distro and pb.suite = y.suite and pb.package = y.package + where x.distro = ? and y.distro = ?`}, + {"name", `from packages pa join packages pb on pa.package = pb.package where pa.distro = ? and pb.distro = ?`}, + } + var out []Agreement + for _, j := range joins { + rows, err := db.QueryContext(ctx, `select distinct pa.package, pb.package, coalesce(pa.upstream_key,''), coalesce(pb.upstream_key,'') `+j.from, a, b) + if err != nil { + return nil, err + } + ag := Agreement{Join: j.name} + for rows.Next() { + var pa, pb, ka, kb string + if err := rows.Scan(&pa, &pb, &ka, &kb); err != nil { + rows.Close() + return nil, err + } + ag.Pairs++ + switch { + case ka != "" && kb != "": + ag.BothResolved++ + if ka == kb { + ag.Agree++ + } else { + ag.Disagree = append(ag.Disagree, [4]string{pa, pb, ka, kb}) + } + case ka != "": + ag.OnlyA++ + case kb != "": + ag.OnlyB++ + default: + ag.Neither++ + } + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, err + } + out = append(out, ag) + } + return out, nil +} diff --git a/internal/bulk/compare_test.go b/internal/bulk/compare_test.go new file mode 100644 index 0000000..99343b9 --- /dev/null +++ b/internal/bulk/compare_test.go @@ -0,0 +1,51 @@ +package bulk + +import ( + "context" + "path/filepath" + "testing" + "time" + + "tangled.org/odd.computer/isomorph/internal/model" +) + +func TestCompareMatrix(t *testing.T) { + ctx := context.Background() + db, err := Open(ctx, filepath.Join(t.TempDir(), "p.sqlite")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + pkg := func(d model.Distro, name, key string) Package { + return Package{Distro: d, Suite: "s", Name: name, UpstreamKey: key} + } + a := []Package{ + pkg(model.Debian, "same", "github.com/x/same"), + pkg(model.Debian, "differ", "github.com/x/one"), + pkg(model.Debian, "only-a", "github.com/x/only-a"), + pkg(model.Debian, "only-b", ""), + pkg(model.Debian, "neither", ""), + pkg(model.Debian, "unpaired", "github.com/x/unpaired"), + } + b := []Package{ + pkg(model.Arch, "same", "github.com/x/same"), + pkg(model.Arch, "differ", "github.com/x/other"), + pkg(model.Arch, "only-a", ""), + pkg(model.Arch, "only-b", "github.com/x/only-b"), + pkg(model.Arch, "neither", ""), + } + Replace(ctx, db, Source{Distro: model.Debian, Suite: "s", Origin: "t", FetchedAt: time.Now()}, a) + Replace(ctx, db, Source{Distro: model.Arch, Suite: "s", Origin: "t", FetchedAt: time.Now()}, b) + ags, err := Compare(ctx, db, "debian", "arch") + if err != nil { + t.Fatal(err) + } + got := ags[1] + if got.Join != "name" || got.Pairs != 5 || got.BothResolved != 2 || got.Agree != 1 || + len(got.Disagree) != 1 || got.OnlyA != 1 || got.OnlyB != 1 || got.Neither != 1 { + t.Errorf("name join: %+v", got) + } + if d := got.Disagree[0]; d != [4]string{"differ", "differ", "github.com/x/one", "github.com/x/other"} { + t.Errorf("disagreement: %v", d) + } +} diff --git a/internal/bulk/debian.go b/internal/bulk/debian.go new file mode 100644 index 0000000..9f85a54 --- /dev/null +++ b/internal/bulk/debian.go @@ -0,0 +1,152 @@ +package bulk + +import ( + "sort" + "strings" + + "tangled.org/odd.computer/isomorph/internal/distro" + "tangled.org/odd.computer/isomorph/internal/distro/debian" + "tangled.org/odd.computer/isomorph/internal/model" + "tangled.org/odd.computer/isomorph/internal/repourl" +) + +// FromUDD turns UDD rows of one suite into association rows. +func FromUDD(rows []debian.UDDRow, suite string) []Package { + out := make([]Package, 0, len(rows)) + for _, r := range rows { + p := Package{ + Distro: model.Debian, + Suite: suite, + Name: r.Source, + Version: r.Version, + OSVEcosystem: "Debian:" + suite, + PURL: "pkg:deb/debian/" + r.Source + "?arch=source", + Binaries: r.Binaries, + Homepage: r.Homepage, + Status: StatusUnresolved, + SourceSHA256: r.OrigSHA256, + } + cands := debian.ResolveRow(r, nil) + fill(&p, cands) + out = append(out, p) + } + return out +} + +// fill sets the upstream fields from ranked candidates. +func fill(p *Package, cands []distro.Candidate) { + for _, c := range cands { + p.Evidence = append(p.Evidence, c.Evidence...) + } + sort.SliceStable(p.Evidence, func(i, j int) bool { return p.Evidence[i].Weight > p.Evidence[j].Weight }) + if best := choose(cands); best != nil { + p.UpstreamKey, p.Repo, p.Status = best.Key, best.URL, StatusResolved + p.Basis = basis(*best) + } +} + +// choose picks the upstream among ranked candidates: the first repository, unless it rests on +// a homepage on a staleHomeHosts host and a release or VCS source names another repository. +func choose(cands []distro.Candidate) *distro.Candidate { + var best *distro.Candidate + for i := range cands { + c := &cands[i] + if c.Kind != repourl.KindRepo || basis(*c) == BasisSecondary { + continue + } + if best == nil { + best = c + if basis(*c) != BasisHomepage || !onStaleHomeHost(c.Key) { + break + } + } else if b := basis(*c); b == BasisReleaseURL || b == BasisVCSSource { + return c // see staleHomeHosts + } + } + return best +} + +// staleHomeHosts are hosts whose project pages outlive a project's move elsewhere. A homepage +// there loses to a release or VCS source of the same package that names another repository: +// on Debian, where the Homepage is on one of these hosts and the watch URL names a different +// repository, DEP-12 Repository sides with the watch URL 22 times out of 23 (GitHub homepages, +// by contrast, split 16 to 17, so the rule stays narrow; docs/EVALUATION.md). +var staleHomeHosts = []string{"sourceforge.net/", "launchpad.net/"} + +func onStaleHomeHost(key string) bool { + for _, h := range staleHomeHosts { + if strings.HasPrefix(key, h) { + return true + } + } + return false +} + +// basis is the strongest kind of evidence supporting a candidate, by the fixed precedence of +// basisRank (not by the ranking weights, which scale hints down). +func basis(c distro.Candidate) string { + best := "" + for _, e := range c.Evidence { + if b := BasisOf(e.Source); best == "" || basisRank[b] < basisRank[best] { + best = b + } + } + return best +} + +// HeldOut measures Debian's other fields against DEP-12 Repository. Among packages whose +// Repository resolves to a repository, it resolves again without Repository and +// Repository-Browse and counts how often the remaining fields (watch URL, Homepage, +// Bug-Database) resolve at all, and to the same key. +type HeldOut struct { + WithRepository int + // FallbackResolved counts packages the other fields still resolve; Agree counts those that + // resolve to the Repository key. + FallbackResolved int + Agree int + // ByBasis counts, per basis of the fallback's decision, [resolved, agreeing]. + ByBasis map[string][2]int + // Disagree lists (package, repository key, fallback key, fallback basis) for inspection. + Disagree [][4]string +} + +// DebianHeldOut runs the held-out comparison. +func DebianHeldOut(rows []debian.UDDRow) HeldOut { + skip := map[string]bool{ + "debian/upstream/metadata:Repository": true, + "debian/upstream/metadata:Repository-Browse": true, + } + h := HeldOut{ByBasis: map[string][2]int{}} + for _, r := range rows { + full := choose(debian.ResolveRow(r, nil)) + if full == nil || !fromRepository(*full) { + continue + } + h.WithRepository++ + held := choose(debian.ResolveRow(r, skip)) + if held == nil { + continue + } + h.FallbackResolved++ + hb := basis(*held) + b := h.ByBasis[hb] + b[0]++ + if held.Key == full.Key { + h.Agree++ + b[1]++ + } else { + h.Disagree = append(h.Disagree, [4]string{r.Source, full.Key, held.Key, hb}) + } + h.ByBasis[hb] = b + } + return h +} + +func fromRepository(c distro.Candidate) bool { + for _, e := range c.Evidence { + if e.Source == "debian/upstream/metadata:Repository" { + return true + } + } + return false +} diff --git a/internal/bulk/github.go b/internal/bulk/github.go new file mode 100644 index 0000000..50fe2a5 --- /dev/null +++ b/internal/bulk/github.go @@ -0,0 +1,61 @@ +package bulk + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "tangled.org/odd.computer/isomorph/internal/fetch" + "tangled.org/odd.computer/isomorph/internal/repourl" +) + +const ( + defaultGitHubAPI = "https://api.github.com" + githubAccept = "application/vnd.github+json" + // githubTTL is how long a repository lookup is cached. Renames are rare and GitHub keeps + // redirecting the old name for as long as it is unclaimed, so a stale answer is harmless. + githubTTL = 30 * 24 * time.Hour +) + +// githubAPI overrides the GitHub REST base URL; tests point it at an httptest server. Empty +// means defaultGitHubAPI. +var githubAPI string + +// LookupGitHub resolves a github.com key through the repositories API, which follows renames +// and ownership transfers: the key derived from the response's full_name is the current one. +// gone is true (with key returned unchanged) when the API answers 404, the repository no longer +// exists. Rate limits, offline mode and network failures are returned as errors. Keys on other +// hosts are returned as is. +func LookupGitHub(ctx context.Context, client *fetch.Client, key string) (canonical string, gone bool, err error) { + if !strings.HasPrefix(key, "github.com/") { + return key, false, nil + } + c, err := repourl.Parse("https://" + key) + if err != nil || c.Host != "github.com" || c.Kind != repourl.KindRepo { + return key, false, nil + } + if client == nil { + return key, false, errors.New("bulk: rename lookup needs a fetch.Client") + } + api := githubAPI + if api == "" { + api = defaultGitHubAPI + } + var resp struct { + FullName string `json:"full_name"` + } + u := api + "/repos/" + c.Path + err = client.GetJSON(ctx, u, &resp, fetch.WithHeader("Accept", githubAccept), fetch.WithTTL(githubTTL)) + if fetch.IsNotFound(err) { + return key, true, nil + } + if err != nil { + return key, false, fmt.Errorf("bulk: %s: rename lookup: %w", key, err) + } + if resp.FullName == "" { + return key, false, fmt.Errorf("bulk: %s: rename lookup: response has no full_name", key) + } + return "github.com/" + strings.ToLower(resp.FullName), false, nil +} diff --git a/internal/bulk/github_test.go b/internal/bulk/github_test.go new file mode 100644 index 0000000..3994b33 --- /dev/null +++ b/internal/bulk/github_test.go @@ -0,0 +1,63 @@ +package bulk + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "tangled.org/odd.computer/isomorph/internal/fetch" +) + +func TestLookupGitHub(t *testing.T) { + mux := http.NewServeMux() + // GitHub answers a renamed repository with a 301 to its id-based URL; the client follows. + mux.HandleFunc("/repos/bagder/curl", func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "/repositories/1", http.StatusMovedPermanently) + }) + mux.HandleFunc("/repositories/1", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{"id":1,"full_name":"Curl/Curl"}`)) + }) + mux.HandleFunc("/repos/curl/curl", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{"id":1,"full_name":"curl/curl"}`)) + }) + mux.HandleFunc("/repos/gone/repo", func(w http.ResponseWriter, r *http.Request) { + http.Error(w, `{"message":"Not Found"}`, http.StatusNotFound) + }) + mux.HandleFunc("/repos/limited/repo", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-RateLimit-Remaining", "0") + w.Header().Set("X-RateLimit-Reset", "1700000000") + http.Error(w, `{"message":"API rate limit exceeded"}`, http.StatusForbidden) + }) + srv := httptest.NewServer(mux) + defer srv.Close() + client := fetch.New(t.TempDir()) + client.HTTP = srv.Client() + client.MinInterval = 0 + client.Retries = 0 + client.Offline = false + client.Tokens = map[string]string{} + old := githubAPI + githubAPI = srv.URL + defer func() { githubAPI = old }() + ctx := context.Background() + + for _, c := range []struct { + key, want string + gone bool + }{ + {"github.com/bagder/curl", "github.com/curl/curl", false}, + {"github.com/curl/curl", "github.com/curl/curl", false}, + {"github.com/gone/repo", "github.com/gone/repo", true}, + {"gitlab.com/gnutls/gnutls", "gitlab.com/gnutls/gnutls", false}, // other hosts pass through + {"github.com/curl", "github.com/curl", false}, // not a repository + } { + got, gone, err := LookupGitHub(ctx, client, c.key) + if err != nil || got != c.want || gone != c.gone { + t.Errorf("%s: %q gone=%v err=%v, want %q gone=%v", c.key, got, gone, err, c.want, c.gone) + } + } + if got, _, err := LookupGitHub(ctx, client, "github.com/limited/repo"); err == nil { + t.Errorf("rate limit: expected an error, got %q", got) + } +} diff --git a/internal/bulk/opensuse.go b/internal/bulk/opensuse.go new file mode 100644 index 0000000..0724513 --- /dev/null +++ b/internal/bulk/opensuse.go @@ -0,0 +1,58 @@ +package bulk + +import ( + "sort" + + "tangled.org/odd.computer/isomorph/internal/distro/opensuse" + "tangled.org/odd.computer/isomorph/internal/model" +) + +// FromOpenSUSE builds one row per source package of Tumbleweed's src-oss repository. bins (the +// oss repository) supplies binary names through each binary's source RPM; giturls (package -> +// git URLs, rb.zq1.de) corroborates the spec's Url:. +func FromOpenSUSE(srcs, bins []opensuse.RPM, giturls map[string][]string, files map[string]*opensuse.OBSFiles) []Package { + binaries := map[string][]string{} + for _, b := range bins { + if b.Arch == "src" || b.Arch == "nosrc" || b.SourceRPM == "" { + continue + } + if n := opensuse.SourceName(b.SourceRPM); n != "" { + binaries[n] = append(binaries[n], b.Name) + } + } + seen := map[string]bool{} + var out []Package + for _, s := range srcs { + if (s.Arch != "src" && s.Arch != "nosrc") || seen[s.Name] { + continue + } + seen[s.Name] = true + bs := dedupe(binaries[s.Name]) + p := Package{ + Distro: model.OpenSUSE, + Suite: "tumbleweed", + Name: s.Name, + Version: s.Version, + OSVEcosystem: "openSUSE:Tumbleweed", + PURL: "pkg:rpm/opensuse/" + s.Name + "?arch=src", + Binaries: bs, + Homepage: s.URL, + Status: StatusUnresolved, + } + fill(&p, opensuse.Candidates(s.URL, giturls[s.Name], files[s.Name])) + out = append(out, p) + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out +} + +func dedupe(s []string) []string { + sort.Strings(s) + var out []string + for i, v := range s { + if i == 0 || v != s[i-1] { + out = append(out, v) + } + } + return out +} diff --git a/internal/bulk/pypi.go b/internal/bulk/pypi.go new file mode 100644 index 0000000..38d33fe --- /dev/null +++ b/internal/bulk/pypi.go @@ -0,0 +1,128 @@ +package bulk + +import ( + "regexp" + "slices" + "strings" +) + +// The PyPI repository selection below is adapted from oss-rebuild's pypi.InferRepo and +// uri.FindCommonRepo (github.com/google/oss-rebuild, Apache-2.0), where it was validated on PyPI +// rebuilds. PyPI metadata names repositories inconsistently (a "Source" link, a GitHub home +// page, only a mention in the description), so links are tried in order of how reliably they +// name the source: +// +// source links: +// - the home page, on a known repo host +// - a link named like a source link (commonRepoLinks), on a known repo host +// - a link named like a source link, on any host +// other mentions: +// - the first known repo host cited in the description +// - a link under any other name, on a known repo host, sponsor pages excluded +// +// Unlike oss-rebuild, only the project's current metadata is read (no per-release metadata), +// and the result goes through repourl like every other candidate. + +// pypiInfo is the part of PyPI's /pypi//json "info" object the selection reads. +type pypiInfo struct { + HomePage string `json:"home_page"` + ProjectURLs map[string]string `json:"project_urls"` + Description string `json:"description"` +} + +// commonRepoLinks are project_urls names (lower-cased, spaces removed) that usually name the +// source repository. +var commonRepoLinks = []string{"source", "sourcecode", "repository", "project", "github", "code"} + +var ( + pypiGitHubRE = regexp.MustCompile(`(?i)\bgithub(?:\.com[:/]|:)([\w-]+/[\w.-]+)`) + pypiGitLabRE = regexp.MustCompile(`(?i)\bgitlab(?:\.com[:/]|:)([\w-]+(?:/[\w.-]+)+)`) + pypiBitbucketRE = regexp.MustCompile(`(?i)\bbitbucket(?:\.org[:/]|:)([\w-]+/[\w.-]+)`) + pypiCodebergRE = regexp.MustCompile(`(?i)\bcodeberg(?:\.org[:/]|:)([\w-]+/[\w.-]+)`) + pypiCommonRepos = []*regexp.Regexp{pypiGitHubRE, pypiGitLabRE, pypiBitbucketRE, pypiCodebergRE} + // gitlabProjectRoutes directly follow a GitLab project path in older links without /-/. + gitlabProjectRoutes = []string{"badges", "blob", "commits", "container_registry", "issues", "packages", + "pipelines", "raw", "releases", "tags", "tree", "uploads", "wikis"} +) + +// pypiRepo returns the repository URL a PyPI project's metadata names, or "". +func pypiRepo(info pypiInfo) string { + if r := repoFromSourceLinks(info); r != "" { + return r + } + return repoFromOtherLinks(info) +} + +func repoFromSourceLinks(info pypiInfo) string { + byName := make(map[string]string, len(info.ProjectURLs)) + for name, u := range info.ProjectURLs { + byName[strings.ReplaceAll(strings.ToLower(name), " ", "")] = u + } + var sources []string + for _, name := range commonRepoLinks { + if u, ok := byName[name]; ok { + sources = append(sources, u) + } + } + for _, u := range append([]string{info.HomePage, byName["homepage"]}, sources...) { + if r := findCommonRepo(u); r != "" { + return r + } + } + if len(sources) != 0 { + return sources[0] + } + return "" +} + +func repoFromOtherLinks(info pypiInfo) string { + if r := findCommonRepo(info.Description); r != "" && !strings.Contains(r, "sponsors") { + return r + } + names := make([]string, 0, len(info.ProjectURLs)) + for name := range info.ProjectURLs { + names = append(names, name) + } + slices.Sort(names) // map order is random; make the choice deterministic + for _, name := range names { + u := info.ProjectURLs[name] + if strings.Contains(u, "sponsors") { + continue + } + if r := findCommonRepo(u); r != "" { + return r + } + } + return "" +} + +// findCommonRepo returns the first known repository host path in text ("github.com/o/r"), or "". +func findCommonRepo(text string) string { + for _, re := range pypiCommonRepos { + if r := re.FindString(text); r != "" { + if re == pypiGitLabRE { + r = trimGitLabRoute(r) + } + return r + } + } + return "" +} + +// trimGitLabRoute drops a project route from a matched GitLab path. +func trimGitLabRoute(repo string) string { + if root, _, found := strings.Cut(repo, "/-/"); found { + return root + } + start := strings.IndexAny(repo, ":/") + if start < 0 { + return repo + } + segs := strings.Split(repo[start+1:], "/") + for i := 2; i < len(segs); i++ { + if slices.Contains(gitlabProjectRoutes, segs[i]) { + return repo[:start+1] + strings.Join(segs[:i], "/") + } + } + return repo +} diff --git a/internal/bulk/pypi_test.go b/internal/bulk/pypi_test.go new file mode 100644 index 0000000..20808e1 --- /dev/null +++ b/internal/bulk/pypi_test.go @@ -0,0 +1,23 @@ +package bulk + +import "testing" + +func TestPyPIRepo(t *testing.T) { + for _, tc := range []struct { + name string + info pypiInfo + want string + }{ + {"homepage on a repo host wins", pypiInfo{HomePage: "https://github.com/o/home", ProjectURLs: map[string]string{"Source": "https://gitlab.com/o/src"}}, "github.com/o/home"}, + {"source link, spaces and case folded", pypiInfo{ProjectURLs: map[string]string{"Source Code": "https://gitlab.com/g/sub/p/-/tree/main", "Documentation": "https://docs.example.org"}}, "gitlab.com/g/sub/p"}, + {"legacy gitlab route trimmed", pypiInfo{ProjectURLs: map[string]string{"Repository": "https://gitlab.com/g/p/tree/main"}}, "gitlab.com/g/p"}, + {"source link on any host", pypiInfo{ProjectURLs: map[string]string{"Source": "https://git.example.org/p.git"}}, "https://git.example.org/p.git"}, + {"description mention", pypiInfo{HomePage: "https://p.readthedocs.io", Description: "Clone https://github.com/o/desc to hack."}, "github.com/o/desc"}, + {"sponsors skipped", pypiInfo{ProjectURLs: map[string]string{"Funding": "https://github.com/sponsors/o", "Tracker": "https://github.com/o/real/issues"}}, "github.com/o/real"}, + {"nothing", pypiInfo{HomePage: "https://example.org"}, ""}, + } { + if got := pypiRepo(tc.info); got != tc.want { + t.Errorf("%s: pypiRepo = %q, want %q", tc.name, got, tc.want) + } + } +} diff --git a/internal/bulk/sqlite.go b/internal/bulk/sqlite.go new file mode 100644 index 0000000..ffb66f0 --- /dev/null +++ b/internal/bulk/sqlite.go @@ -0,0 +1,208 @@ +package bulk + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "strings" + + _ "modernc.org/sqlite" // pure-Go driver, registered as "sqlite" + + "tangled.org/odd.computer/isomorph/internal/model" +) + +// schema is the published SQLite layout. Rows for one (distro, suite) are replaced as a unit, +// so each slice can be refreshed independently. +const schema = ` +create table if not exists sources ( + distro text not null, suite text not null, origin text not null, fetched_at text not null, + packages integer not null, primary key (distro, suite)); +create table if not exists packages ( + distro text not null, suite text not null, package text not null, version text not null, + osv_ecosystem text, purl text, homepage text, upstream_key text, repo text, + status text not null, basis text, + primary key (distro, suite, package)); +create index if not exists packages_upstream on packages (upstream_key); +create table if not exists evidence ( + distro text not null, suite text not null, package text not null, + source text not null, value text not null, key text, weight real not null); +create index if not exists evidence_pkg on evidence (distro, suite, package); +create table if not exists binaries ( + distro text not null, suite text not null, binary text not null, package text not null); +create index if not exists binaries_name on binaries (binary); +create table if not exists source_sha256 ( + distro text not null, suite text not null, package text not null, sha256 text not null); +create index if not exists source_sha256_hash on source_sha256 (sha256); +create table if not exists aliases ( + from_key text primary key, to_key text not null, source text not null, checked_at text not null); +create view if not exists counterparts as + select a.distro, a.suite, a.package, a.upstream_key, + b.distro as other_distro, b.suite as other_suite, b.package as other_package + from packages a join packages b + on a.upstream_key = b.upstream_key + and not (a.distro = b.distro and a.suite = b.suite and a.package = b.package) + where a.upstream_key is not null and a.upstream_key != ''; +` + +// schemaVersion is stored in SQLite's user_version; bump it whenever schema changes shape. +const schemaVersion = 2 + +// Open opens (creating if needed) the SQLite database at path. A database written with another +// schema version is refused rather than silently mixed: the file is derived, so rebuild it. +func Open(ctx context.Context, path string) (*sql.DB, error) { + db, err := sql.Open("sqlite", path) + if err != nil { + return nil, err + } + var version, tables int + if err := db.QueryRowContext(ctx, `pragma user_version`).Scan(&version); err != nil { + db.Close() + return nil, err + } + if err := db.QueryRowContext(ctx, `select count(*) from sqlite_master where type = 'table'`).Scan(&tables); err != nil { + db.Close() + return nil, err + } + if tables > 0 && version != schemaVersion { + db.Close() + return nil, fmt.Errorf("bulk: %s has schema version %d, want %d; delete it and rebuild", path, version, schemaVersion) + } + if _, err := db.ExecContext(ctx, fmt.Sprintf("pragma user_version = %d", schemaVersion)); err != nil { + db.Close() + return nil, err + } + if _, err := db.ExecContext(ctx, schema); err != nil { + db.Close() + return nil, fmt.Errorf("bulk: schema: %w", err) + } + return db, nil +} + +// Replace writes pkgs as the complete (src.Distro, src.Suite) slice, removing what was there. +func Replace(ctx context.Context, db *sql.DB, src Source, pkgs []Package) error { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + for _, t := range []string{"packages", "evidence", "binaries", "source_sha256"} { + if _, err := tx.ExecContext(ctx, "delete from "+t+" where distro = ? and suite = ?", src.Distro, src.Suite); err != nil { + return err + } + } + ins := func(q string) (*sql.Stmt, error) { return tx.PrepareContext(ctx, q) } + pkgStmt, err := ins(`insert into packages values (?,?,?,?,?,?,?,?,?,?,?)`) + if err != nil { + return err + } + evStmt, err := ins(`insert into evidence values (?,?,?,?,?,?,?)`) + if err != nil { + return err + } + binStmt, err := ins(`insert into binaries values (?,?,?,?)`) + if err != nil { + return err + } + shaStmt, err := ins(`insert into source_sha256 values (?,?,?,?)`) + if err != nil { + return err + } + for _, p := range pkgs { + if _, err := pkgStmt.ExecContext(ctx, p.Distro, p.Suite, p.Name, p.Version, p.OSVEcosystem, p.PURL, p.Homepage, + p.UpstreamKey, p.Repo, p.Status, p.Basis); err != nil { + return fmt.Errorf("bulk: %s/%s: %w", p.Distro, p.Name, err) + } + for _, e := range p.Evidence { + if _, err := evStmt.ExecContext(ctx, p.Distro, p.Suite, p.Name, e.Source, e.Value, e.Key, e.Weight); err != nil { + return err + } + } + for _, b := range p.Binaries { + if _, err := binStmt.ExecContext(ctx, p.Distro, p.Suite, b, p.Name); err != nil { + return err + } + } + for _, h := range p.SourceSHA256 { + if _, err := shaStmt.ExecContext(ctx, p.Distro, p.Suite, p.Name, h); err != nil { + return err + } + } + } + if _, err := tx.ExecContext(ctx, `insert or replace into sources values (?,?,?,?,?)`, + src.Distro, src.Suite, src.Origin, src.FetchedAt.UTC().Format("2006-01-02T15:04:05Z"), len(pkgs)); err != nil { + return err + } + return tx.Commit() +} + +// Lookup returns the packages matching ref: "distro:package", "distro/suite:package", or an +// upstream key (every package mapped to it). +func Lookup(ctx context.Context, db *sql.DB, ref string) ([]Package, error) { + q := `select distro, suite, package, version, coalesce(osv_ecosystem,''), coalesce(purl,''), coalesce(homepage,''), + coalesce(upstream_key,''), coalesce(repo,''), status, coalesce(basis,'') from packages where ` + var args []any + if d, name, ok := strings.Cut(ref, ":"); ok && !strings.Contains(d, ".") { + distro, suite, _ := strings.Cut(d, "/") + q += "distro = ? and package = ?" + args = append(args, distro, name) + if suite != "" { + q += " and suite = ?" + args = append(args, suite) + } + } else { + q += "upstream_key = ?" + args = append(args, ref) + } + rows, err := db.QueryContext(ctx, q+" order by distro, suite, package", args...) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Package + for rows.Next() { + var p Package + if err := rows.Scan(&p.Distro, &p.Suite, &p.Name, &p.Version, &p.OSVEcosystem, &p.PURL, &p.Homepage, + &p.UpstreamKey, &p.Repo, &p.Status, &p.Basis); err != nil { + return nil, err + } + out = append(out, p) + } + if err := rows.Err(); err != nil { + return nil, err + } + for i := range out { + if err := loadEvidence(ctx, db, &out[i]); err != nil { + return nil, err + } + } + return out, nil +} + +func loadEvidence(ctx context.Context, db *sql.DB, p *Package) error { + rows, err := db.QueryContext(ctx, `select source, value, coalesce(key,''), weight from evidence + where distro = ? and suite = ? and package = ? order by weight desc`, p.Distro, p.Suite, p.Name) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var e model.Evidence + if err := rows.Scan(&e.Source, &e.Value, &e.Key, &e.Weight); err != nil { + return err + } + p.Evidence = append(p.Evidence, e) + } + return rows.Err() +} + +// WriteJSONL writes one JSON object per package. +func WriteJSONL(w interface{ Write([]byte) (int, error) }, pkgs []Package) error { + enc := json.NewEncoder(w) + for _, p := range pkgs { + if err := enc.Encode(p); err != nil { + return err + } + } + return nil +}