diff --git a/README.md b/README.md index 9385359..c6a1187 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ the other distributions. ## How well it works -Between 75% and 85% of source packages resolve, depending on the distribution. +Between 74% and 86% of source packages resolve, depending on the distribution. Many Debian packages declare their repository in DEP-12 metadata. When isomorph is given the rest of such a package's fields and not that one, it names the same repository 96.8% of the diff --git a/cmd/isomorph/bulk.go b/cmd/isomorph/bulk.go index cd993f0..6f3caa9 100644 --- a/cmd/isomorph/bulk.go +++ b/cmd/isomorph/bulk.go @@ -349,7 +349,7 @@ func bulkCompare(ctx context.Context, env *Env, args []string) error { *samples = 0 } } - db, err := bulk.Open(ctx, *dbPath) + db, err := bulk.OpenReadOnly(ctx, *dbPath) if err != nil { return err } @@ -491,7 +491,7 @@ func bulkCanonicalize(ctx context.Context, env *Env, args []string) error { env.Logf("canonicalize: %d GitHub keys in conflicts across %d distribution pairs", len(keys), len(pairs)) now := time.Now() var aliases []bulk.Alias - failed, gone := 0, 0 + failed, gone, renamed := 0, 0, 0 for i, k := range keys { to, isGone, err := bulk.LookupGitHub(ctx, env.Client, k) if err != nil { @@ -504,6 +504,7 @@ func bulkCanonicalize(ctx context.Context, env *Env, args []string) error { gone++ aliases = append(aliases, bulk.Alias{From: k, To: "", Source: bulk.AliasGitHubGone, CheckedAt: now}) case to != k: + renamed++ aliases = append(aliases, bulk.Alias{From: k, To: to, Source: bulk.AliasGitHubRename, CheckedAt: now}) } if (i+1)%200 == 0 { @@ -539,12 +540,35 @@ func bulkCanonicalize(ctx context.Context, env *Env, args []string) error { } } rk.Close() + // Evidence keeps its keys after aliasing, so an entry aliased before is found through the + // aliases table; recomputing it lets a better answer (or none) replace a stale one. + prev, err := db.QueryContext(ctx, `select from_key from aliases where source = ?`, bulk.AliasRegistry) + if err != nil { + return err + } + for prev.Next() { + var k string + prev.Scan(&k) + if !seenReg[k] { + seenReg[k] = true + registryKeys = append(registryKeys, k) + } + } + prev.Close() sort.Strings(registryKeys) registries, registryFailed := bulk.RegistryAliases(ctx, env.Client, registryKeys, func(host string) string { return "https://" + host }) for i := range registries { registries[i].CheckedAt = now } - aliases = append(aliases, registries...) + // Registry aliases are recomputed in full when every lookup succeeded; otherwise they are + // only added to, so a failed lookup does not drop what it found before. + if registryFailed == 0 { + if err := bulk.ReplaceAliases(ctx, db, bulk.AliasRegistry, registries); err != nil { + return err + } + } else { + aliases = append(aliases, registries...) + } for i := range projects { projects[i].CheckedAt = now } @@ -617,6 +641,6 @@ func bulkCanonicalize(ctx context.Context, env *Env, args []string) error { } } fmt.Fprintf(env.Stdout, "canonicalize: %d GitHub keys checked (%d failed), %d renamed, %d gone; %d GitLab project ids resolved (%d failed); %d registry entries resolved of %d (%d failed); %d mirror aliases; %d KDE aliases; %d package mappings changed\n", - len(keys), failed, len(aliases)-len(projects)-len(registries)-gone, gone, len(projects), projectFailed, len(registries), len(registryKeys), registryFailed, len(mirrors), len(kde), changed) + len(keys), failed, renamed, gone, len(projects), projectFailed, len(registries), len(registryKeys), registryFailed, len(mirrors), len(kde), changed) return nil } diff --git a/cmd/isomorph/export.go b/cmd/isomorph/export.go index fe76103..405a9b5 100644 --- a/cmd/isomorph/export.go +++ b/cmd/isomorph/export.go @@ -24,7 +24,7 @@ func runExport(ctx context.Context, env *Env, args []string) error { if *out == "" { *out = filepath.Join(env.DataDir, "export") } - db, err := bulk.Open(ctx, *dbPath) + db, err := bulk.OpenReadOnly(ctx, *dbPath) if err != nil { return err } diff --git a/cmd/isomorph/lookup.go b/cmd/isomorph/lookup.go index 164d1f1..cf77981 100644 --- a/cmd/isomorph/lookup.go +++ b/cmd/isomorph/lookup.go @@ -31,7 +31,7 @@ func runLookup(ctx context.Context, env *Env, args []string) error { if err := fs.Parse(args); err != nil { return err } - db, err := bulk.Open(ctx, *dbPath) + db, err := bulk.OpenReadOnly(ctx, *dbPath) if err != nil { return err } diff --git a/docs/EVALUATION.md b/docs/EVALUATION.md index 43d91af..13eee28 100644 --- a/docs/EVALUATION.md +++ b/docs/EVALUATION.md @@ -206,6 +206,56 @@ in 12 and `Url:` in 9, so their precedence stays. - **Arch and Alpine source order.** Neither records which source is primary, so a dependency's git source can still outrank the main tarball (telegram-desktop). +## Fixes from the osvrb joins (2026-09-30) + +Joining osvrb's data by upstream (osvrb's `validation/2026-09-30-identity-join.md`) turned up +mappings where a site stands for more projects than the one it names: +- **python.org:** PyPy's releases on `downloads.python.org` and PyPI-era pages on + `packages.python.org` mapped to CPython. +- **gnupg.org:** its pages for gpgme, libgcrypt and gpa mapped to GnuPG. +- **Octave Forge:** every package mapped to one repository. +- **Flagship sites:** `curl.se/trurl` mapped to curl, and `sphinx-doc.org` to Sphinx for the + sphinxcontrib packages. +- **PyPI descriptions:** python-barbicanclient's description, which cites the barbican + server, mapped it to barbican. + +The fixes: +- **Catalogue sites name no repository:** Octave Forge, the Octave package index, PEAR, and + homalg's package pages. Octave Forge's per-package repositories + (`sourceforge.net/p/octave/`) are keyed as their own. +- **python.org's other subdomains:** PyPI-era pages become PyPI entries, and PyPy's releases + map to PyPy. +- **gnupg.org's project pages** map to each project's repository. +- **A project site** decides a package only if the package is named like the site's + repository (`bulk/flagship.go`). A forge page or a source URL still decides any package. +- **PyPI descriptions:** a repository cited in a description counts only if it is named like + the project. +- **OpenStack:** opendev.org is recognised, with OpenStack's GitHub mirrors mapped to it. +- **Registry aliases** are recomputed in full, so stale ones are replaced. + +| | before | after | +|---|---:|---:| +| Debian coverage | 77.9% | 77.8% | +| Arch coverage | 85.6% | 85.6% | +| openSUSE coverage | 75.0% | 74.3% | +| Alpine coverage | 78.3% | 78.2% | +| DEP-12 held-out agreement | 96.8% | 96.8% | +| held-out, homepage basis | 81.2% (112) | 84.9% (106) | +| Arch vs Debian | 93.8% of 4,904 | 94.1% of 4,900 | +| Alpine vs Arch | 93.8% of 3,061 | 94.0% of 3,058 | +| Alpine vs Debian | 93.3% of 2,889 | 93.7% of 2,886 | +| Arch vs openSUSE | 91.6% of 4,255 | 92.1% of 4,276 | +| Alpine vs openSUSE | 91.0% of 2,551 | 91.6% of 2,548 | +| Debian vs openSUSE | 90.7% of 4,300 | 91.2% of 4,357 | + +- **Coverage** falls slightly: a package a site wrongly claimed is now unresolved instead. +- **Agreement** rises in every pair. +- **Known misses:** + - The name guard lets through three openSUSE flavour packages (`python313-pyside6`, + `-shiboken6`, `-packaging`), which still map to CPython. + - openSUSE's giturls still names another project's repository for a few packages, such as + java-11-openj9. + ## How to reproduce ```sh diff --git a/internal/bulk/alias.go b/internal/bulk/alias.go index 364cd77..3031395 100644 --- a/internal/bulk/alias.go +++ b/internal/bulk/alias.go @@ -41,7 +41,8 @@ type Alias struct { // - gitbox.apache.org/[repos/asf/]X -> github.com/apache/X (the GitHub mirror, which the GitHub // API can follow further); // - github.com/gnome/X -> gitlab.gnome.org/gnome/X; -// - github.com/golang/X -> go.googlesource.com/X. +// - github.com/golang/X -> go.googlesource.com/X; +// - github.com/openstack/X -> opendev.org/openstack/X. func MirrorAliases(keys []string) []Alias { var out []Alias for _, k := range keys { @@ -55,6 +56,8 @@ func MirrorAliases(keys []string) []Alias { to = "gitlab.gnome.org/gnome/" + strings.TrimPrefix(k, "github.com/gnome/") case strings.HasPrefix(k, "github.com/golang/"): to = "go.googlesource.com/" + strings.TrimPrefix(k, "github.com/golang/") + case strings.HasPrefix(k, "github.com/openstack/"): + to = "opendev.org/openstack/" + strings.TrimPrefix(k, "github.com/openstack/") } if to != "" && to != k { out = append(out, Alias{From: k, To: to, Source: AliasMirrorRule}) @@ -150,6 +153,9 @@ func Rescore(p *Package, alias map[string]string) { if byKey[k].basis == BasisSecondary { continue // corroborates, never decides } + if siteOnly(p.Evidence, k) && !namedLike(p.Name, k) { + continue // a project site standing for another package's repository (flagship.go) + } if s := (1 - byKey[k].miss) * distro.PackagingPenalty(k); s > bestScore { best, bestScore = k, s } @@ -455,7 +461,7 @@ func RegistryAliases(ctx context.Context, c *fetch.Client, keys []string, base f failed++ continue } - if r := pypiRepo(resp.Info); r != "" { + if r := pypiRepo(name, resp.Info); r != "" { if !strings.Contains(r, "://") { r = "https://" + r } diff --git a/internal/bulk/alias_test.go b/internal/bulk/alias_test.go index b8bd02e..bc4baf5 100644 --- a/internal/bulk/alias_test.go +++ b/internal/bulk/alias_test.go @@ -36,7 +36,7 @@ func TestRescoreMergesAliases(t *testing.T) { func TestMirrorAliases(t *testing.T) { got := MirrorAliases([]string{ - "gitbox.apache.org/repos/asf/activemq", "gitbox.apache.org/zookeeper", "github.com/gnome/glib", "github.com/golang/text", + "gitbox.apache.org/repos/asf/activemq", "gitbox.apache.org/zookeeper", "github.com/gnome/glib", "github.com/golang/text", "github.com/openstack/nova", "github.com/kde/heaptrack", "github.com/other/x", "invent.kde.org/heaptrack", }) want := map[string]string{ @@ -44,6 +44,7 @@ func TestMirrorAliases(t *testing.T) { "gitbox.apache.org/zookeeper": "github.com/apache/zookeeper", "github.com/gnome/glib": "gitlab.gnome.org/gnome/glib", "github.com/golang/text": "go.googlesource.com/text", + "github.com/openstack/nova": "opendev.org/openstack/nova", } if len(got) != len(want) { t.Fatalf("got %+v", got) diff --git a/internal/bulk/debian.go b/internal/bulk/debian.go index 9f85a54..42e1536 100644 --- a/internal/bulk/debian.go +++ b/internal/bulk/debian.go @@ -39,21 +39,26 @@ func fill(p *Package, cands []distro.Candidate) { p.Evidence = append(p.Evidence, c.Evidence...) } sort.SliceStable(p.Evidence, func(i, j int) bool { return p.Evidence[i].Weight > p.Evidence[j].Weight }) - if best := choose(cands); best != nil { + if best := choose(p.Name, cands); best != nil { p.UpstreamKey, p.Repo, p.Status = best.Key, best.URL, StatusResolved p.Basis = basis(*best) } } -// choose picks the upstream among ranked candidates: the first repository, unless it rests on -// a homepage on a staleHomeHosts host and a release or VCS source names another repository. -func choose(cands []distro.Candidate) *distro.Candidate { +// choose picks package name's upstream among ranked candidates: the first repository, unless +// it rests on a homepage on a staleHomeHosts host and a release or VCS source names another +// repository. A candidate that only a project site supports counts only for a package named +// like it (flagship.go). +func choose(name string, cands []distro.Candidate) *distro.Candidate { var best *distro.Candidate for i := range cands { c := &cands[i] if c.Kind != repourl.KindRepo || basis(*c) == BasisSecondary { continue } + if siteOnly(c.Evidence, c.Key) && !namedLike(name, c.Key) { + continue + } if best == nil { best = c if basis(*c) != BasisHomepage || !onStaleHomeHost(c.Key) { @@ -118,12 +123,12 @@ func DebianHeldOut(rows []debian.UDDRow) HeldOut { } h := HeldOut{ByBasis: map[string][2]int{}} for _, r := range rows { - full := choose(debian.ResolveRow(r, nil)) + full := choose(r.Source, debian.ResolveRow(r, nil)) if full == nil || !fromRepository(*full) { continue } h.WithRepository++ - held := choose(debian.ResolveRow(r, skip)) + held := choose(r.Source, debian.ResolveRow(r, skip)) if held == nil { continue } diff --git a/internal/bulk/export_test.go b/internal/bulk/export_test.go index eb4605e..666140f 100644 --- a/internal/bulk/export_test.go +++ b/internal/bulk/export_test.go @@ -85,3 +85,39 @@ func TestExport(t *testing.T) { t.Errorf("re-export: %v %s vs %s", err, again.Snapshot, m.Snapshot) } } + +// TestOpenReadOnly: a reader neither writes the schema nor waits for a writer. +func TestOpenReadOnly(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "p.sqlite") + if _, err := OpenReadOnly(ctx, path); err == nil { + t.Fatal("a missing database must not be created") + } + w, err := Open(ctx, path) + if err != nil { + t.Fatal(err) + } + defer w.Close() + if err := Replace(ctx, w, Source{Distro: model.Debian, Suite: "sid", Origin: "t", FetchedAt: time.Now()}, FromUDD(udd(), "sid")); err != nil { + t.Fatal(err) + } + tx, err := w.BeginTx(ctx, nil) // a writer holding the database + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + if _, err := tx.ExecContext(ctx, `insert into aliases values ('a','b','t','now')`); err != nil { + t.Fatal(err) + } + r, err := OpenReadOnly(ctx, path) + if err != nil { + t.Fatal(err) + } + defer r.Close() + if a, err := Lookup(ctx, r, nil, "debian:json-c", false); err != nil || len(a.Matches) != 1 { + t.Fatalf("read while a writer holds the database: %v %+v", err, a) + } + if _, err := r.ExecContext(ctx, `delete from aliases`); err == nil { + t.Error("a read-only database accepted a write") + } +} diff --git a/internal/bulk/flagship.go b/internal/bulk/flagship.go new file mode 100644 index 0000000..e29d756 --- /dev/null +++ b/internal/bulk/flagship.go @@ -0,0 +1,90 @@ +package bulk + +import ( + "strings" + + "tangled.org/odd.computer/isomorph/internal/model" + "tangled.org/odd.computer/isomorph/internal/repourl" +) + +// A project's website often stands for its repository (curl.se for github.com/curl/curl), but +// the same site also hosts sibling projects: curl.se/trurl is trurl, sphinx-doc.org is the +// homepage every sphinxcontrib-* package gives. So a candidate supported only by a project +// site, rather than by a forge page or a source URL, decides a package only when the package +// is named like the repository (docs/EVALUATION.md). + +// siteOnly reports whether every piece of evidence for key is a homepage on a site other than +// the repository's own host. +func siteOnly(evidence []model.Evidence, key string) bool { + n := 0 + for _, e := range evidence { + if e.Key != key { + continue + } + n++ + if !isSiteHint(e, key) { + return false + } + } + return n > 0 +} + +func isSiteHint(e model.Evidence, key string) bool { + if BasisOf(e.Source) != BasisHomepage { + return false + } + c, err := repourl.Parse(e.Value) + if err == nil && c.Kind == repourl.KindRepo { + return false // a forge page of the repository itself + } + return err != nil || c.Host != key[:strings.IndexByte(key+"/", '/')] +} + +// flagshipNames are the package-name stems of flagship repositories whose name differs from +// their packages'. +var flagshipNames = map[string][]string{ + "github.com/python/cpython": {"python", "cpython"}, + "github.com/apache/httpd": {"apache", "httpd"}, + "git.gnupg.org/gnupg": {"gnupg", "gpg"}, + "git.savannah.gnu.org/libc": {"glibc", "libc"}, + "git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux": {"linux", "kernel"}, +} + +var ecosystemPrefixes = []string{"python3-", "python-", "py3-", "py-", "lib32-", "mingw32-", "mingw64-", + "perl-", "ruby-", "rubygem-", "golang-", "rust-", "nodejs-", "node-"} + +// namedLike reports whether pkg is named like the repository at key: its name (without an +// ecosystem prefix) is the repository's name or a stem of it, followed by nothing, a version +// or a separator. rust1.88, binutils-djgpp and llvm-toolchain-19 are; trurl, lziprecover and +// sphinxcontrib-jsmath are not. +func namedLike(pkg, key string) bool { + name := strings.ToLower(pkg) + for _, p := range ecosystemPrefixes { + name = strings.TrimPrefix(name, p) + } + stems := flagshipNames[key] + if stems == nil { + repo := key[strings.LastIndexByte(key, '/')+1:] + stems = []string{repo} + for _, suf := range []string{"-project", "-portable", "-src"} { + if s := strings.TrimSuffix(repo, suf); s != repo { + stems = append(stems, s) + } + } + } + for _, s := range stems { + if len(s) >= 2 && (startsAtBoundary(name, s) || len(name) >= 3 && startsAtBoundary(s, name)) { + return true + } + } + return false +} + +// startsAtBoundary reports whether s is prefix followed by nothing or a non-letter. +func startsAtBoundary(s, prefix string) bool { + if !strings.HasPrefix(s, prefix) { + return false + } + rest := s[len(prefix):] + return rest == "" || !(rest[0] >= 'a' && rest[0] <= 'z') +} diff --git a/internal/bulk/flagship_test.go b/internal/bulk/flagship_test.go new file mode 100644 index 0000000..804250b --- /dev/null +++ b/internal/bulk/flagship_test.go @@ -0,0 +1,62 @@ +package bulk + +import ( + "testing" + + "tangled.org/odd.computer/isomorph/internal/model" +) + +func TestNamedLike(t *testing.T) { + for _, c := range []struct { + pkg, key string + want bool + }{ + {"curl", "github.com/curl/curl", true}, + {"trurl", "github.com/curl/curl", false}, + {"binutils-djgpp", "git.savannah.gnu.org/binutils", true}, + {"rust1.88", "github.com/rust-lang/rust", true}, + {"llvm-toolchain-19", "github.com/llvm/llvm-project", true}, + {"lit", "github.com/llvm/llvm-project", false}, + {"python-sphinxcontrib-jsmath", "github.com/sphinx-doc/sphinx", false}, + {"python3-sphinx", "github.com/sphinx-doc/sphinx", true}, + {"lziprecover", "git.savannah.gnu.org/lzip", false}, + {"python310", "github.com/python/cpython", true}, + {"python-zope.configuration", "github.com/python/cpython", false}, + {"apache2", "github.com/apache/httpd", true}, + {"gpg2", "git.gnupg.org/gnupg", true}, + {"gpgme", "git.gnupg.org/gnupg", false}, + {"go", "go.googlesource.com/go", true}, + } { + if got := namedLike(c.pkg, c.key); got != c.want { + t.Errorf("namedLike(%q, %q) = %v", c.pkg, c.key, got) + } + } +} + +// TestSiteOnlyGuard: a project site cannot hand its repository to another package, but a +// forge homepage or a source URL still can. +func TestSiteOnlyGuard(t *testing.T) { + ev := func(src, val, key string, w float64) model.Evidence { + return model.Evidence{Source: src, Value: val, Key: key, Weight: w} + } + trurl := Package{Name: "trurl", Evidence: []model.Evidence{ev("spec:URL", "https://curl.se/trurl", "github.com/curl/curl", 0.3)}} + Rescore(&trurl, nil) + if trurl.UpstreamKey != "" { + t.Errorf("trurl took curl's repository from curl.se: %+v", trurl) + } + curl := Package{Name: "curl", Evidence: []model.Evidence{ev("spec:URL", "https://curl.se/", "github.com/curl/curl", 0.3)}} + Rescore(&curl, nil) + if curl.UpstreamKey != "github.com/curl/curl" { + t.Errorf("curl lost its own site: %+v", curl) + } + forge := Package{Name: "trurl", Evidence: []model.Evidence{ev("spec:URL", "https://github.com/curl/trurl", "github.com/curl/trurl", 0.5)}} + Rescore(&forge, nil) + if forge.UpstreamKey != "github.com/curl/trurl" { + t.Errorf("a forge homepage must still count: %+v", forge) + } + src := Package{Name: "lzlib", Evidence: []model.Evidence{ev("debian/watch:upstream_url", "https://download.savannah.gnu.org/releases/lzip/lzlib/lzlib-1.16.tar.lz", "git.savannah.gnu.org/lzip", 0.36)}} + Rescore(&src, nil) + if src.UpstreamKey != "git.savannah.gnu.org/lzip" { + t.Errorf("a source URL is not a site hint: %+v", src) + } +} diff --git a/internal/bulk/pypi.go b/internal/bulk/pypi.go index 38d33fe..114d7d1 100644 --- a/internal/bulk/pypi.go +++ b/internal/bulk/pypi.go @@ -17,11 +17,14 @@ import ( // - a link named like a source link (commonRepoLinks), on a known repo host // - a link named like a source link, on any host // other mentions: -// - the first known repo host cited in the description +// - the first known repo host cited in the description whose repository is named like the +// project // - a link under any other name, on a known repo host, sponsor pages excluded // // Unlike oss-rebuild, only the project's current metadata is read (no per-release metadata), -// and the result goes through repourl like every other candidate. +// the result goes through repourl like every other candidate, and a description mention must +// be named like the project: descriptions cite related projects (python-barbicanclient's +// cites the barbican server) as readily as their own repository. // pypiInfo is the part of PyPI's /pypi//json "info" object the selection reads. type pypiInfo struct { @@ -39,18 +42,19 @@ var ( pypiGitLabRE = regexp.MustCompile(`(?i)\bgitlab(?:\.com[:/]|:)([\w-]+(?:/[\w.-]+)+)`) pypiBitbucketRE = regexp.MustCompile(`(?i)\bbitbucket(?:\.org[:/]|:)([\w-]+/[\w.-]+)`) pypiCodebergRE = regexp.MustCompile(`(?i)\bcodeberg(?:\.org[:/]|:)([\w-]+/[\w.-]+)`) - pypiCommonRepos = []*regexp.Regexp{pypiGitHubRE, pypiGitLabRE, pypiBitbucketRE, pypiCodebergRE} + pypiOpenDevRE = regexp.MustCompile(`(?i)\bopendev(?:\.org[:/]|:)([\w-]+/[\w.-]+)`) + pypiCommonRepos = []*regexp.Regexp{pypiGitHubRE, pypiGitLabRE, pypiBitbucketRE, pypiCodebergRE, pypiOpenDevRE} // gitlabProjectRoutes directly follow a GitLab project path in older links without /-/. gitlabProjectRoutes = []string{"badges", "blob", "commits", "container_registry", "issues", "packages", "pipelines", "raw", "releases", "tags", "tree", "uploads", "wikis"} ) // pypiRepo returns the repository URL a PyPI project's metadata names, or "". -func pypiRepo(info pypiInfo) string { +func pypiRepo(project string, info pypiInfo) string { if r := repoFromSourceLinks(info); r != "" { return r } - return repoFromOtherLinks(info) + return repoFromOtherLinks(project, info) } func repoFromSourceLinks(info pypiInfo) string { @@ -75,8 +79,8 @@ func repoFromSourceLinks(info pypiInfo) string { return "" } -func repoFromOtherLinks(info pypiInfo) string { - if r := findCommonRepo(info.Description); r != "" && !strings.Contains(r, "sponsors") { +func repoFromOtherLinks(project string, info pypiInfo) string { + if r := findNamedRepo(info.Description, project); r != "" { return r } names := make([]string, 0, len(info.ProjectURLs)) @@ -96,6 +100,33 @@ func repoFromOtherLinks(info pypiInfo) string { return "" } +// findNamedRepo returns the first known repository host path in text whose repository is +// named like project, or "". +func findNamedRepo(text, project string) string { + type hit struct { + at int + repo string + } + var hits []hit + for _, re := range pypiCommonRepos { + for _, loc := range re.FindAllStringIndex(text, -1) { + r := text[loc[0]:loc[1]] + if re == pypiGitLabRE { + r = trimGitLabRoute(r) + } + hits = append(hits, hit{loc[0], r}) + } + } + slices.SortFunc(hits, func(a, b hit) int { return a.at - b.at }) + for _, h := range hits { + name := strings.TrimSuffix(h.repo[strings.LastIndexAny(h.repo, "/:")+1:], ".git") + if !strings.Contains(h.repo, "sponsors") && SameProject(project, name) { + return h.repo + } + } + return "" +} + // findCommonRepo returns the first known repository host path in text ("github.com/o/r"), or "". func findCommonRepo(text string) string { for _, re := range pypiCommonRepos { diff --git a/internal/bulk/pypi_test.go b/internal/bulk/pypi_test.go index 20808e1..1169df7 100644 --- a/internal/bulk/pypi_test.go +++ b/internal/bulk/pypi_test.go @@ -4,19 +4,23 @@ import "testing" func TestPyPIRepo(t *testing.T) { for _, tc := range []struct { - name string - info pypiInfo - want string + name string + project string + info pypiInfo + want string }{ - {"homepage on a repo host wins", pypiInfo{HomePage: "https://github.com/o/home", ProjectURLs: map[string]string{"Source": "https://gitlab.com/o/src"}}, "github.com/o/home"}, - {"source link, spaces and case folded", pypiInfo{ProjectURLs: map[string]string{"Source Code": "https://gitlab.com/g/sub/p/-/tree/main", "Documentation": "https://docs.example.org"}}, "gitlab.com/g/sub/p"}, - {"legacy gitlab route trimmed", pypiInfo{ProjectURLs: map[string]string{"Repository": "https://gitlab.com/g/p/tree/main"}}, "gitlab.com/g/p"}, - {"source link on any host", pypiInfo{ProjectURLs: map[string]string{"Source": "https://git.example.org/p.git"}}, "https://git.example.org/p.git"}, - {"description mention", pypiInfo{HomePage: "https://p.readthedocs.io", Description: "Clone https://github.com/o/desc to hack."}, "github.com/o/desc"}, - {"sponsors skipped", pypiInfo{ProjectURLs: map[string]string{"Funding": "https://github.com/sponsors/o", "Tracker": "https://github.com/o/real/issues"}}, "github.com/o/real"}, - {"nothing", pypiInfo{HomePage: "https://example.org"}, ""}, + {"homepage on a repo host wins", "p", pypiInfo{HomePage: "https://github.com/o/home", ProjectURLs: map[string]string{"Source": "https://gitlab.com/o/src"}}, "github.com/o/home"}, + {"source link, spaces and case folded", "p", pypiInfo{ProjectURLs: map[string]string{"Source Code": "https://gitlab.com/g/sub/p/-/tree/main", "Documentation": "https://docs.example.org"}}, "gitlab.com/g/sub/p"}, + {"legacy gitlab route trimmed", "p", pypiInfo{ProjectURLs: map[string]string{"Repository": "https://gitlab.com/g/p/tree/main"}}, "gitlab.com/g/p"}, + {"source link on any host", "p", pypiInfo{ProjectURLs: map[string]string{"Source": "https://git.example.org/p.git"}}, "https://git.example.org/p.git"}, + {"description mention", "desc", pypiInfo{HomePage: "https://p.readthedocs.io", Description: "Clone https://github.com/o/desc to hack."}, "github.com/o/desc"}, + {"sponsors skipped", "real", pypiInfo{ProjectURLs: map[string]string{"Funding": "https://github.com/sponsors/o", "Tracker": "https://github.com/o/real/issues"}}, "github.com/o/real"}, + {"nothing", "p", pypiInfo{HomePage: "https://example.org"}, ""}, + {"description mention of another project skipped", "python-barbicanclient", pypiInfo{HomePage: "https://docs.openstack.org/python-barbicanclient/latest/", + Description: "The server is https://github.com/openstack/barbican; source at https://opendev.org/openstack/python-barbicanclient"}, "opendev.org/openstack/python-barbicanclient"}, + {"only other projects in the description", "p", pypiInfo{Description: "Built on https://github.com/o/framework."}, ""}, } { - if got := pypiRepo(tc.info); got != tc.want { + if got := pypiRepo(tc.project, tc.info); got != tc.want { t.Errorf("%s: pypiRepo = %q, want %q", tc.name, got, tc.want) } } diff --git a/internal/bulk/sqlite.go b/internal/bulk/sqlite.go index f0ad549..0d7c5b3 100644 --- a/internal/bulk/sqlite.go +++ b/internal/bulk/sqlite.go @@ -5,6 +5,7 @@ import ( "database/sql" "encoding/json" "fmt" + "os" _ "modernc.org/sqlite" // pure-Go driver, registered as "sqlite" @@ -47,6 +48,29 @@ create view if not exists counterparts as // schemaVersion is stored in SQLite's user_version; bump it whenever schema changes shape. const schemaVersion = 2 +// OpenReadOnly opens an existing database for reading only, so lookups and exports neither +// need write access nor wait for a writer's lock. It refuses another schema version, as Open +// does. +func OpenReadOnly(ctx context.Context, path string) (*sql.DB, error) { + if _, err := os.Stat(path); err != nil { + return nil, err + } + db, err := sql.Open("sqlite", "file:"+path+"?mode=ro") + if err != nil { + return nil, err + } + var version int + if err := db.QueryRowContext(ctx, `pragma user_version`).Scan(&version); err != nil { + db.Close() + return nil, err + } + if version != schemaVersion { + db.Close() + return nil, fmt.Errorf("bulk: %s has schema version %d, want %d; rebuild it", path, version, schemaVersion) + } + return db, nil +} + // Open opens (creating if needed) the SQLite database at path. A database written with another // schema version is refused rather than silently mixed: the file is derived, so rebuild it. func Open(ctx context.Context, path string) (*sql.DB, error) { diff --git a/internal/distro/distro.go b/internal/distro/distro.go index 5e76c2c..34dd21c 100644 --- a/internal/distro/distro.go +++ b/internal/distro/distro.go @@ -46,7 +46,9 @@ func addCandidate(cands []Candidate, raw string, ev model.Evidence, mapHosts boo return cands } key, url, kind := c.Key, c.URL, c.Kind - if kind != repourl.KindRepo { + if kind != repourl.KindRepo && !(kind == repourl.KindRegistry && c.Hint == "") { + // A registry entry (pypi.org/project/x, packages.python.org/x) stays a registry + // candidate, for canonicalisation to follow to the repository its metadata names. if c.Hint == "" { return cands } diff --git a/internal/distro/hosts.go b/internal/distro/hosts.go index 65b5fa9..9cb3408 100644 --- a/internal/distro/hosts.go +++ b/internal/distro/hosts.go @@ -4,6 +4,8 @@ import ( "net/url" "regexp" "strings" + + "tangled.org/odd.computer/isomorph/internal/repourl" ) // hostRule rewrites a project website / download URL that repourl does not understand into @@ -82,7 +84,9 @@ var hostRules = []hostRule{ // distributions. // docs.xfce.org///start and goodies.xfce.org/projects// -> // gitlab.xfce.org// (the documentation wiki mirrors the GitLab groups). - {regexp.MustCompile(`(?i)^https?://docs\.xfce\.org/(apps|panel-plugins|thunar-plugins|xfce|bindings)/([\w.-]+)`), 0.9, xfceProject}, + // Only a project's own page (/[/start]): deeper pages document plugins and + // parts (docs.xfce.org/xfce/thunar/archive is thunar-archive-plugin, not Thunar). + {regexp.MustCompile(`(?i)^https?://docs\.xfce\.org/(apps|panel-plugins|thunar-plugins|xfce|bindings)/([\w.-]+)(?:/start)?/?(?:$|[#?])`), 0.9, xfceProject}, {regexp.MustCompile(`(?i)^https?://goodies\.xfce\.org/projects/(apps|panel-plugins|thunar-plugins|xfce|bindings)/([\w.-]+)`), 0.9, xfceProject}, // netfilter.org/projects/ -> git.netfilter.org/ (ulogd's repository is ulogd2). {regexp.MustCompile(`(?i)^https?://(?:www\.)?netfilter\.org/projects/([\w.-]+)`), 0.9, @@ -93,6 +97,18 @@ var hostRules = []hostRule{ } return "https://git.netfilter.org/" + name }}, + // gnupg.org/software// and gnupg.org/ftp/gcrypt// -> git.gnupg.org/: each + // of GnuPG's sibling projects (gpgme, libgcrypt, libassuan, gpa) has its own repository. + {regexp.MustCompile(`(?i)^https?://(?:www\.)?gnupg\.org/(?:software|ftp/gcrypt)/([\w.+-]+?)(?:\.html)?(?:/|$)`), 0.9, + func(m []string) string { + if name := strings.ToLower(m[1]); name != "index" && name != "gnupg" { + return "https://git.gnupg.org/" + name + ".git" + } + return "" + }}, + // PyPy's releases are served from downloads.python.org/pypy/; its repository is on GitHub. + {regexp.MustCompile(`(?i)^https?://downloads\.python\.org/pypy/`), 0.9, + func(m []string) string { return "https://github.com/pypy/pypy" }}, // freedesktop.org/software/pulseaudio/ -> gitlab.freedesktop.org/pulseaudio/ // (paprefs, pavucontrol, webrtc-audio-processing; other /software/ paths name documents). {regexp.MustCompile(`(?i)^https?://(?:www\.)?freedesktop\.org/software/pulseaudio/([\w.-]+)`), 0.9, @@ -155,6 +171,9 @@ func mapGitLab(raw string) (string, bool) { // URL, a confidence factor in (0,1] and true when a rule matched. func MapHost(raw string) (string, float64, bool) { raw = strings.TrimSpace(raw) + if repourl.IsCatalogue(raw) { + return "", 0, false + } if out, ok := mapGitLab(raw); ok { return out, 1.0, true } diff --git a/internal/distro/hosts_test.go b/internal/distro/hosts_test.go index e98d1c7..a6ee850 100644 --- a/internal/distro/hosts_test.go +++ b/internal/distro/hosts_test.go @@ -36,6 +36,13 @@ func TestMapHost(t *testing.T) { {"https://netfilter.org/projects/ulogd/", "https://git.netfilter.org/ulogd2", 0.9}, {"https://freedesktop.org/software/pulseaudio/paprefs/", "https://gitlab.freedesktop.org/pulseaudio/paprefs", 0.9}, {"https://www.freedesktop.org/software/appstream/docs/", "", 0}, + {"https://docs.xfce.org/xfce/thunar/archive", "", 0}, + {"https://docs.xfce.org/xfce/thunar/start", "https://gitlab.xfce.org/xfce/thunar", 0.9}, + {"https://gnupg.org/software/gpa/index.html", "https://git.gnupg.org/gpa.git", 0.9}, + {"https://gnupg.org/ftp/gcrypt/libgcrypt/libgcrypt-1.11.2.tar.bz2", "https://git.gnupg.org/libgcrypt.git", 0.9}, + {"https://gnupg.org/software/gnupg/index.html", "", 0}, + {"https://downloads.python.org/pypy/pypy3.11-v8.0.0-src.tar.gz", "https://github.com/pypy/pypy", 0.9}, + {"https://gnu-octave.github.io/packages/geometry/", "", 0}, {"https://ant.apache.org/", "https://github.com/apache/ant", 0.6}, {"https://commons.apache.org/proper/commons-lang/", "", 0}, {"https://www.apache.org/", "", 0}, diff --git a/internal/repourl/repourl.go b/internal/repourl/repourl.go index 08efdcb..70bddf5 100644 --- a/internal/repourl/repourl.go +++ b/internal/repourl/repourl.go @@ -112,6 +112,7 @@ var gitlabHosts = map[string]bool{ // giteaHosts are known Gitea/Forgejo instances. var giteaHosts = map[string]bool{ "codeberg.org": true, "src.opensuse.org": true, "gitea.com": true, "forge.sourceware.org": true, + "opendev.org": true, // OpenStack's forge; github.com/openstack mirrors it "git.sfconservancy.org": false, "gitea.osmocom.org": true, "git.savannah.gnu.org": false, "git.sr.ht": false, "notabug.org": true, "git.disroot.org": true, "code.forgejo.org": true, "git.jami.net": true, "git.ffmpeg.org": true, @@ -227,8 +228,13 @@ func Parse(raw string) (Canon, error) { host = "sourceforge.net" case "code.launchpad.net", "git.launchpad.net", "bazaar.launchpad.net": host = "launchpad.net" - case "files.pythonhosted.org", "pypi.python.org", "pypi.io": + case "files.pythonhosted.org", "pypi.python.org", "pypi.io", "cheeseshop.python.org": host = "pypi.org" + case "packages.python.org": + // PyPI's old documentation hosting: packages.python.org// is a PyPI project. + if parts := splitPath(p); len(parts) > 0 { + return finish(Canon{Host: "pypi.org", Path: normPyPI(parts[0]), Forge: "pypi", Kind: KindRegistry}) + } case "static.crates.io": host = "crates.io" case "registry.npmjs.org", "npmjs.com", "npmjs.org": @@ -238,6 +244,9 @@ func Parse(raw string) (Canon, error) { case "ftpmirror.gnu.org", "ftp.gnu.org", "mirrors.kernel.org", "alpha.gnu.org", "gnu.org": // keep; handled below } + if isCatalogue(host, p) { + return finish(Canon{Host: host, Path: strings.TrimPrefix(p, "/"), Kind: KindOther}) + } if strings.HasSuffix(host, ".sourceforge.net") || strings.HasSuffix(host, ".sf.net") || strings.HasSuffix(host, ".sourceforge.io") { // project.sourceforge.net -> sourceforge.net/project proj := strings.SplitN(host, ".", 2)[0] @@ -640,6 +649,28 @@ func hostsManyProjects(c Canon) bool { return strings.HasPrefix(p, "pub/") && !kernelReleaseRe.MatchString(p) } return true + case isCatalogue(c.Host, "/"+p): + return true + case c.Host == "python.org" || strings.HasSuffix(c.Host, ".python.org"): + // python.org is CPython's site, but its subdomains also serve PyPy's releases + // (downloads.python.org/pypy), wikis and mailing lists of other projects. + switch strings.TrimSuffix(strings.TrimSuffix(c.Host, "python.org"), ".") { + case "", "docs", "bugs", "devguide": + return false + } + return true + case c.Host == "gnupg.org": + // gnupg.org/software// and gnupg.org/ftp/gcrypt// name GnuPG's sibling + // projects (gpgme, libgcrypt, gpa, ...), each with its own repository (a host rule maps + // them); only GnuPG itself stands for git.gnupg.org/gnupg. + parts := strings.Split(p, "/") + switch { + case len(parts) >= 2 && parts[0] == "software": + return parts[1] != "gnupg" && parts[1] != "index.html" + case len(parts) >= 3 && parts[0] == "ftp" && parts[1] == "gcrypt": + return parts[2] != "gnupg" + } + return false case c.Host == "samba.org" || strings.HasSuffix(c.Host, ".samba.org"): switch sub := strings.TrimSuffix(strings.TrimSuffix(c.Host, "samba.org"), "."); sub { case "", "www", "download", "ftp": @@ -656,6 +687,37 @@ func hostsManyProjects(c Canon) bool { return false } +// catalogues are sites that list many projects, one page each: a page there names no +// repository, and taking the site's own repository for it would join unrelated packages. +var catalogues = []struct{ host, prefix string }{ + {"octave.sourceforge.net", "/"}, {"octave.sourceforge.io", "/"}, // Octave Forge + // Octave Forge's SourceForge project, which holds every package's releases. + // (its /p/octave/ repositories are each package's own). + {"sourceforge.net", "/octave/"}, {"sourceforge.net", "/projects/octave/"}, {"sourceforge.net", "/project/octave/"}, {"sourceforge.net", "/sourceforge/octave/"}, + {"gnu-octave.github.io", "/packages/"}, // the Octave package index + {"pear.php.net", "/"}, // PEAR + {"homalg-project.github.io", "/pkg/"}, // homalg's GAP packages +} + +func isCatalogue(host, p string) bool { + for _, c := range catalogues { + if host == c.host && strings.HasPrefix(p+"/", c.prefix) { + return true + } + } + return false +} + +// IsCatalogue reports whether raw is a page of a catalogue site (Octave Forge, PEAR, ...), +// which names no repository of its own. +func IsCatalogue(raw string) bool { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil { + return false + } + return isCatalogue(wwwPrefix.ReplaceAllString(strings.ToLower(u.Hostname()), ""), path.Clean("/"+u.Path)) +} + // sambaTree names the projects developed in Samba's own repository. var sambaTree = map[string]bool{"samba": true, "talloc": true, "tdb": true, "tevent": true, "ldb": true} @@ -876,6 +938,12 @@ func parseCgit(c Canon, p string, u *url.URL) (Canon, error) { func parseSourceForge(c Canon, p string) (Canon, error) { // /projects/NAME/..., /p/NAME/code, /project/NAME/... (downloads) parts := splitPath(p) + // An umbrella project keeps each package in its own repository under /p/PROJECT/REPO + // (Octave Forge under /p/octave/); elsewhere the project is the key, as distributions reach + // it through its downloads as often as through a repository page. + if len(parts) >= 3 && parts[0] == "p" && sfUmbrellas[strings.ToLower(parts[1])] && !sfToolPages[strings.ToLower(parts[2])] && !strings.EqualFold(parts[2], parts[1]) { + return finish(Canon{Host: "sourceforge.net", Path: parts[1] + "/" + parts[2], Forge: "sourceforge", Kind: KindRepo, Ref: c.Ref}) + } if len(parts) >= 2 && (parts[0] == "projects" || parts[0] == "project" || parts[0] == "p") { return finish(Canon{Host: "sourceforge.net", Path: parts[1], Forge: "sourceforge", Kind: KindRepo, Ref: c.Ref}) } @@ -892,6 +960,17 @@ func parseSourceForge(c Canon, p string) (Canon, error) { return Canon{}, ErrUnparseable } +// sfUmbrellas are SourceForge projects that host many packages, each in its own repository. +var sfUmbrellas = map[string]bool{"octave": true} + +// sfToolPages are the SourceForge project pages and default repository names under /p/PROJECT/. +var sfToolPages = map[string]bool{ + "code": true, "git": true, "svn": true, "hg": true, "mercurial": true, "cvs": true, "bzr": true, + "home": true, "wiki": true, "tickets": true, "bugs": true, "feature-requests": true, "patches": true, + "support-requests": true, "discussion": true, "news": true, "blog": true, "mailman": true, + "activity": true, "files": true, "donate": true, "support": true, "reviews": true, +} + func splitPath(p string) []string { var out []string for _, s := range strings.Split(strings.Trim(p, "/"), "/") { diff --git a/internal/repourl/repourl_test.go b/internal/repourl/repourl_test.go index 98328e8..11ec20d 100644 --- a/internal/repourl/repourl_test.go +++ b/internal/repourl/repourl_test.go @@ -54,6 +54,26 @@ func TestParse(t *testing.T) { {"https://apps.kde.org/kasts/", "apps.kde.org/kasts", KindOther, "invent.kde.org/kasts", ""}, {"https://apps.kde.org/de/kate/", "apps.kde.org/de/kate", KindOther, "invent.kde.org/kate", ""}, {"https://kde.org/applications/utilities/org.kde.kmousetool", "kde.org/applications/utilities/org.kde.kmousetool", KindOther, "invent.kde.org/kmousetool", ""}, + // python.org serves CPython, but its subdomains serve PyPI-era pages and PyPy. + {"https://packages.python.org/WSME/", "pypi.org/wsme", KindRegistry, "", ""}, + {"http://cheeseshop.python.org/pypi/simplegeneric", "pypi.org/simplegeneric", KindRegistry, "", ""}, + {"https://downloads.python.org/pypy/pypy3.11-v8.0.0-src.tar.gz", "downloads.python.org/pypy/pypy3.11-v8.0.0-src.tar.gz", KindTarball, "", ""}, + {"https://www.python.org/ftp/python/3.13.1/Python-3.13.1.tar.xz", "python.org/ftp/python/3.13.1/Python-3.13.1.tar.xz", KindTarball, "github.com/python/cpython", ""}, + {"https://wiki.python.org/moin/X", "wiki.python.org/moin/X", KindOther, "", ""}, + // Catalogue sites list many projects; a page names none of their repositories. + {"https://octave.sourceforge.net/geometry/", "octave.sourceforge.net/geometry", KindOther, "", ""}, + {"https://pear.php.net/package/Mail_Mime", "pear.php.net/package/Mail_Mime", KindOther, "", ""}, + {"https://downloads.sourceforge.net/octave/geometry-4.1.0.tar.gz", "sourceforge.net/octave/geometry-4.1.0.tar.gz", KindOther, "", ""}, + {"https://sourceforge.net/p/octave/geometry/", "sourceforge.net/octave/geometry", KindRepo, "", ""}, + {"https://git.code.sf.net/p/octave/geometry", "sourceforge.net/octave/geometry", KindRepo, "", ""}, + {"https://sourceforge.net/p/net-snmp/code/", "sourceforge.net/net-snmp", KindRepo, "", ""}, + {"https://sourceforge.net/p/mingw-w64/mingw-w64/", "sourceforge.net/mingw-w64", KindRepo, "", ""}, + {"https://sourceforge.net/p/gnuplot/gnuplot-main/", "sourceforge.net/gnuplot", KindRepo, "", ""}, + {"https://sourceforge.net/p/foo/tickets/12/", "sourceforge.net/foo", KindRepo, "", ""}, + // gnupg.org's sibling projects are not GnuPG. + {"https://gnupg.org/software/gpa/index.html", "gnupg.org/software/gpa/index.html", KindOther, "", ""}, + {"https://gnupg.org/ftp/gcrypt/gnupg/gnupg-2.4.8.tar.bz2", "gnupg.org/ftp/gcrypt/gnupg/gnupg-2.4.8.tar.bz2", KindTarball, "git.gnupg.org/gnupg", ""}, + {"https://opendev.org/openstack/python-barbicanclient", "opendev.org/openstack/python-barbicanclient", KindRepo, "", ""}, // GitLab API routes that name the project by path. {"https://gitlab.com/api/v4/projects/orcus%2Forcus/packages/generic/source/0.21.0/liborcus-0.21.0.tar.xz", "gitlab.com/orcus/orcus", KindTarball, "gitlab.com/orcus/orcus", ""}, // Packaging fields append refs and branches after the URL.