Something went wrong. Try again.
Maps Linux distribution source packages to their upstream repositories, and through them to each other
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208package distro
import ( "net/url" "regexp" "strings"
"tangled.org/odd.computer/isomorph/internal/repourl")
// hostRule rewrites a project website / download URL that repourl does not understand into// the repository it (very likely) corresponds to. These mirror the mappings in Bernhard// Wiedemann's findvcs script. Confidence < 1 marks guesses (a project homepage naming// convention rather than a structural relationship).type hostRule struct { re *regexp.Regexp confidence float64 repl func(m []string) string}
var apacheNonProjects = map[string]bool{ "www": true, "archive": true, "downloads": true, "dist": true, "people": true, "projects": true, "mail": true, "lists": true, "issues": true, "svn": true, "git": true, "gitbox": true, "cwiki": true, "wiki": true, "repository": true, "commons": true, "attic": true, "incubator": true, "blogs": true, "community": true, "infra": true, "security": true, "whimsy": true, "status": true, "apache": true, "www-us": true, "dlcdn": true,}
// pageFileRe matches a web page file name, as opposed to a repository name.var pageFileRe = regexp.MustCompile(`(?i)\.(?:html?|php|md)$`)
var hostRules = []hostRule{ // owner.github.io/repo -> github.com/owner/repo; a page file at the top level // (owner.github.io/page.html) belongs to the site repository owner/owner.github.io. {regexp.MustCompile(`(?i)^https?://([\w-]+)\.github\.io/([\w.-]+?)/?(?:$|[/#?])`), 1.0, func(m []string) string { if pageFileRe.MatchString(m[2]) { return "https://github.com/" + m[1] + "/" + m[1] + ".github.io" } return "https://github.com/" + m[1] + "/" + m[2] }}, // pagure.io/<project>[/...] and pagure.io/releases/<project>/... {regexp.MustCompile(`(?i)^https?://pagure\.io/(?:releases/)?([\w.-]+?)(?:\.git)?(?:$|[/#?])`), 1.0, func(m []string) string { if m[1] == "fork" || m[1] == "releases" { return "" } return "https://pagure.io/" + m[1] + ".git" }}, // cgit.freedesktop.org/<path> -> gitlab.freedesktop.org/<path> {regexp.MustCompile(`(?i)^https?://cgit\.freedesktop\.org/([\w.-]+(?:/[\w.-]+)*?)/?(?:$|/(?:tree|log|commit|snapshot|tag|plain|refs|diff)\b|[#?])`), 1.0, func(m []string) string { return "https://gitlab.freedesktop.org/" + strings.TrimSuffix(m[1], ".git") }}, // archive.xfce.org/src/<category>/<name>/... -> gitlab.xfce.org/<category>/<name> {regexp.MustCompile(`(?i)^https?://archive\.xfce\.org/src/([\w-]+)/([\w.-]+)/`), 1.0, func(m []string) string { return "https://gitlab.xfce.org/" + m[1] + "/" + m[2] }}, // kernel.org/pub/software/scm/<name>/ -> git.kernel.org/pub/scm/<name>/<name>.git {regexp.MustCompile(`(?i)^https?://(?:www\.|mirrors\.)?kernel\.org/pub/software/scm/([\w-]+)/`), 1.0, func(m []string) string { return "https://git.kernel.org/pub/scm/" + m[1] + "/" + m[1] + ".git" }}, // hg.mozilla.org/projects/<p> {regexp.MustCompile(`(?i)^https?://hg\.mozilla\.org/projects/([\w.-]+)`), 1.0, func(m []string) string { return "hg+https://hg.mozilla.org/projects/" + m[1] }}, // www.gnu.org/software/<x> -> savannah (guess: most, not all, GNU projects live there) {regexp.MustCompile(`(?i)^https?://(?:www\.)?gnu\.org/software/([\w.+-]+)`), 0.6, func(m []string) string { return "https://git.savannah.gnu.org/git/" + m[1] + ".git" }}, // www.nongnu.org/<x> and download.savannah.(non)gnu.org/releases/<x> {regexp.MustCompile(`(?i)^https?://(?:www\.)?nongnu\.org/([\w.+-]+)/`), 0.6, func(m []string) string { return "https://git.savannah.nongnu.org/git/" + m[1] + ".git" }}, {regexp.MustCompile(`(?i)^https?://download\.savannah\.(?:non)?gnu\.org/releases/([\w.+-]+)/`), 0.8, func(m []string) string { return "https://git.savannah.gnu.org/git/" + m[1] + ".git" }}, // Apache: dist/downloads mirrors and <project>.apache.org homepages -> github.com/apache/<project> // (the GitHub organisation mirrors gitbox.apache.org one-to-one). {regexp.MustCompile(`(?i)^https?://(?:www\.|archive\.)?apache\.org/dist/([\w-]+)/(?:([\w-]+)/)?`), 0.8, apacheProject}, {regexp.MustCompile(`(?i)^https?://(?:downloads|dlcdn)\.apache\.org/([\w-]+)/(?:([\w-]+)/)?`), 0.8, apacheProject}, {regexp.MustCompile(`(?i)^https?://([\w-]+)\.apache\.org/?(?:$|[#?])`), 0.6, func(m []string) string { if apacheNonProjects[strings.ToLower(m[1])] { return "" } return "https://github.com/apache/" + strings.ToLower(m[1]) }},
// The rules below are ours rather than findvcs's; each covers homepages that an error // analysis of the association table (2026-09-28) found unresolved in several // distributions. // docs.xfce.org/<group>/<name>/start and goodies.xfce.org/projects/<group>/<name> -> // gitlab.xfce.org/<group>/<name> (the documentation wiki mirrors the GitLab groups). // Only a project's own page (<group>/<name>[/start]): deeper pages document plugins and // parts (docs.xfce.org/xfce/thunar/archive is thunar-archive-plugin, not Thunar). {regexp.MustCompile(`(?i)^https?://docs\.xfce\.org/(apps|panel-plugins|thunar-plugins|xfce|bindings)/([\w.-]+)(?:/start)?/?(?:$|[#?])`), 0.9, xfceProject}, {regexp.MustCompile(`(?i)^https?://goodies\.xfce\.org/projects/(apps|panel-plugins|thunar-plugins|xfce|bindings)/([\w.-]+)`), 0.9, xfceProject}, // netfilter.org/projects/<name> -> git.netfilter.org/<name> (ulogd's repository is ulogd2). {regexp.MustCompile(`(?i)^https?://(?:www\.)?netfilter\.org/projects/([\w.-]+)`), 0.9, func(m []string) string { name := strings.ToLower(m[1]) if name == "ulogd" { name = "ulogd2" } return "https://git.netfilter.org/" + name }}, // gnupg.org/software/<name>/ and gnupg.org/ftp/gcrypt/<name>/ -> git.gnupg.org/<name>: each // of GnuPG's sibling projects (gpgme, libgcrypt, libassuan, gpa) has its own repository. {regexp.MustCompile(`(?i)^https?://(?:www\.)?gnupg\.org/(?:software|ftp/gcrypt)/([\w.+-]+?)(?:\.html)?(?:/|$)`), 0.9, func(m []string) string { if name := strings.ToLower(m[1]); name != "index" && name != "gnupg" { return "https://git.gnupg.org/" + name + ".git" } return "" }}, // PyPy's releases are served from downloads.python.org/pypy/; its repository is on GitHub. {regexp.MustCompile(`(?i)^https?://downloads\.python\.org/pypy/`), 0.9, func(m []string) string { return "https://github.com/pypy/pypy" }}, // freedesktop.org/software/pulseaudio/<name> -> gitlab.freedesktop.org/pulseaudio/<name> // (paprefs, pavucontrol, webrtc-audio-processing; other /software/ paths name documents). {regexp.MustCompile(`(?i)^https?://(?:www\.)?freedesktop\.org/software/pulseaudio/([\w.-]+)`), 0.9, func(m []string) string { return "https://gitlab.freedesktop.org/pulseaudio/" + m[1] }},}
func xfceProject(m []string) string { if strings.EqualFold(m[2], "start") { return "" } return "https://gitlab.xfce.org/" + strings.ToLower(m[1]) + "/" + m[2]}
func apacheProject(m []string) string { proj := strings.ToLower(m[1]) if proj == "commons" && len(m) > 2 && m[2] != "" { return "https://github.com/apache/commons-" + strings.ToLower(m[2]) } if apacheNonProjects[proj] { return "" } return "https://github.com/apache/" + proj}
// gitlabVerbs end the project path on a GitLab instance.var gitlabVerbs = map[string]bool{ "-": true, "tree": true, "blob": true, "raw": true, "commit": true, "commits": true, "tags": true, "archive": true, "releases": true, "issues": true, "merge_requests": true, "wikis": true, "pipelines": true, "compare": true, "graphs": true, "network": true, "uploads": true,}
// mapGitLab handles GitLab instances repourl does not know (gitlab.<anything>): the project// is every path segment before "/-/" or a browsing verb, with subgroups kept.func mapGitLab(raw string) (string, bool) { u, err := url.Parse(raw) if err != nil || u.Host == "" || !strings.HasPrefix(strings.ToLower(u.Hostname()), "gitlab.") { return "", false } var segs []string for _, s := range strings.Split(u.Path, "/") { if s == "" { continue } if gitlabVerbs[s] { break } segs = append(segs, s) } if len(segs) < 2 { return "", false } segs[len(segs)-1] = strings.TrimSuffix(segs[len(segs)-1], ".git") if IsArchiveURL(segs[len(segs)-1]) { return "", false } return "https://" + strings.ToLower(u.Hostname()) + "/" + strings.Join(segs, "/") + ".git", true}
// MapHost applies the findvcs-style host rules to raw. It returns the rewritten repository// URL, a confidence factor in (0,1] and true when a rule matched.func MapHost(raw string) (string, float64, bool) { raw = strings.TrimSpace(raw) if repourl.IsCatalogue(raw) { return "", 0, false } if out, ok := mapGitLab(raw); ok { return out, 1.0, true } for _, r := range hostRules { m := r.re.FindStringSubmatch(raw) if m == nil { continue } if out := r.repl(m); out != "" { return out, r.confidence, true } } return "", 0, false}
var archiveExts = []string{".tar.gz", ".tgz", ".tar.bz2", ".tbz2", ".tar.xz", ".txz", ".tar.zst", ".tar.lz", ".tar", ".zip", ".crate", ".gem", ".whl", ".jar", ".7z", ".tar.lzma", ".obscpio"}
// IsArchiveURL reports whether the URL path names a release archive.func IsArchiveURL(raw string) bool { p := raw if i := strings.IndexAny(p, "#?"); i >= 0 { p = p[:i] } p = strings.ToLower(p) for _, e := range archiveExts { if strings.HasSuffix(p, e) { return true } } return false}