package distro import ( "net/url" "regexp" "strings" "tangled.org/odd.computer/isomorph/internal/repourl" ) // hostRule rewrites a project website / download URL that repourl does not understand into // the repository it (very likely) corresponds to. These mirror the mappings in Bernhard // Wiedemann's findvcs script. Confidence < 1 marks guesses (a project homepage naming // convention rather than a structural relationship). type hostRule struct { re *regexp.Regexp confidence float64 repl func(m []string) string } var apacheNonProjects = map[string]bool{ "www": true, "archive": true, "downloads": true, "dist": true, "people": true, "projects": true, "mail": true, "lists": true, "issues": true, "svn": true, "git": true, "gitbox": true, "cwiki": true, "wiki": true, "repository": true, "commons": true, "attic": true, "incubator": true, "blogs": true, "community": true, "infra": true, "security": true, "whimsy": true, "status": true, "apache": true, "www-us": true, "dlcdn": true, } // pageFileRe matches a web page file name, as opposed to a repository name. var pageFileRe = regexp.MustCompile(`(?i)\.(?:html?|php|md)$`) var hostRules = []hostRule{ // owner.github.io/repo -> github.com/owner/repo; a page file at the top level // (owner.github.io/page.html) belongs to the site repository owner/owner.github.io. {regexp.MustCompile(`(?i)^https?://([\w-]+)\.github\.io/([\w.-]+?)/?(?:$|[/#?])`), 1.0, func(m []string) string { if pageFileRe.MatchString(m[2]) { return "https://github.com/" + m[1] + "/" + m[1] + ".github.io" } return "https://github.com/" + m[1] + "/" + m[2] }}, // pagure.io/[/...] and pagure.io/releases//... {regexp.MustCompile(`(?i)^https?://pagure\.io/(?:releases/)?([\w.-]+?)(?:\.git)?(?:$|[/#?])`), 1.0, func(m []string) string { if m[1] == "fork" || m[1] == "releases" { return "" } return "https://pagure.io/" + m[1] + ".git" }}, // cgit.freedesktop.org/ -> gitlab.freedesktop.org/ {regexp.MustCompile(`(?i)^https?://cgit\.freedesktop\.org/([\w.-]+(?:/[\w.-]+)*?)/?(?:$|/(?:tree|log|commit|snapshot|tag|plain|refs|diff)\b|[#?])`), 1.0, func(m []string) string { return "https://gitlab.freedesktop.org/" + strings.TrimSuffix(m[1], ".git") }}, // archive.xfce.org/src///... -> gitlab.xfce.org// {regexp.MustCompile(`(?i)^https?://archive\.xfce\.org/src/([\w-]+)/([\w.-]+)/`), 1.0, func(m []string) string { return "https://gitlab.xfce.org/" + m[1] + "/" + m[2] }}, // kernel.org/pub/software/scm// -> git.kernel.org/pub/scm//.git {regexp.MustCompile(`(?i)^https?://(?:www\.|mirrors\.)?kernel\.org/pub/software/scm/([\w-]+)/`), 1.0, func(m []string) string { return "https://git.kernel.org/pub/scm/" + m[1] + "/" + m[1] + ".git" }}, // hg.mozilla.org/projects/

{regexp.MustCompile(`(?i)^https?://hg\.mozilla\.org/projects/([\w.-]+)`), 1.0, func(m []string) string { return "hg+https://hg.mozilla.org/projects/" + m[1] }}, // www.gnu.org/software/ -> savannah (guess: most, not all, GNU projects live there) {regexp.MustCompile(`(?i)^https?://(?:www\.)?gnu\.org/software/([\w.+-]+)`), 0.6, func(m []string) string { return "https://git.savannah.gnu.org/git/" + m[1] + ".git" }}, // www.nongnu.org/ and download.savannah.(non)gnu.org/releases/ {regexp.MustCompile(`(?i)^https?://(?:www\.)?nongnu\.org/([\w.+-]+)/`), 0.6, func(m []string) string { return "https://git.savannah.nongnu.org/git/" + m[1] + ".git" }}, {regexp.MustCompile(`(?i)^https?://download\.savannah\.(?:non)?gnu\.org/releases/([\w.+-]+)/`), 0.8, func(m []string) string { return "https://git.savannah.gnu.org/git/" + m[1] + ".git" }}, // Apache: dist/downloads mirrors and .apache.org homepages -> github.com/apache/ // (the GitHub organisation mirrors gitbox.apache.org one-to-one). {regexp.MustCompile(`(?i)^https?://(?:www\.|archive\.)?apache\.org/dist/([\w-]+)/(?:([\w-]+)/)?`), 0.8, apacheProject}, {regexp.MustCompile(`(?i)^https?://(?:downloads|dlcdn)\.apache\.org/([\w-]+)/(?:([\w-]+)/)?`), 0.8, apacheProject}, {regexp.MustCompile(`(?i)^https?://([\w-]+)\.apache\.org/?(?:$|[#?])`), 0.6, func(m []string) string { if apacheNonProjects[strings.ToLower(m[1])] { return "" } return "https://github.com/apache/" + strings.ToLower(m[1]) }}, // The rules below are ours rather than findvcs's; each covers homepages that an error // analysis of the association table (2026-09-28) found unresolved in several // distributions. // docs.xfce.org///start and goodies.xfce.org/projects// -> // gitlab.xfce.org// (the documentation wiki mirrors the GitLab groups). // Only a project's own page (/[/start]): deeper pages document plugins and // parts (docs.xfce.org/xfce/thunar/archive is thunar-archive-plugin, not Thunar). {regexp.MustCompile(`(?i)^https?://docs\.xfce\.org/(apps|panel-plugins|thunar-plugins|xfce|bindings)/([\w.-]+)(?:/start)?/?(?:$|[#?])`), 0.9, xfceProject}, {regexp.MustCompile(`(?i)^https?://goodies\.xfce\.org/projects/(apps|panel-plugins|thunar-plugins|xfce|bindings)/([\w.-]+)`), 0.9, xfceProject}, // netfilter.org/projects/ -> git.netfilter.org/ (ulogd's repository is ulogd2). {regexp.MustCompile(`(?i)^https?://(?:www\.)?netfilter\.org/projects/([\w.-]+)`), 0.9, func(m []string) string { name := strings.ToLower(m[1]) if name == "ulogd" { name = "ulogd2" } return "https://git.netfilter.org/" + name }}, // gnupg.org/software// and gnupg.org/ftp/gcrypt// -> git.gnupg.org/: each // of GnuPG's sibling projects (gpgme, libgcrypt, libassuan, gpa) has its own repository. {regexp.MustCompile(`(?i)^https?://(?:www\.)?gnupg\.org/(?:software|ftp/gcrypt)/([\w.+-]+?)(?:\.html)?(?:/|$)`), 0.9, func(m []string) string { if name := strings.ToLower(m[1]); name != "index" && name != "gnupg" { return "https://git.gnupg.org/" + name + ".git" } return "" }}, // PyPy's releases are served from downloads.python.org/pypy/; its repository is on GitHub. {regexp.MustCompile(`(?i)^https?://downloads\.python\.org/pypy/`), 0.9, func(m []string) string { return "https://github.com/pypy/pypy" }}, // freedesktop.org/software/pulseaudio/ -> gitlab.freedesktop.org/pulseaudio/ // (paprefs, pavucontrol, webrtc-audio-processing; other /software/ paths name documents). {regexp.MustCompile(`(?i)^https?://(?:www\.)?freedesktop\.org/software/pulseaudio/([\w.-]+)`), 0.9, func(m []string) string { return "https://gitlab.freedesktop.org/pulseaudio/" + m[1] }}, } func xfceProject(m []string) string { if strings.EqualFold(m[2], "start") { return "" } return "https://gitlab.xfce.org/" + strings.ToLower(m[1]) + "/" + m[2] } func apacheProject(m []string) string { proj := strings.ToLower(m[1]) if proj == "commons" && len(m) > 2 && m[2] != "" { return "https://github.com/apache/commons-" + strings.ToLower(m[2]) } if apacheNonProjects[proj] { return "" } return "https://github.com/apache/" + proj } // gitlabVerbs end the project path on a GitLab instance. var gitlabVerbs = map[string]bool{ "-": true, "tree": true, "blob": true, "raw": true, "commit": true, "commits": true, "tags": true, "archive": true, "releases": true, "issues": true, "merge_requests": true, "wikis": true, "pipelines": true, "compare": true, "graphs": true, "network": true, "uploads": true, } // mapGitLab handles GitLab instances repourl does not know (gitlab.): the project // is every path segment before "/-/" or a browsing verb, with subgroups kept. func mapGitLab(raw string) (string, bool) { u, err := url.Parse(raw) if err != nil || u.Host == "" || !strings.HasPrefix(strings.ToLower(u.Hostname()), "gitlab.") { return "", false } var segs []string for _, s := range strings.Split(u.Path, "/") { if s == "" { continue } if gitlabVerbs[s] { break } segs = append(segs, s) } if len(segs) < 2 { return "", false } segs[len(segs)-1] = strings.TrimSuffix(segs[len(segs)-1], ".git") if IsArchiveURL(segs[len(segs)-1]) { return "", false } return "https://" + strings.ToLower(u.Hostname()) + "/" + strings.Join(segs, "/") + ".git", true } // MapHost applies the findvcs-style host rules to raw. It returns the rewritten repository // URL, a confidence factor in (0,1] and true when a rule matched. func MapHost(raw string) (string, float64, bool) { raw = strings.TrimSpace(raw) if repourl.IsCatalogue(raw) { return "", 0, false } if out, ok := mapGitLab(raw); ok { return out, 1.0, true } for _, r := range hostRules { m := r.re.FindStringSubmatch(raw) if m == nil { continue } if out := r.repl(m); out != "" { return out, r.confidence, true } } return "", 0, false } var archiveExts = []string{".tar.gz", ".tgz", ".tar.bz2", ".tbz2", ".tar.xz", ".txz", ".tar.zst", ".tar.lz", ".tar", ".zip", ".crate", ".gem", ".whl", ".jar", ".7z", ".tar.lzma", ".obscpio"} // IsArchiveURL reports whether the URL path names a release archive. func IsArchiveURL(raw string) bool { p := raw if i := strings.IndexAny(p, "#?"); i >= 0 { p = p[:i] } p = strings.ToLower(p) for _, e := range archiveExts { if strings.HasSuffix(p, e) { return true } } return false }