Something went wrong. Try again.
Maps Linux distribution source packages to their upstream repositories, and through them to each other
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160package bulk
import ( "regexp" "slices" "strings")
// The PyPI repository selection below is adapted from oss-rebuild's pypi.InferRepo and// uri.FindCommonRepo (github.com/google/oss-rebuild, Apache-2.0), where it was validated on PyPI// rebuilds. PyPI metadata names repositories inconsistently (a "Source" link, a GitHub home// page, only a mention in the description), so links are tried in order of how reliably they// name the source://// source links:// - the home page, on a known repo host// - a link named like a source link (commonRepoLinks), on a known repo host// - a link named like a source link, on any host// other mentions:// - the first known repo host cited in the description whose repository is named like the// project// - a link under any other name, on a known repo host, sponsor pages excluded//// Unlike oss-rebuild, only the project's current metadata is read (no per-release metadata),// the result goes through repourl like every other candidate, and a description mention must// be named like the project: descriptions cite related projects (python-barbicanclient's// cites the barbican server) as readily as their own repository.
// pypiInfo is the part of PyPI's /pypi/<name>/json "info" object the selection reads.type pypiInfo struct { HomePage string `json:"home_page"` ProjectURLs map[string]string `json:"project_urls"` Description string `json:"description"`}
// commonRepoLinks are project_urls names (lower-cased, spaces removed) that usually name the// source repository.var commonRepoLinks = []string{"source", "sourcecode", "repository", "project", "github", "code"}
var ( pypiGitHubRE = regexp.MustCompile(`(?i)\bgithub(?:\.com[:/]|:)([\w-]+/[\w.-]+)`) pypiGitLabRE = regexp.MustCompile(`(?i)\bgitlab(?:\.com[:/]|:)([\w-]+(?:/[\w.-]+)+)`) pypiBitbucketRE = regexp.MustCompile(`(?i)\bbitbucket(?:\.org[:/]|:)([\w-]+/[\w.-]+)`) pypiCodebergRE = regexp.MustCompile(`(?i)\bcodeberg(?:\.org[:/]|:)([\w-]+/[\w.-]+)`) pypiOpenDevRE = regexp.MustCompile(`(?i)\bopendev(?:\.org[:/]|:)([\w-]+/[\w.-]+)`) pypiCommonRepos = []*regexp.Regexp{pypiGitHubRE, pypiGitLabRE, pypiBitbucketRE, pypiCodebergRE, pypiOpenDevRE} // gitlabProjectRoutes directly follow a GitLab project path in older links without /-/. gitlabProjectRoutes = []string{"badges", "blob", "commits", "container_registry", "issues", "packages", "pipelines", "raw", "releases", "tags", "tree", "uploads", "wikis"})
// pypiRepo returns the repository URL a PyPI project's metadata names, or "".func pypiRepo(project string, info pypiInfo) string { if r := repoFromSourceLinks(info); r != "" { return r } return repoFromOtherLinks(project, info)}
func repoFromSourceLinks(info pypiInfo) string { byName := make(map[string]string, len(info.ProjectURLs)) for name, u := range info.ProjectURLs { byName[strings.ReplaceAll(strings.ToLower(name), " ", "")] = u } var sources []string for _, name := range commonRepoLinks { if u, ok := byName[name]; ok { sources = append(sources, u) } } for _, u := range append([]string{info.HomePage, byName["homepage"]}, sources...) { if r := findCommonRepo(u); r != "" { return r } } if len(sources) != 0 { return sources[0] } return ""}
func repoFromOtherLinks(project string, info pypiInfo) string { if r := findNamedRepo(info.Description, project); r != "" { return r } names := make([]string, 0, len(info.ProjectURLs)) for name := range info.ProjectURLs { names = append(names, name) } slices.Sort(names) // map order is random; make the choice deterministic for _, name := range names { u := info.ProjectURLs[name] if strings.Contains(u, "sponsors") { continue } if r := findCommonRepo(u); r != "" { return r } } return ""}
// findNamedRepo returns the first known repository host path in text whose repository is// named like project, or "".func findNamedRepo(text, project string) string { type hit struct { at int repo string } var hits []hit for _, re := range pypiCommonRepos { for _, loc := range re.FindAllStringIndex(text, -1) { r := text[loc[0]:loc[1]] if re == pypiGitLabRE { r = trimGitLabRoute(r) } hits = append(hits, hit{loc[0], r}) } } slices.SortFunc(hits, func(a, b hit) int { return a.at - b.at }) for _, h := range hits { name := strings.TrimSuffix(h.repo[strings.LastIndexAny(h.repo, "/:")+1:], ".git") if !strings.Contains(h.repo, "sponsors") && SameProject(project, name) { return h.repo } } return ""}
// findCommonRepo returns the first known repository host path in text ("github.com/o/r"), or "".func findCommonRepo(text string) string { for _, re := range pypiCommonRepos { if r := re.FindString(text); r != "" { if re == pypiGitLabRE { r = trimGitLabRoute(r) } return r } } return ""}
// trimGitLabRoute drops a project route from a matched GitLab path.func trimGitLabRoute(repo string) string { if root, _, found := strings.Cut(repo, "/-/"); found { return root } start := strings.IndexAny(repo, ":/") if start < 0 { return repo } segs := strings.Split(repo[start+1:], "/") for i := 2; i < len(segs); i++ { if slices.Contains(gitlabProjectRoutes, segs[i]) { return repo[:start+1] + strings.Join(segs[:i], "/") } } return repo}