package bulk import ( "regexp" "slices" "strings" ) // The PyPI repository selection below is adapted from oss-rebuild's pypi.InferRepo and // uri.FindCommonRepo (github.com/google/oss-rebuild, Apache-2.0), where it was validated on PyPI // rebuilds. PyPI metadata names repositories inconsistently (a "Source" link, a GitHub home // page, only a mention in the description), so links are tried in order of how reliably they // name the source: // // source links: // - the home page, on a known repo host // - a link named like a source link (commonRepoLinks), on a known repo host // - a link named like a source link, on any host // other mentions: // - the first known repo host cited in the description whose repository is named like the // project // - a link under any other name, on a known repo host, sponsor pages excluded // // Unlike oss-rebuild, only the project's current metadata is read (no per-release metadata), // the result goes through repourl like every other candidate, and a description mention must // be named like the project: descriptions cite related projects (python-barbicanclient's // cites the barbican server) as readily as their own repository. // pypiInfo is the part of PyPI's /pypi//json "info" object the selection reads. type pypiInfo struct { HomePage string `json:"home_page"` ProjectURLs map[string]string `json:"project_urls"` Description string `json:"description"` } // commonRepoLinks are project_urls names (lower-cased, spaces removed) that usually name the // source repository. var commonRepoLinks = []string{"source", "sourcecode", "repository", "project", "github", "code"} var ( pypiGitHubRE = regexp.MustCompile(`(?i)\bgithub(?:\.com[:/]|:)([\w-]+/[\w.-]+)`) pypiGitLabRE = regexp.MustCompile(`(?i)\bgitlab(?:\.com[:/]|:)([\w-]+(?:/[\w.-]+)+)`) pypiBitbucketRE = regexp.MustCompile(`(?i)\bbitbucket(?:\.org[:/]|:)([\w-]+/[\w.-]+)`) pypiCodebergRE = regexp.MustCompile(`(?i)\bcodeberg(?:\.org[:/]|:)([\w-]+/[\w.-]+)`) pypiOpenDevRE = regexp.MustCompile(`(?i)\bopendev(?:\.org[:/]|:)([\w-]+/[\w.-]+)`) pypiCommonRepos = []*regexp.Regexp{pypiGitHubRE, pypiGitLabRE, pypiBitbucketRE, pypiCodebergRE, pypiOpenDevRE} // gitlabProjectRoutes directly follow a GitLab project path in older links without /-/. gitlabProjectRoutes = []string{"badges", "blob", "commits", "container_registry", "issues", "packages", "pipelines", "raw", "releases", "tags", "tree", "uploads", "wikis"} ) // pypiRepo returns the repository URL a PyPI project's metadata names, or "". func pypiRepo(project string, info pypiInfo) string { if r := repoFromSourceLinks(info); r != "" { return r } return repoFromOtherLinks(project, info) } func repoFromSourceLinks(info pypiInfo) string { byName := make(map[string]string, len(info.ProjectURLs)) for name, u := range info.ProjectURLs { byName[strings.ReplaceAll(strings.ToLower(name), " ", "")] = u } var sources []string for _, name := range commonRepoLinks { if u, ok := byName[name]; ok { sources = append(sources, u) } } for _, u := range append([]string{info.HomePage, byName["homepage"]}, sources...) { if r := findCommonRepo(u); r != "" { return r } } if len(sources) != 0 { return sources[0] } return "" } func repoFromOtherLinks(project string, info pypiInfo) string { if r := findNamedRepo(info.Description, project); r != "" { return r } names := make([]string, 0, len(info.ProjectURLs)) for name := range info.ProjectURLs { names = append(names, name) } slices.Sort(names) // map order is random; make the choice deterministic for _, name := range names { u := info.ProjectURLs[name] if strings.Contains(u, "sponsors") { continue } if r := findCommonRepo(u); r != "" { return r } } return "" } // findNamedRepo returns the first known repository host path in text whose repository is // named like project, or "". func findNamedRepo(text, project string) string { type hit struct { at int repo string } var hits []hit for _, re := range pypiCommonRepos { for _, loc := range re.FindAllStringIndex(text, -1) { r := text[loc[0]:loc[1]] if re == pypiGitLabRE { r = trimGitLabRoute(r) } hits = append(hits, hit{loc[0], r}) } } slices.SortFunc(hits, func(a, b hit) int { return a.at - b.at }) for _, h := range hits { name := strings.TrimSuffix(h.repo[strings.LastIndexAny(h.repo, "/:")+1:], ".git") if !strings.Contains(h.repo, "sponsors") && SameProject(project, name) { return h.repo } } return "" } // findCommonRepo returns the first known repository host path in text ("github.com/o/r"), or "". func findCommonRepo(text string) string { for _, re := range pypiCommonRepos { if r := re.FindString(text); r != "" { if re == pypiGitLabRE { r = trimGitLabRoute(r) } return r } } return "" } // trimGitLabRoute drops a project route from a matched GitLab path. func trimGitLabRoute(repo string) string { if root, _, found := strings.Cut(repo, "/-/"); found { return root } start := strings.IndexAny(repo, ":/") if start < 0 { return repo } segs := strings.Split(repo[start+1:], "/") for i := 2; i < len(segs); i++ { if slices.Contains(gitlabProjectRoutes, segs[i]) { return repo[:start+1] + strings.Join(segs[:i], "/") } } return repo }