package bulk import ( "sort" "strings" "tangled.org/odd.computer/isomorph/internal/distro" "tangled.org/odd.computer/isomorph/internal/distro/debian" "tangled.org/odd.computer/isomorph/internal/model" "tangled.org/odd.computer/isomorph/internal/repourl" ) // FromUDD turns UDD rows of one suite into association rows. func FromUDD(rows []debian.UDDRow, suite string) []Package { out := make([]Package, 0, len(rows)) for _, r := range rows { p := Package{ Distro: model.Debian, Suite: suite, Name: r.Source, Version: r.Version, OSVEcosystem: "Debian:" + suite, PURL: "pkg:deb/debian/" + r.Source + "?arch=source", Binaries: r.Binaries, Homepage: r.Homepage, Status: StatusUnresolved, SourceSHA256: r.OrigSHA256, } cands := debian.ResolveRow(r, nil) fill(&p, cands) out = append(out, p) } return out } // fill sets the upstream fields from ranked candidates. func fill(p *Package, cands []distro.Candidate) { for _, c := range cands { p.Evidence = append(p.Evidence, c.Evidence...) } sort.SliceStable(p.Evidence, func(i, j int) bool { return p.Evidence[i].Weight > p.Evidence[j].Weight }) if best := choose(p.Name, cands); best != nil { p.UpstreamKey, p.Repo, p.Status = best.Key, best.URL, StatusResolved p.Basis = basis(*best) } } // choose picks package name's upstream among ranked candidates: the first repository, unless // it rests on a homepage on a staleHomeHosts host and a release or VCS source names another // repository. A candidate that only a project site supports counts only for a package named // like it (flagship.go). func choose(name string, cands []distro.Candidate) *distro.Candidate { var best *distro.Candidate for i := range cands { c := &cands[i] if c.Kind != repourl.KindRepo || basis(*c) == BasisSecondary { continue } if siteOnly(c.Evidence, c.Key) && !namedLike(name, c.Key) { continue } if best == nil { best = c if basis(*c) != BasisHomepage || !onStaleHomeHost(c.Key) { break } } else if b := basis(*c); b == BasisReleaseURL || b == BasisVCSSource { return c // see staleHomeHosts } } return best } // staleHomeHosts are hosts whose project pages outlive a project's move elsewhere. A homepage // there loses to a release or VCS source of the same package that names another repository: // on Debian, where the Homepage is on one of these hosts and the watch URL names a different // repository, DEP-12 Repository sides with the watch URL 22 times out of 23 (GitHub homepages, // by contrast, split 16 to 17, so the rule stays narrow; docs/EVALUATION.md). var staleHomeHosts = []string{"sourceforge.net/", "launchpad.net/"} func onStaleHomeHost(key string) bool { for _, h := range staleHomeHosts { if strings.HasPrefix(key, h) { return true } } return false } // basis is the strongest kind of evidence supporting a candidate, by the fixed precedence of // basisRank (not by the ranking weights, which scale hints down). func basis(c distro.Candidate) string { best := "" for _, e := range c.Evidence { if b := BasisOf(e.Source); best == "" || basisRank[b] < basisRank[best] { best = b } } return best } // HeldOut measures Debian's other fields against DEP-12 Repository. Among packages whose // Repository resolves to a repository, it resolves again without Repository and // Repository-Browse and counts how often the remaining fields (watch URL, Homepage, // Bug-Database) resolve at all, and to the same key. type HeldOut struct { WithRepository int // FallbackResolved counts packages the other fields still resolve; Agree counts those that // resolve to the Repository key. FallbackResolved int Agree int // ByBasis counts, per basis of the fallback's decision, [resolved, agreeing]. ByBasis map[string][2]int // Disagree lists (package, repository key, fallback key, fallback basis) for inspection. Disagree [][4]string } // DebianHeldOut runs the held-out comparison. func DebianHeldOut(rows []debian.UDDRow) HeldOut { skip := map[string]bool{ "debian/upstream/metadata:Repository": true, "debian/upstream/metadata:Repository-Browse": true, } h := HeldOut{ByBasis: map[string][2]int{}} for _, r := range rows { full := choose(r.Source, debian.ResolveRow(r, nil)) if full == nil || !fromRepository(*full) { continue } h.WithRepository++ held := choose(r.Source, debian.ResolveRow(r, skip)) if held == nil { continue } h.FallbackResolved++ hb := basis(*held) b := h.ByBasis[hb] b[0]++ if held.Key == full.Key { h.Agree++ b[1]++ } else { h.Disagree = append(h.Disagree, [4]string{r.Source, full.Key, held.Key, hb}) } h.ByBasis[hb] = b } return h } func fromRepository(c distro.Candidate) bool { for _, e := range c.Evidence { if e.Source == "debian/upstream/metadata:Repository" { return true } } return false }