Something went wrong. Try again.
Maps Linux distribution source packages to their upstream repositories, and through them to each other
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131package bulk
import ( "context" "sort" "strings" "time"
"tangled.org/odd.computer/isomorph/internal/distro" "tangled.org/odd.computer/isomorph/internal/distro/arch" "tangled.org/odd.computer/isomorph/internal/model")
// ArchSource provides .SRCINFO files at released packaging commits (arch.Resolver does).type ArchSource interface { SRCINFOAt(ctx context.Context, pkgbase, commit string) (*arch.SRCINFO, error) FetchSRCINFO(ctx context.Context, pkgbase, commit string, web bool) (*arch.SRCINFO, error) HasSRCINFO(pkgbase, commit string) bool}
// CrawlInterval paces .SRCINFO fetches. gitlab.archlinux.org allows anonymous clients about 600// requests per ten minutes on each of the API and web routes; alternating the two at this// interval stays under both even when a third of the packages need a second request (the// PKGBUILD fallback).var CrawlInterval = 700 * time.Millisecond
// ArchStats describes how an Arch slice was built.type ArchStats struct { PkgBases int // FromSRCINFO counts rows resolved from .SRCINFO (cached or fetched this run); the rest use // the repository databases' url field only. FromSRCINFO int // Fetched and FetchErrors count .SRCINFO requests made this run. Fetched, FetchErrors int // Stopped is set when the crawl hit a rate limit and stopped early. Stopped error // Errors samples fetch errors by pkgbase (at most 20). Errors map[string]string}
// FromArch builds one row per pkgbase of the repository databases. Each row starts from the// databases (version, binary names, url). A pkgbase whose .SRCINFO at its released commit (from// the state repository) is cached, or can be fetched within the crawl budget, is resolved from// .SRCINFO instead: its source array names git repositories and release tarballs (with// sha256), which the url field does not.func FromArch(ctx context.Context, src ArchSource, dbs []arch.SyncPackage, state []arch.StateEntry, crawl int, progress func(fetched int)) ([]Package, ArchStats) { type base struct { repo, name, version, url string bins []string } bases := map[[2]string]*base{} var order [][2]string for _, p := range dbs { k := [2]string{p.Repo, p.Base} b := bases[k] if b == nil { b = &base{repo: p.Repo, name: p.Base, version: p.Version} bases[k] = b order = append(order, k) } if b.url == "" { b.url = p.URL } b.bins = append(b.bins, p.Name) } sort.Slice(order, func(i, j int) bool { if order[i][0] != order[j][0] { return order[i][0] < order[j][0] } return order[i][1] < order[j][1] }) commits := map[[2]string]string{} for _, e := range state { commits[[2]string{e.Repo, e.PkgBase}] = e.Commit } var st ArchStats var last time.Time out := make([]Package, 0, len(order)) for _, k := range order { b := bases[k] sort.Strings(b.bins) p := Package{Distro: model.Arch, Suite: b.repo, Name: b.name, Version: b.version, Binaries: b.bins, Homepage: b.url, Status: StatusUnresolved} var si *arch.SRCINFO if c := commits[k]; c != "" && ctx.Err() == nil { switch { case src.HasSRCINFO(b.name, c): si, _ = src.SRCINFOAt(ctx, b.name, c) case st.Fetched < crawl && st.Stopped == nil: if wait := CrawlInterval - time.Since(last); wait > 0 { select { case <-ctx.Done(): case <-time.After(wait): } } last = time.Now() st.Fetched++ var err error if si, err = src.FetchSRCINFO(ctx, b.name, c, st.Fetched%2 == 0); err != nil { si = nil st.FetchErrors++ if st.Errors == nil { st.Errors = map[string]string{} } if len(st.Errors) < 20 { st.Errors[b.name] = err.Error() } if strings.Contains(err.Error(), "rate limited") { st.Stopped = err } } if progress != nil { progress(st.Fetched) } } } if si != nil { st.FromSRCINFO++ if si.URL != "" { p.Homepage = si.URL } p.SourceSHA256 = arch.SourceSHA256(si) fill(&p, distro.RankCandidates(arch.Candidates(si))) } else if b.url != "" { fill(&p, distro.RankCandidates(distro.AddCandidate(nil, b.url, model.Evidence{Source: "pacman-db:url", Value: b.url, Weight: 0.5}))) } out = append(out, p) } st.PkgBases = len(out) return out, st}