// Package opensuse resolves openSUSE Tumbleweed source packages to their upstream // repositories. // // Data sources (all anonymous, all through fetch.Client): // // - the Tumbleweed repository metadata (download.opensuse.org): the src-oss primary.xml // lists every source package with its version and spec Url:, the oss one every binary // with the source RPM it was built from (bulk.go); // - rb.zq1.de giturls.good, Bernhard Wiedemann's findvcs output, as corroborating git URLs // (giturls.go); // - the OBS public API (api.opensuse.org/public/source/openSUSE:Factory/): the // package's spec and _service, fetched per package at a polite pace and pinned to the // package revision (obsfiles.go). // // Upstream candidates, by weight: _service obs_scm/tar_scm url (0.9), #Git-Clone (0.9), // giturls (0.7), Source git URLs (0.7), forge release archives (0.6, as a hint), plain // tarballs (0.4, through their host hint or a findvcs-style host rule), Url: (0.5). package opensuse import ( "context" "encoding/xml" "fmt" "net/url" "sort" "strings" "tangled.org/odd.computer/isomorph/internal/distro" "tangled.org/odd.computer/isomorph/internal/fetch" "tangled.org/odd.computer/isomorph/internal/model" "tangled.org/odd.computer/isomorph/internal/repourl" ) // Resolver reads openSUSE's package sources. type Resolver struct { c *fetch.Client // OBSBase is the anonymous OBS API root (directory listings and raw files). OBSBase string // ZQ1Base is Bernhard Wiedemann's results directory (giturls.good lives there). ZQ1Base string // Project is the OBS project, normally openSUSE:Factory. Project string } // New returns a Resolver with production defaults. func New(c *fetch.Client) *Resolver { return &Resolver{ c: c, OBSBase: "https://api.opensuse.org/public", ZQ1Base: "https://rb.zq1.de/compare.factory", Project: "openSUSE:Factory", } } // OBS XML shapes. type obsDirectory struct { XMLName xml.Name `xml:"directory"` Name string `xml:"name,attr"` Rev string `xml:"rev,attr"` Vrev string `xml:"vrev,attr"` SrcMD5 string `xml:"srcmd5,attr"` LinkInfo *struct { Project string `xml:"project,attr"` Package string `xml:"package,attr"` Error string `xml:"error,attr"` } `xml:"linkinfo"` Entries []obsEntry `xml:"entry"` } type obsEntry struct { Name string `xml:"name,attr"` MD5 string `xml:"md5,attr"` Size int64 `xml:"size,attr"` MTime int64 `xml:"mtime,attr"` OriginPackage string `xml:"originpackage,attr"` } func (d *obsDirectory) has(name string) bool { for _, e := range d.Entries { if e.Name == name { return true } } return false } func (r *Resolver) pkgURL(pkg string) string { return strings.TrimSuffix(r.OBSBase, "/") + "/source/" + url.PathEscape(r.Project) + "/" + url.PathEscape(pkg) } // fileURL is the raw-file URL of one file in the package (used as Change.URL). func (r *Resolver) fileURL(pkg, file string) string { return r.pkgURL(pkg) + "/" + url.PathEscape(file) } // serviceCandidates adds the URLs a _service file fetches from: a source-control service's // url is the most precise upstream statement a package makes (0.9); download services count // as source URLs. It returns the URLs it added. func (r *Resolver) serviceCandidates(cands []distro.Candidate, service []byte) ([]distro.Candidate, []string, error) { svcs, err := ParseService(service) if err != nil { return cands, nil, err } var urls []string for _, s := range svcs { u := s.URL() if u == "" { continue } val := u if rev := s.Params["revision"]; rev != "" { val += " revision=" + rev } if vf := s.Params["versionformat"]; vf != "" { val += " versionformat=" + vf } if s.IsSCM() { cands = distro.AddCandidate(cands, u, model.Evidence{Source: "_service:" + s.Name + ":url", Value: val, Weight: 0.9}) } else { cands = r.addSourceURL(cands, u, "_service:"+s.Name+":url", val) } urls = append(urls, u) } return cands, urls, nil } // specCandidates adds the primary spec's Git-Clone and every spec's SourceN URLs. It returns // the source URLs it added. func (r *Resolver) specCandidates(cands []distro.Candidate, primary *Spec, specs []*Spec) ([]distro.Candidate, []string) { if primary != nil && primary.GitClone != "" { cands = distro.AddCandidate(cands, primary.GitClone, model.Evidence{Source: "spec:Git-Clone", Value: primary.GitClone, Weight: 0.9}) } var urls []string seen := map[string]bool{} for _, s := range specs { for _, src := range s.Sources { if !src.IsURL || seen[src.Value] { continue } seen[src.Value] = true urls = append(urls, src.Value) cands = r.addSourceURL(cands, src.Value, "spec:"+src.Tag, src.Value) } } return cands, urls } // addSourceURL feeds a download URL (SourceN, download_url service) into the candidates. // Git URLs count as strong evidence; forge archives (github/gitlab/codeberg release tarballs) // are good hints; other tarballs contribute through their repourl host hint or a local host // mapping. func (r *Resolver) addSourceURL(cands []distro.Candidate, raw, source, value string) []distro.Candidate { weight := 0.4 c, err := repourl.Parse(raw) if err == nil { switch { case c.Kind == repourl.KindRepo: weight = 0.7 case c.Kind == repourl.KindTarball && (c.Host == "github.com" || strings.HasPrefix(c.Host, "gitlab.") || c.Host == "codeberg.org" || c.Host == "bitbucket.org" || c.Forge == "gitlab" || c.Forge == "github"): weight = 0.6 } } if mapped, conf, ok := distro.MapHost(raw); ok && (err != nil || c.Kind != repourl.KindRepo) { w := weight * conf if distro.IsArchiveURL(raw) { w *= 0.6 // a tarball only hints at its repository, as in distro.AddCandidate } return distro.AddCandidate(cands, mapped, model.Evidence{Source: source, Value: value, Weight: w}) } return distro.AddCandidate(cands, raw, model.Evidence{Source: source, Value: value, Weight: weight}) } // addHomepage feeds the URL: tag into the candidates (weight 0.5, mapped hosts scaled by // the rule's confidence). func (r *Resolver) addHomepage(cands []distro.Candidate, raw, source string) []distro.Candidate { if c, err := repourl.Parse(raw); err == nil && c.Kind == repourl.KindRepo { return distro.AddCandidate(cands, raw, model.Evidence{Source: source, Value: raw, Weight: 0.5}) } if mapped, conf, ok := distro.MapHost(raw); ok { return distro.AddCandidate(cands, mapped, model.Evidence{Source: source, Value: raw, Weight: 0.5 * conf}) } return distro.AddCandidate(cands, raw, model.Evidence{Source: source, Value: raw, Weight: 0.5}) } // ListPackages returns every real source package in the project // (multibuild flavors and 000*/_ metadata entries dropped). func (r *Resolver) ListPackages(ctx context.Context) ([]string, error) { b, err := r.c.Get(ctx, strings.TrimSuffix(r.OBSBase, "/")+"/source/"+url.PathEscape(r.Project)) if err != nil { return nil, fmt.Errorf("opensuse: list %s: %w", r.Project, err) } var d obsDirectory if err := xml.Unmarshal(b, &d); err != nil { return nil, fmt.Errorf("opensuse: list %s: %w", r.Project, err) } var out []string for _, e := range d.Entries { if e.OriginPackage != "" || strings.HasPrefix(e.Name, "000") || strings.HasPrefix(e.Name, "_") || strings.Contains(e.Name, ":") { continue } out = append(out, e.Name) } sort.Strings(out) return out, nil }