Something went wrong. Try again.
Maps Linux distribution source packages to their upstream repositories, and through them to each other
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170// Package debian resolves Debian source packages to their upstream repositories from the// Ultimate Debian Database: every package's Homepage, DEP-12 upstream metadata, watch URL and// .orig tarball checksums, in one query.package debian
import ( "context" "encoding/json" "fmt" "os" "strings"
"github.com/jackc/pgx/v5"
"tangled.org/odd.computer/isomorph/internal/distro" "tangled.org/odd.computer/isomorph/internal/model")
// Resolution reads the Ultimate Debian Database instead of fetching each package's files: one// query returns, per source package in a suite, the fields of debian/control,// debian/upstream/metadata (DEP-12) and debian/watch that name an upstream. The public mirror// is documented at https://udd-mirror.debian.net/.
// DefaultUDDDSN is the public, read-only UDD mirror. ISOMORPH_UDD_DSN overrides it.const DefaultUDDDSN = "postgres://udd-mirror:udd-mirror@udd-mirror.debian.net:5432/udd?sslmode=prefer"
// Evidence weights: the prior trust in each packaging field.const ( weightMetadataRepo = 0.95 weightMetadataBrowse = 0.8 weightWatchForge = 0.6 weightHomepage = 0.4)
// weightBugDatabase is the prior for a DEP-12 Bug-Database URL: a forge issue tracker names the// repository, but projects also file bugs elsewhere (a GitHub mirror of a GitLab project).const weightBugDatabase = 0.5
// UDDRow is one source package as UDD describes it.type UDDRow struct { Source string Version string Homepage string // VcsURL is the packaging repository (usually salsa), not the upstream. VcsURL string Binaries []string // Metadata holds DEP-12 fields (Repository, Repository-Browse, Bug-Database). Metadata map[string]string // WatchURL is the newest upstream release URL that uscan found through debian/watch. WatchURL string // OrigSHA256 lists the sha256 of the .orig tarballs (a fingerprint for cross-distro joins). OrigSHA256 []string}
// UDDDSN returns the DSN to use.func UDDDSN() string { if v := os.Getenv("ISOMORPH_UDD_DSN"); v != "" { return v } return DefaultUDDDSN}
const uddQuery = `select s.source, s.version, coalesce(s.homepage, ''), coalesce(s.vcs_url, ''), coalesce(s.bin, ''), coalesce(s.checksums_sha256, ''), coalesce((select json_object_agg(m.key, m.value) from upstream_metadata m where m.source = s.source and m.key in ('Repository', 'Repository-Browse', 'Bug-Database'))::text, '{}'), coalesce((select u.upstream_url from upstream u where u.source = s.source and u.release = s.release and u.upstream_url is not null order by u.version desc limit 1), '')from sources swhere s.release = $1 and s.component = $2order by s.source, s.version`
// QueryUDD returns every source package of suite/component (e.g. "sid", "main"). When a source// has several versions in the suite, the highest version (last in UDD's order) wins.func QueryUDD(ctx context.Context, dsn, suite, component string) ([]UDDRow, error) { conn, err := pgx.Connect(ctx, dsn) if err != nil { return nil, fmt.Errorf("udd: connect: %w", err) } defer conn.Close(ctx) rows, err := conn.Query(ctx, uddQuery, suite, component) if err != nil { return nil, fmt.Errorf("udd: query: %w", err) } defer rows.Close() bySource := map[string]int{} var out []UDDRow for rows.Next() { var r UDDRow var bin, sums, meta string if err := rows.Scan(&r.Source, &r.Version, &r.Homepage, &r.VcsURL, &bin, &sums, &meta, &r.WatchURL); err != nil { return nil, fmt.Errorf("udd: scan: %w", err) } r.Binaries = splitList(bin) r.OrigSHA256 = origSums(sums) r.Metadata = parseMeta(meta) if i, ok := bySource[r.Source]; ok { if CompareVersions(r.Version, out[i].Version) > 0 { out[i] = r } continue } bySource[r.Source] = len(out) out = append(out, r) } return out, rows.Err()}
// ResolveRow turns a UDD row into ranked upstream candidates with the evidence// weights. skip names evidence sources to leave out// (used to measure the fallbacks against DEP-12 Repository).func ResolveRow(r UDDRow, skip map[string]bool) []distro.Candidate { var cands []distro.Candidate add := func(source, value string, w float64) { if value == "" || skip[source] { return } cands = distro.AddCandidate(cands, value, model.Evidence{Source: source, Value: value, Weight: w}) } add("debian/upstream/metadata:Repository", r.Metadata["Repository"], weightMetadataRepo) add("debian/upstream/metadata:Repository-Browse", r.Metadata["Repository-Browse"], weightMetadataBrowse) add("debian/upstream/metadata:Bug-Database", r.Metadata["Bug-Database"], weightBugDatabase) add("debian/watch:upstream_url", r.WatchURL, weightWatchForge) add("debian/control:Homepage", r.Homepage, weightHomepage) return distro.RankCandidates(cands)}
func splitList(s string) []string { var out []string for _, f := range strings.Split(s, ",") { if f = strings.TrimSpace(f); f != "" { out = append(out, f) } } return out}
// origSums extracts the sha256 of .orig tarballs from a Checksums-Sha256 field// ("<sha256> <size> <file>" per line).func origSums(s string) []string { var out []string for _, line := range strings.Split(s, "\n") { f := strings.Fields(line) if len(f) == 3 && strings.Contains(f[2], ".orig.") && !strings.HasSuffix(f[2], ".asc") { out = append(out, f[0]) } } return out}
// parseMeta decodes the {"key": "value"} object built by uddQuery. UDD values are free text,// so only the first whitespace-separated token is kept (some fields carry "URL -b branch").func parseMeta(s string) map[string]string { out := map[string]string{} var m map[string]any if err := json.Unmarshal([]byte(s), &m); err != nil { return out } for k, v := range m { if str, ok := v.(string); ok { if f := strings.Fields(str); len(f) > 0 { out[k] = f[0] } } } return out}