From a9665ca0310b8c091884a5da984fab57094b67bd Mon Sep 17 00:00:00 2001 From: Matthew Suozzo Date: Wed, 30 Sep 2026 11:00:51 -0400 Subject: [PATCH] export, lookup: snapshots with schemas, and queries by package, binary or repository isomorph export writes a snapshot directory: one JSONL file per distribution suite (the canonical, diffable record), the SQLite database built from the same run, the JSON Schemas of a row and of the manifest, and manifest.json with every file's sha256 and where each slice came from. The snapshot id is the UTC date and a hash of the JSONL files, so the same content re-exported gets the same id; LATEST names the newest. isomorph lookup answers distro[/suite]:package, a source or binary package name in any distribution, or a repository URL or key (mapped through the stored aliases), with every counterpart that shares the upstream. Queries come from the arguments or, one per line, from stdin; -json writes one answer (or error) per line. schema/ holds the JSON Schemas; a test keeps them in step with the Go types. README.md describes the tool, its output and its limits. --- README.md | 176 +++++++++++++++++++++++++++++ cmd/isomorph/export.go | 41 +++++++ cmd/isomorph/lookup.go | 116 +++++++++++++++++++ internal/bulk/bulk_test.go | 33 ++++-- internal/bulk/export.go | 195 ++++++++++++++++++++++++++++++++ internal/bulk/export_test.go | 87 +++++++++++++++ internal/bulk/lookup.go | 208 +++++++++++++++++++++++++++++++++++ internal/bulk/sqlite.go | 44 -------- schema/manifest.schema.json | 50 +++++++++ schema/package.schema.json | 43 ++++++++ schema/schema.go | 11 ++ schema/schema_test.go | 130 ++++++++++++++++++++++ 12 files changed, 1082 insertions(+), 52 deletions(-) create mode 100644 README.md create mode 100644 cmd/isomorph/export.go create mode 100644 cmd/isomorph/lookup.go create mode 100644 internal/bulk/export.go create mode 100644 internal/bulk/export_test.go create mode 100644 internal/bulk/lookup.go create mode 100644 schema/manifest.schema.json create mode 100644 schema/package.schema.json create mode 100644 schema/schema.go create mode 100644 schema/schema_test.go diff --git a/README.md b/README.md new file mode 100644 index 0000000..f17a1f3 --- /dev/null +++ b/README.md @@ -0,0 +1,176 @@ +# isomorph + +isomorph maps every source package in four Linux distributions to the upstream repository +it is built from. Through that repository it finds the same software in the other +distributions: +- Debian sid; +- Arch (core, extra, multilib); +- openSUSE Tumbleweed; +- Alpine edge. + +The mapping never relies on package names. `py3-requests`, `python-requests` and +`python3-requests` meet at `github.com/psf/requests`, and two unrelated projects that happen +to share a name stay apart. + +Every mapping is made from the distribution's own packaging data, and records the evidence +and the kind of evidence behind it. [docs/EVALUATION.md](docs/EVALUATION.md) measures how far +each kind can be trusted. + +## What you get + +`isomorph export` writes a snapshot: + +``` +export/ + LATEST the newest snapshot id + 20260928-946df9558260/ + manifest.json every file with its sha256, plus where each slice came from + debian-sid.jsonl one row per source package, per distribution suite + arch-core.jsonl arch-extra.jsonl arch-multilib.jsonl + opensuse-tumbleweed.jsonl + alpine-edge-main.jsonl alpine-edge-community.jsonl + packages.sqlite the same rows as a SQLite database + package.schema.json manifest.schema.json +``` + +- **The JSONL files are the canonical record.** Each line is one source package + ([schema](schema/package.schema.json)): + + ```json + {"distro": "arch", "suite": "core", "package": "json-c", "version": "0.19-1", + "binaries": ["json-c"], "homepage": "https://github.com/json-c/json-c/wiki", + "upstream_key": "github.com/json-c/json-c", "repo": "https://github.com/json-c/json-c", + "status": "resolved", "basis": "vcs-source", + "evidence": [ + {"source": "SRCINFO:source:git", "value": "git+https://github.com/json-c/json-c#tag=json-c-0.19-20260627", + "key": "github.com/json-c/json-c", "weight": 0.9}, + {"source": "SRCINFO:url", "value": "https://github.com/json-c/json-c/wiki", + "key": "github.com/json-c/json-c", "weight": 0.5}]} + ``` + + - `upstream_key` identifies the repository: packages with equal keys are the same software. + - `repo` is a clone URL, ready for an OSV `GIT` range. + - `osv_ecosystem` is set where OSV defines one (`Debian:sid`, `openSUSE:Tumbleweed`). +- **The SQLite database** is a convenience built from the same run. Python reads it with the + standard library. + - Its `counterparts` view pairs every package with the packages that share its upstream. + - `binaries` maps binary package names to source packages. + - `aliases` records which keys were merged, and why. +- **The snapshot id** is the export's UTC date and a hash of the JSONL files. Re-exporting + unchanged data gives the same id. +- **Publishing a snapshot** means copying its directory (and `LATEST`) wherever consumers + read from, such as `gcloud storage rsync -r data/export gs://BUCKET/snapshots`. Consumers + check files against `manifest.json`. + +`isomorph lookup` answers queries against the database. A query can be: +- `distro[/suite]:package`; +- a source or binary package name, in any distribution; +- a repository URL or key. + +It reads the queries from its arguments, or one per line on stdin. With `-json` it writes one +answer per line. + +``` +$ isomorph lookup libjson-c5 +libjson-c5: + debian/sid:json-c 0.19+ds-1 github.com/json-c/json-c (release-url) [via binary] + opensuse/tumbleweed:json-c 0.19-1.2 github.com/json-c/json-c (derived) [via binary] + = alpine/edge-main:json-c 0.19-r1 github.com/json-c/json-c (homepage) + = arch/core:json-c 0.19-1 github.com/json-c/json-c (vcs-source) + = arch/multilib:lib32-json-c 0.19-1 github.com/json-c/json-c (vcs-source) +``` + +## Building the table + +```sh +go build ./cmd/isomorph +isomorph bulk debian # one query of the Ultimate Debian Database +isomorph bulk arch -crawl 20000 # .SRCINFO per pkgbase, paced; later runs read the cache +isomorph bulk opensuse -crawl 7000 # spec and _service per unresolved package, paced +isomorph bulk alpine # APKINDEX and one streamed download of aports +GITHUB_TOKEN=... isomorph bulk canonicalize +isomorph bulk compare -all # agreement between distributions, as confusion matrices +isomorph export +``` + +- **Storage:** the database is `data/packages.sqlite` (`-data DIR` moves it). HTTP responses + are cached in the user cache directory (`-cache DIR`). +- **Slices:** each `bulk ` run replaces that distribution's slices, so they can be + refreshed independently. +- **After a slice changes:** run `canonicalize` again. It maps keys that name the same + repository onto one: + - GitHub renames, and repositories that no longer exist; + - GitLab project ids; + - PyPI and crates.io metadata; + - KDE's project list; + - known mirrors. + +## Sources and evidence + +| distribution | where the mapping comes from | +|---|---| +| Debian | UDD: `Homepage`, DEP-12 `Repository`/`Repository-Browse`/`Bug-Database`, the watch URL, `.orig` tarball checksums | +| Arch | pacman databases, the packaging state repository (released commit per pkgbase), `.SRCINFO` or `PKGBUILD` at that commit | +| openSUSE | repository metadata (spec `Url:`), rb.zq1.de giturls, the OBS spec (`Source`, `#Git-Clone`) and `_service` | +| Alpine | the edge APKINDEX, and each APKBUILD's `url`, `source` and `_repo_url` | + +Each observation is weighted by how much its field can be trusted, and observations of the +same repository combine. A project site or download area that names no repository is mapped +by host rules (GNU and nongnu to Savannah, cgit to GitLab, Xfce, KDE, netfilter, Apache). + +A row's `basis` is the strongest kind of evidence behind its mapping: + +| basis | example | +|---|---| +| metadata | DEP-12 `Repository` | +| vcs-source | a git source in `.SRCINFO`, an openSUSE `_service` | +| release-url | a watch URL, a release tarball | +| derived | rb.zq1.de giturls | +| bug-tracker | DEP-12 `Bug-Database` | +| homepage | `Homepage`, `url` | + +Other distributions never decide a mapping. They are used only to measure agreement, and to +choose which keys are worth a GitHub lookup. + +## How well it works + +From [docs/EVALUATION.md](docs/EVALUATION.md), 2026-09-28: + +- **Coverage:** Arch 85.3%, Alpine 78.3%, Debian 77.9%, openSUSE 75.0% of source packages map + to a repository. +- **Debian's DEP-12 `Repository` as ground truth:** without it, the other fields resolve 94.6% + of those packages and agree with it on 96.8%. +- **Same-named packages** in two distributions map to the same repository 90.7% to 93.8% of + the time. The name join also pairs unrelated software, so this understates agreement. +- **Packages built from the identical release tarball** (Debian and Arch) agree on 96.4%. + +## Crawling politely + +- **Pacing:** every request goes through one client with a disk cache and per-host pacing. + crates.io and the OBS API get one request a second, and gitlab.archlinux.org stays under + its published anonymous limit. +- **Rate limits:** the client waits out short rate-limit windows and stops at long ones + rather than retrying into them. +- **Contact:** set `ISOMORPH_CONTACT`, or write `~/.config/isomorph/contact`, so + operators can reach you. It is appended to the User-Agent, and crates.io is not queried + without it. +- **Tokens:** + - `GITHUB_TOKEN` lifts GitHub's anonymous limit of 60 lookups an hour. + - `ARCH_GITLAB_TOKEN` lifts gitlab.archlinux.org's anonymous limit. The crawl does not need + it. + +## Limitations + +- **Packages whose only URL is a project website stay unresolved.** Nothing in their + packaging names a repository. +- **Stale packaging data wins.** When a distribution's data points at a project's old home + (a SourceForge download, a repository that moved), the mapping follows it. The evaluation + counts these cases. +- **One repository can build many packages,** and a package built from a monorepo maps to the + monorepo. +- **Arch and Alpine keep no source order,** so a dependency's git source can outrank the main + tarball. + +## Licence + +MIT; see [LICENSE](LICENSE). diff --git a/cmd/isomorph/export.go b/cmd/isomorph/export.go new file mode 100644 index 0000000..fe76103 --- /dev/null +++ b/cmd/isomorph/export.go @@ -0,0 +1,41 @@ +package main + +import ( + "context" + "flag" + "fmt" + "path/filepath" + "time" + + "tangled.org/odd.computer/isomorph/internal/bulk" +) + +func init() { + register(command{name: "export", usage: "[-db FILE] [-out DIR]: write a snapshot (JSONL per distribution suite, SQLite, schemas, manifest.json)", run: runExport}) +} + +func runExport(ctx context.Context, env *Env, args []string) error { + fs := flag.NewFlagSet("export", flag.ContinueOnError) + dbPath := fs.String("db", defaultDB(env), "SQLite database to export") + out := fs.String("out", "", "directory that holds the snapshots (default /export)") + if err := fs.Parse(args); err != nil { + return err + } + if *out == "" { + *out = filepath.Join(env.DataDir, "export") + } + db, err := bulk.Open(ctx, *dbPath) + if err != nil { + return err + } + defer db.Close() + m, dir, err := bulk.Export(ctx, db, *out, time.Now()) + if err != nil { + return err + } + for _, s := range m.Slices { + fmt.Fprintf(env.Stdout, "%-24s %6d packages, %6d resolved (%.1f%%)\n", s.Path, s.Packages, s.Resolved, pct(s.Resolved, s.Packages)) + } + fmt.Fprintf(env.Stdout, "snapshot %s in %s\n", m.Snapshot, dir) + return nil +} diff --git a/cmd/isomorph/lookup.go b/cmd/isomorph/lookup.go new file mode 100644 index 0000000..164d1f1 --- /dev/null +++ b/cmd/isomorph/lookup.go @@ -0,0 +1,116 @@ +package main + +import ( + "bufio" + "context" + "encoding/json" + "flag" + "fmt" + "io" + "os" + "strings" + + "tangled.org/odd.computer/isomorph/internal/bulk" +) + +func init() { + register(command{name: "lookup", usage: "[-db FILE] [-json] [-evidence] [query...]: find packages and their counterparts; queries are distro[/suite]:package, a package or binary name, or a repository URL, read from stdin when none are given", run: runLookup}) +} + +// lookupResult is one line of `lookup -json`: the answer, or why the query failed. +type lookupResult struct { + bulk.Answer + Error string `json:"error,omitempty"` +} + +func runLookup(ctx context.Context, env *Env, args []string) error { + fs := flag.NewFlagSet("lookup", flag.ContinueOnError) + dbPath := fs.String("db", defaultDB(env), "SQLite database") + asJSON := fs.Bool("json", false, "write one JSON object per query (JSONL)") + evidence := fs.Bool("evidence", false, "include each row's evidence") + if err := fs.Parse(args); err != nil { + return err + } + db, err := bulk.Open(ctx, *dbPath) + if err != nil { + return err + } + defer db.Close() + alias, err := bulk.LoadAliases(ctx, db) + if err != nil { + return err + } + queries := fs.Args() + var next func() (string, bool) + if len(queries) > 0 { + i := 0 + next = func() (string, bool) { + if i == len(queries) { + return "", false + } + i++ + return queries[i-1], true + } + } else { + sc := bufio.NewScanner(os.Stdin) + next = func() (string, bool) { + for sc.Scan() { + if q := strings.TrimSpace(sc.Text()); q != "" && !strings.HasPrefix(q, "#") { + return q, true + } + } + return "", false + } + } + enc := json.NewEncoder(env.Stdout) + failed := 0 + for q, ok := next(); ok; q, ok = next() { + a, err := bulk.Lookup(ctx, db, alias, q, *evidence) + if err != nil { + failed++ + } + if *asJSON { + r := lookupResult{Answer: a} + if err != nil { + r.Error = err.Error() + } + if err := enc.Encode(r); err != nil { + return err + } + continue + } + printAnswer(env.Stdout, a, err) + } + if failed > 0 && !*asJSON { + return fmt.Errorf("%d queries failed", failed) + } + return nil +} + +func printAnswer(w io.Writer, a bulk.Answer, err error) { + fmt.Fprintf(w, "%s:\n", a.Query) + if err != nil { + fmt.Fprintf(w, " error: %v\n", err) + return + } + if len(a.Matches) == 0 { + fmt.Fprintln(w, " no match") + return + } + line := func(prefix string, p bulk.Package, via string) { + up := "unresolved" + if p.UpstreamKey != "" { + up = p.UpstreamKey + " (" + p.Basis + ")" + } + if via != "" && via != "package" { + up += " [via " + via + "]" + } + fmt.Fprintf(w, "%s%s/%s:%s %s %s\n", prefix, p.Distro, p.Suite, p.Name, p.Version, up) + } + for _, m := range a.Matches { + line(" ", m.Package, m.Via) + } + for _, c := range a.Counterparts { + line(" = ", c, "") + } +} diff --git a/internal/bulk/bulk_test.go b/internal/bulk/bulk_test.go index caba262..01e8195 100644 --- a/internal/bulk/bulk_test.go +++ b/internal/bulk/bulk_test.go @@ -74,21 +74,38 @@ func TestSQLiteRoundTrip(t *testing.T) { if err := Replace(ctx, db, Source{Distro: model.Arch, Suite: "extra", Origin: "test", FetchedAt: time.Now()}, arch); err != nil { t.Fatal(err) } - got, err := Lookup(ctx, db, "debian/sid:json-c") - if err != nil || len(got) != 1 || got[0].UpstreamKey != "github.com/json-c/json-c" || len(got[0].Evidence) == 0 { + got, err := Lookup(ctx, db, nil, "debian/sid:json-c", true) + if err != nil || len(got.Matches) != 1 || got.Matches[0].UpstreamKey != "github.com/json-c/json-c" || len(got.Matches[0].Evidence) == 0 || + got.Matches[0].Via != "package" || len(got.Counterparts) != 1 || got.Counterparts[0].Distro != model.Arch { t.Fatalf("lookup by package: %v %+v", err, got) } - both, err := Lookup(ctx, db, "github.com/json-c/json-c") - if err != nil || len(both) != 2 { - t.Fatalf("lookup by upstream: %v %+v", err, both) + bin, err := Lookup(ctx, db, nil, "libjson-c5", false) + if err != nil || len(bin.Matches) != 1 || bin.Matches[0].Name != "json-c" || bin.Matches[0].Via != "binary" || + len(bin.Matches[0].Binaries) == 0 || len(bin.Matches[0].Evidence) != 0 { + t.Fatalf("lookup by binary: %v %+v", err, bin) + } + for _, q := range []string{"github.com/json-c/json-c", "https://github.com/json-c/json-c.git", "https://github.com/json-c/json-c/archive/refs/tags/v1.tar.gz"} { + both, err := Lookup(ctx, db, nil, q, false) + if err != nil || len(both.Matches) != 2 || both.Matches[0].Via != "upstream" || len(both.Counterparts) != 0 { + t.Fatalf("lookup by upstream %q: %v %+v", q, err, both) + } + } + renamed, err := Lookup(ctx, db, map[string]string{"github.com/old/json-c": "github.com/json-c/json-c"}, "github.com/old/json-c", false) + if err != nil || len(renamed.Matches) != 2 { + t.Fatalf("lookup through an alias: %v %+v", err, renamed) + } + for _, q := range []string{"nosuch:json-c", "https://example.org/", ""} { + if _, err := Lookup(ctx, db, nil, q, false); err == nil { + t.Errorf("%q: expected an error", q) + } } var n int if err := db.QueryRowContext(ctx, `select count(*) from counterparts where package = 'json-c'`).Scan(&n); err != nil || n != 2 { t.Fatalf("counterparts view: %v %d", err, n) } - var bin string - if err := db.QueryRowContext(ctx, `select package from binaries where binary = 'libjson-c5'`).Scan(&bin); err != nil || bin != "json-c" { - t.Fatalf("binaries: %v %q", err, bin) + var binPkg string + if err := db.QueryRowContext(ctx, `select package from binaries where binary = 'libjson-c5'`).Scan(&binPkg); err != nil || binPkg != "json-c" { + t.Fatalf("binaries: %v %q", err, binPkg) } back, err := LoadSlice(ctx, db, model.Debian, "sid") if err != nil || len(back) != len(deb) { diff --git a/internal/bulk/export.go b/internal/bulk/export.go new file mode 100644 index 0000000..e52e9e5 --- /dev/null +++ b/internal/bulk/export.go @@ -0,0 +1,195 @@ +package bulk + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "time" + + schemas "tangled.org/odd.computer/isomorph/schema" +) + +// An export snapshot is a directory holding one JSONL file per (distro, suite) slice, a copy +// of the SQLite database, the JSON Schemas, and manifest.json listing every file with its +// sha256. The JSONL files are the canonical, diffable record; the database is a convenience +// built from the same run. The snapshot id is the UTC date and a hash of the slice files, so +// the same content exported on the same day gets the same id. + +// ManifestFormat versions the manifest and the row format. +const ManifestFormat = 1 + +// Manifest is a snapshot's manifest.json (schema/manifest.schema.json). +type Manifest struct { + Format int `json:"format"` + Snapshot string `json:"snapshot"` + Created time.Time `json:"created"` + Slices []SliceFile `json:"slices"` + Database DatabaseFile `json:"database"` + Schemas ManifestFiles `json:"schemas"` +} + +// SliceFile describes one slice's JSONL file. +type SliceFile struct { + Distro string `json:"distro"` + Suite string `json:"suite"` + Origin string `json:"origin"` + FetchedAt time.Time `json:"fetched_at"` + Packages int `json:"packages"` + Resolved int `json:"resolved"` + Path string `json:"path"` + SHA256 string `json:"sha256"` +} + +// DatabaseFile describes the snapshot's SQLite database. +type DatabaseFile struct { + Path string `json:"path"` + SHA256 string `json:"sha256"` + SchemaVersion int `json:"schema_version"` +} + +// ManifestFiles names the schema files in the snapshot. +type ManifestFiles struct { + Package string `json:"package"` + Manifest string `json:"manifest"` +} + +// Export writes a snapshot of db into outDir// and points outDir/LATEST at it. It +// returns the manifest and the snapshot directory. An identical snapshot that already exists +// is kept as is. +func Export(ctx context.Context, db *sql.DB, outDir string, now time.Time) (*Manifest, string, error) { + if err := os.MkdirAll(outDir, 0o755); err != nil { + return nil, "", err + } + tmp, err := os.MkdirTemp(outDir, ".export-") + if err != nil { + return nil, "", err + } + defer os.RemoveAll(tmp) + m := &Manifest{ + Format: ManifestFormat, + Created: now.UTC().Truncate(time.Second), + Schemas: ManifestFiles{Package: "package.schema.json", Manifest: "manifest.schema.json"}, + } + srcs, err := Sources(ctx, db) + if err != nil { + return nil, "", err + } + id := sha256.New() + for _, s := range srcs { + pkgs, err := LoadSlice(ctx, db, s.Distro, s.Suite) + if err != nil { + return nil, "", err + } + sf := SliceFile{Distro: string(s.Distro), Suite: s.Suite, Origin: s.Origin, FetchedAt: s.FetchedAt.UTC(), + Packages: len(pkgs), Path: string(s.Distro) + "-" + s.Suite + ".jsonl"} + for _, p := range pkgs { + if p.Status == StatusResolved { + sf.Resolved++ + } + } + if sf.SHA256, err = writeFile(filepath.Join(tmp, sf.Path), func(w io.Writer) error { return WriteJSONL(w, pkgs) }); err != nil { + return nil, "", err + } + fmt.Fprintf(id, "%s %s\n", sf.Path, sf.SHA256) + m.Slices = append(m.Slices, sf) + } + m.Snapshot = m.Created.Format("20060102") + "-" + hex.EncodeToString(id.Sum(nil))[:12] + + m.Database = DatabaseFile{Path: "packages.sqlite", SchemaVersion: schemaVersion} + dbPath := filepath.Join(tmp, m.Database.Path) + if _, err := db.ExecContext(ctx, `vacuum into ?`, dbPath); err != nil { + return nil, "", fmt.Errorf("export: copy database: %w", err) + } + if m.Database.SHA256, err = hashFile(dbPath); err != nil { + return nil, "", err + } + for _, name := range []string{m.Schemas.Package, m.Schemas.Manifest} { + b, err := fs.ReadFile(schemas.FS, name) + if err != nil { + return nil, "", err + } + if err := os.WriteFile(filepath.Join(tmp, name), b, 0o644); err != nil { + return nil, "", err + } + } + if _, err := writeFile(filepath.Join(tmp, "manifest.json"), func(w io.Writer) error { + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(m) + }); err != nil { + return nil, "", err + } + + if err := os.Chmod(tmp, 0o755); err != nil { // MkdirTemp makes it private + return nil, "", err + } + dir := filepath.Join(outDir, m.Snapshot) + if _, err := os.Stat(dir); err == nil { + // Same content, same day: the id already names it. + } else if err := os.Rename(tmp, dir); err != nil { + return nil, "", err + } + if err := os.WriteFile(filepath.Join(outDir, "LATEST"), []byte(m.Snapshot+"\n"), 0o644); err != nil { + return nil, "", err + } + return m, dir, nil +} + +// Sources lists the slices in db with where they came from, in (distro, suite) order. +func Sources(ctx context.Context, db *sql.DB) ([]Source, error) { + rows, err := db.QueryContext(ctx, `select distro, suite, origin, fetched_at, packages from sources order by distro, suite`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Source + for rows.Next() { + var s Source + var at string + if err := rows.Scan(&s.Distro, &s.Suite, &s.Origin, &at, &s.Packages); err != nil { + return nil, err + } + if s.FetchedAt, err = time.Parse(time.RFC3339, at); err != nil { + return nil, fmt.Errorf("sources: %s/%s fetched_at %q: %w", s.Distro, s.Suite, at, err) + } + out = append(out, s) + } + return out, rows.Err() +} + +// writeFile writes path through fill and returns the sha256 of what was written. +func writeFile(path string, fill func(io.Writer) error) (string, error) { + f, err := os.Create(path) + if err != nil { + return "", err + } + h := sha256.New() + if err := fill(io.MultiWriter(f, h)); err != nil { + f.Close() + return "", err + } + if err := f.Close(); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +func hashFile(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/internal/bulk/export_test.go b/internal/bulk/export_test.go new file mode 100644 index 0000000..eb4605e --- /dev/null +++ b/internal/bulk/export_test.go @@ -0,0 +1,87 @@ +package bulk + +import ( + "bufio" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "tangled.org/odd.computer/isomorph/internal/model" +) + +func TestExport(t *testing.T) { + ctx := context.Background() + db, err := Open(ctx, filepath.Join(t.TempDir(), "p.sqlite")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + at := time.Date(2026, 9, 28, 12, 0, 0, 0, time.UTC) + if err := Replace(ctx, db, Source{Distro: model.Debian, Suite: "sid", Origin: "udd", FetchedAt: at}, FromUDD(udd(), "sid")); err != nil { + t.Fatal(err) + } + arch := []Package{{Distro: model.Arch, Suite: "extra", Name: "json-c", Version: "0.18-1", UpstreamKey: "github.com/json-c/json-c", + Repo: "https://github.com/json-c/json-c", Status: StatusResolved, Basis: BasisVCSSource}} + if err := Replace(ctx, db, Source{Distro: model.Arch, Suite: "extra", Origin: "pacman", FetchedAt: at}, arch); err != nil { + t.Fatal(err) + } + out := t.TempDir() + m, dir, err := Export(ctx, db, out, at) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(m.Snapshot, "20260928-") || dir != filepath.Join(out, m.Snapshot) || len(m.Slices) != 2 { + t.Fatalf("manifest: %+v in %s", m, dir) + } + if m.Slices[0].Distro != "arch" || m.Slices[0].Packages != 1 || m.Slices[0].Resolved != 1 || m.Slices[1].Path != "debian-sid.jsonl" { + t.Errorf("slices: %+v", m.Slices) + } + // Every file is where the manifest says, with the hash it says. + for path, want := range map[string]string{m.Slices[0].Path: m.Slices[0].SHA256, m.Slices[1].Path: m.Slices[1].SHA256, m.Database.Path: m.Database.SHA256} { + b, err := os.ReadFile(filepath.Join(dir, path)) + if err != nil { + t.Fatal(err) + } + if sum := sha256.Sum256(b); hex.EncodeToString(sum[:]) != want { + t.Errorf("%s: sha256 differs from the manifest", path) + } + } + var back Manifest + if b, err := os.ReadFile(filepath.Join(dir, "manifest.json")); err != nil || json.Unmarshal(b, &back) != nil || back.Snapshot != m.Snapshot { + t.Errorf("manifest.json: %v %+v", err, back) + } + for _, f := range []string{"package.schema.json", "manifest.schema.json"} { + if _, err := os.Stat(filepath.Join(dir, f)); err != nil { + t.Error(err) + } + } + if b, _ := os.ReadFile(filepath.Join(out, "LATEST")); strings.TrimSpace(string(b)) != m.Snapshot { + t.Errorf("LATEST = %q", b) + } + // Rows round-trip. + f, err := os.Open(filepath.Join(dir, "debian-sid.jsonl")) + if err != nil { + t.Fatal(err) + } + defer f.Close() + n := 0 + for sc := bufio.NewScanner(f); sc.Scan(); n++ { + var p Package + if err := json.Unmarshal(sc.Bytes(), &p); err != nil || p.Distro != model.Debian { + t.Fatalf("row %d: %v %+v", n, err, p) + } + } + if n != m.Slices[1].Packages { + t.Errorf("%d rows, manifest says %d", n, m.Slices[1].Packages) + } + // The same content exported again gets the same id. + if again, _, err := Export(ctx, db, out, at.Add(time.Hour)); err != nil || again.Snapshot != m.Snapshot { + t.Errorf("re-export: %v %s vs %s", err, again.Snapshot, m.Snapshot) + } +} diff --git a/internal/bulk/lookup.go b/internal/bulk/lookup.go new file mode 100644 index 0000000..a12f1ce --- /dev/null +++ b/internal/bulk/lookup.go @@ -0,0 +1,208 @@ +package bulk + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + + "tangled.org/odd.computer/isomorph/internal/model" + "tangled.org/odd.computer/isomorph/internal/repourl" +) + +// Match is one package a query found, and how. +type Match struct { + Package + // Via is "package" (the source package's name), "binary" (the name of one of its binary + // packages) or "upstream" (its upstream repository). + Via string `json:"via"` +} + +// Answer is what Lookup returns for one query. +type Answer struct { + Query string `json:"query"` + Matches []Match `json:"matches"` + // Counterparts are the other packages, in any distribution, that share an upstream with a + // match. + Counterparts []Package `json:"counterparts"` +} + +// Lookup answers one query: +// - distro[/suite]:name, a source package of one distribution; +// - name, a source package of any distribution; +// - an upstream repository, as a URL or key (github.com/curl/curl). +// +// A name that is no source package is looked up among the binary packages (libcurl4 finds +// Debian's curl). A repository is mapped through alias (LoadAliases) onto the table's keys. +// Rows carry their evidence only when evidence is set. +func Lookup(ctx context.Context, db *sql.DB, alias map[string]string, query string, evidence bool) (Answer, error) { + a := Answer{Query: query} + q := strings.TrimSpace(query) + if q == "" { + return a, errors.New("empty query") + } + var matches []Package + via := "package" + if isRepoQuery(q) { + key, err := queryKey(q) + if err != nil { + return a, err + } + via = "upstream" + if matches, err = queryPackages(ctx, db, "upstream_key = ?", Resolve(alias, key)); err != nil { + return a, err + } + } else { + name, where, args := q, "", []any{} + if d, n, ok := strings.Cut(q, ":"); ok { + distro, suite, _ := strings.Cut(d, "/") + if !knownDistro(distro) { + return a, fmt.Errorf("unknown distribution %q (want one of %v)", distro, model.AllDistros) + } + name, where, args = n, " and distro = ?", []any{distro} + if suite != "" { + where += " and suite = ?" + args = append(args, suite) + } + } + var err error + if matches, err = queryPackages(ctx, db, "package = ?"+where, append([]any{name}, args...)...); err != nil { + return a, err + } + if len(matches) == 0 { + via = "binary" + bin := "(distro, suite, package) in (select distro, suite, package from binaries where binary = ?" + where + ")" + if matches, err = queryPackages(ctx, db, bin, append([]any{name}, args...)...); err != nil { + return a, err + } + } + } + seen := map[[3]string]bool{} + for _, p := range matches { + seen[[3]string{string(p.Distro), p.Suite, p.Name}] = true + a.Matches = append(a.Matches, Match{Package: p, Via: via}) + } + keys := map[string]bool{} + for _, p := range matches { + if p.UpstreamKey == "" || keys[p.UpstreamKey] { + continue + } + keys[p.UpstreamKey] = true + others, err := queryPackages(ctx, db, "upstream_key = ?", p.UpstreamKey) + if err != nil { + return a, err + } + for _, o := range others { + if k := [3]string{string(o.Distro), o.Suite, o.Name}; !seen[k] { + seen[k] = true + a.Counterparts = append(a.Counterparts, o) + } + } + } + for _, ps := range [][]*Package{matchPtrs(a.Matches), pkgPtrs(a.Counterparts)} { + for _, p := range ps { + if err := loadBinaries(ctx, db, p); err != nil { + return a, err + } + if evidence { + if err := loadEvidence(ctx, db, p); err != nil { + return a, err + } + } + } + } + return a, nil +} + +// isRepoQuery reports whether q names a repository rather than a package: a URL, or a key +// whose first segment is a host name. +func isRepoQuery(q string) bool { + if strings.Contains(q, "://") { + return true + } + host, _, ok := strings.Cut(q, "/") + return ok && strings.Contains(host, ".") +} + +// queryKey canonicalizes a repository URL or key; a release or project URL counts through the +// repository it hints at. +func queryKey(q string) (string, error) { + u := q + if !strings.Contains(u, "://") { + u = "https://" + u + } + c, err := repourl.Parse(u) + if err != nil { + return "", fmt.Errorf("%q: %w", q, err) + } + switch { + case c.Kind == repourl.KindRepo: + return c.Key, nil + case c.Hint != "": + return c.Hint, nil + } + return "", fmt.Errorf("%q names no repository", q) +} + +func knownDistro(d string) bool { + for _, k := range model.AllDistros { + if string(k) == d { + return true + } + } + return false +} + +func queryPackages(ctx context.Context, db *sql.DB, where string, args ...any) ([]Package, error) { + rows, err := db.QueryContext(ctx, `select distro, suite, package, version, coalesce(osv_ecosystem,''), coalesce(purl,''), + coalesce(homepage,''), coalesce(upstream_key,''), coalesce(repo,''), status, coalesce(basis,'') + from packages where `+where+` order by distro, suite, package`, args...) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Package + for rows.Next() { + var p Package + if err := rows.Scan(&p.Distro, &p.Suite, &p.Name, &p.Version, &p.OSVEcosystem, &p.PURL, &p.Homepage, + &p.UpstreamKey, &p.Repo, &p.Status, &p.Basis); err != nil { + return nil, err + } + out = append(out, p) + } + return out, rows.Err() +} + +func loadBinaries(ctx context.Context, db *sql.DB, p *Package) error { + rows, err := db.QueryContext(ctx, `select binary from binaries where distro = ? and suite = ? and package = ? order by rowid`, + p.Distro, p.Suite, p.Name) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var b string + if err := rows.Scan(&b); err != nil { + return err + } + p.Binaries = append(p.Binaries, b) + } + return rows.Err() +} + +func matchPtrs(ms []Match) []*Package { + out := make([]*Package, len(ms)) + for i := range ms { + out[i] = &ms[i].Package + } + return out +} + +func pkgPtrs(ps []Package) []*Package { + out := make([]*Package, len(ps)) + for i := range ps { + out[i] = &ps[i] + } + return out +} diff --git a/internal/bulk/sqlite.go b/internal/bulk/sqlite.go index ffb66f0..f0ad549 100644 --- a/internal/bulk/sqlite.go +++ b/internal/bulk/sqlite.go @@ -5,7 +5,6 @@ import ( "database/sql" "encoding/json" "fmt" - "strings" _ "modernc.org/sqlite" // pure-Go driver, registered as "sqlite" @@ -136,49 +135,6 @@ func Replace(ctx context.Context, db *sql.DB, src Source, pkgs []Package) error return tx.Commit() } -// Lookup returns the packages matching ref: "distro:package", "distro/suite:package", or an -// upstream key (every package mapped to it). -func Lookup(ctx context.Context, db *sql.DB, ref string) ([]Package, error) { - q := `select distro, suite, package, version, coalesce(osv_ecosystem,''), coalesce(purl,''), coalesce(homepage,''), - coalesce(upstream_key,''), coalesce(repo,''), status, coalesce(basis,'') from packages where ` - var args []any - if d, name, ok := strings.Cut(ref, ":"); ok && !strings.Contains(d, ".") { - distro, suite, _ := strings.Cut(d, "/") - q += "distro = ? and package = ?" - args = append(args, distro, name) - if suite != "" { - q += " and suite = ?" - args = append(args, suite) - } - } else { - q += "upstream_key = ?" - args = append(args, ref) - } - rows, err := db.QueryContext(ctx, q+" order by distro, suite, package", args...) - if err != nil { - return nil, err - } - defer rows.Close() - var out []Package - for rows.Next() { - var p Package - if err := rows.Scan(&p.Distro, &p.Suite, &p.Name, &p.Version, &p.OSVEcosystem, &p.PURL, &p.Homepage, - &p.UpstreamKey, &p.Repo, &p.Status, &p.Basis); err != nil { - return nil, err - } - out = append(out, p) - } - if err := rows.Err(); err != nil { - return nil, err - } - for i := range out { - if err := loadEvidence(ctx, db, &out[i]); err != nil { - return nil, err - } - } - return out, nil -} - func loadEvidence(ctx context.Context, db *sql.DB, p *Package) error { rows, err := db.QueryContext(ctx, `select source, value, coalesce(key,''), weight from evidence where distro = ? and suite = ? and package = ? order by weight desc`, p.Distro, p.Suite, p.Name) diff --git a/schema/manifest.schema.json b/schema/manifest.schema.json new file mode 100644 index 0000000..5cd358d --- /dev/null +++ b/schema/manifest.schema.json @@ -0,0 +1,50 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "isomorph export manifest", + "description": "manifest.json of one export snapshot: what each file holds and its sha256.", + "type": "object", + "required": ["format", "snapshot", "created", "slices", "database", "schemas"], + "properties": { + "format": {"const": 1, "description": "Version of this manifest and of the row format."}, + "snapshot": {"type": "string", "pattern": "^[0-9]{8}-[0-9a-f]{12}$", "description": "-: the same content gives the same id on the same day."}, + "created": {"type": "string", "format": "date-time"}, + "slices": { + "type": "array", + "items": { + "type": "object", + "required": ["distro", "suite", "origin", "fetched_at", "packages", "resolved", "path", "sha256"], + "properties": { + "distro": {"enum": ["debian", "opensuse", "arch", "alpine"]}, + "suite": {"type": "string"}, + "origin": {"type": "string", "description": "Where the slice's data came from."}, + "fetched_at": {"type": "string", "format": "date-time"}, + "packages": {"type": "integer", "minimum": 0}, + "resolved": {"type": "integer", "minimum": 0}, + "path": {"type": "string", "description": "The slice's JSONL file, relative to the manifest."}, + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"} + }, + "additionalProperties": false + } + }, + "database": { + "type": "object", + "required": ["path", "sha256", "schema_version"], + "properties": { + "path": {"type": "string", "description": "The SQLite database, relative to the manifest; a convenience built from the same data as the JSONL files."}, + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "schema_version": {"type": "integer", "description": "The database's PRAGMA user_version."} + }, + "additionalProperties": false + }, + "schemas": { + "type": "object", + "required": ["package", "manifest"], + "properties": { + "package": {"type": "string"}, + "manifest": {"type": "string"} + }, + "additionalProperties": false + } + }, + "additionalProperties": false +} diff --git a/schema/package.schema.json b/schema/package.schema.json new file mode 100644 index 0000000..9c9bd15 --- /dev/null +++ b/schema/package.schema.json @@ -0,0 +1,43 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "isomorph package row", + "description": "One source package of one distribution suite, mapped to its upstream repository. One JSON object per line in -.jsonl.", + "type": "object", + "required": ["distro", "suite", "package", "version", "status"], + "properties": { + "distro": {"enum": ["debian", "opensuse", "arch", "alpine"]}, + "suite": {"type": "string", "description": "sid, tumbleweed, core/extra/multilib, edge-main/edge-community."}, + "package": {"type": "string", "description": "The source package: Debian source, openSUSE source RPM name, Arch pkgbase, Alpine origin."}, + "version": {"type": "string", "description": "The distribution's version string."}, + "osv_ecosystem": {"type": "string", "description": "The OSV ecosystem (Debian:sid, openSUSE:Tumbleweed); absent where OSV defines none."}, + "purl": {"type": "string", "description": "Advisory package URL of the source package; distribution qualifiers are not standardised, so key on (osv_ecosystem, package)."}, + "binaries": {"type": "array", "items": {"type": "string"}, "description": "Names of the binary packages built from this source package."}, + "homepage": {"type": "string"}, + "upstream_key": {"type": "string", "description": "Canonical repository key (github.com/curl/curl): equal keys mean the same upstream."}, + "repo": {"type": "string", "description": "Clone URL of the upstream repository."}, + "status": {"enum": ["resolved", "unresolved"]}, + "basis": { + "enum": ["metadata", "vcs-source", "release-url", "derived", "bug-tracker", "homepage"], + "description": "The strongest kind of evidence behind the mapping; docs/EVALUATION.md measures each." + }, + "evidence": { + "type": "array", + "description": "Every observation considered, highest weight first.", + "items": { + "type": "object", + "required": ["source", "value", "weight"], + "properties": { + "source": {"type": "string", "description": "The field observed (debian/upstream/metadata:Repository, SRCINFO:source:git, spec:Source0, ...)."}, + "value": {"type": "string"}, + "key": {"type": "string", "description": "The repository key the value names or hints at, after aliasing."}, + "weight": {"type": "number", "minimum": 0, "maximum": 1} + }, + "additionalProperties": false + } + }, + "source_sha256": {"type": "array", "items": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, "description": "sha256 of the upstream release archives the package builds from."} + }, + "additionalProperties": false, + "if": {"properties": {"status": {"const": "resolved"}}}, + "then": {"required": ["upstream_key", "repo", "basis"]} +} diff --git a/schema/schema.go b/schema/schema.go new file mode 100644 index 0000000..d277743 --- /dev/null +++ b/schema/schema.go @@ -0,0 +1,11 @@ +// Package schema holds the JSON Schemas of the export: package.schema.json for each row of a +// -.jsonl file, manifest.schema.json for a snapshot's manifest.json. Each export +// snapshot carries a copy of both. +package schema + +import "embed" + +// FS holds package.schema.json and manifest.schema.json. +// +//go:embed package.schema.json manifest.schema.json +var FS embed.FS diff --git a/schema/schema_test.go b/schema/schema_test.go new file mode 100644 index 0000000..0aef20b --- /dev/null +++ b/schema/schema_test.go @@ -0,0 +1,130 @@ +package schema_test + +import ( + "encoding/json" + "io/fs" + "sort" + "testing" + "time" + + "tangled.org/odd.computer/isomorph/internal/bulk" + "tangled.org/odd.computer/isomorph/internal/model" + "tangled.org/odd.computer/isomorph/schema" +) + +type node struct { + Required []string `json:"required"` + Properties map[string]*node `json:"properties"` + Items *node `json:"items"` + Enum []string `json:"enum"` +} + +func load(t *testing.T, name string) *node { + t.Helper() + b, err := fs.ReadFile(schema.FS, name) + if err != nil { + t.Fatal(err) + } + var n node + if err := json.Unmarshal(b, &n); err != nil { + t.Fatalf("%s: %v", name, err) + } + return &n +} + +// conforms checks that every key of v is declared in n and every required key is present, +// recursing into objects and arrays of objects. +func conforms(t *testing.T, path string, n *node, v any) { + t.Helper() + switch v := v.(type) { + case map[string]any: + for _, r := range n.Required { + if _, ok := v[r]; !ok { + t.Errorf("%s: required %q missing", path, r) + } + } + for k, sub := range v { + p, ok := n.Properties[k] + if !ok { + t.Errorf("%s: %q is not in the schema", path, k) + continue + } + conforms(t, path+"."+k, p, sub) + } + case []any: + if n.Items != nil { + for _, sub := range v { + conforms(t, path+"[]", n.Items, sub) + } + } + } +} + +func roundTrip(t *testing.T, v any) any { + b, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + var out any + json.Unmarshal(b, &out) + return out +} + +func sorted(s []string) []string { + s = append([]string(nil), s...) + sort.Strings(s) + return s +} + +func TestPackageSchema(t *testing.T) { + n := load(t, "package.schema.json") + full := bulk.Package{ + Distro: model.Debian, Suite: "sid", Name: "json-c", Version: "0.18+ds-1", OSVEcosystem: "Debian:sid", + PURL: "pkg:deb/debian/json-c?arch=source", Binaries: []string{"libjson-c5"}, Homepage: "https://github.com/json-c/json-c/wiki", + UpstreamKey: "github.com/json-c/json-c", Repo: "https://github.com/json-c/json-c", Status: bulk.StatusResolved, + Basis: bulk.BasisMetadata, + Evidence: []model.Evidence{{Source: "debian/upstream/metadata:Repository", Value: "https://github.com/json-c/json-c.git", Key: "github.com/json-c/json-c", Weight: 0.95}}, + SourceSHA256: []string{"0000000000000000000000000000000000000000000000000000000000000000"}, + } + conforms(t, "package", n, roundTrip(t, full)) + conforms(t, "package", n, roundTrip(t, bulk.Package{Distro: model.Arch, Suite: "extra", Name: "x", Version: "1-1", Status: bulk.StatusUnresolved})) + + var distros []string + for _, d := range model.AllDistros { + distros = append(distros, string(d)) + } + if got := sorted(n.Properties["distro"].Enum); !equal(got, sorted(distros)) { + t.Errorf("distro enum %v, want %v", got, distros) + } + bases := []string{bulk.BasisMetadata, bulk.BasisVCSSource, bulk.BasisReleaseURL, bulk.BasisDerived, bulk.BasisBugTracker, bulk.BasisHomepage} + if got := sorted(n.Properties["basis"].Enum); !equal(got, sorted(bases)) { + t.Errorf("basis enum %v, want %v (a secondary source never decides a mapping)", got, bases) + } + if got := sorted(n.Properties["status"].Enum); !equal(got, sorted([]string{bulk.StatusResolved, bulk.StatusUnresolved})) { + t.Errorf("status enum %v", got) + } +} + +func TestManifestSchema(t *testing.T) { + n := load(t, "manifest.schema.json") + m := bulk.Manifest{ + Format: bulk.ManifestFormat, Snapshot: "20260928-0123456789ab", Created: time.Now(), + Slices: []bulk.SliceFile{{Distro: "debian", Suite: "sid", Origin: "udd", FetchedAt: time.Now(), Packages: 2, Resolved: 1, + Path: "debian-sid.jsonl", SHA256: "00"}}, + Database: bulk.DatabaseFile{Path: "packages.sqlite", SHA256: "00", SchemaVersion: 2}, + Schemas: bulk.ManifestFiles{Package: "package.schema.json", Manifest: "manifest.schema.json"}, + } + conforms(t, "manifest", n, roundTrip(t, m)) +} + +func equal(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} -- 2.51.2