Something went wrong. Try again.
Maps Linux distribution source packages to their upstream repositories, and through them to each other
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189package bulk
import ( "context" "database/sql" "encoding/json" "fmt" "os"
_ "modernc.org/sqlite" // pure-Go driver, registered as "sqlite"
"tangled.org/odd.computer/isomorph/internal/model")
// schema is the published SQLite layout. Rows for one (distro, suite) are replaced as a unit,// so each slice can be refreshed independently.const schema = `create table if not exists sources ( distro text not null, suite text not null, origin text not null, fetched_at text not null, packages integer not null, primary key (distro, suite));create table if not exists packages ( distro text not null, suite text not null, package text not null, version text not null, osv_ecosystem text, purl text, homepage text, upstream_key text, repo text, status text not null, basis text, primary key (distro, suite, package));create index if not exists packages_upstream on packages (upstream_key);create table if not exists evidence ( distro text not null, suite text not null, package text not null, source text not null, value text not null, key text, weight real not null);create index if not exists evidence_pkg on evidence (distro, suite, package);create table if not exists binaries ( distro text not null, suite text not null, binary text not null, package text not null);create index if not exists binaries_name on binaries (binary);create table if not exists source_sha256 ( distro text not null, suite text not null, package text not null, sha256 text not null);create index if not exists source_sha256_hash on source_sha256 (sha256);create table if not exists aliases ( from_key text primary key, to_key text not null, source text not null, checked_at text not null);create view if not exists counterparts as select a.distro, a.suite, a.package, a.upstream_key, b.distro as other_distro, b.suite as other_suite, b.package as other_package from packages a join packages b on a.upstream_key = b.upstream_key and not (a.distro = b.distro and a.suite = b.suite and a.package = b.package) where a.upstream_key is not null and a.upstream_key != '';`
// schemaVersion is stored in SQLite's user_version; bump it whenever schema changes shape.const schemaVersion = 2
// OpenReadOnly opens an existing database for reading only, so lookups and exports neither// need write access nor wait for a writer's lock. It refuses another schema version, as Open// does.func OpenReadOnly(ctx context.Context, path string) (*sql.DB, error) { if _, err := os.Stat(path); err != nil { return nil, err } db, err := sql.Open("sqlite", "file:"+path+"?mode=ro") if err != nil { return nil, err } var version int if err := db.QueryRowContext(ctx, `pragma user_version`).Scan(&version); err != nil { db.Close() return nil, err } if version != schemaVersion { db.Close() return nil, fmt.Errorf("bulk: %s has schema version %d, want %d; rebuild it", path, version, schemaVersion) } return db, nil}
// Open opens (creating if needed) the SQLite database at path. A database written with another// schema version is refused rather than silently mixed: the file is derived, so rebuild it.func Open(ctx context.Context, path string) (*sql.DB, error) { db, err := sql.Open("sqlite", path) if err != nil { return nil, err } var version, tables int if err := db.QueryRowContext(ctx, `pragma user_version`).Scan(&version); err != nil { db.Close() return nil, err } if err := db.QueryRowContext(ctx, `select count(*) from sqlite_master where type = 'table'`).Scan(&tables); err != nil { db.Close() return nil, err } if tables > 0 && version != schemaVersion { db.Close() return nil, fmt.Errorf("bulk: %s has schema version %d, want %d; delete it and rebuild", path, version, schemaVersion) } if _, err := db.ExecContext(ctx, fmt.Sprintf("pragma user_version = %d", schemaVersion)); err != nil { db.Close() return nil, err } if _, err := db.ExecContext(ctx, schema); err != nil { db.Close() return nil, fmt.Errorf("bulk: schema: %w", err) } return db, nil}
// Replace writes pkgs as the complete (src.Distro, src.Suite) slice, removing what was there.func Replace(ctx context.Context, db *sql.DB, src Source, pkgs []Package) error { tx, err := db.BeginTx(ctx, nil) if err != nil { return err } defer tx.Rollback() for _, t := range []string{"packages", "evidence", "binaries", "source_sha256"} { if _, err := tx.ExecContext(ctx, "delete from "+t+" where distro = ? and suite = ?", src.Distro, src.Suite); err != nil { return err } } ins := func(q string) (*sql.Stmt, error) { return tx.PrepareContext(ctx, q) } pkgStmt, err := ins(`insert into packages values (?,?,?,?,?,?,?,?,?,?,?)`) if err != nil { return err } evStmt, err := ins(`insert into evidence values (?,?,?,?,?,?,?)`) if err != nil { return err } binStmt, err := ins(`insert into binaries values (?,?,?,?)`) if err != nil { return err } shaStmt, err := ins(`insert into source_sha256 values (?,?,?,?)`) if err != nil { return err } for _, p := range pkgs { if _, err := pkgStmt.ExecContext(ctx, p.Distro, p.Suite, p.Name, p.Version, p.OSVEcosystem, p.PURL, p.Homepage, p.UpstreamKey, p.Repo, p.Status, p.Basis); err != nil { return fmt.Errorf("bulk: %s/%s: %w", p.Distro, p.Name, err) } for _, e := range p.Evidence { if _, err := evStmt.ExecContext(ctx, p.Distro, p.Suite, p.Name, e.Source, e.Value, e.Key, e.Weight); err != nil { return err } } for _, b := range p.Binaries { if _, err := binStmt.ExecContext(ctx, p.Distro, p.Suite, b, p.Name); err != nil { return err } } for _, h := range p.SourceSHA256 { if _, err := shaStmt.ExecContext(ctx, p.Distro, p.Suite, p.Name, h); err != nil { return err } } } if _, err := tx.ExecContext(ctx, `insert or replace into sources values (?,?,?,?,?)`, src.Distro, src.Suite, src.Origin, src.FetchedAt.UTC().Format("2006-01-02T15:04:05Z"), len(pkgs)); err != nil { return err } return tx.Commit()}
func loadEvidence(ctx context.Context, db *sql.DB, p *Package) error { rows, err := db.QueryContext(ctx, `select source, value, coalesce(key,''), weight from evidence where distro = ? and suite = ? and package = ? order by weight desc`, p.Distro, p.Suite, p.Name) if err != nil { return err } defer rows.Close() for rows.Next() { var e model.Evidence if err := rows.Scan(&e.Source, &e.Value, &e.Key, &e.Weight); err != nil { return err } p.Evidence = append(p.Evidence, e) } return rows.Err()}
// WriteJSONL writes one JSON object per package.func WriteJSONL(w interface{ Write([]byte) (int, error) }, pkgs []Package) error { enc := json.NewEncoder(w) for _, p := range pkgs { if err := enc.Encode(p); err != nil { return err } } return nil}