Something went wrong. Try again.
Maps Linux distribution source packages to their upstream repositories, and through them to each other
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196package bulk
import ( "context" "crypto/sha256" "database/sql" "encoding/hex" "encoding/json" "fmt" "io" "io/fs" "os" "path/filepath" "time"
schemas "tangled.org/odd.computer/isomorph/schema")
// An export snapshot is a directory holding one JSONL file per (distro, suite) slice, a copy// of the SQLite database, the JSON Schemas, and manifest.json listing every file with its// sha256. The JSONL files are the canonical, diffable record; the database is a convenience// built from the same run. The snapshot id is the UTC date and a hash of the slice files, so// the same content exported on the same day gets the same id.
// ManifestFormat versions the manifest and the row format.const ManifestFormat = 1
// Manifest is a snapshot's manifest.json (schema/manifest.schema.json).type Manifest struct { Format int `json:"format"` Snapshot string `json:"snapshot"` Created time.Time `json:"created"` Slices []SliceFile `json:"slices"` Database DatabaseFile `json:"database"` Schemas ManifestFiles `json:"schemas"`}
// SliceFile describes one slice's JSONL file.type SliceFile struct { Distro string `json:"distro"` Suite string `json:"suite"` Origin string `json:"origin"` FetchedAt time.Time `json:"fetched_at"` Packages int `json:"packages"` Resolved int `json:"resolved"` Path string `json:"path"` SHA256 string `json:"sha256"`}
// DatabaseFile describes the snapshot's SQLite database.type DatabaseFile struct { Path string `json:"path"` SHA256 string `json:"sha256"` SchemaVersion int `json:"schema_version"`}
// ManifestFiles names the schema files in the snapshot.type ManifestFiles struct { Package string `json:"package"` Manifest string `json:"manifest"`}
// Export writes a snapshot of db into outDir/<snapshot>/ and points outDir/LATEST at it. It// returns the manifest and the snapshot directory. An identical snapshot that already exists// is kept as is.func Export(ctx context.Context, db *sql.DB, outDir string, now time.Time) (*Manifest, string, error) { if err := os.MkdirAll(outDir, 0o755); err != nil { return nil, "", err } tmp, err := os.MkdirTemp(outDir, ".export-") if err != nil { return nil, "", err } defer os.RemoveAll(tmp) m := &Manifest{ Format: ManifestFormat, Created: now.UTC().Truncate(time.Second), Schemas: ManifestFiles{Package: "package.schema.json", Manifest: "manifest.schema.json"}, } srcs, err := Sources(ctx, db) if err != nil { return nil, "", err } id := sha256.New() for _, s := range srcs { pkgs, err := LoadSlice(ctx, db, s.Distro, s.Suite) if err != nil { return nil, "", err } sf := SliceFile{Distro: string(s.Distro), Suite: s.Suite, Origin: s.Origin, FetchedAt: s.FetchedAt.UTC(), Packages: len(pkgs), Path: string(s.Distro) + "-" + s.Suite + ".jsonl"} for _, p := range pkgs { if p.Status == StatusResolved { sf.Resolved++ } } if sf.SHA256, err = writeFile(filepath.Join(tmp, sf.Path), func(w io.Writer) error { return WriteJSONL(w, pkgs) }); err != nil { return nil, "", err } fmt.Fprintf(id, "%s %s\n", sf.Path, sf.SHA256) m.Slices = append(m.Slices, sf) } m.Snapshot = m.Created.Format("20060102") + "-" + hex.EncodeToString(id.Sum(nil))[:12]
m.Database = DatabaseFile{Path: "packages.sqlite", SchemaVersion: schemaVersion} dbPath := filepath.Join(tmp, m.Database.Path) if _, err := db.ExecContext(ctx, `vacuum into ?`, dbPath); err != nil { return nil, "", fmt.Errorf("export: copy database: %w", err) } if m.Database.SHA256, err = hashFile(dbPath); err != nil { return nil, "", err } for _, name := range []string{m.Schemas.Package, m.Schemas.Manifest} { b, err := fs.ReadFile(schemas.FS, name) if err != nil { return nil, "", err } if err := os.WriteFile(filepath.Join(tmp, name), b, 0o644); err != nil { return nil, "", err } } if _, err := writeFile(filepath.Join(tmp, "manifest.json"), func(w io.Writer) error { enc := json.NewEncoder(w) enc.SetIndent("", " ") return enc.Encode(m) }); err != nil { return nil, "", err }
if err := os.Chmod(tmp, 0o755); err != nil { // MkdirTemp makes it private return nil, "", err } dir := filepath.Join(outDir, m.Snapshot) if _, err := os.Stat(dir); err == nil { // Same content, same day: the id already names it. } else if err := os.Rename(tmp, dir); err != nil { return nil, "", err } if err := os.WriteFile(filepath.Join(outDir, "LATEST"), []byte(m.Snapshot+"\n"), 0o644); err != nil { return nil, "", err } return m, dir, nil}
// Sources lists the slices in db with where they came from, in (distro, suite) order.func Sources(ctx context.Context, db *sql.DB) ([]Source, error) { rows, err := db.QueryContext(ctx, `select distro, suite, origin, fetched_at, packages from sources order by distro, suite`) if err != nil { return nil, err } defer rows.Close() var out []Source for rows.Next() { var s Source var at string if err := rows.Scan(&s.Distro, &s.Suite, &s.Origin, &at, &s.Packages); err != nil { return nil, err } if s.FetchedAt, err = time.Parse(time.RFC3339, at); err != nil { return nil, fmt.Errorf("sources: %s/%s fetched_at %q: %w", s.Distro, s.Suite, at, err) } out = append(out, s) } return out, rows.Err()}
// writeFile writes path through fill and returns the sha256 of what was written.func writeFile(path string, fill func(io.Writer) error) (string, error) { f, err := os.Create(path) if err != nil { return "", err } h := sha256.New() if err := fill(io.MultiWriter(f, h)); err != nil { f.Close() return "", err } if err := f.Close(); err != nil { return "", err } return hex.EncodeToString(h.Sum(nil)), nil}
func hashFile(path string) (string, error) { f, err := os.Open(path) if err != nil { return "", err } defer f.Close() h := sha256.New() if _, err := io.Copy(h, f); err != nil { return "", err } return hex.EncodeToString(h.Sum(nil)), nil}