package bulk import ( "context" "crypto/sha256" "database/sql" "encoding/hex" "encoding/json" "fmt" "io" "io/fs" "os" "path/filepath" "time" schemas "tangled.org/odd.computer/isomorph/schema" ) // An export snapshot is a directory holding one JSONL file per (distro, suite) slice, a copy // of the SQLite database, the JSON Schemas, and manifest.json listing every file with its // sha256. The JSONL files are the canonical, diffable record; the database is a convenience // built from the same run. The snapshot id is the UTC date and a hash of the slice files, so // the same content exported on the same day gets the same id. // ManifestFormat versions the manifest and the row format. const ManifestFormat = 1 // Manifest is a snapshot's manifest.json (schema/manifest.schema.json). type Manifest struct { Format int `json:"format"` Snapshot string `json:"snapshot"` Created time.Time `json:"created"` Slices []SliceFile `json:"slices"` Database DatabaseFile `json:"database"` Schemas ManifestFiles `json:"schemas"` } // SliceFile describes one slice's JSONL file. type SliceFile struct { Distro string `json:"distro"` Suite string `json:"suite"` Origin string `json:"origin"` FetchedAt time.Time `json:"fetched_at"` Packages int `json:"packages"` Resolved int `json:"resolved"` Path string `json:"path"` SHA256 string `json:"sha256"` } // DatabaseFile describes the snapshot's SQLite database. type DatabaseFile struct { Path string `json:"path"` SHA256 string `json:"sha256"` SchemaVersion int `json:"schema_version"` } // ManifestFiles names the schema files in the snapshot. type ManifestFiles struct { Package string `json:"package"` Manifest string `json:"manifest"` } // Export writes a snapshot of db into outDir// and points outDir/LATEST at it. It // returns the manifest and the snapshot directory. An identical snapshot that already exists // is kept as is. func Export(ctx context.Context, db *sql.DB, outDir string, now time.Time) (*Manifest, string, error) { if err := os.MkdirAll(outDir, 0o755); err != nil { return nil, "", err } tmp, err := os.MkdirTemp(outDir, ".export-") if err != nil { return nil, "", err } defer os.RemoveAll(tmp) m := &Manifest{ Format: ManifestFormat, Created: now.UTC().Truncate(time.Second), Schemas: ManifestFiles{Package: "package.schema.json", Manifest: "manifest.schema.json"}, } srcs, err := Sources(ctx, db) if err != nil { return nil, "", err } id := sha256.New() for _, s := range srcs { pkgs, err := LoadSlice(ctx, db, s.Distro, s.Suite) if err != nil { return nil, "", err } sf := SliceFile{Distro: string(s.Distro), Suite: s.Suite, Origin: s.Origin, FetchedAt: s.FetchedAt.UTC(), Packages: len(pkgs), Path: string(s.Distro) + "-" + s.Suite + ".jsonl"} for _, p := range pkgs { if p.Status == StatusResolved { sf.Resolved++ } } if sf.SHA256, err = writeFile(filepath.Join(tmp, sf.Path), func(w io.Writer) error { return WriteJSONL(w, pkgs) }); err != nil { return nil, "", err } fmt.Fprintf(id, "%s %s\n", sf.Path, sf.SHA256) m.Slices = append(m.Slices, sf) } m.Snapshot = m.Created.Format("20060102") + "-" + hex.EncodeToString(id.Sum(nil))[:12] m.Database = DatabaseFile{Path: "packages.sqlite", SchemaVersion: schemaVersion} dbPath := filepath.Join(tmp, m.Database.Path) if _, err := db.ExecContext(ctx, `vacuum into ?`, dbPath); err != nil { return nil, "", fmt.Errorf("export: copy database: %w", err) } if m.Database.SHA256, err = hashFile(dbPath); err != nil { return nil, "", err } for _, name := range []string{m.Schemas.Package, m.Schemas.Manifest} { b, err := fs.ReadFile(schemas.FS, name) if err != nil { return nil, "", err } if err := os.WriteFile(filepath.Join(tmp, name), b, 0o644); err != nil { return nil, "", err } } if _, err := writeFile(filepath.Join(tmp, "manifest.json"), func(w io.Writer) error { enc := json.NewEncoder(w) enc.SetIndent("", " ") return enc.Encode(m) }); err != nil { return nil, "", err } if err := os.Chmod(tmp, 0o755); err != nil { // MkdirTemp makes it private return nil, "", err } dir := filepath.Join(outDir, m.Snapshot) if _, err := os.Stat(dir); err == nil { // Same content, same day: the id already names it. } else if err := os.Rename(tmp, dir); err != nil { return nil, "", err } if err := os.WriteFile(filepath.Join(outDir, "LATEST"), []byte(m.Snapshot+"\n"), 0o644); err != nil { return nil, "", err } return m, dir, nil } // Sources lists the slices in db with where they came from, in (distro, suite) order. func Sources(ctx context.Context, db *sql.DB) ([]Source, error) { rows, err := db.QueryContext(ctx, `select distro, suite, origin, fetched_at, packages from sources order by distro, suite`) if err != nil { return nil, err } defer rows.Close() var out []Source for rows.Next() { var s Source var at string if err := rows.Scan(&s.Distro, &s.Suite, &s.Origin, &at, &s.Packages); err != nil { return nil, err } if s.FetchedAt, err = time.Parse(time.RFC3339, at); err != nil { return nil, fmt.Errorf("sources: %s/%s fetched_at %q: %w", s.Distro, s.Suite, at, err) } out = append(out, s) } return out, rows.Err() } // writeFile writes path through fill and returns the sha256 of what was written. func writeFile(path string, fill func(io.Writer) error) (string, error) { f, err := os.Create(path) if err != nil { return "", err } h := sha256.New() if err := fill(io.MultiWriter(f, h)); err != nil { f.Close() return "", err } if err := f.Close(); err != nil { return "", err } return hex.EncodeToString(h.Sum(nil)), nil } func hashFile(path string) (string, error) { f, err := os.Open(path) if err != nil { return "", err } defer f.Close() h := sha256.New() if _, err := io.Copy(h, f); err != nil { return "", err } return hex.EncodeToString(h.Sum(nil)), nil }