Something went wrong. Try again.
[READ-ONLY] Mirror of https://github.com/nperez0111/bookhive. Track your books, share your shelves, see what others are reading bookhive.buzz
atproto bluesky books bookshelf goodreads management-system
Something went wrong. Try again.
TypeScript
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489import { spawn } from "node:child_process";import crypto from "node:crypto";import fs from "node:fs";import fsp from "node:fs/promises";import path from "node:path";import { Database as DatabaseSync } from "bun:sqlite";import { Readable } from "node:stream";
type ExportResult = { archivePath: string; filename: string; tmpDir: string };
type ExportManifest = { createdAt: string; version: string; files: Array<{ name: string; md5: string; size: number }>; excludedKvTables: string[]; schema?: { tables: string[]; views: string[] };};
function toError(err: unknown): Error { return err instanceof Error ? err : new Error(String(err));}
function timingSafeEqualString(a: string, b: string): boolean { const aBuf = Buffer.from(a); const bBuf = Buffer.from(b); if (aBuf.length !== bBuf.length) { crypto.timingSafeEqual(aBuf, aBuf); return false; } return crypto.timingSafeEqual(aBuf, bBuf);}
export function isAuthorizedExportRequest(opts: { authorizationHeader?: string; sharedSecret: string;}) { const { authorizationHeader, sharedSecret } = opts; if (!sharedSecret) return false; if (!authorizationHeader) return false; const match = authorizationHeader.match(/^Bearer\s+(.+)$/i); if (!match) return false; return timingSafeEqualString(match[1]!, sharedSecret);}
async function sqliteBackup({ sourcePath, destPath }: { sourcePath: string; destPath: string }) { const db = new DatabaseSync(sourcePath); try { db.exec("PRAGMA busy_timeout = 5000"); // VACUUM INTO copies the database to a new file (SQLite 3.27+) const escaped = path.resolve(destPath).replace(/'/g, "''"); db.exec(`VACUUM INTO '${escaped}'`); } finally { db.close(); }}
function shouldExcludeTable(name: string): boolean { return name === "auth_sessions" || name === "auth_state" || name.startsWith("auth_");}
async function createSanitizedKvCopy({ sourcePath, destPath,}: { sourcePath: string; destPath: string;}) { const src = new DatabaseSync(sourcePath, { readonly: true }); const dest = new DatabaseSync(destPath);
try { src.exec("PRAGMA busy_timeout = 10000"); dest.exec("PRAGMA busy_timeout = 5000");
const objects = src .prepare( ` SELECT type, name, tbl_name as tblName, sql FROM sqlite_master WHERE sql IS NOT NULL ORDER BY CASE type WHEN 'table' THEN 0 WHEN 'index' THEN 1 WHEN 'trigger' THEN 2 WHEN 'view' THEN 3 ELSE 4 END, name `, ) .all() as Array<{ type: string; name: string; tblName: string; sql: string; }>;
const tablesToCopy: Array<{ name: string; colList: string; colNames: string[]; quotedTable: string; }> = []; const tableSql: string[] = []; const otherSql: string[] = [];
for (const obj of objects) { const name = obj.name; const tbl = obj.tblName; if (obj.type === "table") { if (name.startsWith("sqlite_")) continue; if (shouldExcludeTable(name)) continue; tableSql.push(obj.sql); const quotedTable = `"${name.replace(/"/g, '""')}"`; const cols = src.prepare(`PRAGMA table_info(${quotedTable})`).all() as Array<{ name: string; }>; if (cols.length === 0) { throw new Error(`Failed to retrieve column info for table: ${name}`); } tablesToCopy.push({ name, colList: cols.map((c) => `"${c.name}"`).join(", "), colNames: cols.map((c) => c.name), quotedTable, }); continue; } if (tbl && shouldExcludeTable(tbl)) continue; otherSql.push(obj.sql); }
dest.exec("BEGIN IMMEDIATE"); try { for (const sql of tableSql) { dest.exec(sql); }
// Copy data by reading from src and inserting into dest (no ATTACH — // the app may have the source DB open, which would lock it) for (const { name, colList, colNames, quotedTable } of tablesToCopy) { const rows = src.prepare(`SELECT ${colList} FROM ${quotedTable}`).all() as Record< string, unknown >[]; if (rows.length === 0) continue; const placeholders = colNames.map(() => "?").join(", "); const safeName = `"${name.replace(/"/g, '""')}"`; const insert = dest.prepare( `INSERT INTO main.${safeName} (${colList}) VALUES (${placeholders})`, ); for (const row of rows) { insert.run(...colNames.map((col) => row[col] as string | number | bigint | null)); } }
for (const sql of otherSql) { dest.exec(sql); }
dest.exec("COMMIT"); } catch (e) { dest.exec("ROLLBACK"); throw new Error(`Failed to create sanitized KV copy: ${toError(e).message}`); }
dest.exec("VACUUM"); } finally { src.close(); dest.close(); }}
function createTgz(cwd: string, outputFile: string, files: string[]): Promise<void> { return new Promise((resolve, reject) => { const proc = spawn("tar", ["-czf", outputFile, "-C", cwd, ...files], { stdio: ["ignore", "ignore", "pipe"], }); const stderr: string[] = []; proc.stderr.on("data", (d) => stderr.push(d.toString())); proc.on("error", reject); proc.on("close", (code) => code === 0 ? resolve() : reject(new Error(`tar exited ${code}: ${stderr.join("").trim() || "no stderr"}`)), ); });}
export function createTgzReadStream( cwd: string, files: string[], callbacks?: { onClose?: () => void; onError?: (err: Error) => void },): ReadableStream<Uint8Array> { const proc = spawn("tar", ["-czf", "-", "-C", cwd, ...files], { stdio: ["ignore", "pipe", "pipe"], }); const stderr: string[] = []; if (proc.stderr) { proc.stderr.on("data", (d: Buffer) => stderr.push(d.toString())); } proc.on("error", (err: Error) => callbacks?.onError?.(err)); proc.on("close", (code: number | null) => { if (code === 0) { callbacks?.onClose?.(); } else { callbacks?.onError?.( new Error(`tar exited ${code}: ${stderr.join("").trim() || "no stderr"}`), ); } }); return Readable.toWeb(proc.stdout!) as unknown as ReadableStream<Uint8Array>;}
function computeFileMd5(filePath: string): string { const content = fs.readFileSync(filePath); return crypto.createHash("md5").update(content).digest("hex");}
async function getFileStats(filePath: string): Promise<{ md5: string; size: number }> { const { size } = await fsp.stat(filePath); return { md5: computeFileMd5(filePath), size };}
type PrepareResult = { tmpDir: string; filename: string; files: string[];};
export async function prepareSanitizedExportFiles(opts: { dbPath: string; kvPath?: string; exportDir: string; includeKv: boolean;}): Promise<PrepareResult> { const { dbPath, kvPath, exportDir, includeKv } = opts;
await cleanupStaleExports(exportDir);
const now = new Date(); const stamp = now.toISOString().replace(/[:.]/g, "-"); const runId = crypto.randomUUID(); const tmpDir = path.join(exportDir, `bookhive-export-${stamp}-${runId}`);
try { await fsp.mkdir(tmpDir, { recursive: true }); } catch (err) { throw new Error( `Failed to create export temporary directory at ${tmpDir}: ${toError(err).message}`, ); }
const filename = `bookhive-export-${stamp}-${runId.slice(0, 8)}.tgz`;
try { const dbOut = path.join(tmpDir, "db.sqlite"); try { await sqliteBackup({ sourcePath: dbPath, destPath: dbOut }); } catch (err) { throw new Error(`Failed to backup main database from ${dbPath}: ${toError(err).message}`); }
const includedFiles: ExportManifest["files"] = []; const tables: string[] = []; const views: string[] = [];
const dbStats = await getFileStats(dbOut); includedFiles.push({ name: "db.sqlite", md5: dbStats.md5, size: dbStats.size });
try { const db = new DatabaseSync(dbOut, { readonly: true }); try { const schemaObjects = db .prepare( `SELECT type, name FROM sqlite_master WHERE type IN ('table', 'view') AND name NOT LIKE 'sqlite_%' ORDER BY type, name`, ) .all() as Array<{ type: string; name: string }>; for (const obj of schemaObjects) { (obj.type === "table" ? tables : views).push(obj.name); } } finally { db.close(); } } catch (err) { throw new Error( `Failed to extract schema information from main database: ${toError(err).message}`, ); }
if (includeKv && kvPath) { const kvOut = path.join(tmpDir, "kv.sqlite"); try { await createSanitizedKvCopy({ sourcePath: kvPath, destPath: kvOut }); } catch (err) { throw new Error( `Failed to create sanitized KV copy from ${kvPath}: ${toError(err).message}`, ); } const kvStats = await getFileStats(kvOut); includedFiles.push({ name: "kv.sqlite", md5: kvStats.md5, size: kvStats.size }); }
const manifest: ExportManifest = { createdAt: now.toISOString(), version: "1.0", files: includedFiles, excludedKvTables: ["auth_sessions", "auth_state"], schema: { tables, views }, };
const manifestPath = path.join(tmpDir, "manifest.json"); try { await fsp.writeFile(manifestPath, JSON.stringify(manifest, null, 2) + "\n", "utf8"); } catch (err) { throw new Error(`Failed to write manifest file: ${toError(err).message}`); }
const manifestStats = await getFileStats(manifestPath); includedFiles.push({ name: "manifest.json", md5: manifestStats.md5, size: manifestStats.size });
return { tmpDir, filename, files: includedFiles.map((f) => f.name) }; } catch (err) { await cleanupExportPaths({ tmpDir }); throw err; }}
export async function createSanitizedExportArchive(opts: { dbPath: string; kvPath?: string; exportDir: string; includeKv: boolean;}): Promise<ExportResult> { const { dbPath, kvPath, exportDir, includeKv } = opts;
await cleanupStaleExports(exportDir);
const now = new Date(); const stamp = now.toISOString().replace(/[:.]/g, "-"); const runId = crypto.randomUUID(); const tmpDir = path.join(exportDir, `bookhive-export-${stamp}-${runId}`);
try { await fsp.mkdir(tmpDir, { recursive: true }); } catch (err) { throw new Error( `Failed to create export temporary directory at ${tmpDir}: ${toError(err).message}`, ); }
const filename = `bookhive-export-${stamp}-${runId.slice(0, 8)}.tgz`; const archivePath = path.join(exportDir, filename);
try { // Backup main database const dbOut = path.join(tmpDir, "db.sqlite"); try { await sqliteBackup({ sourcePath: dbPath, destPath: dbOut }); } catch (err) { throw new Error(`Failed to backup main database from ${dbPath}: ${toError(err).message}`); }
const includedFiles: ExportManifest["files"] = []; const tables: string[] = []; const views: string[] = [];
// Add db.sqlite file info const dbStats = await getFileStats(dbOut); includedFiles.push({ name: "db.sqlite", md5: dbStats.md5, size: dbStats.size, });
// Extract schema info from main database try { const db = new DatabaseSync(dbOut, { readonly: true }); try { const schemaObjects = db .prepare( `SELECT type, name FROM sqlite_master WHERE type IN ('table', 'view') AND name NOT LIKE 'sqlite_%' ORDER BY type, name`, ) .all() as Array<{ type: string; name: string }>;
for (const obj of schemaObjects) { (obj.type === "table" ? tables : views).push(obj.name); } } finally { db.close(); } } catch (err) { throw new Error( `Failed to extract schema information from main database: ${toError(err).message}`, ); }
// Handle KV database if (includeKv && kvPath) { const kvOut = path.join(tmpDir, "kv.sqlite"); try { await createSanitizedKvCopy({ sourcePath: kvPath, destPath: kvOut }); } catch (err) { throw new Error( `Failed to create sanitized KV copy from ${kvPath}: ${toError(err).message}`, ); }
const kvStats = await getFileStats(kvOut); includedFiles.push({ name: "kv.sqlite", md5: kvStats.md5, size: kvStats.size, }); }
// Create and add manifest const manifest: ExportManifest = { createdAt: now.toISOString(), version: "1.0", files: includedFiles, excludedKvTables: ["auth_sessions", "auth_state"], schema: { tables, views, }, };
const manifestPath = path.join(tmpDir, "manifest.json"); try { await fsp.writeFile(manifestPath, JSON.stringify(manifest, null, 2) + "\n", "utf8"); } catch (err) { throw new Error(`Failed to write manifest file: ${toError(err).message}`); }
const manifestStats = await getFileStats(manifestPath); includedFiles.push({ name: "manifest.json", md5: manifestStats.md5, size: manifestStats.size, });
// Create archive try { await createTgz( tmpDir, archivePath, includedFiles.map((f) => f.name), ); } catch (err) { throw new Error(`Failed to create tar archive: ${toError(err).message}`); }
return { archivePath, filename, tmpDir }; } catch (err) { // Clean up on error await cleanupExportPaths({ archivePath, tmpDir }); throw err; }}
async function cleanupStaleExports(exportDir: string): Promise<void> { let entries: string[]; try { entries = await fsp.readdir(exportDir); } catch { return; // exportDir doesn't exist yet — nothing to clean } const stale = entries.filter((e) => e.startsWith("bookhive-export-")); await Promise.allSettled( stale.map((e) => fsp.rm(path.join(exportDir, e), { recursive: true, force: true })), );}
export async function cleanupExportPaths(paths: { archivePath?: string; tmpDir?: string;}): Promise<void> { const promises: Promise<unknown>[] = []; if (paths.archivePath) promises.push(fsp.rm(paths.archivePath, { force: true })); if (paths.tmpDir) promises.push(fsp.rm(paths.tmpDir, { recursive: true, force: true })); await Promise.allSettled(promises);}
export function createExportReadStream( filePath: string, callbacks?: { onClose?: () => void; onError?: (err: Error) => void },): ReadableStream<Uint8Array> { const stream = fs.createReadStream(filePath, { highWaterMark: 64 * 1024 }); stream.on("close", () => callbacks?.onClose?.()); stream.on("error", (err) => callbacks?.onError?.(err)); return Readable.toWeb(stream) as unknown as ReadableStream<Uint8Array>;}