Something went wrong. Try again.
This repository has no description
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218import test from "node:test";import assert from "node:assert/strict";import { randomUUID } from "node:crypto";
// Run against an isolated local Odre instance, never production.// ODRE_TEST_KEEP_FIXTURES=1 leaves uniquely named materials for browser QA.// ODRE_TEST_BASE_URL=http://127.0.0.1:4119 rtk node --test scripts/reader-integration.test.mjsconst base = process.env.ODRE_TEST_BASE_URL;const target = base ? new URL(base) : null;if (target && !["127.0.0.1", "localhost", "[::1]"].includes(target.hostname)) { throw new Error("Reader integration checks require a loopback API URL.");}
const crcTable = Array.from({ length: 256 }, (_, byte) => { for (let bit = 0; bit < 8; bit++) byte = byte & 1 ? 0xedb88320 ^ (byte >>> 1) : byte >>> 1; return byte >>> 0;});function crc32(bytes) { let value = 0xffffffff; for (const byte of bytes) value = crcTable[(value ^ byte) & 255] ^ (value >>> 8); return (value ^ 0xffffffff) >>> 0;}function zip(files) { const local = [], central = []; let offset = 0; for (const [path, content] of Object.entries(files)) { const name = Buffer.from(path), bytes = Buffer.from(content), checksum = crc32(bytes); const header = Buffer.alloc(30); header.writeUInt32LE(0x04034b50, 0); header.writeUInt16LE(20, 4); header.writeUInt32LE(checksum, 14); header.writeUInt32LE(bytes.length, 18); header.writeUInt32LE(bytes.length, 22); header.writeUInt16LE(name.length, 26); local.push(header, name, bytes); const entry = Buffer.alloc(46); entry.writeUInt32LE(0x02014b50, 0); entry.writeUInt16LE(20, 4); entry.writeUInt16LE(20, 6); entry.writeUInt32LE(checksum, 16); entry.writeUInt32LE(bytes.length, 20); entry.writeUInt32LE(bytes.length, 24); entry.writeUInt16LE(name.length, 28); entry.writeUInt32LE(offset, 42); central.push(entry, name); offset += header.length + name.length + bytes.length; } const directory = Buffer.concat(central), end = Buffer.alloc(22); end.writeUInt32LE(0x06054b50, 0); end.writeUInt16LE(central.length / 2, 8); end.writeUInt16LE(central.length / 2, 10); end.writeUInt32LE(directory.length, 12); end.writeUInt32LE(offset, 16); return Buffer.concat([...local, directory, end]);}async function unzip(bytes) { // Use Node's zlib for ZIP method 8; the server independently uses Rust zip. const { inflateRawSync } = await import("node:zlib"); const files = {}, buffer = Buffer.from(bytes); let end = buffer.length - 22; while (end >= Math.max(0, buffer.length - 65557) && buffer.readUInt32LE(end) !== 0x06054b50) end--; assert.ok(end >= 0, "ZIP end directory exists"); let offset = buffer.readUInt32LE(end + 16); const count = buffer.readUInt16LE(end + 10); for (let i = 0; i < count; i++) { assert.equal(buffer.readUInt32LE(offset), 0x02014b50); const method = buffer.readUInt16LE(offset + 10), compressedSize = buffer.readUInt32LE(offset + 20); const nameSize = buffer.readUInt16LE(offset + 28), extra = buffer.readUInt16LE(offset + 30), comment = buffer.readUInt16LE(offset + 32); const path = buffer.subarray(offset + 46, offset + 46 + nameSize).toString(); const localOffset = buffer.readUInt32LE(offset + 42); const dataOffset = localOffset + 30 + buffer.readUInt16LE(localOffset + 26) + buffer.readUInt16LE(localOffset + 28); const compressed = buffer.subarray(dataOffset, dataOffset + compressedSize); files[path] = method === 0 ? compressed : inflateRawSync(compressed); offset += 46 + nameSize + extra + comment; } return files;}function pdf() { const text = (lines) => `BT /F1 14 Tf 50 750 Td ${lines.map((line, i) => `${i ? "0 -25 Td " : ""}(${line}) Tj`).join("\n")} ET\n`; const one = text(["Reader QA First page", "Virtual addresses map to physical frames."]), two = text(["Reader QA Second page", "QaSearchPageTwo describes page offsets."]); const objects = [ "<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [4 0 R 6 0 R] /Count 2 >>", "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >> /Contents 5 0 R >>", `<< /Length ${Buffer.byteLength(one)} >>\nstream\n${one}endstream`, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >> /Contents 7 0 R >>", `<< /Length ${Buffer.byteLength(two)} >>\nstream\n${two}endstream`, ]; let content = "%PDF-1.4\n", offsets = [0]; for (let i = 0; i < objects.length; i++) { offsets.push(Buffer.byteLength(content)); content += `${i + 1} 0 obj\n${objects[i]}\nendobj\n`; } const xref = Buffer.byteLength(content); content += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n${offsets.slice(1).map((n) => `${String(n).padStart(10, "0")} 00000 n \n`).join("")}trailer\n<< /Size ${objects.length + 1} /Root 1 0 R >>\nstartxref\n${xref}\n%%EOF\n`; return Buffer.from(content);}async function json(path, { method = "GET", body, status = 200 } = {}) { const response = await fetch(base + path, { method, headers: body === undefined ? {} : { "Content-Type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) }); const text = await response.text(); assert.equal(response.status, status, `${method} ${path}: ${text.slice(0, 600)}`); return text ? JSON.parse(text) : null;}async function upload(bytes, filename, expected = 201) { const body = new FormData(); body.append("file", new Blob([bytes]), filename); const response = await fetch(base + "/api/reader/import", { method: "POST", body }); const text = await response.text(); assert.equal(response.status, expected, `import ${filename}: ${text.slice(0, 600)}`); return text ? JSON.parse(text) : null;}async function waitBook(id) { const deadline = Date.now() + 15000; do { const material = await json(`/api/reader/materials/${id}`); if (material.status !== "indexing") { assert.equal(material.status, "ready", material.error || "book indexing status"); return material; } await new Promise((resolve) => setTimeout(resolve, 50)); } while (Date.now() < deadline); assert.fail("PDF indexing did not finish within 15 seconds");}
test("reader preserves real PostgreSQL state, legacy formats and PDF behavior", { skip: !base && "Set ODRE_TEST_BASE_URL to an isolated local API before running integration checks." }, async (t) => { const label = `QA Reader ${randomUUID().slice(0, 8)}`, materialIds = new Set(), nativeIds = new Set(), assetIds = new Set(); const providersBefore = new Set((await json("/api/reader/providers")).map((p) => p.id)); const fixture = {}; const keep = process.env.ODRE_TEST_KEEP_FIXTURES === "1"; t.after(async () => { if (keep) { console.log("Reader browser fixtures:", JSON.stringify(fixture)); return; } for (const id of materialIds) await fetch(`${base}/api/reader/materials/${id}`, { method: "DELETE" }); for (const id of nativeIds) await fetch(`${base}/api/notes/${id}`, { method: "DELETE" }); for (const id of assetIds) await fetch(`${base}/api/assets/${id}`, { method: "DELETE" }); for (const p of await json("/api/reader/providers")) if (!providersBefore.has(p.id) && p.name?.includes(label)) await fetch(`${base}/api/reader/providers/${p.id}`, { method: "DELETE" }); }); const question = "Virtual page 3 maps to frame 8. Translate address 3 × 4096 + 96.", key = `Translate address\n${question}`; const body = `Virtual memory is a mapping between virtual addresses and physical frames.\n\n## Translation\nThe page offset remains unchanged. QaArticleUnique explains this invariant.\n\n$address = page \\times 4096 + offset$\n\n\n\n\`\`\`mermaid\nflowchart LR\n Address --> PageTable --> Frame\n\`\`\`\n\n\`\`\`exercise\n${JSON.stringify({ title: "Translate address", question, hint: "Preserve the offset.", solution: "8 × 4096 + 96 = 32864." }, null, 2)}\n\`\`\`\n`; const svg = Buffer.from('<svg xmlns="http://www.w3.org/2000/svg" width="440" height="100"><rect x="5" y="10" width="430" height="80" rx="8" fill="#f0f4f8"/><text x="30" y="57" font-size="20">Virtual page → Physical frame</text></svg>'); let article, book, native;
await t.test("lesson ZIP keeps Markdown and relative assets, unsafe archives roll back", async () => { article = await upload(zip({ "lesson.json": JSON.stringify({ version: 1, title: `${label} Article`, subject: "Operating Systems", contentFile: "content.md", sources: [{ title: "QA source", url: "https://example.org/qa", excerpt: "An original QA summary of page translation." }] }), "content.md": body, "assets/figure.svg": svg }), "lesson.zip"); materialIds.add(article.id); fixture.article = article.id; assert.equal(article.body, body); assert.equal(article.kind, "article"); const asset = await fetch(`${base}/api/reader/files/${article.id}/assets/figure.svg`); assert.equal(asset.status, 200); assert.equal(await asset.text(), svg.toString()); assert.match(asset.headers.get("content-security-policy"), /sandbox/); const exported = await unzip(await (await fetch(`${base}/api/reader/materials/${article.id}/export`)).arrayBuffer()); assert.equal(exported["content.md"].toString(), body); assert.deepEqual(exported["assets/figure.svg"], svg); const count = (await json("/api/reader/materials")).length; await upload(zip({ "lesson.json": JSON.stringify({ title: `${label} Invalid` }), "content.md": "safe", "../escape.png": "unsafe" }), "unsafe.zip", 400); assert.equal((await json("/api/reader/materials")).length, count); });
await t.test("CAS revisions preserve contents and position patches stay independent", async () => { const edited = await json(`/api/reader/materials/${article.id}`, { method: "PATCH", body: { expected_version: article.version, content: body + "\n## Added\nA temporary revision.\n" } }); assert.equal(edited.version, article.version + 1); await json(`/api/reader/materials/${article.id}`, { method: "PATCH", body: { expected_version: article.version, content: "Stale overwrite" }, status: 409 }); const positioned = await json(`/api/reader/materials/${article.id}`, { method: "PATCH", body: { position: { scrollRatio: 0.72, zoom: 1.2 } } }); assert.equal(positioned.version, edited.version); assert.equal(positioned.position.scrollRatio, 0.72); const history = await json(`/api/reader/materials/${article.id}/revisions`); assert.equal(history[0].body, body); article = await json(`/api/reader/materials/${article.id}/revisions/${history[0].id}/restore`, { method: "POST", body: { expected_version: edited.version } }); assert.equal(article.body, body); assert.equal(article.position.scrollRatio, 0.72); const shelf = await json("/api/shelf"); assert.equal(shelf.items.filter((item) => item.metadata.reader_material_id === article.id).length, 0); const native = await json(`/api/notes/${article.note_id}`); assert.equal(native.reader_material_id, article.id); assert.equal(native.title, article.title); assert.equal(native.version, article.version); });
await t.test("highlights, notes capture, attempts and review cards retain their relationships", async () => { const highlight = await json(`/api/reader/materials/${article.id}/highlights`, { method: "POST", status: 201, body: { quote: "The page offset remains unchanged.", prefix: "Translation", note: "Check why this holds." } }); await json(`/api/reader/highlights/${highlight.id}`, { method: "PATCH", body: { note: "Confirmed: only the page number changes." }, status: 204 }); assert.equal((await json(`/api/reader/materials/${article.id}/highlights`))[0].note, "Confirmed: only the page number changes."); const note = await json(`/api/reader/materials/${article.id}/notes`, { method: "POST", status: 201, body: { title: `${label} Note`, body: "**A frame** stores a page. $offset$ remains unchanged.", quote: highlight.quote } }); const captured = await json(`/api/reader/notes/${note.id}/capture`, { method: "POST", body: {} }); nativeIds.add(captured.id); fixture.nativeNote = captured.id; assert.equal((await json(`/api/reader/notes/${note.id}/capture`, { method: "POST", body: {} })).id, captured.id); native = await json(`/api/notes/${captured.id}`); assert.equal(native.document.content.content[0].type, "paragraph", "API binary must include readable Markdown capture conversion"); assert.equal(native.document.origin.reader_note_id, note.id); assert.equal(native.document.origin.markdown_source, note.body); await json(`/api/reader/materials/${article.id}/attempts`, { method: "POST", body: { exerciseKey: key, answer: "32864" }, status: 204 }); await json(`/api/reader/materials/${article.id}/attempts`, { method: "POST", body: { exerciseKey: key, answer: "32864; preserve offset 96." }, status: 204 }); const card = await json(`/api/reader/materials/${article.id}/cards`, { method: "POST", status: 201, body: { question: "Which bits remain unchanged?", answer: "The offset bits.", noteId: note.id } }); const graded = await json(`/api/reader/cards/${card.id}/review`, { method: "POST", body: { grade: "good" } }); assert.equal(graded.interval_days, 1); assert.equal(graded.repetitions, 1); await json(`/api/reader/cards/${card.id}/review`, { method: "POST", body: { grade: "good" }, status: 409 }); const learning = await json(`/api/reader/materials/${article.id}/learning`); assert.equal(learning.attempts[0].answer, "32864; preserve offset 96."); assert.equal(learning.cards[0].note_id, note.id); });
await t.test("Poppler indexes actual PDF pages, Range is bounded and Shelf reuses the asset", async () => { const bytes = pdf(); book = await upload(bytes, `${label} Book.pdf`); materialIds.add(book.id); assetIds.add(book.asset_id); fixture.book = book.id; book = await waitBook(book.id); assert.equal(book.page_count, 2); assert.equal(book.progress, 2); const search = await json(`/api/reader/materials/${book.id}/search?q=QaSearchPageTwo`); assert.equal(search[0].page, 2); const position = await json(`/api/reader/materials/${book.id}`, { method: "PATCH", body: { position: { page: 2, pageOffset: 0.72, zoom: 1.25 } } }); assert.equal(position.position.pageOffset, 0.72); const ranged = await fetch(`${base}/api/reader/files/${book.id}/book.pdf`, { headers: { Range: "bytes=0-63" } }); assert.equal(ranged.status, 206); assert.deepEqual(Buffer.from(await ranged.arrayBuffer()), bytes.subarray(0, 64)); assert.match(ranged.headers.get("content-range"), /^bytes 0-63\//); const invalid = await fetch(`${base}/api/reader/files/${book.id}/book.pdf`, { headers: { Range: "bytes=999999-" } }); assert.equal(invalid.status, 416); const full = await fetch(`${base}/api/reader/materials/${book.id}/export`); assert.deepEqual(Buffer.from(await full.arrayBuffer()), bytes); const shelf = (await json("/api/shelf")).items.find((item) => item.metadata.reader_material_id === book.id); assert.equal((await json(`/api/reader/from-shelf/${shelf.id}`, { method: "POST", body: {} })).id, book.id); // A native Shelf PDF that has never been opened in the reader. const form = new FormData(); form.append("kind", "book"); form.append("file", new Blob([bytes], { type: "application/pdf" }), `${label} Native.pdf`); const uploadResponse = await fetch(`${base}/api/assets`, { method: "POST", body: form }); assert.equal(uploadResponse.status, 200); const asset = await uploadResponse.json(); assetIds.add(asset.id); const original = await json("/api/shelf/items", { method: "POST", body: { kind: "book", title: `${label} From Shelf`, asset_id: asset.id } }); const linked = await json(`/api/reader/from-shelf/${original.id}`, { method: "POST", body: {} }); materialIds.add(linked.id); fixture.fromShelf = linked.id; assert.equal(linked.asset_id, asset.id); await waitBook(linked.id); const current = (await json("/api/shelf")).items.find((item) => item.id === original.id); assert.equal(current.metadata.reader_material_id, linked.id); });
await t.test("old Odre backup remaps chats and source notes without changing schedules or keys", async () => { const created = new Date(Date.now() - 2000).toISOString(), answered = new Date(Date.now() - 1000).toISOString(); const rawMaterial = (id, kind) => ({ id, kind, title: `${label} Legacy ${kind}`, subject: "QA", body: kind === "article" ? "## Legacy\nQaLegacyUnique records source provenance." : "", sources: JSON.stringify([{ title: "Current source", url: "https://example.org/current" }]), filename: kind === "book" ? "legacy.pdf" : null, page_count: kind === "book" ? 2 : 0, status: "ready", progress: kind === "book" ? 2 : 0, error: null, position: JSON.stringify(kind === "book" ? { page: 2, pageOffset: 0.31, zoom: 1.1 } : { scrollRatio: 0.33 }), created_at: created, updated_at: created }); const snapshot = [{ source: 2, label: "Deleted original source", url: "https://example.org/original" }, { comparison: true, materialId: "legacy-book", page: 2, label: "Book page 2", excerpt: "Saved comparison excerpt" }]; const legacy = { version: 1, createdAt: created, materials: [rawMaterial("legacy-article", "article"), rawMaterial("legacy-book", "book")], chunks: [{ material_id: "legacy-book", page: 2, heading: "Page 2", text: "QaLegacyBookUnique from a retained index" }], providers: [{ id: "legacy-provider", name: `${label} Provider`, baseUrl: "https://example.invalid/v1", models: ["qa-disabled-model"], options: { contextTokens: 1000, maxTokens: 1024, vision: false } }], chats: [{ id: "legacy-chat", material_id: "legacy-article", title: `${label} Conversation`, provider_id: "legacy-provider", model: "qa-disabled-model", created_at: created, updated_at: answered }], messages: [{ id: "legacy-question", chat_id: "legacy-chat", role: "user", content: "Compare the two source passages.", status: "complete", model: null, citations: "[]", context: JSON.stringify({ selection: "QaLegacyUnique", comparison: { materialId: "legacy-book", page: 2 } }), created_at: created }, { id: "legacy-answer", chat_id: "legacy-chat", role: "assistant", content: "A preserved completed answer.", status: "complete", model: "qa-disabled-model", citations: JSON.stringify(snapshot), context: JSON.stringify({ mode: "Two materials + local retrieval", estimatedTokens: 100 }), created_at: answered }], highlights: [{ id: "legacy-highlight", material_id: "legacy-article", quote: "QaLegacyUnique", prefix: "Legacy", suffix: " records", page: null, rects: "[]", note: "Retained annotation", created_at: created }], notes: [{ id: "legacy-note", material_id: "legacy-article", title: `${label} Provenance Note`, body: "A retained source-linked note.", quote: "QaLegacyUnique", page: null, chat_id: "legacy-chat", message_id: "legacy-answer", citations: JSON.stringify(snapshot), created_at: created, updated_at: answered }], review_cards: [{ id: "legacy-card", material_id: "legacy-article", question: "What did the source claim?", answer: "Its explicit retained claim.", quote: "QaLegacyUnique", page: null, note_id: "legacy-note", due_at: answered, interval_days: 3.4, repetitions: 2, created_at: created, updated_at: answered }], exercise_attempts: [{ material_id: "legacy-article", exercise_key: "Legacy exercise\nExplain the source", answer: "Retained personal answer", updated_at: answered }] }; assert.equal((await upload(zip({ "odre-backup.json": JSON.stringify(legacy), "files/legacy-book/book.pdf": pdf() }), "legacy-backup.zip", 200)).restored, 2); const materials = await json("/api/reader/materials"), a = materials.find((m) => m.title === `${label} Legacy article`), b = materials.find((m) => m.title === `${label} Legacy book`); materialIds.add(a.id); materialIds.add(b.id); assetIds.add(b.asset_id); fixture.legacyArticle = a.id; fixture.legacyBook = b.id; const chats = await json(`/api/reader/materials/${a.id}/chats`); assert.equal(chats.length, 1); assert.notEqual(chats[0].id, "legacy-chat"); const messages = await json(`/api/reader/chats/${chats[0].id}/messages`); assert.equal(messages[0].context.comparison.materialId, b.id); assert.equal(messages[1].citations[0].source, 2); assert.equal(messages[1].citations[1].materialId, b.id); const learning = await json(`/api/reader/materials/${a.id}/learning`); assert.equal(learning.notes[0].chat_id, chats[0].id); assert.equal(learning.notes[0].message_id, messages[1].id); assert.equal(learning.cards[0].note_id, learning.notes[0].id); assert.equal(learning.cards[0].interval_days, 3.4); assert.equal(learning.cards[0].repetitions, 2); assert.equal(Date.parse(learning.cards[0].due_at), Date.parse(answered)); assert.equal(learning.attempts[0].exercise_key, legacy.exercise_attempts[0].exercise_key); const count = materials.length; legacy.materials[0].sources = "{}"; await upload(zip({ "odre-backup.json": JSON.stringify(legacy), "files/legacy-book/book.pdf": pdf() }), "invalid-backup.zip", 400); assert.equal((await json("/api/reader/materials")).length, count); });
await t.test("reader backup round-trips only this test's data and global search has source locations", async () => { const response = await fetch(base + "/api/reader/backup"); assert.equal(response.status, 200); const files = await unzip(await response.arrayBuffer()), manifest = JSON.parse(files["odre-backup.json"]); const selected = new Set(materialIds); manifest.materials = manifest.materials.filter((m) => selected.has(m.id)); manifest.chats = manifest.chats.filter((c) => selected.has(c.material_id)); const chatIds = new Set(manifest.chats.map((c) => c.id)); manifest.messages = manifest.messages.filter((m) => chatIds.has(m.chat_id)); for (const key of ["chunks", "highlights", "notes", "review_cards", "exercise_attempts", "reader_material_revisions"]) manifest[key] = manifest[key].filter((row) => selected.has(row.material_id)); manifest.providers = manifest.providers.filter((p) => manifest.chats.some((c) => c.provider_id === p.id)); assert.ok(manifest.providers.every((p) => !Object.hasOwn(p, "apiKey") && !Object.hasOwn(p, "secret") && !Object.hasOwn(p, "api_key_ciphertext"))); assert.equal(typeof manifest.materials[0].position, "string"); assert.equal(typeof manifest.messages[0].citations, "string"); const ownFiles = { "odre-backup.json": JSON.stringify(manifest) }; for (const [name, bytes] of Object.entries(files)) if (name.startsWith("files/") && selected.has(name.split("/")[1])) ownFiles[name] = bytes; const before = new Set((await json("/api/reader/materials")).map((m) => m.id)); assert.equal((await upload(zip(ownFiles), "roundtrip-backup.zip", 200)).restored, selected.size); const copied = (await json("/api/reader/materials")).filter((m) => !before.has(m.id)); for (const m of copied) { materialIds.add(m.id); if (m.asset_id) assetIds.add(m.asset_id); } const a = copied.find((m) => m.title === article.title); assert.equal((await json(`/api/reader/materials/${a.id}`)).body, body); assert.equal((await json(`/api/reader/materials/${a.id}/highlights`))[0].quote, "The page offset remains unchanged."); assert.ok((await json(`/api/reader/materials/${a.id}/revisions`)).length >= 2); const hits = await json("/api/reader/search?q=QaLegacyUnique"); assert.ok(hits.some((h) => h.type === "material" || h.type === "passage")); assert.ok(hits.some((h) => h.type === "highlight")); assert.ok(hits.every((h) => h.material_id && h.item_id && h.excerpt)); });});