From 4b2b06b5ba9c72e5f79f1aadbe2dff719fba229f Mon Sep 17 00:00:00 2001 From: juprodh Date: Sun, 16 Aug 2026 13:42:32 +0800 Subject: [PATCH] Rewrite markdown-style internal links on import and report unresolved ones --- src/lib/import-export/import.ts | 196 ++++++++++++++---- src/lib/import-export/markdown-transform.ts | 55 ++++- tests/lib/import-export/import.test.ts | 62 ++++++ .../import-export/markdown-transform.test.ts | 121 ++++++++--- 4 files changed, 364 insertions(+), 70 deletions(-) diff --git a/src/lib/import-export/import.ts b/src/lib/import-export/import.ts index 8f03bfa..ce10250 100644 --- a/src/lib/import-export/import.ts +++ b/src/lib/import-export/import.ts @@ -1,6 +1,10 @@ import { ok } from "@atcute/client"; import * as TID from "@atcute/tid"; -import { writeNoteRecord, writeRevisionRecord } from "../../atproto/pds.ts"; +import { + type RevisionBlob, + writeNoteRecord, + writeRevisionRecord, +} from "../../atproto/pds.ts"; import type { getAgent } from "../../atproto/session.ts"; import { createNote } from "../../server/db/queries/index.ts"; import type { RequestContext } from "../access.ts"; @@ -49,56 +53,164 @@ export async function importWikiAction( } // Create notes sequentially (PDS-first pattern) + const unresolved: UnresolvedLinks[] = []; for (const note of notes) { - const content = rewriteForImport(note.content, slugMap, imageMap); - const blobs = buildBlobsForContent(content, blobMeta); - - const noteTid = TID.now(); - const revisionTid = TID.now(); - const noteAtUri = `at://${did}/wiki.lichen.note/${noteTid}`; - const revisionAtUri = `at://${did}/wiki.lichen.noteRevision/${revisionTid}`; - - if (agent) { - const diff = createDiff("", content); - await withPdsError(`import note "${note.title}"`, async () => { - await writeNoteRecord( - agent, - did, - noteTid, - note.slug, - note.title, - wikiAtUri, - now, - ); - await writeRevisionRecord( - agent, - did, - revisionTid, - noteAtUri, - null, - diff, - "Imported", - now, - blobs.length > 0 ? blobs : undefined, - ); - }); + const { content, unresolvedLinks } = rewriteForImport( + note.content, + slugMap, + imageMap, + ); + if (unresolvedLinks.length > 0) { + unresolved.push({ filename: note.filename, links: unresolvedLinks }); } - createNote( - noteAtUri, - revisionAtUri, + await writeImportedNote({ + agent, + did, wikiAtUri, wikiSlug, - note.slug, - note.title, - did, + now, + slug: note.slug, + title: note.title, content, - "Imported", - blobs.length > 0 ? blobs : undefined, + blobs: buildBlobsForContent(content, blobMeta), + }); + } + + // The route redirects to the new wiki, so a returned warning would never be seen. + // A note in the wiki is the only delivery that survives the redirect. + const report = buildWarningsReport(unresolved, warnings); + let noteCount = notes.length; + if (report) { + await writeImportedNote({ + agent, + did, + wikiAtUri, + wikiSlug, + now, + slug: freeSlug(WARNINGS_SLUG, new Set(notes.map((n) => n.slug))), + title: WARNINGS_TITLE, + content: report, + blobs: [], + }); + noteCount += 1; + } + + return { wikiSlug, noteCount, warnings }; +} + +interface UnresolvedLinks { + filename: string; + links: string[]; +} + +interface ImportedNoteWrite { + agent: ReturnType; + did: string; + wikiAtUri: string; + wikiSlug: string; + now: string; + slug: string; + title: string; + content: string; + blobs: RevisionBlob[]; +} + +async function writeImportedNote(note: ImportedNoteWrite): Promise { + const { agent, did, wikiAtUri, wikiSlug, now, slug, title, content, blobs } = + note; + const noteTid = TID.now(); + const revisionTid = TID.now(); + const noteAtUri = `at://${did}/wiki.lichen.note/${noteTid}`; + const revisionAtUri = `at://${did}/wiki.lichen.noteRevision/${revisionTid}`; + + if (agent) { + const diff = createDiff("", content); + await withPdsError(`import note "${title}"`, async () => { + await writeNoteRecord(agent, did, noteTid, slug, title, wikiAtUri, now); + await writeRevisionRecord( + agent, + did, + revisionTid, + noteAtUri, + null, + diff, + "Imported", + now, + blobs.length > 0 ? blobs : undefined, + ); + }); + } + + createNote( + noteAtUri, + revisionAtUri, + wikiAtUri, + wikiSlug, + slug, + title, + did, + content, + "Imported", + blobs.length > 0 ? blobs : undefined, + ); +} + +const WARNINGS_TITLE = "Import warnings"; +const WARNINGS_SLUG = "import-warnings"; +const MAX_REPORTED_LINKS = 100; + +function freeSlug(base: string, taken: Set): string { + if (!taken.has(base)) return base; + let suffix = 2; + while (taken.has(`${base}-${suffix}`)) suffix++; + return `${base}-${suffix}`; +} + +// English only, matching the warnings parseImportZip already produces. +function buildWarningsReport( + unresolved: UnresolvedLinks[], + warnings: string[], +): string | null { + if (unresolved.length === 0 && warnings.length === 0) return null; + + const sections = [ + "The importer created this note. Delete it once you have read it.", + ]; + + if (unresolved.length > 0) { + const total = unresolved.reduce((sum, e) => sum + e.links.length, 0); + sections.push( + `## ${total} link${total === 1 ? "" : "s"} could not be resolved`, + "Nothing was lost — every link below was left exactly as written — but none of them matches a note in this wiki.", ); + let budget = MAX_REPORTED_LINKS; + for (const entry of unresolved) { + if (budget <= 0) break; + const shown = entry.links.slice(0, budget); + budget -= shown.length; + sections.push( + `### ${code(entry.filename)}\n\n${shown.map((link) => `- ${code(link)}`).join("\n")}`, + ); + } + if (total > MAX_REPORTED_LINKS) { + sections.push(`…and ${total - MAX_REPORTED_LINKS} more.`); + } } - return { wikiSlug, noteCount: notes.length, warnings }; + if (warnings.length > 0) { + sections.push( + "## Files the importer renamed or skipped", + warnings.map((warning) => `- ${warning}`).join("\n"), + ); + } + + return `${sections.join("\n\n")}\n`; +} + +// Vault paths are arbitrary text; a stray backtick would break out of the span. +function code(value: string): string { + return `\`${value.replace(/`/g, "")}\``; } async function uploadImages( diff --git a/src/lib/import-export/markdown-transform.ts b/src/lib/import-export/markdown-transform.ts index 644f6a6..d6bfd34 100644 --- a/src/lib/import-export/markdown-transform.ts +++ b/src/lib/import-export/markdown-transform.ts @@ -9,12 +9,21 @@ interface ExportResult { blobRefs: { did: string; cid: string }[]; } +interface ImportRewriteResult { + content: string; + // Hrefs of `.md` links that matched no note. An unresolved [[wikilink]] is a normal + // Obsidian state (the note may not exist yet); a markdown link to a missing file is + // a dead link, so only those are reported. + unresolvedLinks: string[]; +} + // Obsidian → Lichen: note-title wikilinks become slugs, local image refs become blob URLs. export function rewriteForImport( content: string, slugMap: Map, imageMap: Map, -): string { +): ImportRewriteResult { + const unresolvedLinks: string[] = []; // 1. Rewrite Obsidian image embeds: ![[image.png]] let result = content.replace( /!\[\[([^\]]+)\]\]/g, @@ -59,7 +68,41 @@ export function rewriteForImport( return _match; // leave as-is if no matching note }); - return result; + // 4. Rewrite markdown-style internal links: [text](Some%20Note.md), the other format + // Obsidian writes when "Use [[Wikilinks]]" is off. The lookbehind keeps image embeds + // (handled in step 2) out of this. + result = result.replace( + /(? { + const trimmed = href.trim(); + if (isAbsoluteUrl(trimmed) || trimmed.startsWith("/")) return _match; + + const hashIdx = trimmed.indexOf("#"); + const path = hashIdx >= 0 ? trimmed.slice(0, hashIdx) : trimmed; + const section = hashIdx >= 0 ? trimmed.slice(hashIdx) : ""; + if (!/\.(md|markdown)$/i.test(path)) return _match; + + const title = basename(decodePath(path)).replace(/\.(md|markdown)$/i, ""); + const slug = findSlug(slugMap, title); + if (!slug) { + unresolvedLinks.push(trimmed); + return _match; + } + return `[[${slug}${section}${text ? `|${text}` : ""}]]`; + }, + ); + + return { content: result, unresolvedLinks }; +} + +// A vault path is percent-encoded by Obsidian but may also contain a literal `%`, +// which makes decodeURIComponent throw rather than round-trip. +function decodePath(path: string): string { + try { + return decodeURIComponent(path); + } catch { + return path; + } } // Lichen → Obsidian: slug wikilinks become note titles, /blob/ refs become attachments/ paths. @@ -169,7 +212,8 @@ function findImageUrl( return undefined; } -// Exact match, then case-insensitive fallback. +// Exact match, then case-insensitive, then without the extension — Obsidian accepts +// `[[Note.md]]` as a link to the same note as `[[Note]]`. function findSlug( slugMap: Map, title: string, @@ -180,5 +224,8 @@ function findSlug( for (const [key, value] of slugMap) { if (key.toLowerCase() === lower) return value; } - return undefined; + const withoutExtension = title.replace(/\.(md|markdown)$/i, ""); + return withoutExtension === title + ? undefined + : findSlug(slugMap, withoutExtension); } diff --git a/tests/lib/import-export/import.test.ts b/tests/lib/import-export/import.test.ts index 25a6895..4a6f9db 100644 --- a/tests/lib/import-export/import.test.ts +++ b/tests/lib/import-export/import.test.ts @@ -279,4 +279,66 @@ describe("importWikiAction", () => { expect(notes).toHaveLength(1); expect(notes[0]?.slug).toBe("alpha"); }); + + test("rewrites markdown-style links between imported notes", async () => { + const zip = makeZip({ + "Alpha.md": "See [Beta](Beta.md) for more.", + "Beta.md": "Back to [Alpha](Alpha.md).", + }); + + const result = await importWikiAction( + makeCtx(), + { + name: "Import Md Link Test", + language: "en", + visibility: "public", + description: "", + zipBuffer: zip, + }, + dummyMsg, + ); + + createdSlugs.push(result.wikiSlug); + + const db = getDb(); + const alpha = db + .query( + "SELECT content FROM current_note WHERE note_at_uri IN (SELECT at_uri FROM notes WHERE wiki_slug = ? AND slug = 'alpha')", + ) + .get(result.wikiSlug) as { content: string } | null; + expect(alpha?.content).toContain("[[beta|Beta]]"); + }); + + test("reports links that did not resolve in an import warnings note", async () => { + const zip = makeZip({ + "Alpha.md": "See [Ghost](Ghost%20Note.md) and [Beta](Beta.md).", + "Beta.md": "Content.", + }); + + const result = await importWikiAction( + makeCtx(), + { + name: "Import Unresolved Test", + language: "en", + visibility: "public", + description: "", + zipBuffer: zip, + }, + dummyMsg, + ); + + createdSlugs.push(result.wikiSlug); + expect(result.noteCount).toBe(3); + + const db = getDb(); + const report = db + .query( + "SELECT content FROM current_note WHERE note_at_uri IN (SELECT at_uri FROM notes WHERE wiki_slug = ? AND slug = 'import-warnings')", + ) + .get(result.wikiSlug) as { content: string } | null; + expect(report?.content).toContain("Ghost%20Note.md"); + expect(report?.content).toContain("Alpha.md"); + // The link that resolved must not be listed as a failure. + expect(report?.content).not.toContain("Beta.md"); + }); }); diff --git a/tests/lib/import-export/markdown-transform.test.ts b/tests/lib/import-export/markdown-transform.test.ts index 1494660..ae7f860 100644 --- a/tests/lib/import-export/markdown-transform.test.ts +++ b/tests/lib/import-export/markdown-transform.test.ts @@ -18,79 +18,152 @@ describe("rewriteForImport", () => { test("rewrites Obsidian wikilinks to slugs", () => { const input = "See [[Getting Started]] for info."; - const result = rewriteForImport(input, slugMap, imageMap); - expect(result).toBe("See [[getting-started]] for info."); + const { content } = rewriteForImport(input, slugMap, imageMap); + expect(content).toBe("See [[getting-started]] for info."); }); test("rewrites wikilinks with display text", () => { const input = "Check [[My Notes|my personal notes]] here."; - const result = rewriteForImport(input, slugMap, imageMap); - expect(result).toBe("Check [[my-notes|my personal notes]] here."); + const { content } = rewriteForImport(input, slugMap, imageMap); + expect(content).toBe("Check [[my-notes|my personal notes]] here."); }); test("keeps heading and block-id sections attached to the slug", () => { - const result = rewriteForImport( + const { content } = rewriteForImport( "See [[Getting Started#Install]] and [[My Notes#^ref-1|notes]].", slugMap, imageMap, ); - expect(result).toBe( + expect(content).toBe( "See [[getting-started#Install]] and [[my-notes#^ref-1|notes]].", ); }); test("leaves unmatched wikilinks as-is", () => { const input = "See [[Unknown Page]] for info."; - const result = rewriteForImport(input, slugMap, imageMap); - expect(result).toBe("See [[Unknown Page]] for info."); + const { content } = rewriteForImport(input, slugMap, imageMap); + expect(content).toBe("See [[Unknown Page]] for info."); }); test("preserves cross-wiki wikilinks on import", () => { const input = "See [[other-wiki/shared-note|Shared Note]] for info."; - const result = rewriteForImport(input, slugMap, imageMap); - expect(result).toBe(input); + const { content } = rewriteForImport(input, slugMap, imageMap); + expect(content).toBe(input); }); test("rewrites Obsidian image embeds", () => { const input = "Here is ![[photo.png]] in text."; - const result = rewriteForImport(input, slugMap, imageMap); - expect(result).toBe("Here is ![photo](/blob/did:plc:abc/bafk1) in text."); + const { content } = rewriteForImport(input, slugMap, imageMap); + expect(content).toBe("Here is ![photo](/blob/did:plc:abc/bafk1) in text."); }); test("rewrites relative image paths", () => { const input = "![my diagram](assets/diagram.jpg)"; - const result = rewriteForImport(input, slugMap, imageMap); - expect(result).toBe("![my diagram](/blob/did:plc:abc/bafk2)"); + const { content } = rewriteForImport(input, slugMap, imageMap); + expect(content).toBe("![my diagram](/blob/did:plc:abc/bafk2)"); }); test("leaves absolute URLs untouched", () => { const input = "![ext](https://example.com/img.png)"; - const result = rewriteForImport(input, slugMap, imageMap); - expect(result).toBe("![ext](https://example.com/img.png)"); + const { content } = rewriteForImport(input, slugMap, imageMap); + expect(content).toBe("![ext](https://example.com/img.png)"); }); test("leaves /blob/ refs untouched", () => { const input = "![img](/blob/did:plc:xyz/bafkabc)"; - const result = rewriteForImport(input, slugMap, imageMap); - expect(result).toBe("![img](/blob/did:plc:xyz/bafkabc)"); + const { content } = rewriteForImport(input, slugMap, imageMap); + expect(content).toBe("![img](/blob/did:plc:xyz/bafkabc)"); }); test("leaves unmatched image embeds as-is", () => { const input = "![[missing.png]]"; - const result = rewriteForImport(input, slugMap, imageMap); - expect(result).toBe("![[missing.png]]"); + const { content } = rewriteForImport(input, slugMap, imageMap); + expect(content).toBe("![[missing.png]]"); }); test("handles case-insensitive image match", () => { const input = "![[Photo.PNG]]"; - const result = rewriteForImport(input, slugMap, imageMap); - expect(result).toBe("![Photo](/blob/did:plc:abc/bafk1)"); + const { content } = rewriteForImport(input, slugMap, imageMap); + expect(content).toBe("![Photo](/blob/did:plc:abc/bafk1)"); }); test("handles case-insensitive wikilink match", () => { const input = "See [[home]] page."; - const result = rewriteForImport(input, slugMap, imageMap); - expect(result).toBe("See [[home]] page."); + const { content } = rewriteForImport(input, slugMap, imageMap); + expect(content).toBe("See [[home]] page."); + }); + + test("rewrites markdown-style internal links", () => { + const { content } = rewriteForImport( + "See [Getting Started](Getting%20Started.md) for info.", + slugMap, + imageMap, + ); + expect(content).toBe("See [[getting-started|Getting Started]] for info."); + }); + + test("resolves markdown-style links through a folder path", () => { + const { content } = rewriteForImport( + "[notes](vault/sub/My Notes.markdown)", + slugMap, + imageMap, + ); + expect(content).toBe("[[my-notes|notes]]"); + }); + + test("keeps the section suffix on a markdown-style link", () => { + const { content } = rewriteForImport( + "[install](Getting%20Started.md#Install)", + slugMap, + imageMap, + ); + expect(content).toBe("[[getting-started#Install|install]]"); + }); + + test("resolves a wikilink written with the .md extension", () => { + const { content } = rewriteForImport( + "See [[My Notes.md]] here.", + slugMap, + imageMap, + ); + expect(content).toBe("See [[my-notes]] here."); + }); + + test("leaves external and non-markdown links untouched", () => { + const input = "[site](https://example.com/page.md) and [pdf](report.pdf)"; + const { content, unresolvedLinks } = rewriteForImport( + input, + slugMap, + imageMap, + ); + expect(content).toBe(input); + expect(unresolvedLinks).toEqual([]); + }); + + test("reports markdown-style links that match no note", () => { + const input = "[gone](Missing%20Note.md) and [ok](My%20Notes.md)"; + const { content, unresolvedLinks } = rewriteForImport( + input, + slugMap, + imageMap, + ); + expect(content).toBe("[gone](Missing%20Note.md) and [[my-notes|ok]]"); + expect(unresolvedLinks).toEqual(["Missing%20Note.md"]); + }); + + test("does not turn an image ref into a link", () => { + const input = "![embed](Getting%20Started.md)"; + const { content } = rewriteForImport(input, slugMap, imageMap); + expect(content).toBe(input); + }); + + test("does not report unresolved wikilinks", () => { + const { unresolvedLinks } = rewriteForImport( + "See [[Unknown Page]] for info.", + slugMap, + imageMap, + ); + expect(unresolvedLinks).toEqual([]); }); }); -- 2.51.2