diff --git a/package-lock.json b/package-lock.json index af272e2..40069cf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,7 +18,9 @@ "html-to-text": "^9.0.5", "ora": "^8.1.0", "picocolors": "^1.1.0", - "sharp": "^0.33.0" + "sharp": "^0.33.0", + "turndown": "^7.2.4", + "turndown-plugin-gfm": "^1.0.2" }, "bin": { "ghoststandard": "dist/cli.js", @@ -1403,6 +1405,12 @@ } } }, + "node_modules/@mixmark-io/domino": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@mixmark-io/domino/-/domino-2.2.0.tgz", + "integrity": "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==", + "license": "BSD-2-Clause" + }, "node_modules/@selderee/plugin-htmlparser2": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/@selderee/plugin-htmlparser2/-/plugin-htmlparser2-0.11.0.tgz", @@ -2958,6 +2966,25 @@ "node": "*" } }, + "node_modules/turndown": { + "version": "7.2.4", + "resolved": "https://registry.npmjs.org/turndown/-/turndown-7.2.4.tgz", + "integrity": "sha512-I8yFsfRzmzK0WV1pNNOA4A7y4RDfFxPRxb3t+e3ui14qSGOxGtiSP6GjeX+Y6CHb7HYaFj7ECUD7VE5kQMZWGQ==", + "license": "MIT", + "dependencies": { + "@mixmark-io/domino": "^2.2.0" + }, + "engines": { + "node": ">=18", + "npm": ">=9" + } + }, + "node_modules/turndown-plugin-gfm": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/turndown-plugin-gfm/-/turndown-plugin-gfm-1.0.2.tgz", + "integrity": "sha512-vwz9tfvF7XN/jE0dGoBei3FXWuvll78ohzCZQuOb+ZjWrs3a0XhQVomJEb2Qh4VHTPNRO4GPZh0V7VRbiWwkRg==", + "license": "MIT" + }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", diff --git a/package.json b/package.json index d5abe0a..458dfe0 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,9 @@ "html-to-text": "^9.0.5", "ora": "^8.1.0", "picocolors": "^1.1.0", - "sharp": "^0.33.0" + "sharp": "^0.33.0", + "turndown": "^7.2.4", + "turndown-plugin-gfm": "^1.0.2" }, "devDependencies": { "@atproto/lexicon": "^0.7.5", diff --git a/src/lib/content-shape.test.ts b/src/lib/content-shape.test.ts index 53afa84..ba32ead 100644 --- a/src/lib/content-shape.test.ts +++ b/src/lib/content-shape.test.ts @@ -3,16 +3,92 @@ import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; import { Lexicons } from '@atproto/lexicon'; +import { buildMarkpubContent, MARKPUB_MARKDOWN_TYPE } from './content-shape.js'; + // Vendored fixtures are loaded via fs (not import) so no tsconfig JSON-module // setting is load-bearing for the test suite. const lexDoc = (name: string) => JSON.parse(readFileSync(new URL(`../lexicons/markpub/${name}`, import.meta.url), 'utf8')); const lexicons = new Lexicons([lexDoc('markdown.json'), lexDoc('text.json')]); +test('buildMarkpubContent output validates against the vendored at.markpub.markdown lexicon', () => { + const content = buildMarkpubContent('

Hello

Body with a link.

'); + assert.ok(content, 'expected a content member, got null'); + const result = lexicons.validate(MARKPUB_MARKDOWN_TYPE, content); + assert.ok(result.success, result.success ? '' : String(result.error)); +}); + test('the vendored lexicon genuinely rejects invalid shapes (negative control)', () => { - const result = lexicons.validate('at.markpub.markdown', { - $type: 'at.markpub.markdown', + const result = lexicons.validate(MARKPUB_MARKDOWN_TYPE, { + $type: MARKPUB_MARKDOWN_TYPE, flavor: 'gfm', }); assert.ok(!result.success); // required `text` field missing — validation must fail }); + +test('buildMarkpubContent carries exactly the declared fields', () => { + const content = buildMarkpubContent('

body

') as Record & { text: { markdown: string } }; + assert.equal(content.$type, MARKPUB_MARKDOWN_TYPE); + assert.equal(content.text.markdown, 'body'); + assert.equal(content.flavor, 'gfm'); + // renderingRules deliberately absent (derived markdown — spec "The wrinkle"), + // and nothing extra sneaks into records: textBlob/facets/lenses/frontMatter + // stay omitted (spec decision 4). + assert.deepEqual(Object.keys(content).sort(), ['$type', 'flavor', 'text']); +}); + +test('prose, headings, and links convert exactly (atx headings)', () => { + const md = buildMarkpubContent( + '

Heading

Some bold text and a link.

', + ) as { text: { markdown: string } }; + assert.equal(md.text.markdown, '## Heading\n\nSome **bold** text and a [link](https://example.com/).'); +}); + +test('Ghost kg-image-card figures become image + caption lines', () => { + const md = buildMarkpubContent( + '
' + + 'A pic' + + '
The caption
' + + '
', + ) as { text: { markdown: string } }; + assert.equal(md.text.markdown, '![A pic](https://blog.example.com/content/images/pic.jpg)\n\n*The caption*'); +}); + +test('code cards become fenced blocks with the language tag', () => { + const md = buildMarkpubContent( + '
const x = 1;\n
', + ) as { text: { markdown: string } }; + assert.match(md.text.markdown, /^```javascript\nconst x = 1;\n```$/); +}); + +test('tables convert via the GFM plugin', () => { + const md = buildMarkpubContent( + '' + + '
ab
12
', + ) as { text: { markdown: string } }; + assert.match(md.text.markdown, /\| a \| b \|/); + assert.match(md.text.markdown, /\| 1 \| 2 \|/); +}); + +test('embed iframes drop out of the markdown', () => { + const md = buildMarkpubContent( + '

Watch this:

' + + '
', + ) as { text: { markdown: string } }; + assert.match(md.text.markdown, /Watch this:/); + assert.doesNotMatch(md.text.markdown, /youtube|iframe/); +}); + +test('an embed-only post converts to nothing → null', () => { + assert.equal( + buildMarkpubContent( + '
', + ), + null, + ); +}); + +test('empty and whitespace-only HTML → null', () => { + assert.equal(buildMarkpubContent(''), null); + assert.equal(buildMarkpubContent(' \n\t '), null); +}); diff --git a/src/lib/content-shape.ts b/src/lib/content-shape.ts new file mode 100644 index 0000000..ac2075e --- /dev/null +++ b/src/lib/content-shape.ts @@ -0,0 +1,62 @@ +// The document `content` union member: Ghost post HTML → at.markpub.markdown. +// site.standard.document's content union is open (`closed: false`), so the +// member needs no lexicon registration — but a shared $type is a shared +// contract: at.markpub.markdown (markpub.at, AramZS) is a community lexicon +// for markdown-in-a-union that standard.site viewers understand. The markdown +// here is *derived* from Ghost's HTML (Ghost emits html/lexical/plaintext, +// never markdown), which is why `renderingRules` is omitted: nothing ever +// renders our pages from it. +// Design: docs/superpowers/specs/2026-07-06-markpub-content-design.md + +import TurndownService from 'turndown'; +import { gfm } from 'turndown-plugin-gfm'; + +export const MARKPUB_MARKDOWN_TYPE = 'at.markpub.markdown'; + +const turndown = new TurndownService({ + headingStyle: 'atx', + codeBlockStyle: 'fenced', +}); +turndown.use(gfm); + +// Ghost wraps every editor card in
. Only +// captioned images need a rule (the defaults would smear the figcaption into +// the surrounding text); every other card type degrades through the generic +// rules — bookmark cards to links, embed iframes to nothing. +turndown.addRule('kgImageCard', { + filter: (node) => + node.nodeName === 'FIGURE' && + /\bkg-image-card\b/.test(node.getAttribute('class') ?? ''), + replacement: (_content, node) => { + const img = node.querySelector('img'); + if (!img) return ''; + const image = `![${img.getAttribute('alt') ?? ''}](${img.getAttribute('src') ?? ''})`; + const caption = node.querySelector('figcaption')?.textContent?.trim(); + return caption ? `\n\n${image}\n\n*${caption}*\n\n` : `\n\n${image}\n\n`; + }, +}); + +/** + * Record-ready `content` union member for a post body, or null when there is + * nothing to say: empty HTML, HTML that converts to empty markdown (e.g. an + * embed-only post), or a converter throw. Null means the record simply omits + * `content` — a conversion bug must never fail a sync. + */ +export function buildMarkpubContent(html: string): Record | null { + if (!html.trim()) return null; + + let markdown: string; + try { + markdown = turndown.turndown(html); + } catch (err) { + console.error(' markdown conversion failed, omitting content:', (err as Error).message); + return null; + } + if (!markdown.trim()) return null; + + return { + $type: MARKPUB_MARKDOWN_TYPE, + text: { markdown }, + flavor: 'gfm', + }; +} diff --git a/src/types.d.ts b/src/types.d.ts index 1e50d0d..db370f8 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -11,3 +11,30 @@ declare module '@tryghost/admin-api' { declare module 'html-to-text' { export function convert(html: string, options?: Record): string; } + +declare module 'turndown' { + // Minimal stand-in for the DOM node turndown hands to rules (it uses a + // bundled DOM implementation in Node) — tsconfig lib is ES2022-only, so + // real DOM types are unavailable here. + export interface TurndownNode { + nodeName: string; + textContent: string | null; + getAttribute(name: string): string | null; + querySelector(selector: string): TurndownNode | null; + } + export interface TurndownRule { + filter: string[] | ((node: TurndownNode) => boolean); + replacement: (content: string, node: TurndownNode) => string; + } + export default class TurndownService { + constructor(options?: Record); + use(plugin: (service: TurndownService) => void): this; + addRule(key: string, rule: TurndownRule): this; + turndown(html: string): string; + } +} + +declare module 'turndown-plugin-gfm' { + import type TurndownService from 'turndown'; + export const gfm: (service: TurndownService) => void; +}