import * as fs from "node:fs/promises"; import * as path from "node:path"; import { glob } from "glob"; import yaml from "js-yaml"; import { minimatch } from "minimatch"; import * as toml from "smol-toml"; import type { BlogPost, FrontmatterMapping, PostFrontmatter } from "./types"; import { resolveImagePath } from "./atproto"; import { remark } from "remark"; import gfm from "remark-gfm"; import html from "./remark-html"; import strip from "strip-markdown"; export function parseFrontmatter( content: string, mapping?: FrontmatterMapping, ): { frontmatter: PostFrontmatter; body: string; rawFrontmatter: Record; } { // Support multiple frontmatter delimiters: // --- (YAML) - Jekyll, Astro, most SSGs // +++ (TOML) - Hugo // *** - Alternative format const frontmatterRegex = /^(---|\+\+\+|\*\*\*)\r?\n([\s\S]*?)\r?\n\1\r?\n([\s\S]*)$/; const match = content.match(frontmatterRegex); if (!match) { throw new Error("Could not parse frontmatter"); } const delimiter = match[1]; const frontmatterStr = match[2] ?? ""; const body = match[3] ?? ""; // Determine format based on delimiter: // +++ uses TOML (key = value) // --- and *** use YAML (key: value) const isToml = delimiter === "+++"; // Parse frontmatter using the appropriate library let raw: Record; if (isToml) { raw = toml.parse(frontmatterStr) as Record; } else { // Use CORE_SCHEMA to keep dates as strings rather than Date objects raw = (yaml.load(frontmatterStr, { schema: yaml.CORE_SCHEMA }) as Record< string, unknown >) ?? {}; } // Apply field mappings to normalize to standard PostFrontmatter fields const frontmatter: Record = {}; // Title mapping const titleField = mapping?.title || "title"; frontmatter.title = raw[titleField] || raw.title; // Description mapping const descField = mapping?.description || "description"; frontmatter.description = raw[descField] || raw.description; // Publish date mapping - check custom field first, then fallbacks const dateField = mapping?.publishDate; if (dateField && raw[dateField]) { frontmatter.publishDate = raw[dateField]; } else if (raw.publishDate) { frontmatter.publishDate = raw.publishDate; } else { // Fallback to common date field names const dateFields = ["pubDate", "date", "createdAt", "created_at"]; for (const field of dateFields) { if (raw[field]) { frontmatter.publishDate = raw[field]; break; } } } // Updated date mapping - check custom field first, then fallbacks const updatedAtField = mapping?.updatedAt; if (updatedAtField && raw[updatedAtField]) { frontmatter.updatedAt = raw[updatedAtField]; } else if (raw.updatedAt) { frontmatter.updatedAt = raw.updatedAt; } else { // Fallback to common date field names const updatedAtFields = ["updated_at", "modifiedAt", "modified_at"]; for (const field of updatedAtFields) { if (raw[field]) { frontmatter.updatedAt = raw[field]; break; } } } // Cover image mapping const coverField = mapping?.coverImage || "ogImage"; frontmatter.ogImage = raw[coverField] || raw.ogImage; // Tags mapping const tagsField = mapping?.tags || "tags"; frontmatter.tags = raw[tagsField] || raw.tags; // Draft mapping const draftField = mapping?.draft || "draft"; const draftValue = raw[draftField] ?? raw.draft; if (draftValue !== undefined) { frontmatter.draft = draftValue === true || draftValue === "true"; } // Always preserve atUri (internal field) frontmatter.atUri = raw.atUri; return { frontmatter: frontmatter as unknown as PostFrontmatter, body, rawFrontmatter: raw, }; } export function getSlugFromFilename(filename: string): string { return filename .replace(/\.mdx?$/, "") .toLowerCase() .replace(/\s+/g, "-"); } export interface SlugOptions { slugField?: string; removeIndexFromSlug?: boolean; stripDatePrefix?: boolean; } export function getSlugFromOptions( relativePath: string, rawFrontmatter: Record, options: SlugOptions = {}, ): string { const { slugField, removeIndexFromSlug = false, stripDatePrefix = false, } = options; let slug: string; // If slugField is set, try to get the value from frontmatter if (slugField) { const frontmatterValue = rawFrontmatter[slugField]; if (frontmatterValue && typeof frontmatterValue === "string") { // Remove leading slash if present slug = frontmatterValue .replace(/^\//, "") .toLowerCase() .replace(/\s+/g, "-"); } else { // Fallback to filepath if frontmatter field not found slug = relativePath .replace(/\.mdx?$/, "") .toLowerCase() .replace(/\s+/g, "-"); } } else { // Default: use filepath slug = relativePath .replace(/\.mdx?$/, "") .toLowerCase() .replace(/\s+/g, "-"); } // Remove /index or /_index suffix if configured if (removeIndexFromSlug) { slug = slug.replace(/\/_?index$/, ""); } // Strip Jekyll-style date prefix (YYYY-MM-DD-) from filename if (stripDatePrefix) { slug = slug.replace(/(^|\/)(\d{4}-\d{2}-\d{2})-/g, "$1"); } return slug; } export function resolvePathTemplate(template: string, post: BlogPost): string { const publishDate = new Date(post.frontmatter.publishDate); const year = String(publishDate.getFullYear()); const yearUTC = String(publishDate.getUTCFullYear()); const month = String(publishDate.getMonth() + 1).padStart(2, "0"); const monthUTC = String(publishDate.getUTCMonth() + 1).padStart(2, "0"); const day = String(publishDate.getDate()).padStart(2, "0"); const dayUTC = String(publishDate.getUTCDate()).padStart(2, "0"); const slugifiedTitle = (post.frontmatter.title || "") .toLowerCase() .replace(/\s+/g, "-") .replace(/[^\w-]/g, ""); // Replace known tokens let result = template .replace(/\{slug\}/g, post.slug) .replace(/\{year\}/g, year) .replace(/\{yearUTC\}/g, yearUTC) .replace(/\{month\}/g, month) .replace(/\{monthUTC\}/g, monthUTC) .replace(/\{day\}/g, day) .replace(/\{dayUTC\}/g, dayUTC) .replace(/\{title\}/g, slugifiedTitle); // Replace any remaining {field} tokens with raw frontmatter values result = result.replace(/\{(\w+)\}/g, (_match, field: string) => { const value = post.rawFrontmatter[field]; if (value != null && typeof value === "string") { return value; } return ""; }); // Ensure leading slash if (!result.startsWith("/")) { result = `/${result}`; } return result; } export function resolvePostPath( post: BlogPost, pathPrefix?: string, pathTemplate?: string, ): string { if (pathTemplate) { return resolvePathTemplate(pathTemplate, post); } const prefix = pathPrefix ?? "/posts"; return prefix ? `${prefix}/${post.slug}` : `/${post.slug}`; } export async function getContentHash(post: Partial): Promise { const encoder = new TextEncoderStream(); const writer = encoder.writable.getWriter(); const dataPromise = new Response(encoder.readable).arrayBuffer(); await writer.write(post?.rawContent); if (post.coverImagePath) { const coverImage = await fs.readFile(post.coverImagePath); await writer.write(coverImage.toString("base64")); } await writer.close(); const data = await dataPromise; const hashBuffer = await crypto.subtle.digest("SHA-256", data); const hashArray = Array.from(new Uint8Array(hashBuffer)); return hashArray.map((b) => b.toString(16).padStart(2, "0")).join(""); } function shouldIgnore(relativePath: string, ignorePatterns: string[]): boolean { for (const pattern of ignorePatterns) { if (minimatch(relativePath, pattern)) { return true; } } return false; } export interface ScanOptions { frontmatterMapping?: FrontmatterMapping; ignorePatterns?: string[]; slugField?: string; removeIndexFromSlug?: boolean; stripDatePrefix?: boolean; } export async function scanContentDirectory( contentDir: string, imagesDir: string | undefined, frontmatterMappingOrOptions?: FrontmatterMapping | ScanOptions, ignorePatterns: string[] = [], ): Promise { // Handle both old signature (frontmatterMapping, ignorePatterns) and new signature (options) let options: ScanOptions; if ( frontmatterMappingOrOptions && ("frontmatterMapping" in frontmatterMappingOrOptions || "ignorePatterns" in frontmatterMappingOrOptions || "slugField" in frontmatterMappingOrOptions) ) { options = frontmatterMappingOrOptions as ScanOptions; } else { // Old signature: (contentDir, frontmatterMapping?, ignorePatterns?) options = { frontmatterMapping: frontmatterMappingOrOptions as | FrontmatterMapping | undefined, ignorePatterns, }; } const { frontmatterMapping, ignorePatterns: ignore = [], slugField, removeIndexFromSlug, stripDatePrefix, } = options; const patterns = ["**/*.md", "**/*.mdx"]; const posts: BlogPost[] = []; for (const pattern of patterns) { const files = await glob(pattern, { cwd: contentDir, absolute: false, }); for (const relativePath of files) { // Skip files matching ignore patterns if (shouldIgnore(relativePath, ignore)) { continue; } const filePath = path.join(contentDir, relativePath); const rawContent = await fs.readFile(filePath, "utf-8"); try { const { frontmatter, body, rawFrontmatter } = parseFrontmatter( rawContent, frontmatterMapping, ); const slug = getSlugFromOptions(relativePath, rawFrontmatter, { slugField, removeIndexFromSlug, stripDatePrefix, }); // Resolve the cover image to include it in the hash. let coverImagePath: string | undefined; if (frontmatter.ogImage) { coverImagePath = await resolveImagePath( frontmatter.ogImage, imagesDir, contentDir, ); } posts.push({ filePath, slug, frontmatter, content: body, rawContent, rawFrontmatter, coverImagePath, }); } catch (error) { console.error(`Error parsing ${relativePath}:`, error); } } } // Sort by publish date (newest first) posts.sort((a, b) => { const dateA = new Date(a.frontmatter.publishDate); const dateB = new Date(b.frontmatter.publishDate); return dateB.getTime() - dateA.getTime(); }); return posts; } export function updateFrontmatterWithAtUri( rawContent: string, atUri: string, ): string { // Detect which delimiter is used (---, +++, or ***) const delimiterMatch = rawContent.match(/^(---|\+\+\+|\*\*\*)/); const delimiter = delimiterMatch?.[1] ?? "---"; const isToml = delimiter === "+++"; const { body, rawFrontmatter } = parseFrontmatter(rawContent); rawFrontmatter.atUri = atUri; const serializedFrontmatter = isToml ? toml.stringify(rawFrontmatter).trimEnd() : yaml.dump(rawFrontmatter, { lineWidth: -1 }).trimEnd(); return `${delimiter}\n${serializedFrontmatter}\n${delimiter}\n${body}`; } export async function stripMarkdownForText(markdown: string): Promise { const rawContent = await remark() // Process GitHub-flavored markdown. .use(gfm) // Process remaining HTML into markdown. .use(html) // Strip markdown. .use(strip) .process(markdown); return String(rawContent).trim(); }