Something went wrong. Try again.
A CLI for publishing standard.site documents to ATProto
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409import * as fs from "node:fs/promises";import * as path from "node:path";import { glob } from "glob";import yaml from "yaml";import { minimatch } from "minimatch";import * as toml from "smol-toml";import type { BlogPost, FrontmatterMapping, PostFrontmatter } from "./types";import { resolveImagePath } from "./atproto";
export function parseFrontmatter( content: string, mapping?: FrontmatterMapping,): { frontmatter: PostFrontmatter; body: string; rawFrontmatter: Record<string, unknown>;} { // Support multiple frontmatter delimiters: // --- (YAML) - Jekyll, Astro, most SSGs // +++ (TOML) - Hugo // *** - Alternative format const frontmatterRegex = /^(---|\+\+\+|\*\*\*)\r?\n([\s\S]*?)\r?\n\1\r?\n([\s\S]*)$/; const match = content.match(frontmatterRegex);
if (!match) { throw new Error("Could not parse frontmatter"); }
const delimiter = match[1]; const frontmatterStr = match[2] ?? ""; const body = match[3] ?? "";
// Determine format based on delimiter: // +++ uses TOML (key = value) // --- and *** use YAML (key: value) const isToml = delimiter === "+++";
// Parse frontmatter using the appropriate library let raw: Record<string, unknown>; if (isToml) { raw = toml.parse(frontmatterStr) as Record<string, unknown>; } else { // Use CORE_SCHEMA to keep dates as strings rather than Date objects raw = (yaml.parse(frontmatterStr) as Record< string, unknown >) ?? {}; }
// Apply field mappings to normalize to standard PostFrontmatter fields const frontmatter: Record<string, unknown> = {};
// Title mapping const titleField = mapping?.title || "title"; frontmatter.title = raw[titleField] || raw.title;
// Description mapping const descField = mapping?.description || "description"; frontmatter.description = raw[descField] || raw.description;
// Publish date mapping - check custom field first, then fallbacks const dateField = mapping?.publishDate; if (dateField && raw[dateField]) { frontmatter.publishDate = raw[dateField]; } else if (raw.publishDate) { frontmatter.publishDate = raw.publishDate; } else { // Fallback to common date field names const dateFields = ["pubDate", "date", "createdAt", "created_at"]; for (const field of dateFields) { if (raw[field]) { frontmatter.publishDate = raw[field]; break; } } }
// Updated date mapping - check custom field first, then fallbacks const updatedAtField = mapping?.updatedAt; if (updatedAtField && raw[updatedAtField]) { frontmatter.updatedAt = raw[updatedAtField]; } else if (raw.updatedAt) { frontmatter.updatedAt = raw.updatedAt; } else { // Fallback to common date field names const updatedAtFields = ["updated_at", "modifiedAt", "modified_at"]; for (const field of updatedAtFields) { if (raw[field]) { frontmatter.updatedAt = raw[field]; break; } } }
// Cover image mapping const coverField = mapping?.coverImage || "ogImage"; frontmatter.ogImage = raw[coverField] || raw.ogImage;
// Tags mapping const tagsField = mapping?.tags || "tags"; frontmatter.tags = raw[tagsField] || raw.tags;
// Draft mapping const draftField = mapping?.draft || "draft"; const draftValue = raw[draftField] ?? raw.draft; if (draftValue !== undefined) { frontmatter.draft = draftValue === true || draftValue === "true"; }
// Always preserve atUri (internal field) frontmatter.atUri = raw.atUri;
return { frontmatter: frontmatter as unknown as PostFrontmatter, body, rawFrontmatter: raw, };}
export function getSlugFromFilename(filename: string): string { return filename .replace(/\.mdx?$/, "") .toLowerCase() .replace(/\s+/g, "-");}
export interface SlugOptions { slugField?: string; removeIndexFromSlug?: boolean; stripDatePrefix?: boolean;}
export function getSlugFromOptions( relativePath: string, rawFrontmatter: Record<string, unknown>, options: SlugOptions = {},): string { const { slugField, removeIndexFromSlug = false, stripDatePrefix = false, } = options;
let slug: string;
// If slugField is set, try to get the value from frontmatter if (slugField) { const frontmatterValue = rawFrontmatter[slugField]; if (frontmatterValue && typeof frontmatterValue === "string") { // Remove leading slash if present slug = frontmatterValue .replace(/^\//, "") .toLowerCase() .replace(/\s+/g, "-"); } else { // Fallback to filepath if frontmatter field not found slug = relativePath .replace(/\.mdx?$/, "") .toLowerCase() .replace(/\s+/g, "-"); } } else { // Default: use filepath slug = relativePath .replace(/\.mdx?$/, "") .toLowerCase() .replace(/\s+/g, "-"); }
// Remove /index or /_index suffix if configured if (removeIndexFromSlug) { slug = slug.replace(/\/_?index$/, ""); }
// Strip Jekyll-style date prefix (YYYY-MM-DD-) from filename if (stripDatePrefix) { slug = slug.replace(/(^|\/)(\d{4}-\d{2}-\d{2})-/g, "$1"); }
return slug;}
export function resolvePathTemplate(template: string, post: BlogPost): string { const publishDate = new Date(post.frontmatter.publishDate); const year = String(publishDate.getFullYear()); const yearUTC = String(publishDate.getUTCFullYear()); const month = String(publishDate.getMonth() + 1).padStart(2, "0"); const monthUTC = String(publishDate.getUTCMonth() + 1).padStart(2, "0"); const day = String(publishDate.getDate()).padStart(2, "0"); const dayUTC = String(publishDate.getUTCDate()).padStart(2, "0");
const slugifiedTitle = (post.frontmatter.title || "") .toLowerCase() .replace(/\s+/g, "-") .replace(/[^\w-]/g, "");
// Replace known tokens let result = template .replace(/\{slug\}/g, post.slug) .replace(/\{year\}/g, year) .replace(/\{yearUTC\}/g, yearUTC) .replace(/\{month\}/g, month) .replace(/\{monthUTC\}/g, monthUTC) .replace(/\{day\}/g, day) .replace(/\{dayUTC\}/g, dayUTC) .replace(/\{title\}/g, slugifiedTitle);
// Replace any remaining {field} tokens with raw frontmatter values result = result.replace(/\{(\w+)\}/g, (_match, field: string) => { const value = post.rawFrontmatter[field]; if (value != null && typeof value === "string") { return value; } return ""; });
// Ensure leading slash if (!result.startsWith("/")) { result = `/${result}`; }
return result;}
export function resolvePostPath( post: BlogPost, pathPrefix?: string, pathTemplate?: string,): string { if (pathTemplate) { return resolvePathTemplate(pathTemplate, post); } const prefix = pathPrefix ?? "/posts"; return prefix ? `${prefix}/${post.slug}` : `/${post.slug}`;}
export async function getContentHash(post: Partial<BlogPost>): Promise<string> { const encoder = new TextEncoderStream(); const writer = encoder.writable.getWriter(); const dataPromise = new Response(encoder.readable).arrayBuffer();
await writer.write(post?.rawContent); if (post.coverImagePath) { const coverImage = await fs.readFile(post.coverImagePath); await writer.write(coverImage.toString("base64")); } await writer.close();
const data = await dataPromise; const hashBuffer = await crypto.subtle.digest("SHA-256", data); const hashArray = Array.from(new Uint8Array(hashBuffer)); return hashArray.map((b) => b.toString(16).padStart(2, "0")).join("");}
function shouldIgnore(relativePath: string, ignorePatterns: string[]): boolean { for (const pattern of ignorePatterns) { if (minimatch(relativePath, pattern)) { return true; } } return false;}
export interface ScanOptions { frontmatterMapping?: FrontmatterMapping; ignorePatterns?: string[]; slugField?: string; removeIndexFromSlug?: boolean; stripDatePrefix?: boolean;}
export async function scanContentDirectory( contentDir: string, imagesDir: string | undefined, frontmatterMappingOrOptions?: FrontmatterMapping | ScanOptions, ignorePatterns: string[] = [],): Promise<BlogPost[]> { // Handle both old signature (frontmatterMapping, ignorePatterns) and new signature (options) let options: ScanOptions; if ( frontmatterMappingOrOptions && ("frontmatterMapping" in frontmatterMappingOrOptions || "ignorePatterns" in frontmatterMappingOrOptions || "slugField" in frontmatterMappingOrOptions) ) { options = frontmatterMappingOrOptions as ScanOptions; } else { // Old signature: (contentDir, frontmatterMapping?, ignorePatterns?) options = { frontmatterMapping: frontmatterMappingOrOptions as | FrontmatterMapping | undefined, ignorePatterns, }; }
const { frontmatterMapping, ignorePatterns: ignore = [], slugField, removeIndexFromSlug, stripDatePrefix, } = options;
const patterns = ["**/*.md", "**/*.mdx"]; const posts: BlogPost[] = [];
for (const pattern of patterns) { const files = await glob(pattern, { cwd: contentDir, absolute: false, });
for (const relativePath of files) { // Skip files matching ignore patterns if (shouldIgnore(relativePath, ignore)) { continue; }
const filePath = path.join(contentDir, relativePath); const rawContent = await fs.readFile(filePath, "utf-8");
try { const { frontmatter, body, rawFrontmatter } = parseFrontmatter( rawContent, frontmatterMapping, ); const slug = getSlugFromOptions(relativePath, rawFrontmatter, { slugField, removeIndexFromSlug, stripDatePrefix, });
// Resolve the cover image to include it in the hash. let coverImagePath: string | undefined; if (frontmatter.ogImage) { coverImagePath = await resolveImagePath( frontmatter.ogImage, imagesDir, contentDir, ); }
posts.push({ filePath, slug, frontmatter, content: body, rawContent, rawFrontmatter, coverImagePath, }); } catch (error) { console.error(`Error parsing ${relativePath}:`, error); } } }
// Sort by publish date (newest first) posts.sort((a, b) => { const dateA = new Date(a.frontmatter.publishDate); const dateB = new Date(b.frontmatter.publishDate); return dateB.getTime() - dateA.getTime(); });
return posts;}
export function updateFrontmatterWithAtUri( rawContent: string, atUri: string,): string { // Detect which delimiter is used (---, +++, or ***) const delimiterMatch = rawContent.match(/^(---|\+\+\+|\*\*\*)/); const delimiter = delimiterMatch?.[1] ?? "---"; const isToml = delimiter === "+++"; const { body, rawFrontmatter } = parseFrontmatter(rawContent);
rawFrontmatter.atUri = atUri;
const serializedFrontmatter = isToml ? toml.stringify(rawFrontmatter).trimEnd() : yaml.stringify(rawFrontmatter, { lineWidth: 0 }).trimEnd();
return `${delimiter}\n${serializedFrontmatter}\n${delimiter}\n${body}`;}
export function stripMarkdownForText(markdown: string): string { return ( markdown .replace(/#{1,6}\s/g, "") // Remove headers .replace(/\*\*([^*]+)\*\*/g, "$1") // Remove bold .replace(/\*([^*]+)\*/g, "$1") // Remove italic // Must run before link regex: image syntax  shares [alt](url) with links .replace(/!\[.*?\]\(.*?\)/g, "") // Remove images .replace(/\[([^\]]+)\]\([^)]+\)/g, "$1") // Remove links, keep text .replace(/`{3}[\s\S]*?`{3}/g, "") // Remove code blocks .replace(/`([^`]+)`/g, "$1") // Remove inline code formatting .replace(/\n{3,}/g, "\n\n") // Normalize multiple newlines .replace(/(~{1,2})([^~]+?)\1/g, "$2") // Remove strikethrough for ~text~ and ~~text~~ .replace(/<!--[\s\S]*?-->/g, "") // Remove any html comments .replace(/^\s*\[\^[^\]]+\]:[^\r\n]*(?:\r?\n[ \t]+[^\r\n]*)*/gm, "") // Remove footnote definitions .replace(/\[\^\S+\]/g, "") // Remove footnote references (we do after definitions so that those can be matched first) .trim() );}