From 9ba6e7b6dc088f3e016b8f5f0b7edbe630dd0339 Mon Sep 17 00:00:00 2001 From: Seth Etter Date: Wed, 21 Jan 2026 22:24:59 -0600 Subject: [PATCH] Site export config, with init command --- README.md | 215 ++++++++++++++++++++++++++++++++- packages/cli/src/index.ts | 127 ++++++++++++++++++- packages/core/package.json | 24 ++-- packages/core/src/config.ts | 61 ++++++++++ packages/core/src/export.ts | 79 ++++++++++-- packages/core/src/index.ts | 16 ++- packages/core/src/templates.ts | 2 +- packages/core/src/types.ts | 46 ++++++- 8 files changed, 532 insertions(+), 38 deletions(-) create mode 100644 packages/core/src/config.ts diff --git a/README.md b/README.md index 9c72476..88f9b84 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,222 @@ # sitebase -To install dependencies: +Export content from [standard.site](https://standard.site) publications to markdown files. + +Sitebase connects to ATProto Personal Data Servers (PDS) to fetch documents from `site.standard.publication` collections and exports them as markdown files with configurable templates and filtering. + +## Installation ```bash bun install ``` -To run: +## Packages + +This is a monorepo with three packages: + +- **@sitebase/core** - Library for exporting publications and template utilities +- **@sitebase/cli** - Command-line interface for running exports +- **@sitebase/web** - Web UI for managing publications and documents + +## CLI Usage ```bash -bun run index.ts +# Auto-discover sitebase.config.ts in current directory +sitebase export + +# Specify a config file +sitebase export --config ./my-config.ts +``` + +The CLI looks for `sitebase.config.ts` or `sitebase.config.js` in the current directory. Use `-c` or `--config` to specify a different path. + +## Configuration + +Create a `sitebase.config.ts` file in your project root: + +```typescript +import type { ExportConfig } from "@sitebase/core"; +import { slugify } from "@sitebase/core"; + +const config: ExportConfig = { + // AT URI of your publication + publicationUri: "at://did:plc:xyz/site.standard.publication/rkey", + + // One or more export targets + exports: [ + { + outputDir: "./content/posts", + includeTags: ["post"], + excludeTags: ["draft"], + filename: (data) => { + const date = data.publishedAt?.slice(0, 10) || "undated"; + return `${date}_${slugify(data.title)}.md`; + }, + contentTemplate: "./templates/post.hbs", + }, + ], +}; + +export default config; ``` -This project was created using `bun init` in bun v1.3.5. [Bun](https://bun.com) is a fast all-in-one JavaScript runtime. +### Config Reference + +#### `ExportConfig` + +| Field | Type | Description | +|-------|------|-------------| +| `publicationUri` | `string` | AT URI of the publication (`at://did:plc:.../site.standard.publication/rkey`) | +| `exports` | `ExportTarget[]` | Array of export targets | + +#### `ExportTarget` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `outputDir` | `string` | Yes | Directory to write output files | +| `filename` | `(data: TemplateData) => string` | Yes | Function to generate filename | +| `includeTags` | `string[]` | No | Only include documents with ANY of these tags | +| `excludeTags` | `string[]` | No | Exclude documents with ANY of these tags | +| `contentTemplate` | `string` | No | Path to Handlebars template file | +| `content` | `(data: TemplateData) => string` | No | Function to generate content (overrides `contentTemplate`) | + +### Template Data + +Both `filename` and `content` functions receive a `TemplateData` object: + +```typescript +interface TemplateData { + title: string; + path?: string; + description?: string; + content: string; // Markdown content + tags: string[]; + publishedAt?: string; // ISO 8601 date + updatedAt?: string; // ISO 8601 date + publication: { + name: string; + url: string; + description?: string; + }; +} +``` + +### Tag Filtering + +- **includeTags**: Only documents with at least one matching tag are included +- **excludeTags**: Documents with any matching tag are excluded +- When neither is specified, documents tagged "draft" are excluded by default + +### Handlebars Templates + +Content templates use [Handlebars](https://handlebarsjs.com/) with these custom helpers: + +| Helper | Usage | Description | +|--------|-------|-------------| +| `slug` | `{{slug text}}` | Convert text to URL-safe slug | +| `dateFormat` | `{{dateFormat date "YYYY-MM-DD"}}` | Format date (supports YYYY, MM, DD) | +| `default` | `{{default value fallback}}` | Use fallback if value is empty | + +Example template (`templates/post.hbs`): + +```handlebars +--- +title: "{{title}}" +date: {{publishedAt}} +slug: {{slug (default path title)}} +{{#if tags.length}} +tags: +{{#each tags}} + - {{this}} +{{/each}} +{{/if}} +--- + +{{content}} +``` + +### Using the Content Function + +For full control, use a `content` function instead of a template: + +```typescript +{ + outputDir: "./content", + filename: (data) => `${slugify(data.title)}.md`, + content: (data) => { + return [ + "---", + `title: "${data.title}"`, + `date: ${data.publishedAt}`, + "---", + "", + data.content, + ].join("\n"); + }, +} +``` + +### Multiple Export Targets + +Export the same publication to different locations with different filters: + +```typescript +export default { + publicationUri: "at://did:plc:xyz/site.standard.publication/rkey", + exports: [ + { + outputDir: "./content/notes", + includeTags: ["note"], + filename: (data) => `${slugify(data.title)}.md`, + }, + { + outputDir: "./content/posts", + includeTags: ["post"], + excludeTags: ["draft"], + filename: (data) => `${data.publishedAt?.slice(0, 10)}_${slugify(data.title)}.md`, + contentTemplate: "./templates/post.hbs", + }, + ], +} as ExportConfig; +``` + +## Core Library + +Use `@sitebase/core` directly in your own scripts: + +```typescript +import { exportPublication, slugify, createHandlebars } from "@sitebase/core"; + +const result = await exportPublication({ + publicationUri: "at://did:plc:xyz/site.standard.publication/rkey", + outputDir: "./output", + filename: (data) => `${slugify(data.title)}.md`, +}); + +console.log(`Wrote ${result.filesWritten.length} files`); +``` + +### Exports + +- `exportPublication(options)` - Export a publication to markdown files +- `exportFromConfig(config, baseDir)` - Export using a config object +- `findConfigFile(dir)` - Find `sitebase.config.{ts,js}` in directory +- `loadExportConfig(path)` - Load and validate a config file +- `slugify(text)` - Convert text to URL-safe slug +- `createHandlebars()` - Create Handlebars instance with helpers registered +- `generateContent(hbs, template, data)` - Render a Handlebars template + +## Web UI + +The web package provides a management interface for publications and documents with ATProto OAuth authentication. + +```bash +cd packages/web +bun run dev +``` + +See `packages/web/CLAUDE.md` for web package details. + +## License + +MIT diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index ca80520..4cca600 100755 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -1,5 +1,6 @@ #!/usr/bin/env bun -import { dirname, resolve } from "node:path"; +import { access, mkdir, writeFile } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; import { Command } from "commander"; import { exportFromConfig, @@ -76,4 +77,128 @@ program } }); +program + .command("init") + .description("Create a sitebase.config.ts file with starter configuration") + .action(async () => { + const cwd = process.cwd(); + const configPath = join(cwd, "sitebase.config.ts"); + const templatesDir = join(cwd, "templates"); + const templatePath = join(templatesDir, "post.hbs"); + + // Check if config already exists + try { + await access(configPath); + console.error("Error: sitebase.config.ts already exists"); + process.exit(1); + } catch { + // File doesn't exist, we can proceed + } + + // Config file template with comments + const configContent = `// sitebase.config.ts +import type { ExportConfig } from "@sitebase/core"; +import { slugify } from "@sitebase/core"; + +/** + * Sitebase Export Configuration + * + * This file configures how your AT Protocol publication is exported to markdown files. + * For more information, see: https://github.com/sethetter/sitebase + */ +const config: ExportConfig = { + // The AT URI of your publication (required) + // Format: at://did:plc:xxx/site.standard.publication/rkey + // Find this in your PDS or use the standard.site dashboard + publicationUri: "at://YOUR_DID/site.standard.publication/YOUR_RKEY", + + // Export targets - each entry exports documents to a different location/format + // You can have multiple targets to export the same publication different ways + exports: [ + { + // Directory where markdown files will be written (relative to this config file) + outputDir: "./content", + + // Optional: Only include documents with at least one of these tags + // If not specified, all documents are included (except those matching excludeTags) + // includeTags: ["post", "article"], + + // Optional: Exclude documents with any of these tags + // Defaults to ["draft"] if includeTags is not specified + // excludeTags: ["draft", "private"], + + // Function to generate the filename for each document (required) + // Receives an object with: title, path, publishedAt, updatedAt, tags, content, etc. + filename: (data) => { + // Example: "2024-01-15_my-post-title.md" + const date = data.publishedAt?.slice(0, 10) || "undated"; + return \`\${date}_\${slugify(data.title)}.md\`; + }, + + // Optional: Path to a Handlebars template for content generation + // The template receives the same data object as the filename function + contentTemplate: "./templates/post.hbs", + + // Optional: Function to generate content (overrides contentTemplate if both specified) + // content: (data) => \`--- + // title: "\${data.title}" + // date: \${data.publishedAt} + // --- + // + // \${data.content} + // \`, + }, + ], +}; + +export default config; +`; + + // Handlebars template for posts + const templateContent = `--- +title: "{{title}}" +{{#if description}} +description: "{{description}}" +{{/if}} +{{#if publishedAt}} +date: {{publishedAt}} +{{/if}} +{{#if updatedAt}} +updated: {{updatedAt}} +{{/if}} +{{#if tags.length}} +tags: +{{#each tags}} + - {{this}} +{{/each}} +{{/if}} +--- + +{{{content}}} +`; + + try { + // Write config file + await writeFile(configPath, configContent, "utf-8"); + console.log("Created sitebase.config.ts"); + + // Create templates directory and template file + await mkdir(templatesDir, { recursive: true }); + await writeFile(templatePath, templateContent, "utf-8"); + console.log("Created templates/post.hbs"); + + console.log("\nNext steps:"); + console.log( + " 1. Update publicationUri in sitebase.config.ts with your publication's AT URI", + ); + console.log(" 2. Customize the export targets as needed"); + console.log(" 3. Run: sitebase export"); + } catch (error) { + console.error( + `Error: ${error instanceof Error ? error.message : String(error)}`, + ); + process.exit(1); + } + }); + program.parse(); diff --git a/packages/core/package.json b/packages/core/package.json index d0a6f05..a97814f 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,14 +1,14 @@ { - "name": "@sitebase/core", - "type": "module", - "version": "0.0.1", - "exports": { - ".": "./src/index.ts" - }, - "scripts": { - "typecheck": "tsc --noEmit" - }, - "dependencies": { - "handlebars": "^4.7.8" - } + "name": "@sitebase/core", + "type": "module", + "version": "0.0.1", + "exports": { + ".": "./src/index.ts" + }, + "scripts": { + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "handlebars": "^4.7.8" + } } diff --git a/packages/core/src/config.ts b/packages/core/src/config.ts new file mode 100644 index 0000000..3fba3ef --- /dev/null +++ b/packages/core/src/config.ts @@ -0,0 +1,61 @@ +import { access } from "node:fs/promises"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import type { ExportConfig } from "./types.ts"; + +const CONFIG_FILENAMES = ["sitebase.config.ts", "sitebase.config.js"]; + +/** + * Find config file in directory (auto-discovery) + */ +export async function findConfigFile(dir: string): Promise { + for (const filename of CONFIG_FILENAMES) { + const configPath = join(dir, filename); + try { + await access(configPath); + return configPath; + } catch { + // File doesn't exist, try next + } + } + return null; +} + +/** + * Load export config from a JS/TS file + */ +export async function loadExportConfig( + configPath: string, +): Promise { + const configUrl = pathToFileURL(configPath).href; + const module = await import(configUrl); + const config = module.default as ExportConfig; + + // Validate required fields + if (!config.publicationUri) { + throw new Error("Config missing required field: publicationUri"); + } + if ( + !config.exports || + !Array.isArray(config.exports) || + config.exports.length === 0 + ) { + throw new Error( + "Config missing required field: exports (must be non-empty array)", + ); + } + + // Validate each export target + for (const [i, target] of config.exports.entries()) { + if (!target.outputDir) { + throw new Error(`Export target ${i} missing required field: outputDir`); + } + if (!target.filename || typeof target.filename !== "function") { + throw new Error( + `Export target ${i} missing required field: filename (must be a function)`, + ); + } + } + + return config; +} diff --git a/packages/core/src/export.ts b/packages/core/src/export.ts index 6c2acb3..0c9a870 100644 --- a/packages/core/src/export.ts +++ b/packages/core/src/export.ts @@ -1,15 +1,14 @@ -import { mkdir, writeFile } from "node:fs/promises"; -import { join } from "node:path"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; import { fetchPublicationWithDocuments } from "./atproto.ts"; import { createHandlebars, DEFAULT_CONTENT_TEMPLATE, - DEFAULT_FILENAME_TEMPLATE, generateContent, - generateFilename, } from "./templates.ts"; import type { Document, + ExportConfig, ExportOptions, ExportResult, Publication, @@ -96,6 +95,13 @@ function buildTemplateData( }; } +/** + * Sanitize a filename by removing path traversal and invalid characters + */ +function sanitizeFilename(filename: string): string { + return filename.replace(/\.\./g, "").replace(/[<>:"|?*]/g, ""); +} + /** * Export a publication to markdown files */ @@ -105,8 +111,9 @@ export async function exportPublication( const { publicationUri, outputDir, - contentTemplate = DEFAULT_CONTENT_TEMPLATE, - filenameTemplate = DEFAULT_FILENAME_TEMPLATE, + filename: filenameFunction, + contentTemplate, + contentFunction, includeTags, excludeTags, } = options; @@ -136,7 +143,7 @@ export async function exportPublication( // Track filenames to detect conflicts const usedFilenames = new Set(); - // Set up Handlebars + // Set up Handlebars (only needed if using content template) const hbs = createHandlebars(); // Process each document @@ -145,8 +152,9 @@ export async function exportPublication( const data = buildTemplateData(doc, publication); - // Generate filename - const filename = generateFilename(hbs, filenameTemplate, data); + // Generate filename using function + let filename = filenameFunction(data); + filename = sanitizeFilename(filename); if (!filename) { result.warnings.push( @@ -166,14 +174,61 @@ export async function exportPublication( } usedFilenames.add(filename); - // Generate content - const content = generateContent(hbs, contentTemplate, data); + // Generate content using function or template + let content: string; + if (contentFunction) { + content = contentFunction(data); + } else { + content = generateContent( + hbs, + contentTemplate || DEFAULT_CONTENT_TEMPLATE, + data, + ); + } - // Write file + // Write file (creating subdirectories if needed) const filePath = join(outputDir, filename); + const fileDir = dirname(filePath); + await mkdir(fileDir, { recursive: true }); await writeFile(filePath, content, "utf-8"); result.filesWritten.push(filePath); } return result; } + +/** + * Export a publication using a config with multiple export targets + * @param config - The export configuration + * @param configDir - Directory containing the config file (for resolving relative paths) + */ +export async function exportFromConfig( + config: ExportConfig, + configDir: string, +): Promise { + const results: ExportResult[] = []; + + for (const target of config.exports) { + // Load content template from file if specified + let contentTemplate: string | undefined; + if (target.contentTemplate) { + const templatePath = resolve(configDir, target.contentTemplate); + contentTemplate = await readFile(templatePath, "utf-8"); + } + + const options: ExportOptions = { + publicationUri: config.publicationUri, + outputDir: resolve(configDir, target.outputDir), + includeTags: target.includeTags, + excludeTags: target.excludeTags, + filename: target.filename, + contentTemplate, + contentFunction: target.content, + }; + + const result = await exportPublication(options); + results.push(result); + } + + return results; +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 1638bf7..54235b9 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,14 +1,16 @@ -// Main export function -export { exportPublication } from "./export.ts"; +// Main export functions +export { exportFromConfig, exportPublication } from "./export.ts"; -// Templates +// Config utilities +export { findConfigFile, loadExportConfig } from "./config.ts"; + +// Templates and helpers export { DEFAULT_CONTENT_TEMPLATE, - DEFAULT_FILENAME_TEMPLATE, createHandlebars, generateContent, - generateFilename, renderTemplate, + slugify, } from "./templates.ts"; // AT Protocol utilities @@ -22,9 +24,13 @@ export { // Types export type { + ContentFunction, Document, + ExportConfig, ExportOptions, ExportResult, + ExportTarget, + FilenameFunction, Publication, TemplateData, } from "./types.ts"; diff --git a/packages/core/src/templates.ts b/packages/core/src/templates.ts index 0fb7a1a..9dc67d0 100644 --- a/packages/core/src/templates.ts +++ b/packages/core/src/templates.ts @@ -26,7 +26,7 @@ tags: /** * Convert a string to a URL-safe slug */ -function slugify(text: string): string { +export function slugify(text: string): string { return text .toString() .toLowerCase() diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 9a7fcd5..0a7b250 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -1,3 +1,13 @@ +/** + * Function to generate a filename from template data + */ +export type FilenameFunction = (data: TemplateData) => string; + +/** + * Function to generate file content from template data + */ +export type ContentFunction = (data: TemplateData) => string; + /** * Options for exporting a publication to markdown files */ @@ -6,14 +16,44 @@ export interface ExportOptions { publicationUri: string; /** Directory to write output files */ outputDir: string; - /** Handlebars template for file content (uses default if not provided) */ + /** Only include documents with ANY of these tags */ + includeTags?: string[]; + /** Exclude documents with ANY of these tags */ + excludeTags?: string[]; + /** Function to generate filename (required) */ + filename: FilenameFunction; + /** Handlebars template string for file content */ contentTemplate?: string; - /** Handlebars template for filename (uses default if not provided) */ - filenameTemplate?: string; + /** Function to generate content (takes precedence over contentTemplate) */ + contentFunction?: ContentFunction; +} + +/** + * Single export target configuration + */ +export interface ExportTarget { + /** Directory to write output files */ + outputDir: string; /** Only include documents with ANY of these tags */ includeTags?: string[]; /** Exclude documents with ANY of these tags */ excludeTags?: string[]; + /** Function to generate filename (required) */ + filename: FilenameFunction; + /** Path to Handlebars template file for content */ + contentTemplate?: string; + /** Function to generate content (takes precedence over contentTemplate) */ + content?: ContentFunction; +} + +/** + * Configuration for exporting a publication (config file format) + */ +export interface ExportConfig { + /** AT URI of the publication */ + publicationUri: string; + /** Export targets */ + exports: ExportTarget[]; } /** -- 2.51.2