diff --git a/packages/opal-web/src/lib/core/import.ts b/packages/opal-web/src/lib/core/import.ts index 68baa6d..799ae4c 100644 --- a/packages/opal-web/src/lib/core/import.ts +++ b/packages/opal-web/src/lib/core/import.ts @@ -5,7 +5,7 @@ import type { Agent } from '@atproto/api'; import type { Platform, MicroblogPost, ConvertResult } from '@ewanc26/opal'; -import { convertTwitter, convertMastodon, convertThreads, convertNostr, publishRecords, splitToThread } from '@ewanc26/opal'; +import { convertData, parseTwitterArchive, publishRecords } from '@ewanc26/opal'; export interface ImportResult { success: number; @@ -28,32 +28,11 @@ export async function parseExport( platform: Platform, ): Promise { const text = await file.text(); - let data: unknown; + const data = platform === 'twitter' + ? parseTwitterArchive(text) + : JSON.parse(text); - if (platform === 'twitter') { - const match = text.match(/\[[\s\S]*\]/); - if (!match) throw new Error('Could not extract tweet array from Twitter archive'); - data = JSON.parse(match[0]); - } else { - data = JSON.parse(text); - } - - const parsers: Record ConvertResult> = { - twitter: convertTwitter, - mastodon: convertMastodon, - threads: convertThreads, - nostr: convertNostr, - }; - - const result = parsers[platform](data); - - // Split long posts into threads - const posts: MicroblogPost[] = []; - for (const post of result.posts) { - posts.push(...splitToThread(post)); - } - - return { posts, skipped: result.skipped, errors: result.errors }; + return convertData(platform, data); } /** diff --git a/packages/opal/src/cli.ts b/packages/opal/src/cli.ts index 65f9d93..fcb3621 100644 --- a/packages/opal/src/cli.ts +++ b/packages/opal/src/cli.ts @@ -6,8 +6,8 @@ */ import { resolve } from 'node:path'; -import { writeFile } from 'node:fs/promises'; -import { convert } from './convert.js'; +import { readFile, writeFile } from 'node:fs/promises'; +import { convertData, parseTwitterArchive } from './convert.js'; import type { ConvertOptions, Platform } from './types.js'; const VALID_SOURCES: Platform[] = ['twitter', 'mastodon', 'threads', 'nostr']; @@ -44,9 +44,9 @@ Usage: Platforms: twitter Twitter/X archive (tweets.js) - mastodon Mastodon outbox (outbox.json) - threads Threads export (JSON) - nostr Nostr events (JSON array) + mastodon Mastodon outbox (outbox.json) + threads Threads export (JSON) + nostr Nostr events (JSON array) Options: --source Source platform (required) @@ -93,7 +93,13 @@ export async function runCLI(): Promise { console.log(`Opal — converting ${opts.source} export…`); console.log(` Input: ${opts.input}`); - const result = await convert(opts); + // Read and parse the input file + const raw = await readFile(opts.input, 'utf-8'); + const data = opts.source === 'twitter' + ? parseTwitterArchive(raw) + : JSON.parse(raw); + + const result = convertData(opts.source, data); console.log(` Posts: ${result.posts.length}`); console.log(` Skipped: ${result.skipped}`); diff --git a/packages/opal/src/convert.ts b/packages/opal/src/convert.ts index 621c48f..c5ee8f8 100644 --- a/packages/opal/src/convert.ts +++ b/packages/opal/src/convert.ts @@ -1,10 +1,11 @@ /** - * Main conversion dispatcher — reads input, delegates to the correct - * platform parser, and runs post-processing (thread splitting). + * Main conversion dispatcher — delegates to the correct platform parser + * and runs post-processing (thread splitting). + * + * This module has no Node.js dependencies and is safe for browser use. */ -import { readFile } from 'node:fs/promises'; -import type { ConvertOptions, ConvertResult, MicroblogPost, Platform } from './types.js'; +import type { ConvertResult, MicroblogPost, Platform } from './types.js'; import { convertTwitter } from './twitter.js'; import { convertMastodon } from './mastodon.js'; import { convertThreads } from './threads.js'; @@ -19,16 +20,16 @@ const parsers: Record ConvertResult> = { }; /** - * Parse a platform export file and convert to normalised MicroblogPost[]. + * Convert pre-parsed platform data to normalised MicroblogPost[]. * Long posts (>300 graphemes) are automatically split into Bluesky threads. + * + * For Twitter archives, the caller must extract the JSON array from the + * JS wrapper before passing it here. Use `parseTwitterArchive()` from + * the CLI module for that. */ -export async function convert(opts: ConvertOptions): Promise { - const raw = await readFile(opts.input, 'utf-8'); - const data = parseRaw(raw, opts.source); - const parser = parsers[opts.source]; +export function convertData(source: Platform, data: unknown): ConvertResult { + const parser = parsers[source]; const result = parser(data); - - // Split long posts into threads return splitLongPosts(result); } @@ -51,21 +52,16 @@ function splitLongPosts(result: ConvertResult): ConvertResult { } /** - * Pre-process raw file content before passing to the parser. + * Pre-process raw Twitter archive content. * Twitter archives wrap data in `window.YTD.tweets.part0 = [...]` * which isn't valid JSON on its own. */ -function parseRaw(raw: string, source: Platform): unknown { - if (source === 'twitter') { - // Twitter archives assign to a JS variable — extract the JSON array - const match = raw.match(/\[[\s\S]*\]/); - if (!match) { - throw new Error('Could not extract tweet array from Twitter archive file'); - } - return JSON.parse(match[0]); +export function parseTwitterArchive(raw: string): unknown { + const match = raw.match(/\[[\s\S]*\]/); + if (!match) { + throw new Error('Could not extract tweet array from Twitter archive file'); } - - return JSON.parse(raw); + return JSON.parse(match[0]); } export { convertTwitter } from './twitter.js'; diff --git a/packages/opal/src/index.ts b/packages/opal/src/index.ts index 6d89587..02697ee 100644 --- a/packages/opal/src/index.ts +++ b/packages/opal/src/index.ts @@ -6,7 +6,7 @@ */ // Conversion -export { convert } from './convert.js'; +export { convertData, parseTwitterArchive } from './convert.js'; export { convertTwitter } from './twitter.js'; export { convertMastodon } from './mastodon.js'; export { convertThreads } from './threads.js';