diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 93fbfca..694712c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ repos: hooks: - id: prettier name: prettier - entry: pnpm exec prettier --write --ignore-unknown + entry: ./scripts/prettier-hook.sh language: system types_or: [javascript, jsx, ts, tsx, json, yaml, markdown, css, svelte] pass_filenames: true diff --git a/package.json b/package.json index 2a19262..a3f3e9d 100644 --- a/package.json +++ b/package.json @@ -25,5 +25,10 @@ "engines": { "node": ">=18", "pnpm": ">=9" + }, + "devDependencies": { + "prettier": "^3.8.3", + "prettier-plugin-svelte": "^3.5.1", + "prettier-plugin-tailwindcss": "^0.7.2" } } diff --git a/packages/jasper-web/package.json b/packages/jasper-web/package.json index 50dc9de..f51cfe4 100644 --- a/packages/jasper-web/package.json +++ b/packages/jasper-web/package.json @@ -47,6 +47,7 @@ "@sveltejs/kit": "^2.50.2", "@sveltejs/vite-plugin-svelte": "^6.2.4", "@tailwindcss/vite": "^4.1.18", + "@types/node": "^25.6.0", "prettier": "^3.8.1", "prettier-plugin-svelte": "^3.4.1", "prettier-plugin-tailwindcss": "^0.7.2", diff --git a/packages/jasper-web/src/routes/import/+page.svelte b/packages/jasper-web/src/routes/import/+page.svelte index e5fda26..fac0025 100644 --- a/packages/jasper-web/src/routes/import/+page.svelte +++ b/packages/jasper-web/src/routes/import/+page.svelte @@ -4,6 +4,7 @@ import { cubicOut } from 'svelte/easing'; import type { Agent } from '@atproto/api'; import { initOAuth, signInWithOAuth } from '$lib/core/oauth'; + import { runImport } from '@ewanc26/jasper/browser'; import logo from '$lib/assets/favicon.svg'; import { Upload, Loader2, ArrowRight } from '@lucide/svelte'; @@ -53,17 +54,31 @@ result = null; goTo(2); - // Placeholder for actual import logic - logs.push({ level: 'info', message: 'Import functionality coming soon...' }); - logs.push({ level: 'info', message: `File: ${file.name}` }); - logs.push({ level: 'info', message: `Agent: ${agent.did}` }); - - // Simulate completion - setTimeout(() => { - result = { success: 0, errors: 0 }; - logs.push({ level: 'success', message: dryRun ? 'Dry run complete' : 'Import complete' }); + try { + const importResult = await runImport( + agent, + file, + dryRun, + (current, total) => { + progress = { current, total }; + }, + (level, message) => { + logs = [...logs, { level, message }]; + } + ); + + result = { success: importResult.success, errors: importResult.errors }; + logs = [ + ...logs, + { level: 'success', message: dryRun ? 'Dry run complete' : 'Import complete' } + ]; + } catch (error) { + logs = [...logs, { level: 'error', message: `Import failed: ${error}` }]; + result = { success: 0, errors: 1 }; + } finally { isRunning = false; - }, 2000); + progress = null; + } } function handleReset() { @@ -179,6 +194,15 @@
Processing... + {#if progress} +
+
+
+ {progress.current} / {progress.total} + {/if}
{/if} @@ -320,12 +344,32 @@ .running-indicator { display: flex; + flex-direction: column; align-items: center; gap: 0.5rem; color: var(--accent); margin-bottom: 1rem; } + .progress-bar { + width: 100%; + height: 4px; + background: var(--border); + border-radius: 2px; + overflow: hidden; + } + + .progress-fill { + height: 100%; + background: var(--accent); + transition: width 0.3s ease; + } + + .progress-text { + font-size: 0.8rem; + font-family: 'JetBrains Mono', monospace; + } + .logs { font-family: 'JetBrains Mono', monospace; font-size: 0.8rem; diff --git a/packages/jasper/package.json b/packages/jasper/package.json index c502f44..cecbfca 100644 --- a/packages/jasper/package.json +++ b/packages/jasper/package.json @@ -1,6 +1,6 @@ { "name": "@ewanc26/jasper", - "version": "0.1.0", + "version": "0.2.0", "description": "Convert Instagram data exports into posts on Grain.social", "author": "Ewan Croft", "license": "AGPL-3.0-only", @@ -34,7 +34,8 @@ "LICENCE" ], "exports": { - ".": "./dist/index.js" + ".": "./dist/index.js", + "./browser": "./dist/lib/browser.js" }, "main": "./dist/index.js", "types": "./dist/index.d.ts", diff --git a/packages/jasper/src/core/config.ts b/packages/jasper/src/core/config.ts index 52a628b..1b30c35 100644 --- a/packages/jasper/src/core/config.ts +++ b/packages/jasper/src/core/config.ts @@ -4,7 +4,7 @@ import type { Config } from "./types.js"; -export const VERSION = "0.1.0"; +export const VERSION = "0.2.0"; export const config: Config = { /** Grain lexicon NSID for photos */ diff --git a/packages/jasper/src/core/types.ts b/packages/jasper/src/core/types.ts index a3e15f4..c9d6864 100644 --- a/packages/jasper/src/core/types.ts +++ b/packages/jasper/src/core/types.ts @@ -52,6 +52,8 @@ export interface ParsedMedia { originalUri: string; /** Creation timestamp */ timestamp: Date; + /** Loaded data (for browser) */ + data?: Blob; } export interface ParsedPost { diff --git a/packages/jasper/src/lib/browser.ts b/packages/jasper/src/lib/browser.ts new file mode 100644 index 0000000..4adb6f4 --- /dev/null +++ b/packages/jasper/src/lib/browser.ts @@ -0,0 +1,302 @@ +/** + * Browser-compatible functions for Jasper + * Used by jasper-web to run the import pipeline in-browser + */ + +import { + BlobReader, + BlobWriter, + TextWriter, + ZipReader, + type FileEntry, +} from "@zip.js/zip.js"; +import type { + InstagramExportPost, + ParsedPost, + ParsedMedia, +} from "../core/types.js"; +import { log } from "../utils/logger.js"; +import path from "path"; +import { fixFacebookEncoding } from "./parser.js"; +import { publishPhoto } from "./publisher.js"; +import type { Agent } from "@atproto/api"; + +/** + * Get the file extension from a path + */ +function getExtension(filePath: string): string { + return path.extname(filePath).toLowerCase(); +} + +/** + * Check if a file is a video + */ +function isVideo(filePath: string): boolean { + const ext = getExtension(filePath); + return ext === ".mp4" || ext === ".mov"; +} + +/** + * Determine media type from path + */ +function getMediaType(filePath: string): "image" | "video" { + if (isVideo(filePath)) return "video"; + return "image"; +} + +/** + * Parse Instagram export from a File (browser) + */ +export async function parseExportFromFile(file: File): Promise { + log.info(`Reading ZIP file: ${file.name}`); + + const zipReader = new ZipReader(new BlobReader(file)); + const entries = (await zipReader.getEntries()) as FileEntry[]; + + // Find posts_1.json + const postsEntry = findPostsJsonInZip(entries); + if (!postsEntry) { + await zipReader.close(); + throw new Error("posts_1.json not found in ZIP archive"); + } + + log.debug(`Found posts_1.json at: ${postsEntry.filename}`); + + // Extract and parse + const postsText = await postsEntry.getData(new TextWriter()); + + const posts = parsePostsJson(postsText); + + // Filter and log video posts + const imagePosts: ParsedPost[] = []; + for (const post of posts) { + const hasVideo = post.media.some((m) => m.type === "video"); + if (hasVideo) { + post.skipped = true; + post.skipReason = "Contains video(s) — Grain does not support videos yet"; + log.warn(`Skipping video post from ${post.createdAt.toISOString()}`); + continue; + } + + imagePosts.push(post); + } + + // Load media data for each post + for (const post of imagePosts) { + if (post.skipped) continue; + + for (const media of post.media) { + if (media.type === "image") { + try { + const mediaEntry = findMediaInZip(entries, media.originalUri); + if (mediaEntry) { + media.data = await mediaEntry.getData(new BlobWriter()); + } else { + log.warn(`Media file not found: ${media.originalUri}`); + post.skipped = true; + post.skipReason = `Media file missing: ${media.originalUri}`; + } + } catch (error) { + log.error(`Failed to load media ${media.originalUri}: ${error}`); + post.skipped = true; + post.skipReason = `Failed to load media: ${error}`; + } + } + } + } + + await zipReader.close(); + + log.info(`Parsed ${imagePosts.length} posts from ZIP`); + return imagePosts; +} + +/** + * Find posts_1.json in the ZIP entries + */ +function findPostsJsonInZip(entries: FileEntry[]): FileEntry | null { + for (const entry of entries) { + if (entry.filename.endsWith("posts_1.json")) { + return entry; + } + } + return null; +} + +/** + * Find a media file in the ZIP entries + */ +function findMediaInZip(entries: FileEntry[], relativePath: string): FileEntry | null { + // Instagram exports have media in various subdirectories + // Try different possible paths + const possiblePaths = [ + relativePath, + `media/${relativePath}`, + `photos/${relativePath}`, + `videos/${relativePath}`, + ]; + + for (const path of possiblePaths) { + for (const entry of entries) { + if (entry.filename === path || entry.filename.endsWith(`/${path}`)) { + return entry; + } + } + } + + return null; +} + +/** + * Parse posts_1.json content + */ +function parsePostsJson( + postsText: string, +): ParsedPost[] { + log.debug("Parsing posts_1.json"); + + // Fix encoding issues + const fixedText = fixFacebookEncoding(postsText); + + let rawPosts: InstagramExportPost[]; + try { + rawPosts = JSON.parse(fixedText); + } catch (error) { + throw new Error(`Failed to parse posts_1.json: ${error}`); + } + + log.debug(`Found ${rawPosts.length} raw posts`); + + const posts: ParsedPost[] = []; + + for (const rawPost of rawPosts) { + try { + const post = parsePost(rawPost); + if (post) { + posts.push(post); + } + } catch (error) { + log.warn(`Failed to parse post: ${error}`); + } + } + + return posts; +} + +/** + * Parse a single Instagram post + */ +function parsePost( + rawPost: InstagramExportPost, +): ParsedPost | null { + // Skip if no media + if (!rawPost.media || rawPost.media.length === 0) { + return null; + } + + // Use the first media item's timestamp as the post timestamp + const firstMedia = rawPost.media[0]; + const timestamp = new Date(firstMedia.creation_timestamp * 1000); + + // Generate ID from timestamp + const id = timestamp.toISOString(); + + const media: ParsedMedia[] = []; + + for (const rawMedia of rawPost.media) { + const mediaItem: ParsedMedia = { + path: rawMedia.uri, + type: getMediaType(rawMedia.uri), + originalUri: rawMedia.uri, + timestamp: new Date(rawMedia.creation_timestamp * 1000), + }; + + // In browser, we load data later from ZIP + media.push(mediaItem); + } + + return { + id, + caption: rawPost.title || undefined, + createdAt: timestamp, + media, + }; +} + +/** + * Browser-compatible publishPhoto that accepts Blob + */ +export async function publishPhotoFromBlob( + agent: Agent, + imageBlob: Blob, + aspectRatio: { width: number; height: number }, + createdAt: string, + alt?: string, + dryRun = false, +): Promise<{ success: boolean; uri?: string; cid?: string; error?: string }> { + // Convert Blob to Uint8Array + const arrayBuffer = await imageBlob.arrayBuffer(); + const uint8Array = new Uint8Array(arrayBuffer); + + return publishPhoto(agent, uint8Array, aspectRatio, createdAt, alt, dryRun); +} + +/** + * Run the full import process in browser + */ +export async function runImport( + agent: Agent, + file: File, + dryRun: boolean, + onProgress?: (current: number, total: number) => void, + onLog?: (level: string, message: string) => void, +): Promise<{ success: number; errors: number }> { + const logger = onLog ? { info: (msg: string) => onLog('info', msg), warn: (msg: string) => onLog('warn', msg), error: (msg: string) => onLog('error', msg) } : log; + + try { + logger.info('Parsing Instagram export...'); + const posts = await parseExportFromFile(file); + + const validPosts = posts.filter(p => !p.skipped); + logger.info(`Found ${validPosts.length} posts to import`); + + let success = 0; + let errors = 0; + + for (let i = 0; i < validPosts.length; i++) { + const post = validPosts[i]; + onProgress?.(i + 1, validPosts.length); + + for (const media of post.media) { + if (media.type === 'image' && media.data) { + try { + logger.info(`Publishing photo from ${post.createdAt.toISOString()}`); + const result = await publishPhotoFromBlob( + agent, + media.data, + { width: 1, height: 1 }, // TODO: get actual aspect ratio + post.createdAt.toISOString(), + post.caption, + dryRun + ); + + if (result.success) { + success++; + } else { + errors++; + logger.error(`Failed to publish: ${result.error}`); + } + } catch (error) { + errors++; + logger.error(`Error publishing photo: ${error}`); + } + } + } + } + + return { success, errors }; + } catch (error) { + logger.error(`Import failed: ${error}`); + throw error; + } +} \ No newline at end of file diff --git a/packages/jasper/src/lib/image-utils.ts b/packages/jasper/src/lib/image-utils.ts index f277afc..74abd2f 100644 --- a/packages/jasper/src/lib/image-utils.ts +++ b/packages/jasper/src/lib/image-utils.ts @@ -200,3 +200,80 @@ export async function validateImage( }; } } + +// ============================================ +// Browser-compatible functions +// ============================================ + +/** + * Browser-compatible processImage that accepts Uint8Array + */ +export async function processImageBrowser(data: Uint8Array): Promise { + // For browser, we'll skip resizing for now and just return the original + // TODO: Implement browser-based image resizing using Canvas API + + const mimeType = getMimeTypeFromData(data); + + // Get dimensions using Image + const dimensions = await getImageDimensionsBrowser(data); + + const aspectRatio = calculateAspectRatio(dimensions.width, dimensions.height); + + return { + original: Buffer.from(data), + processed: Buffer.from(data), // No processing for now + dimensions, + aspectRatio, + mimeType, + size: data.length, + wasResized: false, + }; +} + +/** + * Get image dimensions in browser using Image API + */ +export async function getImageDimensionsBrowser(data: Uint8Array): Promise<{ width: number; height: number }> { + return new Promise((resolve, reject) => { + const img = new (globalThis as any).Image(); + const url = URL.createObjectURL(new Blob([data as any])); + + img.onload = () => { + URL.revokeObjectURL(url); + resolve({ width: img.naturalWidth, height: img.naturalHeight }); + }; + + img.onerror = () => { + URL.revokeObjectURL(url); + reject(new Error('Failed to load image')); + }; + + img.src = url; + }); +} + +/** + * Get MIME type from image data + */ +function getMimeTypeFromData(data: Uint8Array): string { + // Check magic bytes + if (data.length >= 2) { + const firstBytes = data[0] << 8 | data[1]; + if (firstBytes === 0xFFD8) return 'image/jpeg'; + if (firstBytes === 0x8950) return 'image/png'; + if (data.length >= 4 && data[0] === 0x52 && data[1] === 0x49 && data[2] === 0x46 && data[3] === 0x46) return 'image/webp'; + } + return 'image/jpeg'; // fallback +} + +/** + * Browser-compatible validateImage + */ +export async function validateImageBrowser(data: Uint8Array): Promise<{ valid: boolean; error?: string }> { + try { + await getImageDimensionsBrowser(data); + return { valid: true }; + } catch (error) { + return { valid: false, error: `Invalid image: ${(error as Error).message}` }; + } +} diff --git a/packages/jasper/src/lib/publisher.ts b/packages/jasper/src/lib/publisher.ts index 87421c5..8d38388 100644 --- a/packages/jasper/src/lib/publisher.ts +++ b/packages/jasper/src/lib/publisher.ts @@ -7,7 +7,7 @@ import { generateTID } from "@ewanc26/tid"; import type { ParsedPost } from "../core/types.js"; import { config } from "../core/config.js"; import { log } from "../utils/logger.js"; -import { processImage, validateImage } from "./image-utils.js"; +import { processImage, processImageBrowser, validateImage } from "./image-utils.js"; /** * Result of publishing a single photo @@ -46,7 +46,7 @@ async function uploadBlob( */ export async function publishPhoto( agent: Agent, - imageData: Buffer, + imageData: Buffer | Uint8Array, _aspectRatio: { width: number; height: number }, createdAt: string, alt?: string, @@ -58,8 +58,11 @@ export async function publishPhoto( } try { - // Process image (resize if needed) - const processed = await processImage(imageData); + // Process image (resize if needed) - use browser or Node version + const isBrowser = typeof globalThis !== 'undefined' && 'window' in globalThis; + const processed = isBrowser + ? await processImageBrowser(imageData as Uint8Array) + : await processImage(imageData as Buffer); // Upload blob const blob = await uploadBlob( diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c25e50a..0bee681 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6,7 +6,17 @@ settings: importers: - .: {} + .: + devDependencies: + prettier: + specifier: ^3.8.3 + version: 3.8.3 + prettier-plugin-svelte: + specifier: ^3.5.1 + version: 3.5.1(prettier@3.8.3)(svelte@5.55.4) + prettier-plugin-tailwindcss: + specifier: ^0.7.2 + version: 0.7.2(prettier-plugin-svelte@3.5.1(prettier@3.8.3)(svelte@5.55.4))(prettier@3.8.3) packages/atproto: devDependencies: @@ -129,6 +139,9 @@ importers: '@tailwindcss/vite': specifier: ^4.1.18 version: 4.2.2(vite@7.3.2(@types/node@25.6.0)(jiti@2.6.1)(lightningcss@1.32.0)(tsx@4.21.0)) + '@types/node': + specifier: ^25.6.0 + version: 25.6.0 prettier: specifier: ^3.8.1 version: 3.8.3 diff --git a/scripts/prettier-hook.sh b/scripts/prettier-hook.sh new file mode 100755 index 0000000..81c2880 --- /dev/null +++ b/scripts/prettier-hook.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Pre-commit hook script to run prettier in the correct directories + +# Get the list of files to check +files="$@" + +# Find packages with prettier configs +packages_with_prettier=$(find packages -name ".prettierrc*" -type f | xargs dirname | sort | uniq) + +# Run prettier for files in packages with configs +for package_dir in $packages_with_prettier; do + package_files=$(echo "$files" | grep "^$package_dir/" || true) + if [ -n "$package_files" ]; then + echo "Running prettier in $package_dir..." + cd "$package_dir" && pnpm exec prettier --write --ignore-unknown $package_files + cd - > /dev/null + fi +done + +# Run prettier at root for files not in packages with configs +root_files=$(echo "$files" | grep -v "^packages/" || true) +if [ -n "$root_files" ]; then + echo "Running prettier at root..." + pnpm exec prettier --write --ignore-unknown $root_files +fi \ No newline at end of file