diff --git a/src/index.tsx b/src/index.tsx index 455c4b7..3d7b4c5 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -3,9 +3,7 @@ import { Route, Router } from "@solidjs/router"; import { render } from "solid-js/web"; import { Layout } from "./layout.tsx"; import "./styles/index.css"; -import { ExploreToolView } from "./views/car/explore.tsx"; -import { CarView } from "./views/car/index.tsx"; -import { UnpackToolView } from "./views/car/unpack.tsx"; +import { CarView } from "./views/car/explore.tsx"; import { CollectionLayout } from "./views/collection.tsx"; import { Home } from "./views/home.tsx"; import { LabelView } from "./views/labels.tsx"; @@ -22,8 +20,6 @@ render( - - diff --git a/src/layout.tsx b/src/layout.tsx index eb2e898..4ca7075 100644 --- a/src/layout.tsx +++ b/src/layout.tsx @@ -153,7 +153,7 @@ const Layout = (props: RouteSectionProps) => { - + diff --git a/src/views/car/explore.tsx b/src/views/car/explore.tsx index fe7f0c0..6b2423c 100644 --- a/src/views/car/explore.tsx +++ b/src/views/car/explore.tsx @@ -4,6 +4,7 @@ import * as CID from "@atcute/cid"; import { Did } from "@atcute/lexicons"; import { fromStream, isCommit } from "@atcute/repo"; import * as TID from "@atcute/tid"; +import { zip, type ZipEntry } from "@mary/zip"; import { useLocation, useNavigate } from "@solidjs/router"; import { createEffect, @@ -18,20 +19,146 @@ import { import { Button } from "../../components/button.jsx"; import { Favicon } from "../../components/favicon.jsx"; import HoverCard from "../../components/hover-card/base"; -import { JSONValue } from "../../components/json.jsx"; +import { JSONValue, type JSONType } from "../../components/json.jsx"; import { TextInput } from "../../components/text-input.jsx"; import { didDocCache, resolveDidDoc } from "../../lib/api.js"; import { createDebouncedValue } from "../../lib/debounced.js"; import { localDateFromTimestamp } from "../../utils/date.js"; import { createDropHandler, createFileChangeHandler, handleDragOver } from "./file-handlers.js"; -import { - type Archive, - type CollectionEntry, - type RecordEntry, - type View, - toJsonValue, - WelcomeView, -} from "./shared.jsx"; + +const isIOS = + /iPad|iPhone|iPod/.test(navigator.userAgent) || + (navigator.platform === "MacIntel" && navigator.maxTouchPoints > 1); + +const toJsonValue = (obj: unknown): JSONType => { + if (obj === null || obj === undefined) return null; + + if (CID.isCidLink(obj)) { + return { $link: obj.$link }; + } + + if ( + obj && + typeof obj === "object" && + "version" in obj && + "codec" in obj && + "digest" in obj && + "bytes" in obj + ) { + try { + return { $link: CID.toString(obj as CID.Cid) }; + } catch {} + } + + if (CBOR.isBytes(obj)) { + return { $bytes: obj.$bytes }; + } + + if (Array.isArray(obj)) { + return obj.map(toJsonValue); + } + + if (typeof obj === "object") { + const result: Record = {}; + for (const [key, value] of Object.entries(obj)) { + result[key] = toJsonValue(value); + } + return result; + } + + return obj as JSONType; +}; + +interface Archive { + file: File; + did: string; + entries: CollectionEntry[]; +} + +interface CollectionEntry { + name: string; + entries: RecordEntry[]; +} + +interface RecordEntry { + key: string; + cid: string; + record: JSONType; +} + +type View = + | { type: "repo" } + | { type: "collection"; collection: CollectionEntry } + | { type: "record"; collection: CollectionEntry; record: RecordEntry }; + +const WelcomeView = (props: { + title: string; + subtitle: string; + loading: boolean; + progress?: number; + error?: string; + onFileChange: (e: Event) => void; + onDrop: (e: DragEvent) => void; + onDragOver: (e: DragEvent) => void; +}) => { + return ( +
+
+
+

{props.title}

+
+

{props.subtitle}

+
+ +
+ + + + Reading CAR file... + + 0}> + + {props.progress?.toLocaleString()} records processed + + +
+ } + > + +
+

+ Drag and drop a CAR file here +

+

or

+
+ + +
+ + +
+ {props.error} +
+ + + ); +}; const viewToHash = (view: View): string => { switch (view.type) { @@ -68,7 +195,19 @@ const hashToView = (hash: string, archive: Archive): View => { return { type: "repo" }; }; -export const ExploreToolView = () => { +// Check if browser natively supports File System Access API +const hasNativeFileSystemAccess = "showSaveFilePicker" in window; + +// HACK: Disable compression on WebKit due to an error being thrown +const isWebKit = + isIOS || (/AppleWebKit/.test(navigator.userAgent) && !/Chrome/.test(navigator.userAgent)); + +const INVALID_CHAR_RE = /[<>:"/\\|?*\x00-\x1F]/g; +const filenamify = (name: string) => { + return name.replace(INVALID_CHAR_RE, "~"); +}; + +export const CarView = () => { const location = useLocation(); const navigate = useNavigate(); @@ -187,15 +326,15 @@ export const ExploreToolView = () => { if (location.hash) navigate(location.pathname, { replace: true }); }; - document.title = "Explore archive - PDSls"; + document.title = "CAR explorer - PDSls"; return ( <> { ); }; +const exportToZip = async ( + archive: Archive, + onProgress: (count: number) => void, + onSaving: () => void, +) => { + const filename = archive.file.name.replace(/\.car$/, ""); + + const entryGenerator = async function* (): AsyncGenerator { + let count = 0; + for (const collection of archive.entries) { + for (const record of collection.entries) { + const path = `${collection.name}/${filenamify(record.key)}.json`; + const data = JSON.stringify(record.record, null, 2); + yield { filename: path, data, compress: isWebKit ? false : "deflate" }; + if (++count % 500 === 0) { + onProgress(count); + await new Promise((resolve) => requestAnimationFrame(resolve)); + } + } + } + onProgress(count); + }; + + if (hasNativeFileSystemAccess) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const fd = await (window as any) + .showSaveFilePicker({ + suggestedName: `${filename}.zip`, + id: "car-unpack", + startIn: "downloads", + types: [ + { + description: "ZIP archive", + accept: { "application/zip": [".zip"] }, + }, + ], + }) + .catch(() => undefined); + + if (!fd) return; + + const writable = await fd.createWritable(); + if (!writable) return; + + let writeCount = 0; + for await (const chunk of zip(entryGenerator())) { + writeCount++; + if (writeCount % 100 === 0) { + await writable.write(chunk as BufferSource); + } else { + writable.write(chunk as BufferSource); + } + } + + onSaving(); + await new Promise((resolve) => requestAnimationFrame(resolve)); + await writable.close(); + } else { + const chunks: BlobPart[] = []; + for await (const chunk of zip(entryGenerator())) { + chunks.push(chunk as BlobPart); + } + + onSaving(); + await new Promise((resolve) => requestAnimationFrame(resolve)); + const blob = new Blob(chunks, { type: "application/zip" }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = `${filename}.zip`; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + URL.revokeObjectURL(url); + } +}; + const ExploreView = (props: { archive: Archive; view: () => View; setView: (view: View) => void; onClose: () => void; }) => { + const [exporting, setExporting] = createSignal(false); + const [exportCount, setExportCount] = createSignal(0); + const [exportSaving, setExportSaving] = createSignal(false); + + const totalRecords = createMemo(() => + props.archive.entries.reduce((sum, entry) => sum + entry.entries.length, 0), + ); + + const handleExport = async () => { + setExporting(true); + setExportCount(0); + setExportSaving(false); + try { + await exportToZip(props.archive, setExportCount, () => { + setExportSaving(true); + }); + } finally { + setExporting(false); + setExportSaving(false); + } + }; + const handle = didDocCache[props.archive.did]?.alsoKnownAs ?.filter((alias) => alias.startsWith("at://"))[0] @@ -265,14 +503,34 @@ const ExploreView = (props: { - +
+ + Export ZIP + + } + > +
+ + + {exportSaving() ? "" : `${Math.round((exportCount() / totalRecords()) * 100)}%`} + +
+ + +
{/* Collection Level */} diff --git a/src/views/car/index.tsx b/src/views/car/index.tsx deleted file mode 100644 index a1bdc63..0000000 --- a/src/views/car/index.tsx +++ /dev/null @@ -1,43 +0,0 @@ -import { A } from "@solidjs/router"; - -export const CarView = () => { - document.title = "Archive tools - PDSls"; - return ( -
-
-

Archive tools

-

- Tools for working with CAR (Content Addressable aRchive) files. -

-
- -
- - -
- Explore archive - - Browse records inside a repository archive - -
- - - - -
- Unpack archive - - Extract records from an archive into a ZIP file - -
- -
-
- ); -}; diff --git a/src/views/car/logger.tsx b/src/views/car/logger.tsx deleted file mode 100644 index dc15701..0000000 --- a/src/views/car/logger.tsx +++ /dev/null @@ -1,152 +0,0 @@ -import { For } from "solid-js"; -import { createMutable } from "solid-js/store"; - -interface LogEntry { - type: "log" | "info" | "warn" | "error"; - at: number; - msg: string; -} - -interface PendingLogEntry { - msg: string; -} - -export const createLogger = () => { - const pending = createMutable([]); - - let backlog: LogEntry[] | undefined = []; - let push = (entry: LogEntry) => { - backlog!.push(entry); - }; - - return { - internal: { - get pending() { - return pending; - }, - attach(fn: (entry: LogEntry) => void) { - if (backlog !== undefined) { - for (let idx = 0, len = backlog.length; idx < len; idx++) { - fn(backlog[idx]); - } - backlog = undefined; - } - push = fn; - }, - }, - log(msg: string) { - push({ type: "log", at: Date.now(), msg }); - }, - info(msg: string) { - push({ type: "info", at: Date.now(), msg }); - }, - warn(msg: string) { - push({ type: "warn", at: Date.now(), msg }); - }, - error(msg: string) { - push({ type: "error", at: Date.now(), msg }); - }, - progress(initialMsg: string, throttleMs = 500) { - pending.unshift({ msg: initialMsg }); - - let entry: PendingLogEntry | undefined = pending[0]; - - return { - update: throttle((msg: string) => { - if (entry !== undefined) { - entry.msg = msg; - } - }, throttleMs), - destroy() { - if (entry !== undefined) { - const index = pending.indexOf(entry); - pending.splice(index, 1); - entry = undefined; - } - }, - [Symbol.dispose]() { - this.destroy(); - }, - }; - }, - }; -}; - -export type Logger = ReturnType; - -const formatter = new Intl.DateTimeFormat("en-US", { timeStyle: "short", hour12: false }); - -export const LoggerView = (props: { logger: Logger }) => { - return ( -
    - - {(entry) => ( -
  • - ----- - {entry.msg} -
    • - )} -     - -
    { - props.logger.internal.attach(({ type, at, msg }) => { - let ecn = `flex gap-2 whitespace-pre-wrap px-4 py-1`; - let tcn = `shrink-0 whitespace-pre-wrap font-medium`; - if (type === "log") { - tcn += ` text-neutral-500`; - } else if (type === "info") { - ecn += ` bg-blue-100 text-blue-800 dark:bg-blue-900/30 dark:text-blue-300`; - tcn += ` text-blue-500`; - } else if (type === "warn") { - ecn += ` bg-amber-100 text-amber-800 dark:bg-amber-900/30 dark:text-amber-300`; - tcn += ` text-amber-500`; - } else if (type === "error") { - ecn += ` bg-red-100 text-red-800 dark:bg-red-900/30 dark:text-red-300`; - tcn += ` text-red-500`; - } - - const item = ( -
  • - {formatter.format(at)} - {msg} -
    • - ); - - if (item instanceof Node) { - node.after(item); - } - }); - }} - /> -
- ); -}; - -const throttle = void>(func: T, wait: number) => { - let timeout: ReturnType | null = null; - let lastArgs: Parameters | null = null; - let lastCallTime = 0; - - const invoke = () => { - func(...lastArgs!); - lastCallTime = Date.now(); - timeout = null; - }; - - return (...args: Parameters) => { - const now = Date.now(); - const timeSinceLastCall = now - lastCallTime; - - lastArgs = args; - - if (timeSinceLastCall >= wait) { - if (timeout !== null) { - clearTimeout(timeout); - } - invoke(); - } else if (timeout === null) { - timeout = setTimeout(invoke, wait - timeSinceLastCall); - } - }; -}; diff --git a/src/views/car/shared.tsx b/src/views/car/shared.tsx deleted file mode 100644 index f3cbcf5..0000000 --- a/src/views/car/shared.tsx +++ /dev/null @@ -1,146 +0,0 @@ -import * as CBOR from "@atcute/cbor"; -import * as CID from "@atcute/cid"; -import { A } from "@solidjs/router"; -import { Show } from "solid-js"; -import { type JSONType } from "../../components/json.jsx"; - -export const isIOS = - /iPad|iPhone|iPod/.test(navigator.userAgent) || - (navigator.platform === "MacIntel" && navigator.maxTouchPoints > 1); - -// Convert CBOR-decoded objects to JSON-friendly format -export const toJsonValue = (obj: unknown): JSONType => { - if (obj === null || obj === undefined) return null; - - if (CID.isCidLink(obj)) { - return { $link: obj.$link }; - } - - if ( - obj && - typeof obj === "object" && - "version" in obj && - "codec" in obj && - "digest" in obj && - "bytes" in obj - ) { - try { - return { $link: CID.toString(obj as CID.Cid) }; - } catch {} - } - - if (CBOR.isBytes(obj)) { - return { $bytes: obj.$bytes }; - } - - if (Array.isArray(obj)) { - return obj.map(toJsonValue); - } - - if (typeof obj === "object") { - const result: Record = {}; - for (const [key, value] of Object.entries(obj)) { - result[key] = toJsonValue(value); - } - return result; - } - - return obj as JSONType; -}; - -export interface Archive { - file: File; - did: string; - entries: CollectionEntry[]; -} - -export interface CollectionEntry { - name: string; - entries: RecordEntry[]; -} - -export interface RecordEntry { - key: string; - cid: string; - record: JSONType; -} - -export type View = - | { type: "repo" } - | { type: "collection"; collection: CollectionEntry } - | { type: "record"; collection: CollectionEntry; record: RecordEntry }; - -export const WelcomeView = (props: { - title: string; - subtitle: string; - loading: boolean; - progress?: number; - error?: string; - onFileChange: (e: Event) => void; - onDrop: (e: DragEvent) => void; - onDragOver: (e: DragEvent) => void; -}) => { - return ( -
-
-
- - - -

{props.title}

-
-

{props.subtitle}

-
- -
- - - - Reading CAR file... - - 0}> - - {props.progress?.toLocaleString()} records processed - - -
- } - > - -
-

- Drag and drop a CAR file here -

-

or

-
- - -
- - -
- {props.error} -
- - - ); -}; diff --git a/src/views/car/unpack.tsx b/src/views/car/unpack.tsx deleted file mode 100644 index d3bb87b..0000000 --- a/src/views/car/unpack.tsx +++ /dev/null @@ -1,249 +0,0 @@ -import { fromStream } from "@atcute/repo"; -import { zip, type ZipEntry } from "@mary/zip"; -import { createSignal, onCleanup } from "solid-js"; -import { createDropHandler, createFileChangeHandler, handleDragOver } from "./file-handlers.js"; -import { createLogger, LoggerView } from "./logger.jsx"; -import { isIOS, toJsonValue, WelcomeView } from "./shared.jsx"; - -// Check if browser natively supports File System Access API -const hasNativeFileSystemAccess = "showSaveFilePicker" in window; - -// HACK: Disable compression on WebKit due to an error being thrown -const isWebKit = - isIOS || (/AppleWebKit/.test(navigator.userAgent) && !/Chrome/.test(navigator.userAgent)); - -const INVALID_CHAR_RE = /[<>:"/\\|?*\x00-\x1F]/g; -const filenamify = (name: string) => { - return name.replace(INVALID_CHAR_RE, "~"); -}; - -export const UnpackToolView = () => { - const logger = createLogger(); - const [pending, setPending] = createSignal(false); - - let abortController: AbortController | undefined; - - onCleanup(() => { - abortController?.abort(); - }); - - const unpackToZip = async (file: File) => { - abortController?.abort(); - abortController = new AbortController(); - const signal = abortController.signal; - - setPending(true); - logger.log(`Starting extraction`); - - let repo: Awaited> | undefined; - - const stream = file.stream(); - repo = fromStream(stream); - - try { - let count = 0; - - // On Safari/browsers without native File System Access API, use blob download - if (!hasNativeFileSystemAccess) { - const chunks: BlobPart[] = []; - - const entryGenerator = async function* (): AsyncGenerator { - const progress = logger.progress(`Unpacking records (0 entries)`); - - try { - for await (const entry of repo) { - if (signal.aborted) return; - - try { - const record = toJsonValue(entry.record); - const filename = `${entry.collection}/${filenamify(entry.rkey)}.json`; - const data = JSON.stringify(record, null, 2); - - yield { filename, data, compress: isWebKit ? false : "deflate" }; - count++; - progress.update(`Unpacking records (${count} entries)`); - } catch { - // Skip entries with invalid data - } - } - } finally { - progress[Symbol.dispose]?.(); - } - }; - - for await (const chunk of zip(entryGenerator())) { - if (signal.aborted) return; - chunks.push(chunk as BlobPart); - } - - if (signal.aborted) return; - - logger.log(`${count} records extracted`); - logger.log(`Creating download...`); - - const blob = new Blob(chunks, { type: "application/zip" }); - const url = URL.createObjectURL(blob); - const a = document.createElement("a"); - a.href = url; - a.download = `${file.name.replace(/\.car$/, "")}.zip`; - document.body.appendChild(a); - a.click(); - document.body.removeChild(a); - URL.revokeObjectURL(url); - - logger.log(`Finished! Download started.`); - setPending(false); - return; - } - - // Native File System Access API path - let writable: FileSystemWritableFileStream | undefined; - - // Create async generator that yields ZipEntry as we read from CAR - const entryGenerator = async function* (): AsyncGenerator { - const progress = logger.progress(`Unpacking records (0 entries)`); - - try { - for await (const entry of repo) { - if (signal.aborted) return; - - // Prompt for save location on first record - if (writable === undefined) { - const waiting = logger.progress(`Waiting for user...`); - - try { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const fd = await (window as any) - .showSaveFilePicker({ - suggestedName: `${file.name.replace(/\.car$/, "")}.zip`, - id: "car-unpack", - startIn: "downloads", - types: [ - { - description: "ZIP archive", - accept: { "application/zip": [".zip"] }, - }, - ], - }) - .catch((err: unknown) => { - if (err instanceof DOMException && err.name === "AbortError") { - logger.warn(`File picker was cancelled`); - } else { - logger.warn(`Something went wrong when opening the file picker`); - } - return undefined; - }); - - if (!fd) { - logger.warn(`No file handle obtained`); - return; - } - - writable = await fd.createWritable(); - - if (writable === undefined) { - logger.warn(`Failed to create writable stream`); - return; - } - } finally { - waiting[Symbol.dispose]?.(); - } - } - - try { - const record = toJsonValue(entry.record); - const filename = `${entry.collection}/${filenamify(entry.rkey)}.json`; - const data = JSON.stringify(record, null, 2); - - yield { filename, data, compress: isWebKit ? false : "deflate" }; - count++; - progress.update(`Unpacking records (${count} entries)`); - } catch { - // Skip entries with invalid data - } - } - } finally { - progress[Symbol.dispose]?.(); - } - }; - - // Stream entries directly to zip, then to file - let writeCount = 0; - for await (const chunk of zip(entryGenerator())) { - if (signal.aborted) { - await writable?.abort(); - return; - } - if (writable === undefined) { - // User cancelled file picker - setPending(false); - return; - } - writeCount++; - // Await every 100th write to apply backpressure - if (writeCount % 100 === 0) { - await writable.write(chunk as BufferSource); - } else { - writable.write(chunk as BufferSource); - } - } - - if (signal.aborted) return; - - if (writable === undefined) { - logger.warn(`CAR file has no records`); - setPending(false); - return; - } - - logger.log(`${count} records extracted`); - - { - const flushProgress = logger.progress(`Flushing writes...`); - try { - await writable.close(); - logger.log(`Finished! File saved successfully.`); - } catch (err) { - logger.error(`Failed to save file: ${err}`); - throw err; // Re-throw to be caught by outer catch - } finally { - flushProgress[Symbol.dispose]?.(); - } - } - } catch (err) { - if (signal.aborted) return; - logger.error(`Error: ${err}\nFile might be malformed, or might not be a CAR archive`); - } finally { - await repo?.dispose(); - if (!signal.aborted) { - setPending(false); - } - } - }; - - const handleFileChange = createFileChangeHandler(unpackToZip); - - // Wrap handleDrop to prevent multiple simultaneous uploads - const baseDrop = createDropHandler(unpackToZip); - const handleDrop = (e: DragEvent) => { - if (pending()) return; - baseDrop(e); - }; - - document.title = "Unpack archive - PDSls"; - return ( - <> - -
- -
- - ); -}; diff --git a/src/views/home.tsx b/src/views/home.tsx index 74ecece..098d2ab 100644 --- a/src/views/home.tsx +++ b/src/views/home.tsx @@ -188,7 +188,7 @@ export const Home = () => { href="/car" icon="lucide--folder-archive" title="Archive" - description="Unpack CAR files" + description="Explore CAR files" accent="violet" /> diff --git a/src/worker.js b/src/worker.js index 4071215..ef1b427 100644 --- a/src/worker.js +++ b/src/worker.js @@ -365,16 +365,8 @@ const STATIC_ROUTES = { }, "/labels": { title: "Labels", description: "Query labels applied to accounts and records." }, "/car": { - title: "Archive tools", - description: "Tools for working with CAR (Content Addressable aRchive) files.", - }, - "/car/explore": { - title: "Explore archive", - description: "Upload a CAR file to explore its contents.", - }, - "/car/unpack": { - title: "Unpack archive", - description: "Upload a CAR file to extract all records into a ZIP archive.", + title: "CAR explorer", + description: "Upload an archive to explore or export its contents.", }, "/settings": { title: "Settings", description: "Browse the public data on atproto" }, };