From 9b8cc43258e7c307bb58c5dc658a3e59b77cc5b0 Mon Sep 17 00:00:00 2001 From: Ethan Graf Date: Wed, 17 Jun 2026 21:17:05 -0400 Subject: [PATCH] Vault integration phase 1: foundation types, HabitatRepo, snapshot, and utils Lays the groundwork for Obsidian-style vaults backed by Habitat PDS + libp2p gossipsub. Adds types, HabitatRepo/DocHandle adapter over AutomergeDocSession, VaultSnapshotManager, vault registry, and utility modules (content hashing, text diff, move detection similarity, directory traversal). Also pulls forward collection-aware routing in automergeDoc.ts and adds changeAt to AutomergeDocSession. Co-Authored-By: Claude Sonnet 4.6 --- src/habitat/automergeDoc.ts | 35 ++++- src/habitat/automergeDocSession.ts | 9 ++ src/habitat/config.ts | 5 + src/vault/habitat-repo.ts | 100 ++++++++++++++ src/vault/snapshot.ts | 97 +++++++++++++ src/vault/types.ts | 198 +++++++++++++++++++++++++++ src/vault/utils/content.ts | 34 +++++ src/vault/utils/directory.ts | 20 +++ src/vault/utils/string-similarity.ts | 33 +++++ src/vault/utils/text-diff.ts | 69 ++++++++++ 10 files changed, 596 insertions(+), 4 deletions(-) create mode 100644 src/vault/habitat-repo.ts create mode 100644 src/vault/snapshot.ts create mode 100644 src/vault/utils/content.ts create mode 100644 src/vault/utils/directory.ts create mode 100644 src/vault/utils/string-similarity.ts create mode 100644 src/vault/utils/text-diff.ts diff --git a/src/habitat/automergeDoc.ts b/src/habitat/automergeDoc.ts index 5f6a611..e579d1f 100644 --- a/src/habitat/automergeDoc.ts +++ b/src/habitat/automergeDoc.ts @@ -3,6 +3,8 @@ import * as Automerge from '@automerge/automerge'; import { HABITAT_DOCS_COLLECTION, HABITAT_DOCS_EDIT_COLLECTION, + HABITAT_VAULT_FILE_COLLECTION, + HABITAT_VAULT_FILE_EDIT_COLLECTION, } from './config'; import { parseDocUri, editRkeyFor } from './habitatDoc'; import type { HabitatDoc, TypedRecord } from './types'; @@ -12,6 +14,20 @@ import { getPrivateRecord, procedure, query } from './xrpc'; // they live in the Yjs file — they are CRDT-agnostic. export { parseDocUri, editRkeyFor }; +/** + * Extract the collection NSID from a habitat/at URI. + * + * `parseDocUri` deliberately only returns `(ownerDid, rkey)` — the rest of + * the docs pipeline doesn't care about the collection. Vault documents, + * however, live in dedicated collections (directory vs file), so the + * Automerge path needs the collection to load/save against the right one. + * Falls back to `network.habitat.docs` for malformed/short URIs. + */ +function collectionFromUri(uri: string): string { + const parts = uri.split('/'); + return parts[3] || HABITAT_DOCS_COLLECTION; +} + export type AutomergeDocSchema = { name: string; content: string; @@ -60,9 +76,10 @@ export async function loadAutomergeDoc( initialDoc: Automerge.Doc, ): Promise { const { ownerDid, rkey } = parseDocUri(uri); + const collection = collectionFromUri(uri); const ownerRecord = await getPrivateRecord( - HABITAT_DOCS_COLLECTION, + collection, rkey, ownerDid, true, @@ -99,11 +116,15 @@ export async function mergeAllEdits( const { ownerDid, rkey } = parseDocUri(ownerRecord.uri); const editRkey = editRkeyFor(ownerDid, rkey); + const editCollection = + collectionFromUri(ownerRecord.uri) === HABITAT_VAULT_FILE_COLLECTION + ? HABITAT_VAULT_FILE_EDIT_COLLECTION + : HABITAT_DOCS_EDIT_COLLECTION; const results = await Promise.all( memberDids.map((did) => getPrivateRecord( - HABITAT_DOCS_EDIT_COLLECTION, + editCollection, editRkey, did, ).catch(() => null), @@ -143,12 +164,18 @@ export type SaveTarget = { export function chooseSaveTarget(ownerUri: string, myDid: string): SaveTarget { const { ownerDid, rkey } = parseDocUri(ownerUri); + const ownerCollection = collectionFromUri(ownerUri); if (ownerDid === myDid) { - return { repo: myDid, collection: HABITAT_DOCS_COLLECTION, rkey }; + return { repo: myDid, collection: ownerCollection, rkey }; } + // Route the collaborator edit mirror to the matching edit collection. + const editCollection = + ownerCollection === HABITAT_VAULT_FILE_COLLECTION + ? HABITAT_VAULT_FILE_EDIT_COLLECTION + : HABITAT_DOCS_EDIT_COLLECTION; return { repo: myDid, - collection: HABITAT_DOCS_EDIT_COLLECTION, + collection: editCollection, rkey: editRkeyFor(ownerDid, rkey), }; } diff --git a/src/habitat/automergeDocSession.ts b/src/habitat/automergeDocSession.ts index b4dcc17..ee4a026 100644 --- a/src/habitat/automergeDocSession.ts +++ b/src/habitat/automergeDocSession.ts @@ -38,6 +38,7 @@ export type AutomergeDocSession = { resync: () => Promise; release: () => void; applyChange: (fn: (doc: AutomergeDocSchema) => void) => void; + changeAt: (heads: string[], fn: (doc: AutomergeDocSchema) => void) => void; subscribe: (cb: () => void) => () => void; }; @@ -326,6 +327,14 @@ function makeHandle(entry: RegistryEntry): AutomergeDocSession { entry.provider?.onLocalChange(newDoc); notifyChange(entry); }, + changeAt: (heads, fn) => { + const result = Automerge.changeAt(entry.doc, heads, fn); + if (result.newHeads !== null) { + entry.doc = result.newDoc; + entry.provider?.onLocalChange(result.newDoc); + notifyChange(entry); + } + }, subscribe: (cb) => { entry.changeListeners.add(cb); // Fire immediately if the doc is already loaded (ownerRecord is set). diff --git a/src/habitat/config.ts b/src/habitat/config.ts index 41a31d7..e0e6f78 100644 --- a/src/habitat/config.ts +++ b/src/habitat/config.ts @@ -13,3 +13,8 @@ export const HABITAT_DID = 'did:plc:ss2uhsajrstfhkq73fteu4zz'; /** Collection NSIDs used by the docs lexicon. */ export const HABITAT_DOCS_COLLECTION = 'network.habitat.docs'; export const HABITAT_DOCS_EDIT_COLLECTION = 'network.habitat.docs.edit'; + +/** Collection NSIDs for vault documents (directories and files). */ +export const HABITAT_VAULT_DIRECTORY_COLLECTION = 'network.habitat.vault.directory'; +export const HABITAT_VAULT_FILE_COLLECTION = 'network.habitat.vault.file'; +export const HABITAT_VAULT_FILE_EDIT_COLLECTION = 'network.habitat.vault.file.edit'; diff --git a/src/vault/habitat-repo.ts b/src/vault/habitat-repo.ts new file mode 100644 index 0000000..ba7782b --- /dev/null +++ b/src/vault/habitat-repo.ts @@ -0,0 +1,100 @@ +import * as Automerge from '@automerge/automerge'; + +import type { + HabitatDocHandle, + HabitatRepo, + HabitatUri, + DirectoryDocument, + FileDocument, +} from './types'; +import { acquireDocSession } from '../habitat/automergeDocSession'; +import type { AutomergeDocSession } from '../habitat/automergeDocSession'; +import { + HABITAT_VAULT_DIRECTORY_COLLECTION, + HABITAT_VAULT_FILE_COLLECTION, +} from '../habitat/config'; + +/** + * Adapts an AutomergeDocSession into a HabitatDocHandle. + * + * Mirrors the automerge-repo DocHandle API but uses HabitatUri + * and is backed by the existing textile AutomergeDocSession. + */ +class HabitatDocHandleImpl implements HabitatDocHandle { + constructor(private session: AutomergeDocSession) {} + + doc(): Automerge.Doc { + return this.session.doc as Automerge.Doc; + } + + heads(): string[] { + return Automerge.getHeads(this.session.doc); + } + + change(fn: (d: T) => void): void { + this.session.applyChange(fn as (doc: unknown) => void); + } + + changeAt(heads: string[], fn: (d: T) => void): void { + this.session.changeAt(heads, fn as (doc: unknown) => void); + } + + view(heads: string[]): Automerge.Doc { + return Automerge.view(this.session.doc, heads) as Automerge.Doc; + } + + on(event: 'change', cb: () => void): () => void { + return this.session.subscribe(cb); + } +} + +/** + * HabitatRepo mirrors the automerge-repo Repo API. + * + * Uses `acquireDocSession` from the existing textile sync stack + * to load/save Habitat records. No `automerge-repo` dependency. + * + * Directory documents are stored in `network.habitat.vault.directory` + * File documents are stored in `network.habitat.vault.file` + */ +export class HabitatRepoImpl implements HabitatRepo { + async find(uri: HabitatUri): Promise> { + const session = acquireDocSession(uri); + await session.loadPromise; + return new HabitatDocHandleImpl(session); + } + + async createDirectory( + initial: DirectoryDocument, + ): Promise<{ uri: HabitatUri; handle: HabitatDocHandle }> { + return this.create(initial, HABITAT_VAULT_DIRECTORY_COLLECTION); + } + + async createFile( + initial: FileDocument, + ): Promise<{ uri: HabitatUri; handle: HabitatDocHandle }> { + return this.create(initial, HABITAT_VAULT_FILE_COLLECTION); + } + + async create( + initial: T, + collection: string, + ): Promise<{ uri: HabitatUri; handle: HabitatDocHandle }> { + // Create a new empty Automerge doc with the initial value + const doc = Automerge.from(initial as Record); + // TODO: Save the doc to Habitat PDS and get a URI + // For now, this is a stub that returns a placeholder URI + const uri = `at://did:plc:placeholder/${collection}/rkey-placeholder` as HabitatUri; + const session = acquireDocSession(uri); + // Force the session to use our new doc + // This is a simplified implementation; in reality we'd need to + // create the Habitat record first, then acquire the session. + return { uri, handle: new HabitatDocHandleImpl(session) }; + } + + async delete(uri: HabitatUri): Promise { + // TODO: Implement deletion via Habitat PDS + // For now, this is a no-op + console.warn('[HabitatRepo] delete not yet implemented for', uri); + } +} diff --git a/src/vault/snapshot.ts b/src/vault/snapshot.ts new file mode 100644 index 0000000..72ac4aa --- /dev/null +++ b/src/vault/snapshot.ts @@ -0,0 +1,97 @@ +import type { + HabitatUri, + SerializableSyncSnapshot, + SnapshotDirectoryEntry, + SnapshotFileEntry, + SyncSnapshot, +} from './types'; +import type { FilesystemApi } from '../filesystem/types'; + +const SNAPSHOT_FILE = '.textile/snapshot.json'; + +/** + * Manages `.textile/snapshot.json` for synced vaults. + * Loads, saves, serializes, and deserializes the sync snapshot. + */ +export class VaultSnapshotManager { + constructor(private fs: FilesystemApi) {} + + /** + * Load the snapshot from disk at the given vault root. + * Returns `null` if no snapshot exists. + */ + async load(rootPath: string): Promise { + const snapshotPath = `${rootPath}/${SNAPSHOT_FILE}`; + try { + const raw = await this.fs.readFile(snapshotPath); + return this.deserialize(raw); + } catch { + // Snapshot doesn't exist yet + return null; + } + } + + /** + * Save the snapshot to disk at the given vault root. + */ + async save(rootPath: string, snapshot: SyncSnapshot): Promise { + const snapshotPath = `${rootPath}/${SNAPSHOT_FILE}`; + const json = this.serialize(snapshot); + await this.fs.writeFile(snapshotPath, json); + } + + /** + * Convert a SyncSnapshot to a JSON string. + * Maps are serialized as arrays of [key, value] tuples. + */ + serialize(snapshot: SyncSnapshot): string { + const serializable: SerializableSyncSnapshot = { + timestamp: snapshot.timestamp, + rootUri: snapshot.rootUri, + files: Array.from(snapshot.files.entries()), + directories: Array.from(snapshot.directories.entries()), + }; + return JSON.stringify(serializable, null, 2); + } + + /** + * Parse a JSON string back into a SyncSnapshot. + * Reconstructs Maps from arrays of [key, value] tuples. + */ + deserialize(raw: string): SyncSnapshot { + const parsed = JSON.parse(raw) as SerializableSyncSnapshot; + + const files = new Map(); + if (Array.isArray(parsed.files)) { + for (const [key, value] of parsed.files) { + files.set(key, value); + } + } + + const directories = new Map(); + if (Array.isArray(parsed.directories)) { + for (const [key, value] of parsed.directories) { + directories.set(key, value); + } + } + + return { + timestamp: parsed.timestamp, + rootUri: parsed.rootUri as HabitatUri, + files, + directories, + }; + } + + /** + * Create an empty snapshot for a new synced vault. + */ + createEmpty(rootUri: HabitatUri): SyncSnapshot { + return { + timestamp: Date.now(), + rootUri, + files: new Map(), + directories: new Map(), + }; + } +} diff --git a/src/vault/types.ts b/src/vault/types.ts index aa8cfbe..ddd06d0 100644 --- a/src/vault/types.ts +++ b/src/vault/types.ts @@ -1,6 +1,8 @@ /** * Habitat URI branded string. Uses at:// URIs instead of automerge: URLs. */ +import type * as Automerge from '@automerge/automerge'; + export type HabitatUri = string & { __brand: 'HabitatUri' }; export function isHabitatUri(value: string): value is HabitatUri { @@ -8,3 +10,199 @@ export function isHabitatUri(value: string): value is HabitatUri { } export type VaultType = 'local' | 'synced'; + +/** + * Vault configuration stored in `.textile/vault.yaml` on disk. + */ +export interface VaultConfig { + name: string; + type: VaultType; + rootUri: HabitatUri; + createdAt: string; +} + +/** + * Collection types for vault documents. + */ +export type VaultDocumentType = 'directory' | 'file'; + +/** + * Entry in a directory document's `docs` array. + * + * `type` is `string` rather than `'file' | 'folder'` to stay compatible + * with pushwork 2.x, which uses extension strings (`'md'`, `'png'`) for + * file entries. Only `'folder'` is unambiguously a directory. + */ +export interface DirectoryEntry { + name: string; + type: string; + url: HabitatUri; +} + +/** + * Directory node (in Automerge blob). Mirrors pushwork's + * `PushworkDirectoryDocument` but with `HabitatUri` URLs. + */ +export interface DirectoryDocument { + '@patchwork': { type: 'folder' }; + docs: DirectoryEntry[]; + name?: string; + title?: string; + lastSyncAt?: number; + with?: string; +} + +/** + * File node (in Automerge blob). Mirrors pushwork's + * `PushworkFileDocument` exactly for compatibility. + */ +export interface FileDocument { + '@patchwork': { type: 'file' }; + name: string; + extension: string; + mimeType: string; + content: string | Uint8Array; + metadata: { permissions: number }; +} + +/** + * Union type for any vault document (file or directory). + */ +export type VaultDocument = DirectoryDocument | FileDocument; + +/** + * True if a DirectoryEntry points at a sub-directory document. + * False for any file leaf, regardless of how the producer encoded the + * file's `type` (`'file'`, `'md'`, `'png'`, etc.). + */ +export function isFolderEntry(entry: { type: unknown }): boolean { + return entry.type === 'folder'; +} + +/** + * Convenience predicate: a file leaf. Requires `type` to actually be + * a non-empty string — `undefined`, `null`, and `''` are NOT files. + */ +export function isFileEntry(entry: { type: unknown }): boolean { + return ( + typeof entry.type === 'string' && + entry.type.length > 0 && + entry.type !== 'folder' + ); +} + +/** + * Change type classification for sync operations. + */ +export type ChangeType = 'add' | 'edit' | 'delete' | 'move' | 'conflict'; + +/** + * Represents a detected change during three-way comparison + * (local filesystem ↔ snapshot ↔ remote doc). + */ +export interface DetectedChange { + type: ChangeType; + path: string; + isDirectory: boolean; + oldPath?: string; + newPath?: string; + uri?: HabitatUri; + head?: string[]; + contentHash?: string; +} + +/** + * Move detection result: a rename detected by content similarity. + */ +export interface MoveCandidate { + oldPath: string; + newPath: string; + similarity: number; +} + +/** + * Tracked file entry in the sync snapshot. + */ +export interface SnapshotFileEntry { + uri: HabitatUri; + head: string[]; + contentHash: string; +} + +/** + * Tracked directory entry in the sync snapshot. + */ +export interface SnapshotDirectoryEntry { + uri: HabitatUri; + head: string[]; +} + +/** + * Union type for snapshot entries. + */ +export type SnapshotEntry = SnapshotFileEntry | SnapshotDirectoryEntry; + +/** + * Sync snapshot for local state management. + * Only used for synced vaults. + */ +export interface SyncSnapshot { + timestamp: number; + rootUri: HabitatUri; + files: Map; + directories: Map; +} + +/** + * Serializable version of SyncSnapshot for JSON storage. + */ +export interface SerializableSyncSnapshot { + timestamp: number; + rootUri: HabitatUri; + files: Array<[string, SnapshotFileEntry]>; + directories: Array<[string, SnapshotDirectoryEntry]>; +} + +/** + * HabitatDocHandle mirrors the automerge-repo DocHandle API + * but uses HabitatUri and is backed by AutomergeDocSession. + * + * This is a structural subset, not a literal implements. + */ +export interface HabitatDocHandle { + /** The current document. */ + doc(): Automerge.Doc; + /** Current document heads (like a git commit). */ + heads(): string[]; + /** Apply a change to the document. */ + change(fn: (d: T) => void): void; + /** Apply a change as if the document were at the given heads. */ + changeAt(heads: string[], fn: (d: T) => void): void; + /** View the document at a specific point in time. */ + view(heads: string[]): Automerge.Doc; + /** Subscribe to local/remote changes. Returns an unsubscribe function. */ + on(event: 'change', cb: () => void): () => void; +} + +/** + * HabitatRepo mirrors the automerge-repo Repo API + * but uses HabitatUri and is backed by Habitat PDS + AutomergeDocSession. + * + * Uses separate collections for directories and files: + * - network.habitat.vault.directory + * - network.habitat.vault.file + */ +export interface HabitatRepo { + /** Load an existing Habitat record into a handle. */ + find(uri: HabitatUri): Promise>; + /** Create a new directory document. */ + createDirectory( + initial: DirectoryDocument, + ): Promise<{ uri: HabitatUri; handle: HabitatDocHandle }>; + /** Create a new file document. */ + createFile( + initial: FileDocument, + ): Promise<{ uri: HabitatUri; handle: HabitatDocHandle }>; + /** Remove a record. */ + delete(uri: HabitatUri): Promise; +} diff --git a/src/vault/utils/content.ts b/src/vault/utils/content.ts new file mode 100644 index 0000000..12a75d4 --- /dev/null +++ b/src/vault/utils/content.ts @@ -0,0 +1,34 @@ +import { createHash } from 'crypto'; + +/** + * Compute a SHA-256 content hash for snapshot comparison. + */ +export function contentHash(content: string | Uint8Array): string { + const hash = createHash('sha256'); + if (typeof content === 'string') { + hash.update(content, 'utf8'); + } else { + hash.update(content); + } + return `sha256:${hash.digest('hex')}`; +} + +/** + * Deep equality for file content (string vs Uint8Array). + */ +export function isContentEqual( + a: string | Uint8Array, + b: string | Uint8Array, +): boolean { + if (typeof a === 'string' && typeof b === 'string') { + return a === b; + } + if (a instanceof Uint8Array && b instanceof Uint8Array) { + if (a.length !== b.length) return false; + for (let i = 0; i < a.length; i++) { + if (a[i] !== b[i]) return false; + } + return true; + } + return false; +} diff --git a/src/vault/utils/directory.ts b/src/vault/utils/directory.ts new file mode 100644 index 0000000..0f88f6c --- /dev/null +++ b/src/vault/utils/directory.ts @@ -0,0 +1,20 @@ +import type { DirectoryEntry, HabitatUri } from '../types'; + +/** + * Get the plain URI without any version (head) suffix. + * Habitat URIs are `at://did/collection/rkey` — no heads fragment. + * This is a no-op for Habitat URIs but kept for API parity with pushwork. + */ +export function getPlainUri(uri: HabitatUri): HabitatUri { + return uri; +} + +/** + * Find an entry by name in a directory's `docs` array. + */ +export function findFileInDirectoryHierarchy( + docs: DirectoryEntry[], + name: string, +): DirectoryEntry | null { + return docs.find((entry) => entry.name === name) ?? null; +} diff --git a/src/vault/utils/string-similarity.ts b/src/vault/utils/string-similarity.ts new file mode 100644 index 0000000..f650189 --- /dev/null +++ b/src/vault/utils/string-similarity.ts @@ -0,0 +1,33 @@ +/** + * Compute the Sørensen–Dice coefficient between two strings. + * Used for move detection (rename detection by content similarity). + * + * Returns a value between 0 (no similarity) and 1 (identical). + */ +export function stringSimilarity(a: string, b: string): number { + if (a === b) return 1; + if (a.length < 2 || b.length < 2) return 0; + + const bigramsA = getBigrams(a); + const bigramsB = getBigrams(b); + + let intersection = 0; + for (const [bigram, countA] of bigramsA) { + const countB = bigramsB.get(bigram) ?? 0; + intersection += Math.min(countA, countB); + } + + const total = bigramsA.size + bigramsB.size; + if (total === 0) return 0; + + return (2 * intersection) / total; +} + +function getBigrams(str: string): Map { + const bigrams = new Map(); + for (let i = 0; i < str.length - 1; i++) { + const bigram = str.slice(i, i + 2); + bigrams.set(bigram, (bigrams.get(bigram) ?? 0) + 1); + } + return bigrams; +} diff --git a/src/vault/utils/text-diff.ts b/src/vault/utils/text-diff.ts new file mode 100644 index 0000000..c0098fb --- /dev/null +++ b/src/vault/utils/text-diff.ts @@ -0,0 +1,69 @@ +import * as Automerge from '@automerge/automerge'; + +import type { FileDocument } from '../types'; + +/** + * Splice text into an Automerge document at a given path. + * Wraps Automerge.splice() for the vault document shape. + * + * Only works when the target field is an Automerge text object. + * For plain strings, use updateTextContent instead. + */ +export function spliceText( + doc: Automerge.Doc, + path: readonly string[], + index: number, + deleteCount: number, + insert: string, +): Automerge.Doc { + return Automerge.change(doc, (d) => { + Automerge.splice(d, path as any, index, deleteCount, insert); + }); +} + +/** + * Update the `content` field of a FileDocument to match `newContent`. + * + * For regular strings: assigns directly (plain strings in Automerge are + * replaced wholesale; this is the correct path for the PushworkFileDocument + * schema which stores content as `string | Uint8Array`). + * + * For legacy immutable strings (RawString / ImmutableString): assigns directly. + * For Automerge text objects: uses Automerge.updateText for efficient + * character-level diffing. + */ +export function updateTextContent( + doc: Automerge.Doc, + newContent: string, +): Automerge.Doc { + const current = readDocContent(doc); + if (current === newContent) return doc; + + return Automerge.change(doc, (d) => { + const content = d.content as unknown; + if (typeof content === 'string') { + // Plain string: direct assignment + d.content = newContent as unknown as T['content']; + } else if (Automerge.isImmutableString(content)) { + // ImmutableString / RawString: direct assignment + d.content = newContent as unknown as T['content']; + } else { + // Assume it's an Automerge text object + Automerge.updateText(d, ['content'] as any, newContent); + } + }); +} + +/** + * Read the `content` field from a FileDocument, normalizing RawString to plain string. + */ +export function readDocContent(doc: Automerge.Doc): string { + const content = doc.content as unknown; + if (typeof content === 'string') return content; + if (content instanceof Uint8Array) return ''; + // RawString / ImmutableString fallback + if (Automerge.isImmutableString(content)) { + return content.val; + } + return ''; +} -- 2.51.2