diff --git a/docs/SEARCH_ENGINES.md b/docs/SEARCH_ENGINES.md deleted file mode 100644 index 9cf3ce12..00000000 --- a/docs/SEARCH_ENGINES.md +++ /dev/null @@ -1,111 +0,0 @@ -# Search Engine Integrations: Google Search Console & IndexNow - -This document outlines Tangled's search engine discovery mechanisms, sitemap architecture, verification methods, and instant re-indexing integrations. - ---- - -## 1. Dynamic XML Sitemap (TAN-737 & TAN-745) - -Tangled serves an automated, dynamically generated XML sitemap adhering to the standard Sitemap Protocol (0.9). - -- **URL**: `https://tangled.org/sitemap.xml` -- **Implementation**: `web/src/routes/sitemap.xml/+server.ts` -- **Cache-Control**: `public, max-age=3600, s-maxage=3600` (1-hour CDN/browser caching) -- **Specification Limit**: Hard cap at 50,000 URLs per sitemap specification. - -### URL Structure & Priority -1. **Core Static Pages**: - - `https://tangled.org/` — priority `1.0`, changefreq `daily` - - `https://tangled.org/about` — priority `0.8`, changefreq `weekly` -2. **Public Profiles**: - - `https://tangled.org/{handle}` — priority `0.7`, changefreq `daily` - - Dynamically gathered from trending actors (`sh.tangled.actor.getTrending`) and public timeline events (`sh.tangled.feed.getTimeline`). -3. **Public Repositories**: - - `https://tangled.org/{handle}/{repo}` — priority `0.8`, changefreq `daily` - - Includes `` derived from the repository's creation timestamp. - - Paginates repository lists for discovered users via `sh.tangled.repo.listRepos` with cursor-based pagination. - -### Resilience & Graceful Degradation -If upstream backend services (Bobbin / XRPC) are temporarily unreachable, degraded, or timing out, the endpoint catches errors silently and serves the static core URLs cleanly with HTTP 200, ensuring search engine crawlers are never served a 5xx error. - ---- - -## 2. Google Search Console Verification (TAN-746) - -Google Search Console (GSC) is used to monitor search performance, coverage, indexing status, and submit sitemaps. - -### Verification Methods -Tangled supports two primary verification methods: - -1. **HTML Meta Tag Verification**: - Add the verification tag provided by Google Search Console into the `` of `web/src/app.html`: - ```html - - ``` -2. **DNS TXT Record (Recommended for domain-level property)**: - Add a TXT record to the DNS configuration for `tangled.org`: - ```text - Record Type: TXT - Host: @ - Value: google-site-verification=YOUR_GSC_VERIFICATION_TOKEN - ``` - -### Sitemap Submission -1. Log into [Google Search Console](https://search.google.com/search-console). -2. Select the `tangled.org` property. -3. In the left navigation, click **Indexing** > **Sitemaps**. -4. Under "Add a new sitemap", enter `sitemap.xml` and click **Submit**. -5. Google will periodically poll and re-index URLs from `https://tangled.org/sitemap.xml`. - ---- - -## 3. IndexNow Instant Indexing Protocol (TAN-747) - -IndexNow allows websites to immediately notify participating search engines (including Microsoft Bing, Yandex, Seznam, and Naver) whenever content is created, updated, or deleted. Submitted URLs are crawled and reflected in search results within minutes. - -### Verification Key -- **Key**: `d76fae270607bfc761b35c14f3182c9e` -- **Key Location**: `https://tangled.org/d76fae270607bfc761b35c14f3182c9e.txt` -- **Static File**: `web/static/d76fae270607bfc761b35c14f3182c9e.txt` - -Search engines verify domain ownership by requesting `https://tangled.org/{key}.txt` and verifying that the body matches the key. - -### IndexNow Submission Utility -Implementation located at `web/src/lib/server/indexnow.ts`. - -#### API Usage -```typescript -import { - submitToIndexNow, - notifyProfileCreated, - notifyRepoCreated -} from "$lib/server/indexnow"; - -// Submit an arbitrary list of changed URLs: -await submitToIndexNow([ - "https://tangled.org/alice.tangled.sh", - "https://tangled.org/alice.tangled.sh/my-repo" -]); - -// Helper when a new public profile is created: -await notifyProfileCreated("alice.tangled.sh"); - -// Helper when a new public repository is created: -await notifyRepoCreated("alice.tangled.sh", "my-repo"); -``` - -#### Protocol Payload -Requests are sent via HTTP POST to `https://api.indexnow.org/indexnow`: -```json -{ - "host": "tangled.org", - "key": "d76fae270607bfc761b35c14f3182c9e", - "keyLocation": "https://tangled.org/d76fae270607bfc761b35c14f3182c9e.txt", - "urlList": [ - "https://tangled.org/alice.tangled.sh/my-repo" - ] -} -``` - -#### Fault-Tolerance & Non-Blocking Design -All IndexNow helper calls catch network and HTTP status errors internally and log warnings to the console without throwing exceptions. User-facing mutation flows (such as repository creation or profile setup) will never fail or be delayed if the external IndexNow API is slow or unavailable. diff --git a/sitemap/migrations/0002_handles.sql b/sitemap/migrations/0002_handles.sql new file mode 100644 index 00000000..e11b0e5b --- /dev/null +++ b/sitemap/migrations/0002_handles.sql @@ -0,0 +1,16 @@ +-- urls stay keyed by did so a handle change never moves one, and rendering swaps the handle in +ALTER TABLE urls ADD COLUMN did TEXT; + +-- the owner did is the one path segment that starts with did: +UPDATE urls SET did = substr(url, instr(url, '/did:') + 1) WHERE instr(url, '/did:') > 0; +UPDATE urls SET did = substr(did, 1, instr(did, '/') - 1) WHERE instr(did, '/') > 0; + +CREATE INDEX urls_by_did ON urls (did); + +-- hydrant's handle for each repo, null when the did doc names none. stale means an +-- identity event came in since, so it gets read again +CREATE TABLE handles ( + did TEXT PRIMARY KEY, + handle TEXT, + stale INTEGER NOT NULL DEFAULT 0 +) WITHOUT ROWID; diff --git a/sitemap/src/db.ts b/sitemap/src/db.ts index 49d8e447..12b82efa 100644 --- a/sitemap/src/db.ts +++ b/sitemap/src/db.ts @@ -1,9 +1,10 @@ import type { ExtractedAction } from './hydrant'; +import { withHandle, type RepoHandle } from './identity'; import type { UrlEntry } from './types'; import { buildUrlsetXml, measureXmlBytes } from './xml'; // static urls have no record behind them -export type Change = ExtractedAction | { type: 'upsert'; uri?: undefined; url: string; entry: UrlEntry }; +export type Change = ExtractedAction | { type: 'upsert'; uri?: undefined; did?: undefined; url: string; entry: UrlEntry }; export interface ShardLimits { maxUrls: number; @@ -56,6 +57,10 @@ function setState(db: D1Database, key: string, value: string): D1PreparedStateme .bind(key, value); } +export async function putState(db: D1Database, key: string, value: string): Promise { + await setState(db, key, value).run(); +} + export async function countUrls(db: D1Database): Promise { const row = await db.prepare('SELECT count(*) AS n FROM urls').first<{ n: number }>(); return row?.n ?? 0; @@ -98,8 +103,16 @@ export async function applyChanges( )).map((row) => [row.url, row.shard]), ); + const handleOf = new Map( + (await lookup( + db, + 'SELECT did, handle FROM handles WHERE did IN (SELECT value FROM json_each(?1))', + changes.flatMap((change) => (change.did ? [change.did] : [])), + )).map((row) => [row.did, row.handle]), + ); + const active = await activeShard(db); - const upserts = new Map(); + const upserts = new Map(); const records = new Map(); const dirty = new Set(); let mutations = 0; @@ -130,7 +143,9 @@ export async function applyChanges( records.set(change.uri, change.url); } - const bytes = urlBytes(change.entry); + // sized as it renders, so the shard caps hold with the handle in + const handle = change.did ? handleOf.get(change.did) : null; + const bytes = urlBytes({ ...change.entry, loc: withHandle(change.url, change.did, handle) }); let shard = urlShard.get(change.url); if (shard == null) { if (active.count >= limits.maxUrls || active.bytes + bytes > limits.maxBytes) { @@ -143,7 +158,7 @@ export async function applyChanges( shard = active.shard; urlShard.set(change.url, shard); } - upserts.set(change.url, { ...change.entry, shard, bytes }); + upserts.set(change.url, { ...change.entry, did: change.did ?? null, shard, bytes }); dirty.add(shard); mutations++; } @@ -153,12 +168,12 @@ export async function applyChanges( ...chunked([...upserts.values()]).map((json) => db .prepare( - `INSERT INTO urls (url, shard, lastmod, changefreq, priority, bytes) + `INSERT INTO urls (url, shard, lastmod, changefreq, priority, bytes, did) SELECT value ->> 'loc', value ->> 'shard', value ->> 'lastmod', value ->> 'changefreq', - value ->> 'priority', value ->> 'bytes' + value ->> 'priority', value ->> 'bytes', value ->> 'did' FROM json_each(?1) WHERE true ON CONFLICT (url) DO UPDATE SET shard = excluded.shard, lastmod = excluded.lastmod, changefreq = excluded.changefreq, - priority = excluded.priority, bytes = excluded.bytes`, + priority = excluded.priority, bytes = excluded.bytes, did = excluded.did`, ) .bind(json), ), @@ -197,11 +212,22 @@ export async function dirtyShards(db: D1Database): Promise<{ shard: number; dirt export async function shardEntries(db: D1Database, shard: number): Promise { const res = await db - .prepare('SELECT url AS loc, lastmod, changefreq, priority FROM urls WHERE shard = ?1 ORDER BY url') + .prepare( + `SELECT u.url AS loc, u.did, h.handle, u.lastmod, u.changefreq, u.priority + FROM urls u LEFT JOIN handles h ON h.did = u.did + WHERE u.shard = ?1 ORDER BY u.url`, + ) .bind(shard) - .all<{ loc: string; lastmod: string; changefreq: UrlEntry['changefreq'] | null; priority: number | null }>(); + .all<{ + loc: string; + did: string | null; + handle: string | null; + lastmod: string; + changefreq: UrlEntry['changefreq'] | null; + priority: number | null; + }>(); return res.results.map((row) => ({ - loc: row.loc, + loc: withHandle(row.loc, row.did, row.handle), lastmod: row.lastmod, changefreq: row.changefreq ?? undefined, priority: row.priority ?? undefined, @@ -221,10 +247,72 @@ export async function allShards(db: D1Database): Promise<{ shard: number; lastmo return res.results; } +// an identity event can mean a new handle, so those get read again +export async function markStale(db: D1Database, dids: string[]): Promise { + const statements = chunked([...new Set(dids)]).map((json) => + db.prepare('UPDATE handles SET stale = 1 WHERE did IN (SELECT value FROM json_each(?1))').bind(json), + ); + if (statements.length > 0) await db.batch(statements); +} + +// owners with no handle read yet, or a stale one, in did order after `after` +export async function staleOwners(db: D1Database, after: string, limit: number): Promise { + const res = await db + .prepare( + `SELECT DISTINCT u.did FROM urls u LEFT JOIN handles h ON h.did = u.did + WHERE u.did > ?1 AND (h.did IS NULL OR h.stale = 1) + ORDER BY u.did LIMIT ?2`, + ) + .bind(after, limit) + .all<{ did: string }>(); + return res.results.map((row) => row.did); +} + +// a changed handle resizes its owner's urls and rerenders their shards +export async function saveHandles(db: D1Database, found: RepoHandle[]): Promise { + const previous = new Map( + (await lookup( + db, + 'SELECT did, handle FROM handles WHERE did IN (SELECT value FROM json_each(?1))', + found.map((repo) => repo.did), + )).map((row) => [row.did, row.handle]), + ); + const changed = found.filter((repo) => repo.handle !== (previous.get(repo.did) ?? null)); + // dids and handles are plain ascii, so characters are bytes + const growth = changed.map((repo) => [ + repo.did, + (repo.handle ?? repo.did).length - (previous.get(repo.did) ?? repo.did).length, + ]); + const statements: D1PreparedStatement[] = [ + ...chunked(found.map((repo) => [repo.did, repo.handle])).map((json) => + db + .prepare( + `INSERT INTO handles (did, handle, stale) SELECT value ->> 0, value ->> 1, 0 FROM json_each(?1) WHERE true + ON CONFLICT (did) DO UPDATE SET handle = excluded.handle, stale = 0`, + ) + .bind(json), + ), + ...chunked(growth).map((json) => + db + .prepare('UPDATE urls SET bytes = bytes + (j.value ->> 1) FROM json_each(?1) AS j WHERE urls.did = j.value ->> 0') + .bind(json), + ), + ...chunked(changed.map((repo) => repo.did)).map((json) => + db + .prepare( + `UPDATE shards SET dirty = dirty + 1 + WHERE shard IN (SELECT shard FROM urls WHERE did IN (SELECT value FROM json_each(?1)))`, + ) + .bind(json), + ), + ]; + if (statements.length > 0) await db.batch(statements); +} + export async function markSynced(db: D1Database, now: Date): Promise { await setState(db, 'last_sync_at', now.toISOString()).run(); } export async function clearDb(db: D1Database): Promise { - await db.batch(['urls', 'records', 'shards', 'state'].map((table) => db.prepare(`DELETE FROM ${table}`))); + await db.batch(['urls', 'records', 'handles', 'shards', 'state'].map((table) => db.prepare(`DELETE FROM ${table}`))); } diff --git a/sitemap/src/hydrant.ts b/sitemap/src/hydrant.ts index 2d159d2a..d26e30db 100644 --- a/sitemap/src/hydrant.ts +++ b/sitemap/src/hydrant.ts @@ -4,8 +4,8 @@ import { sanitizeLastmod } from './xml'; // uri is the record's at-uri, deletes carry no record so it's what finds their url export type ExtractedAction = - | { type: 'delete'; uri: string; url: string } - | { type: 'upsert'; uri: string; url: string; entry: UrlEntry }; + | { type: 'delete'; uri: string; did: string; url: string } + | { type: 'upsert'; uri: string; did: string; url: string; entry: UrlEntry }; interface CollectionSpec { buildPath: (owner: string, rkey: string, record?: RecordValue) => string; @@ -31,8 +31,8 @@ export const COLLECTION_SPECS: Record = { }, }; -// owners are dids rather than handles, so a url never moves when someone -// changes handle, and the web redirects it to the handle anyway +// owners are dids here so a url never moves when someone changes handle, and +// rendering swaps the handle in export function extractUrlAction( event: JetstreamEvent, baseUrl: string, @@ -55,13 +55,14 @@ export function extractUrlAction( const url = `${normalizedBase}/${path}`; if (operation === 'delete') { - return { type: 'delete', uri, url }; + return { type: 'delete', uri, did: event.did, url }; } const lastmod = sanitizeLastmod(record?.createdAt as string | undefined, now); return { type: 'upsert', uri, + did: event.did, url, entry: { loc: url, diff --git a/sitemap/src/identity.ts b/sitemap/src/identity.ts new file mode 100644 index 00000000..4b7ca7da --- /dev/null +++ b/sitemap/src/identity.ts @@ -0,0 +1,49 @@ +export interface RepoHandle { + did: string; + handle: string | null; +} + +// hydrant's management api, which has no auth so it's only reachable over the tunnel +export interface Repos { + // every repo hydrant tracks, in did order, starting after `after` + list(after: string | undefined, limit: number): Promise; + // throws when there's no answer, so a flaky read never drops a good handle + get(did: string): Promise; +} + +// whatever the did doc claims, nobody checked the domain agrees. that's fine for a +// sitemap because a wrong one just 404s +function handleOf(repo: { handle?: string }): string | null { + const handle = repo.handle?.toLowerCase(); + return handle && !handle.endsWith('.invalid') ? handle : null; +} + +export function hydrantRepos(fetcher: Pick, base: string): Repos { + const read = (path: string, params: Record = {}) => { + const url = new URL(path, base); + for (const [key, value] of Object.entries(params)) url.searchParams.set(key, value); + return fetcher.fetch(url, { headers: { accept: 'application/json' }, signal: AbortSignal.timeout(10_000) }); + }; + return { + async list(after, limit) { + const res = await read('/repos', after ? { limit: String(limit), cursor: after } : { limit: String(limit) }); + if (!res.ok) throw new Error(`hydrant /repos: http ${res.status}`); + const repos = await res.json<{ did: string; handle?: string }[]>(); + return repos.map((repo) => ({ did: repo.did, handle: handleOf(repo) })); + }, + async get(did) { + const res = await read(`/repos/${did}`); + if (res.status === 404) return null; + if (!res.ok) throw new Error(`hydrant /repos/${did}: http ${res.status}`); + return handleOf(await res.json<{ handle?: string }>()); + }, + }; +} + +export function withHandle(url: string, did: string | null | undefined, handle: string | null | undefined): string { + if (!did || !handle) return url; + return url + .split('/') + .map((segment) => (segment === did ? handle : segment)) + .join('/'); +} diff --git a/sitemap/src/index.ts b/sitemap/src/index.ts index 19579026..17f35ba7 100644 --- a/sitemap/src/index.ts +++ b/sitemap/src/index.ts @@ -1,6 +1,16 @@ import { WorkflowEntrypoint, type WorkflowEvent, type WorkflowStep, type WorkflowStepConfig } from 'cloudflare:workers'; -import { applyChanges, countUrls, getState, markSynced } from './db'; +import { + applyChanges, + countUrls, + getState, + markStale, + markSynced, + putState, + saveHandles, + staleOwners, +} from './db'; import { COLLECTION_SPECS, extractUrlAction } from './hydrant'; +import { hydrantRepos, type RepoHandle, type Repos } from './identity'; import { readJetstream, subscribeJetstream, type Subscribe } from './jetstream'; import { getStaticUrls, renderShards, resetSitemap } from './sitemap'; import type { Env } from './types'; @@ -14,12 +24,15 @@ function hydrantUrl(env: Env): string { export interface SyncDeps { subscribe: Subscribe; + repos: (env: Env) => Repos; idleMs: number; closeMs: number; } const defaultDeps: SyncDeps = { subscribe: subscribeJetstream, + // the vpc service only takes http and routes by itself, so the host there means nothing + repos: (env) => (env.HYDRANT ? hydrantRepos(env.HYDRANT, 'http://hydrant') : hydrantRepos(globalThis, hydrantUrl(env))), idleMs: 3000, closeMs: 10_000, }; @@ -94,12 +107,80 @@ export interface SyncOptions { export interface SyncResult { syncedEvents: number; mutatedUrls: number; + walkedRepos: number; + lookedUpHandles: number; newCursor: string; totalUrls: number; } const MAX_PAGES = 10; +const REPO_PAGE = 1000; +const WALK_PAGES = 10; +const MAX_WALK_STEPS = 100; + +// a page of reads that all time out still fits well inside a step +const LOOKUP_PAGE = 200; +const LOOKUP_CONCURRENCY = 8; +const MAX_LOOKUP_STEPS = 10; + +// the first run, and the first after a reset, pages through every handle hydrant has +// instead of reading owners one by one. a walk that gets cut off carries on next run +async function walkRepos( + db: D1Database, + repos: Repos, + now: Date, +): Promise<{ repos: number; done: boolean; failed: boolean }> { + if (await getState(db, 'handles_walked')) return { repos: 0, done: true, failed: false }; + let after = await getState(db, 'handles_after'); + let count = 0; + try { + for (let page = 0; page < WALK_PAGES; page++) { + const found = await repos.list(after, REPO_PAGE); + await saveHandles(db, found); + count += found.length; + if (found.length < REPO_PAGE) { + await putState(db, 'handles_walked', now.toISOString()); + return { repos: count, done: true, failed: false }; + } + after = found[found.length - 1].did; + await putState(db, 'handles_after', after); + } + } catch (err) { + // the sitemap still goes out, just with dids where handles are missing + console.warn(`[sitemap] couldn't page through hydrant's repos: ${err}`); + return { repos: count, done: false, failed: true }; + } + return { repos: count, done: false, failed: false }; +} + +// owners new since the walk or touched by an identity event. a failed read stays +// stale for the next run +async function lookUpHandles( + db: D1Database, + repos: Repos, + after: string, +): Promise<{ looked: number; failed: number; last: string }> { + const dids = await staleOwners(db, after, LOOKUP_PAGE); + const found: RepoHandle[] = []; + let failed = 0; + let next = 0; + const worker = async () => { + while (next < dids.length) { + const did = dids[next++]; + try { + found.push({ did, handle: await repos.get(did) }); + } catch { + failed++; + } + } + }; + await Promise.all(Array.from({ length: LOOKUP_CONCURRENCY }, worker)); + await saveHandles(db, found); + if (failed > 0) console.warn(`[sitemap] ${failed} of ${dids.length} handle reads failed`); + return { looked: dids.length, failed, last: dids.at(-1) ?? after }; +} + export async function syncFromHydrant( env: Env, deps: SyncDeps = defaultDeps, @@ -150,6 +231,8 @@ export async function syncFromHydrant( return { events: 0, mutations: 0, more: false }; } const changes = read.events.flatMap((evt) => extractUrlAction(evt, baseUrl, now) ?? []); + // before the cursor moves, so a retried batch can't skip it + await markStale(db, read.events.flatMap((evt) => (evt.kind === 'identity' ? [evt.did] : []))); const mutations = await applyChanges(db, changes, limits, String(read.cursor), now); return { events: read.events.length, mutations, more: read.full && !read.error }; }); @@ -158,6 +241,27 @@ export async function syncFromHydrant( if (!batch.more) break; } + const repos = deps.repos(env); + let walkedRepos = 0; + let walked = false; + for (let step = 1; step <= MAX_WALK_STEPS && !walked; step++) { + const res = await steps.do(`walk ${step}`, () => walkRepos(db, repos, now)); + walkedRepos += res.repos; + walked = res.done; + if (res.failed) break; + } + + // until the walk is through, most owners are still in it + let lookedUpHandles = 0; + let after = ''; + for (let step = 1; walked && step <= MAX_LOOKUP_STEPS; step++) { + const res = await steps.do(`lookups ${step}`, () => lookUpHandles(db, repos, after)); + lookedUpHandles += res.looked; + // nothing answering means hydrant is down, so render with what we have + if (res.looked < LOOKUP_PAGE || res.failed === res.looked) break; + after = res.last; + } + await steps.do('render', () => renderShards(db, env.SITEMAP_BUCKET, baseUrl, now)); return steps.do('finish', async () => { @@ -165,6 +269,8 @@ export async function syncFromHydrant( return { syncedEvents, mutatedUrls, + walkedRepos, + lookedUpHandles, newCursor: (await getState(db, 'cursor')) ?? '0', totalUrls: await countUrls(db), }; diff --git a/sitemap/src/types.ts b/sitemap/src/types.ts index 20425b98..c97d0de1 100644 --- a/sitemap/src/types.ts +++ b/sitemap/src/types.ts @@ -3,6 +3,9 @@ export interface Env { SITEMAP_BUCKET: R2Bucket; BASE_URL?: string; HYDRANT_URL?: string; + // a workers vpc service onto the hydrant node with the tunnel, for /repos, which + // isn't public. without it /repos goes to HYDRANT_URL, so only a local hydrant works + HYDRANT?: Fetcher; SHARD_MAX_URLS?: string; SHARD_MAX_BYTES?: string; SYNC_BATCH_SIZE?: string; diff --git a/sitemap/test/d1.ts b/sitemap/test/d1.ts index 7599cd68..b8a5f984 100644 --- a/sitemap/test/d1.ts +++ b/sitemap/test/d1.ts @@ -1,13 +1,23 @@ -import { readFileSync } from 'node:fs'; +import { readdirSync, readFileSync } from 'node:fs'; import { DatabaseSync } from 'node:sqlite'; +const MIGRATIONS = new URL('../migrations/', import.meta.url); + // enough of d1 over node's sqlite to run the real migration and queries export class TestD1 { sqlite = new DatabaseSync(':memory:'); writes = 0; - constructor() { - this.sqlite.exec(readFileSync(new URL('../migrations/0001_init.sql', import.meta.url), 'utf8')); + // `stopAt` leaves the later migrations out, to test one against older data + constructor(stopAt?: string) { + for (const name of readdirSync(MIGRATIONS).sort()) { + if (name === stopAt) break; + this.migrate(name); + } + } + + migrate(name: string): void { + this.sqlite.exec(readFileSync(new URL(name, MIGRATIONS), 'utf8')); } prepare(sql: string): TestStatement { diff --git a/sitemap/test/node.d.ts b/sitemap/test/node.d.ts index 96d8400d..7032eb5a 100644 --- a/sitemap/test/node.d.ts +++ b/sitemap/test/node.d.ts @@ -18,4 +18,5 @@ interface ImportMeta { declare module 'node:fs' { export function readFileSync(path: URL, encoding: 'utf8'): string; + export function readdirSync(path: URL): string[]; } diff --git a/sitemap/test/sitemap.test.ts b/sitemap/test/sitemap.test.ts index d1ed3fbd..5032728c 100644 --- a/sitemap/test/sitemap.test.ts +++ b/sitemap/test/sitemap.test.ts @@ -1,6 +1,7 @@ import type { JetstreamEvent } from '@atcute/jetstream'; -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import { extractUrlAction } from '../src/hydrant'; +import { hydrantRepos, withHandle } from '../src/identity'; import { formatShardFilename, getStaticUrls } from '../src/sitemap'; import type { ShardInfo, UrlEntry } from '../src/types'; import { commit, identity } from './events'; @@ -82,6 +83,7 @@ describe('Hydrant Event Processing', () => { const action = extractUrlAction(event, baseUrl, now); expect(action?.type).toBe('upsert'); expect(action?.uri).toBe('at://did:plc:alice123/sh.tangled.repo/3mhgx3cxdfq22'); + expect(action?.did).toBe('did:plc:alice123'); expect(action?.url).toBe('https://tangled.org/did:plc:alice123/cool-tool'); expect(action?.type === 'upsert' && action.entry.lastmod).toBe('2026-07-15'); expect(action?.type === 'upsert' && action.entry.priority).toBe(0.8); @@ -135,6 +137,59 @@ describe('Hydrant Event Processing', () => { }); }); +describe('handles', () => { + function hydrant(status: number, body: unknown) { + const fetch = vi.fn(async (_url: URL, _init?: RequestInit) => new Response(JSON.stringify(body), { status })); + return { fetch, repos: hydrantRepos({ fetch } as any, 'https://hydrant.test') }; + } + + it('swaps the owner segment for the handle and nothing else', () => { + expect(withHandle('https://tangled.org/did:plc:alice/tool', 'did:plc:alice', 'alice.test')).toBe( + 'https://tangled.org/alice.test/tool', + ); + expect(withHandle('https://tangled.org/strings/did:plc:alice/3mtid', 'did:plc:alice', 'alice.test')).toBe( + 'https://tangled.org/strings/alice.test/3mtid', + ); + expect(withHandle('https://tangled.org/did:plc:alice/did:plc:alicex', 'did:plc:alice', 'alice.test')).toBe( + 'https://tangled.org/alice.test/did:plc:alicex', + ); + expect(withHandle('https://tangled.org/did:plc:alice', 'did:plc:alice', null)).toBe('https://tangled.org/did:plc:alice'); + expect(withHandle('https://tangled.org/about', undefined, undefined)).toBe('https://tangled.org/about'); + }); + + it('pages through the repos as json', async () => { + const { fetch, repos } = hydrant(200, [ + { did: 'did:plc:alice', handle: 'Alice.Test', status: 'synced' }, + { did: 'did:plc:bob', status: 'synced' }, + { did: 'did:plc:carol', handle: 'handle.invalid' }, + ]); + expect(await repos.list(undefined, 1000)).toEqual([ + { did: 'did:plc:alice', handle: 'alice.test' }, + { did: 'did:plc:bob', handle: null }, + { did: 'did:plc:carol', handle: null }, + ]); + expect(String(fetch.mock.calls[0][0])).toBe('https://hydrant.test/repos?limit=1000'); + expect(fetch.mock.calls[0][1]?.headers).toEqual({ accept: 'application/json' }); + + await repos.list('did:plc:carol', 2); + expect(String(fetch.mock.calls[1][0])).toBe('https://hydrant.test/repos?limit=2&cursor=did%3Aplc%3Acarol'); + }); + + it('reads one repo, with no handle for one hydrant never saw', async () => { + const found = hydrant(200, { did: 'did:plc:alice', handle: 'Alice.Test' }); + expect(await found.repos.get('did:plc:alice')).toBe('alice.test'); + expect(String(found.fetch.mock.calls[0][0])).toBe('https://hydrant.test/repos/did:plc:alice'); + + expect(await hydrant(404, 'repository not found').repos.get('did:plc:alice')).toBeNull(); + }); + + it('throws when hydrant has no answer', async () => { + const { repos } = hydrant(502, 'bad gateway'); + await expect(repos.get('did:plc:alice')).rejects.toThrow('http 502'); + await expect(repos.list(undefined, 1000)).rejects.toThrow('http 502'); + }); +}); + describe('Shard Management', () => { it('formats shard filenames with zero padding', () => { expect(formatShardFilename(1)).toBe('sitemap-0001.xml.gz'); diff --git a/sitemap/test/sync.test.ts b/sitemap/test/sync.test.ts index 403918cc..32218bc8 100644 --- a/sitemap/test/sync.test.ts +++ b/sitemap/test/sync.test.ts @@ -1,10 +1,11 @@ import { describe, expect, it } from 'vitest'; import type { JetstreamEvent } from '@atcute/jetstream'; import { parseConfigLimits, syncFromHydrant, type SyncDeps } from '../src/index'; +import type { Repos } from '../src/identity'; import type { Env } from '../src/types'; import { TestD1 } from './d1'; import { commit, deadJetstream, fakeJetstream, identity } from './events'; -import { gzipDecompress } from '../src/xml'; +import { buildUrlsetXml, gzipDecompress, measureXmlBytes } from '../src/xml'; class MockR2Object { constructor(private buffer: ArrayBuffer, public httpMetadata?: any) {} @@ -49,8 +50,54 @@ class MockR2Bucket { } } -function deps(subscribe: SyncDeps['subscribe']): SyncDeps { - return { subscribe, idleMs: 5, closeMs: 5 }; +// hydrant's repos as the tunnel serves them, where nobody has a handle unless a test +// gives them one. changing `handles` later is someone changing theirs +function hydrant(handles: Record = {}): Repos & { asked: string[]; pages: (string | undefined)[] } { + const fake = { + asked: [] as string[], + pages: [] as (string | undefined)[], + async list(after: string | undefined, limit: number) { + fake.pages.push(after); + return Object.keys(handles) + .sort() + .filter((did) => after === undefined || did > after) + .slice(0, limit) + .map((did) => ({ did, handle: handles[did] })); + }, + async get(did: string) { + fake.asked.push(did); + return handles[did] ?? null; + }, + }; + return fake; +} + +const down: Repos = { + async list() { + throw new Error('hydrant is down'); + }, + async get() { + throw new Error('hydrant is down'); + }, +}; + +function deps(subscribe: SyncDeps['subscribe'], repos: Repos = hydrant()): SyncDeps { + return { subscribe, repos: () => repos, idleMs: 5, closeMs: 5 }; +} + +function users(count: number): Record { + return Object.fromEntries( + Array.from({ length: count }, (_, i) => [`did:plc:user${String(i).padStart(4, '0')}`, `user${i}.test`]), + ); +} + +async function shardXml(bucket: MockR2Bucket, shard: number): Promise { + const obj = await bucket.get(`sitemaps/sitemap-${String(shard).padStart(4, '0')}.xml.gz`); + return gzipDecompress(await obj!.arrayBuffer()); +} + +function handleRow(db: TestD1, did: string): { handle: string | null; stale: number } | undefined { + return db.query<{ handle: string | null; stale: number }>('SELECT handle, stale FROM handles WHERE did = ?', did)[0]; } function shardOf(db: TestD1, url: string): number | undefined { @@ -282,3 +329,248 @@ describe('Full syncFromHydrant workflow with Sharding & Multi-Sync', () => { }); }); }); + +const EMPTY_SHARD = measureXmlBytes(buildUrlsetXml([])); + +// what `applyChanges` thinks a shard weighs has to match what it renders to +async function expectSizesTracked(db: TestD1, bucket: MockR2Bucket): Promise { + for (const { shard, bytes } of db.query<{ shard: number; bytes: number }>( + 'SELECT shard, sum(bytes) AS bytes FROM urls GROUP BY shard', + )) { + // `urlBytes` counts a joining newline for every url, and there's one fewer + expect(measureXmlBytes(await shardXml(bucket, shard))).toBe(EMPTY_SHARD + bytes - 1); + } +} + +describe('handles', () => { + const day = 24 * 60 * 60 * 1000; + const startedAt = new Date('2026-10-01T00:00:00Z'); + const env = (db: TestD1, bucket: MockR2Bucket, vars: Partial = {}): Env => ({ + SITEMAP_DB: db.asD1(), + SITEMAP_BUCKET: bucket as any, + BASE_URL: 'https://tangled.org', + ...vars, + }); + + it('renders owners as their handles and keeps the did when there is none', async () => { + const db = new TestD1(); + const bucket = new MockR2Bucket(); + const jetstream = fakeJetstream(() => [ + commit(1, 'did:plc:alice', 'sh.tangled.actor.profile', 'self'), + commit(2, 'did:plc:alice', 'sh.tangled.repo', '3mtid1', 'create', { name: 'tool' }), + commit(3, 'did:plc:alice', 'sh.tangled.string', '3mtid2'), + commit(4, 'did:plc:bob', 'sh.tangled.repo', 'thing'), + ]); + const repos = hydrant({ 'did:plc:alice': 'alice.test' }); + + const res = await syncFromHydrant(env(db, bucket), deps(jetstream.subscribe, repos)); + expect(res.walkedRepos).toBe(1); + // the walk had alice, bob is the one it didn't know + expect(repos.asked).toEqual(['did:plc:bob']); + expect(res.lookedUpHandles).toBe(1); + + const xml = await shardXml(bucket, 1); + expect(xml).toContain('https://tangled.org/alice.test'); + expect(xml).toContain('https://tangled.org/alice.test/tool'); + expect(xml).toContain('https://tangled.org/strings/alice.test/3mtid2'); + expect(xml).toContain('https://tangled.org/did:plc:bob/thing'); + expect(xml).not.toContain('did:plc:alice'); + // stored by did, so a handle change never moves them + expect(shardOf(db, 'https://tangled.org/did:plc:alice/tool')).toBe(1); + await expectSizesTracked(db, bucket); + }); + + it('walks every repo once, and again after a reset', async () => { + const db = new TestD1(); + const bucket = new MockR2Bucket(); + const handles = users(2500); + const jetstream = fakeJetstream(() => [ + commit(1, 'did:plc:user0042', 'sh.tangled.repo', 'tool'), + commit(2, 'did:plc:user2499', 'sh.tangled.repo', 'tool'), + ]); + const repos = hydrant(handles); + + const res = await syncFromHydrant(env(db, bucket), deps(jetstream.subscribe, repos)); + expect(res.walkedRepos).toBe(2500); + expect(repos.pages).toEqual([undefined, 'did:plc:user0999', 'did:plc:user1999']); + expect(repos.asked).toEqual([]); + expect(await shardXml(bucket, 1)).toContain('https://tangled.org/user2499.test/tool'); + + const again = await syncFromHydrant(env(db, bucket), deps(jetstream.subscribe, repos)); + expect(again.walkedRepos).toBe(0); + expect(repos.pages).toHaveLength(3); + + await syncFromHydrant(env(db, bucket), deps(jetstream.subscribe, repos), undefined, { reset: true }); + expect(repos.pages).toHaveLength(6); + expect(repos.asked).toEqual([]); + }); + + it('carries on a walk that broke off from where it got to', async () => { + const db = new TestD1(); + const bucket = new MockR2Bucket(); + const jetstream = fakeJetstream(() => [commit(1, 'did:plc:user2000', 'sh.tangled.repo', 'tool')]); + const repos = hydrant(users(2500)); + let calls = 0; + const flaky: Repos = { + list: (after, limit) => (++calls === 2 ? down.list(after, limit) : repos.list(after, limit)), + get: (did) => repos.get(did), + }; + + const res = await syncFromHydrant(env(db, bucket), deps(jetstream.subscribe, flaky)); + expect(res.walkedRepos).toBe(1000); + // the rest of the walk will have most owners, so nobody gets read one by one yet + expect(res.lookedUpHandles).toBe(0); + expect(await shardXml(bucket, 1)).toContain('https://tangled.org/did:plc:user2000/tool'); + + const next = await syncFromHydrant(env(db, bucket), deps(jetstream.subscribe, flaky)); + expect(next.walkedRepos).toBe(1500); + expect(repos.pages).toEqual([undefined, 'did:plc:user0999', 'did:plc:user1999']); + expect(db.query('SELECT did FROM handles')).toHaveLength(2500); + expect(await shardXml(bucket, 1)).toContain('https://tangled.org/user2000.test/tool'); + }); + + it('rerenders only the shards of an owner whose handle changed', async () => { + const db = new TestD1(); + const bucket = new MockR2Bucket(); + let events = [ + commit(1, 'did:plc:alice', 'sh.tangled.repo', 'alice-1'), + commit(2, 'did:plc:bob', 'sh.tangled.repo', 'bob-1'), + commit(3, 'did:plc:bob', 'sh.tangled.repo', 'bob-2'), + commit(4, 'did:plc:bob', 'sh.tangled.repo', 'bob-3'), + commit(5, 'did:plc:alice', 'sh.tangled.repo', 'alice-2'), + ]; + const jetstream = fakeJetstream(() => events); + const vars = { SHARD_MAX_URLS: '3' }; + const handles: Record = { 'did:plc:alice': 'alice.test', 'did:plc:bob': 'bob.test' }; + const repos = hydrant(handles); + + await syncFromHydrant(env(db, bucket, vars), deps(jetstream.subscribe, repos), undefined, { startedAt }); + expect(shardCounts(db)).toEqual([3, 3, 1]); + + handles['did:plc:alice'] = 'alice.example.com'; + events = [identity(10, 'did:plc:alice', 'alice.example.com')]; + const puts = bucket.putCount; + await syncFromHydrant(env(db, bucket, vars), deps(jetstream.subscribe, repos), undefined, { + startedAt: new Date(startedAt.getTime() + day), + }); + + expect(repos.asked).toEqual(['did:plc:alice']); + expect(bucket.putCount - puts).toBe(3); + expect(await shardXml(bucket, 1)).toContain('https://tangled.org/alice.example.com/alice-1'); + expect(await shardXml(bucket, 3)).toContain('https://tangled.org/alice.example.com/alice-2'); + expect(await shardXml(bucket, 2)).not.toContain('alice'); + await expectSizesTracked(db, bucket); + }); + + it('keeps the old handle when a read fails and tries again next run', async () => { + const db = new TestD1(); + const bucket = new MockR2Bucket(); + let events = [commit(1, 'did:plc:alice', 'sh.tangled.repo', 'tool')]; + const jetstream = fakeJetstream(() => events); + const handles: Record = { 'did:plc:alice': 'alice.test' }; + const repos = hydrant(handles); + await syncFromHydrant(env(db, bucket), deps(jetstream.subscribe, repos), undefined, { startedAt }); + + handles['did:plc:alice'] = 'alice.example.com'; + events = [identity(10, 'did:plc:alice', 'alice.example.com')]; + await syncFromHydrant(env(db, bucket), deps(jetstream.subscribe, down), undefined, { startedAt }); + expect(await shardXml(bucket, 1)).toContain('https://tangled.org/alice.test/tool'); + expect(handleRow(db, 'did:plc:alice')).toEqual({ handle: 'alice.test', stale: 1 }); + + events = []; + await syncFromHydrant(env(db, bucket), deps(jetstream.subscribe, repos), undefined, { startedAt }); + expect(await shardXml(bucket, 1)).toContain('https://tangled.org/alice.example.com/tool'); + expect(handleRow(db, 'did:plc:alice')).toEqual({ handle: 'alice.example.com', stale: 0 }); + }); + + it('reads each owner at most once a run, even one that keeps failing', async () => { + const db = new TestD1(); + const bucket = new MockR2Bucket(); + const owners = Object.keys(users(450)); + const jetstream = fakeJetstream(() => owners.map((did, i) => commit(i + 1, did, 'sh.tangled.actor.profile', 'self'))); + const repos = hydrant(); + const asked: string[] = []; + const broken: Repos = { + list: (after, limit) => repos.list(after, limit), + async get(did) { + asked.push(did); + if (did === owners[0]) throw new Error('bad doc'); + return null; + }, + }; + + const res = await syncFromHydrant(env(db, bucket), deps(jetstream.subscribe, broken), undefined, { startedAt }); + expect(res.lookedUpHandles).toBe(450); + expect(asked.sort()).toEqual(owners); + expect(handleRow(db, owners[0])).toBeUndefined(); + }); + + it('stops reading after a page where nothing answered', async () => { + const db = new TestD1(); + const bucket = new MockR2Bucket(); + const owners = Object.keys(users(250)); + const jetstream = fakeJetstream(() => owners.map((did, i) => commit(i + 1, did, 'sh.tangled.actor.profile', 'self'))); + const asked: string[] = []; + const flaky: Repos = { + list: async () => [], + async get(did) { + asked.push(did); + throw new Error('hydrant is down'); + }, + }; + + const res = await syncFromHydrant(env(db, bucket), deps(jetstream.subscribe, flaky), undefined, { startedAt }); + expect(asked).toHaveLength(200); + expect(res.lookedUpHandles).toBe(200); + expect(db.query('SELECT did FROM handles')).toEqual([]); + expect(bucket.store.has('sitemaps/sitemap-0001.xml.gz')).toBe(true); + }); + + it('sizes new urls with the handle they render with', async () => { + const db = new TestD1(); + const bucket = new MockR2Bucket(); + const handle = 'a-very-long-handle-that-takes-up-a-lot-of-room.example.com'; + let events = [commit(1, 'did:plc:alice', 'sh.tangled.actor.profile', 'self')]; + const jetstream = fakeJetstream(() => events); + const repos = hydrant({ 'did:plc:alice': handle }); + const vars = { SHARD_MAX_BYTES: '700' }; + await syncFromHydrant(env(db, bucket, vars), deps(jetstream.subscribe, repos), undefined, { startedAt }); + + events = [1, 2, 3, 4].map((i) => commit(10 + i, 'did:plc:alice', 'sh.tangled.repo', `repo-${i}`)); + await syncFromHydrant(env(db, bucket, vars), deps(jetstream.subscribe, repos), undefined, { startedAt }); + + expect(shardCounts(db).length).toBeGreaterThan(2); + for (const { shard } of db.query<{ shard: number }>('SELECT DISTINCT shard FROM urls')) { + const xml = await shardXml(bucket, shard); + expect(measureXmlBytes(xml)).toBeLessThanOrEqual(700); + } + expect(await shardXml(bucket, 1)).toContain(`https://tangled.org/${handle}`); + await expectSizesTracked(db, bucket); + }); +}); + +describe('0002_handles migration', () => { + it('fills in owner dids and leaves the handles to the walk', () => { + const db = new TestD1('0002_handles.sql'); + const urls = [ + 'https://tangled.org/', + 'https://tangled.org/did:plc:alice', + 'https://tangled.org/did:plc:alice/tool', + 'https://tangled.org/strings/did:plc:bob/3mtid', + 'https://tangled.org/did:web:example.com/thing', + ]; + for (const url of urls) { + db.sqlite.prepare("INSERT INTO urls (url, shard, lastmod, bytes) VALUES (?, 1, '2026-10-01', 100)").run(url); + } + db.migrate('0002_handles.sql'); + + expect(db.query('SELECT url, did FROM urls ORDER BY url')).toEqual([ + { url: 'https://tangled.org/', did: null }, + { url: 'https://tangled.org/did:plc:alice', did: 'did:plc:alice' }, + { url: 'https://tangled.org/did:plc:alice/tool', did: 'did:plc:alice' }, + { url: 'https://tangled.org/did:web:example.com/thing', did: 'did:web:example.com' }, + { url: 'https://tangled.org/strings/did:plc:bob/3mtid', did: 'did:plc:bob' }, + ]); + expect(db.query('SELECT did FROM handles')).toEqual([]); + }); +}); diff --git a/sitemap/wrangler.dev.jsonc b/sitemap/wrangler.dev.jsonc index fccb8416..31b29131 100644 --- a/sitemap/wrangler.dev.jsonc +++ b/sitemap/wrangler.dev.jsonc @@ -30,6 +30,15 @@ "bucket_name": "tangled-sitemaps-dev", }, ], + // hydrant's /repos over the tunnel on hyd-gcp-eu-1 (tunnel = true in infra's + // terraform/modules/envs/dev.tf) + "vpc_services": [ + { + "binding": "HYDRANT", + "service_id": "01a11677-64af-78f2-8512-94b028e01487", + "remote": true, + }, + ], "vars": { "BASE_URL": "https://next.tangled.org", "HYDRANT_URL": "https://index.tangled.network", diff --git a/sitemap/wrangler.jsonc b/sitemap/wrangler.jsonc index e75a416f..599bcfa5 100644 --- a/sitemap/wrangler.jsonc +++ b/sitemap/wrangler.jsonc @@ -30,6 +30,15 @@ "bucket_name": "tangled-sitemaps", }, ], + // hydrant's /repos over the tunnel on hyd-gcp-prod-eu-1 (tunnel = true in infra's + // terraform/modules/envs/prod.tf) + "vpc_services": [ + { + "binding": "HYDRANT", + "service_id": "01a11694-d405-7c93-a786-e2483ed74577", + "remote": true, + }, + ], "vars": { "BASE_URL": "https://tangled.org", "HYDRANT_URL": "https://index.tangled.network",