diff --git a/lexicons/wiki.lichen.membership.json b/lexicons/wiki.lichen.membership.json index d23a416..194fbc2 100644 --- a/lexicons/wiki.lichen.membership.json +++ b/lexicons/wiki.lichen.membership.json @@ -12,6 +12,7 @@ "properties": { "memberDid": { "type": "string", + "format": "did", "maxLength": 2048, "description": "DID of the member being granted access." }, diff --git a/src/firehose/handlers.ts b/src/firehose/handlers.ts index f421dd6..709ec6a 100644 --- a/src/firehose/handlers.ts +++ b/src/firehose/handlers.ts @@ -1,7 +1,9 @@ +import { isDid } from "@atcute/lexicons/syntax"; import { canEdit, getAccessLevel } from "../lib/access.ts"; import { COLLECTIONS, normalizeRole } from "../lib/collections.ts"; import { isValidLanguageTag } from "../lib/languages.ts"; import { LIMITS } from "../lib/limits.ts"; +import { isSafeSlug, normalizeSlug } from "../lib/slug.ts"; import { registerContributor } from "../registry/index.ts"; import { applyRevisionFromFirehose, @@ -75,20 +77,17 @@ interface CommunityBookmarkRecord { createdAt: string; } -// Other PDSes don't enforce our lexicon maxima; oversized records are logged and skipped, never thrown. +// Other PDSes enforce neither our lexicon maxima nor our field shapes; records +// that violate either are logged and skipped, never thrown. function logDrop(atUri: string, reason: string): void { console.warn(`[firehose] dropping record ${atUri}: ${reason}`); } -function wikiExceedsLimits(atUri: string, r: WikiRecord): boolean { +function wikiIsInvalid(atUri: string, r: WikiRecord): boolean { if (r.name.length > LIMITS.wiki.name) { logDrop(atUri, "wiki.name exceeds limit"); return true; } - if (r.visibility.length > LIMITS.wiki.visibility) { - logDrop(atUri, "wiki.visibility exceeds limit"); - return true; - } if (r.language !== undefined && !isValidLanguageTag(r.language)) { // Covers both over-length and malformed/HTML-bearing tags. A valid tag is // already capped at LIMITS.wiki.language inside isValidLanguageTag. @@ -105,9 +104,13 @@ function wikiExceedsLimits(atUri: string, r: WikiRecord): boolean { return false; } -function noteExceedsLimits(atUri: string, r: NoteRecord): boolean { - if (r.slug.length > LIMITS.note.slug) { - logDrop(atUri, "note.slug exceeds limit"); +function noteIsInvalid(atUri: string, r: NoteRecord): boolean { + // The slug is interpolated into URLs, so it has to be URL- and attribute-safe. + // isSafeSlug, not isValidSlug: another client may legitimately write a + // Japanese slug, and rejecting it would drop real content for a reason that + // has nothing to do with safety. + if (!isSafeSlug(normalizeSlug(r.slug))) { + logDrop(atUri, "note.slug is not a safe slug"); return true; } if (r.title.length > LIMITS.note.title) { @@ -117,7 +120,7 @@ function noteExceedsLimits(atUri: string, r: NoteRecord): boolean { return false; } -function revisionExceedsLimits(atUri: string, r: RevisionRecord): boolean { +function revisionIsInvalid(atUri: string, r: RevisionRecord): boolean { if (r.diff.length > LIMITS.revision.diff) { logDrop(atUri, "revision.diff exceeds limit"); return true; @@ -137,9 +140,11 @@ function revisionExceedsLimits(atUri: string, r: RevisionRecord): boolean { return false; } -function membershipExceedsLimits(atUri: string, r: MembershipRecord): boolean { - if (r.memberDid.length > LIMITS.membership.memberDid) { - logDrop(atUri, "membership.memberDid exceeds limit"); +function membershipIsInvalid(atUri: string, r: MembershipRecord): boolean { + // memberDid reaches profileUrl() and the member directory's hrefs; only a real + // DID belongs there. isDid also bounds the length. + if (!isDid(r.memberDid)) { + logDrop(atUri, "membership.memberDid is not a DID"); return true; } if (r.role.length > LIMITS.membership.role) { @@ -225,19 +230,19 @@ export function handleCommitEvent(evt: FirehoseCommit): void { switch (evt.collection) { case COLLECTIONS.wiki: - if (isWikiRecord(r) && !wikiExceedsLimits(atUri, r)) + if (isWikiRecord(r) && !wikiIsInvalid(atUri, r)) handleWiki(evt.did, evt.rkey, atUri, r); break; case COLLECTIONS.note: - if (isNoteRecord(r) && !noteExceedsLimits(atUri, r)) + if (isNoteRecord(r) && !noteIsInvalid(atUri, r)) handleNote(evt.did, atUri, r); break; case COLLECTIONS.noteRevision: - if (isRevisionRecord(r) && !revisionExceedsLimits(atUri, r)) + if (isRevisionRecord(r) && !revisionIsInvalid(atUri, r)) handleRevision(evt.did, atUri, r); break; case COLLECTIONS.membership: - if (isMembershipRecord(r) && !membershipExceedsLimits(atUri, r)) + if (isMembershipRecord(r) && !membershipIsInvalid(atUri, r)) handleMembership(evt.did, atUri, r); break; case COLLECTIONS.memberRequest: @@ -307,10 +312,12 @@ function handleNote(did: string, atUri: string, record: NoteRecord): void { if (!wiki) return; if (!canDidEditWiki(wiki, did)) return; + // Store the NFC form: the same slug written in NFD is a different SQLite key, + // so an un-normalized write becomes a note that exists but never resolves. upsertNote( wiki.at_uri, wiki.slug, - record.slug, + normalizeSlug(record.slug), record.title, did, atUri, diff --git a/src/lib/backfill/crawler.ts b/src/lib/backfill/crawler.ts index f46e23d..772a473 100644 --- a/src/lib/backfill/crawler.ts +++ b/src/lib/backfill/crawler.ts @@ -1,5 +1,5 @@ import { type Client, ClientResponseError, ok } from "@atcute/client"; -import type { Did, Nsid } from "@atcute/lexicons/syntax"; +import { type Did, isRecordKey, type Nsid } from "@atcute/lexicons/syntax"; import { type FirehoseCommit, handleCommitEvent, @@ -100,8 +100,12 @@ async function listPage( } } -function rkeyOf(uri: string): string { - return uri.split("/").pop() ?? ""; +// The uri is whatever the remote PDS returned, so the rkey needs the same +// validation jetstream applies to live commits — otherwise backfill is a second, +// unchecked door into handleCommitEvent. +function rkeyOf(uri: string): string | null { + const rkey = uri.split("/").pop() ?? ""; + return isRecordKey(rkey) ? rkey : null; } // Feed one listRecords row into the firehose handler and classify the *outcome* @@ -124,6 +128,12 @@ function dispatchRow( return; } + const rkey = rkeyOf(row.uri); + if (rkey === null) { + countSkip(report, did, collection, "invalid-rkey"); + return; + } + if (collection === COLLECTIONS.noteRevision) { const noteRef = (row.value as { noteRef?: unknown }).noteRef; const noteExists = options.noteExists ?? defaultNoteExists; @@ -138,7 +148,7 @@ function dispatchRow( dispatch({ did, collection, - rkey: rkeyOf(row.uri), + rkey, operation: "create", record: row.value, }); @@ -199,19 +209,19 @@ interface Stage { reverse: boolean; } -// Cross-collection ordering matters: revisions reference notes, notes/memberships -// reference wikis. Crawl every DID through each stage before advancing. +// Cross-collection ordering matters, for two reasons. References: revisions point +// at notes, notes/memberships point at wikis. Authorization: handleNote and +// handleRevision both call canDidEditWiki, which reads memberships — so +// memberships must land before notes or every non-owner's content is dropped. +// Crawl every DID through each stage before advancing. const STAGES: Stage[] = [ { collections: [COLLECTIONS.wiki], reverse: false }, + { collections: [COLLECTIONS.membership], reverse: false }, { collections: [COLLECTIONS.note], reverse: false }, // reverse: true → ascending TID (oldest first) so the diff chain applies in order. { collections: [COLLECTIONS.noteRevision], reverse: true }, { - collections: [ - COLLECTIONS.membership, - COLLECTIONS.memberRequest, - COLLECTIONS.bookmark, - ], + collections: [COLLECTIONS.memberRequest, COLLECTIONS.bookmark], reverse: false, }, ]; diff --git a/src/lib/headings.ts b/src/lib/headings.ts index ee692e6..069307a 100644 --- a/src/lib/headings.ts +++ b/src/lib/headings.ts @@ -1,4 +1,4 @@ -import { slugify } from "./slug.ts"; +import { toSlug } from "./slug.ts"; interface Heading { text: string; @@ -8,7 +8,7 @@ interface Heading { // Disambiguates repeated slugs with -1, -2, …; `seen` counts uses per base slug. export function makeAnchor(text: string, seen: Map): string { - const base = slugify(text); + const base = toSlug(text); if (!base) return base; const n = seen.get(base) ?? 0; seen.set(base, n + 1); diff --git a/src/lib/import-export/zip-parse.ts b/src/lib/import-export/zip-parse.ts index 3397dbf..985343e 100644 --- a/src/lib/import-export/zip-parse.ts +++ b/src/lib/import-export/zip-parse.ts @@ -4,7 +4,7 @@ import { ImportError } from "../errors.ts"; import { IMAGE_EXTENSIONS, MIME_BY_EXT } from "../image-types.ts"; import { LIMITS } from "../limits.ts"; import { normalizeLF } from "../normalize.ts"; -import { isValidSlug, slugify } from "../slug.ts"; +import { isSafeSlug, toSlug } from "../slug.ts"; import { extractLocalImageRefs } from "./markdown-transform.ts"; import type { ImportedImage, ImportedNote, ImportResult } from "./types.ts"; @@ -110,10 +110,10 @@ export function parseImportZip(buffer: ArrayBuffer): ImportResult { if (isHomeName(name) && !usedSlugs.has("home")) { slug = "home"; } else { - slug = slugify(title); + slug = toSlug(title); } - if (!slug || !isValidSlug(slug)) { + if (!slug || !isSafeSlug(slug)) { warnings.push(`Skipped "${name}": could not generate a valid slug.`); continue; } diff --git a/src/lib/limits.ts b/src/lib/limits.ts index 4000e20..52b9cf2 100644 --- a/src/lib/limits.ts +++ b/src/lib/limits.ts @@ -1,4 +1,6 @@ -// Mirrors lexicon maxLength/maxSize (lexicons/wiki.lichen.*.json) — keep in sync. +// Mirrors lexicon maxLength/maxSize (lexicons/wiki.lichen.*.json); the mirror is +// enforced by tests/lib/limits-lexicon.test.ts. Some entries are only referenced +// there, because ingest applies a stricter predicate than the lexicon declares. export const LIMITS = { wiki: { name: 256, diff --git a/src/lib/markdown/wikilink-plugin.ts b/src/lib/markdown/wikilink-plugin.ts index 7fdf647..371853b 100644 --- a/src/lib/markdown/wikilink-plugin.ts +++ b/src/lib/markdown/wikilink-plugin.ts @@ -1,7 +1,7 @@ import type MarkdownIt from "markdown-it"; import type StateInline from "markdown-it/lib/rules_inline/state_inline.mjs"; import { escapeHtml } from "../html.ts"; -import { slugify } from "../slug.ts"; +import { toSlug } from "../slug.ts"; import { noteUrl, wikiUrl } from "../urls.ts"; export interface WikilinkEnv { @@ -114,9 +114,11 @@ export function wikilinkPlugin(mdi: MarkdownIt): void { ); if (!target) return false; - const anchor = target.section ? `#${slugify(target.section)}` : ""; - const noteSlug = target.noteSlug ? slugify(target.noteSlug) : ""; - const wikiSlug = target.wikiSlug ? slugify(target.wikiSlug) : ""; + // Must match what note-validation mints, or [[日本語のノート]] resolves to + // a slug no note has. + const anchor = target.section ? `#${toSlug(target.section)}` : ""; + const noteSlug = target.noteSlug ? toSlug(target.noteSlug) : ""; + const wikiSlug = target.wikiSlug ? toSlug(target.wikiSlug) : ""; let href: string; if (!wikiSlug && !noteSlug && target.section) { // [[#section]] — anchor only diff --git a/src/lib/note-validation.ts b/src/lib/note-validation.ts index 0e399e8..366e41c 100644 --- a/src/lib/note-validation.ts +++ b/src/lib/note-validation.ts @@ -2,7 +2,7 @@ import { getNoteBySlug } from "../server/db/queries/index.ts"; import type { Messages } from "./i18n/index.ts"; import { fmt } from "./i18n/index.ts"; import { LIMITS } from "./limits.ts"; -import { isValidSlug, slugify } from "./slug.ts"; +import { isSafeSlug, toSlug } from "./slug.ts"; export function validateNewNote( wikiAtUri: string, @@ -15,8 +15,8 @@ export function validateNewNote( error: fmt(msg.error.titleTooLong, { max: String(LIMITS.note.title) }), }; } - const noteSlug = slugify(title); - if (!isValidSlug(noteSlug)) + const noteSlug = toSlug(title); + if (!isSafeSlug(noteSlug)) return { error: fmt(msg.error.invalidSlug, { title }) }; const existing = getNoteBySlug(wikiAtUri, noteSlug); if (existing) return { error: fmt(msg.error.slugExists, { slug: noteSlug }) }; diff --git a/src/lib/response.ts b/src/lib/response.ts index bf5a4a1..8dff926 100644 --- a/src/lib/response.ts +++ b/src/lib/response.ts @@ -3,6 +3,16 @@ interface HtmlResponseOptions { edgeCacheSeconds?: number; } +/** + * RFC 6266 attachment header. Note slugs may be Japanese, and a raw non-ASCII + * byte in a header makes Response throw — so send an ASCII-only `filename` for + * old clients plus the real name in `filename*`. + */ +export function attachmentDisposition(filename: string): string { + const ascii = filename.replace(/[^\x20-\x7E]/g, "_").replace(/["\\]/g, "_"); + return `attachment; filename="${ascii}"; filename*=UTF-8''${encodeURIComponent(filename)}`; +} + // Strict CSP: no inline scripts; inline styles allowed for the layout's per-theme