From 023fccedc3e33494ae9556904ffd70195e5a61ff Mon Sep 17 00:00:00 2001 From: juprodh Date: Tue, 21 Jul 2026 13:53:36 +0800 Subject: [PATCH] Change how string limits are handled --- lexicons/wiki.lichen.note.json | 6 +- lexicons/wiki.lichen.noteRevision.json | 3 +- lexicons/wiki.lichen.wiki.json | 6 +- src/firehose/handlers.ts | 19 ++-- src/lib/limits.ts | 26 +++-- src/lib/note-validation.ts | 7 +- src/lib/orchestrators/note.ts | 15 ++- src/lib/orchestrators/wiki.ts | 27 ++++-- src/lib/slug.ts | 16 ++-- src/lib/text-length.ts | 49 ++++++++++ tests/firehose/hostile-records.test.ts | 127 ++++++++++++++++++++++++- tests/lib/limits-lexicon.test.ts | 38 +++++--- tests/lib/slug.test.ts | 7 +- tests/lib/text-length.test.ts | 76 +++++++++++++++ 14 files changed, 355 insertions(+), 67 deletions(-) create mode 100644 src/lib/text-length.ts create mode 100644 tests/lib/text-length.test.ts diff --git a/lexicons/wiki.lichen.note.json b/lexicons/wiki.lichen.note.json index f39e77b..2622c1a 100644 --- a/lexicons/wiki.lichen.note.json +++ b/lexicons/wiki.lichen.note.json @@ -12,12 +12,14 @@ "properties": { "slug": { "type": "string", - "maxLength": 256, + "maxGraphemes": 256, + "maxLength": 2560, "description": "Permanent human-readable identifier, used in URLs. Never changes once set." }, "title": { "type": "string", - "maxLength": 256, + "maxGraphemes": 256, + "maxLength": 2560, "description": "Display name for the note. Can be updated freely." }, "wikiRef": { diff --git a/lexicons/wiki.lichen.noteRevision.json b/lexicons/wiki.lichen.noteRevision.json index 1db3e8c..4e81bd4 100644 --- a/lexicons/wiki.lichen.noteRevision.json +++ b/lexicons/wiki.lichen.noteRevision.json @@ -33,7 +33,8 @@ }, "message": { "type": "string", - "maxLength": 1024, + "maxGraphemes": 1024, + "maxLength": 10240, "description": "Optional commit message describing this revision." }, "createdAt": { diff --git a/lexicons/wiki.lichen.wiki.json b/lexicons/wiki.lichen.wiki.json index 589d8a8..6c45407 100644 --- a/lexicons/wiki.lichen.wiki.json +++ b/lexicons/wiki.lichen.wiki.json @@ -12,7 +12,8 @@ "properties": { "name": { "type": "string", - "maxLength": 256, + "maxGraphemes": 256, + "maxLength": 2560, "description": "Display name of the wiki." }, "visibility": { @@ -33,7 +34,8 @@ }, "description": { "type": "string", - "maxLength": 300, + "maxGraphemes": 300, + "maxLength": 3000, "description": "Short description of the wiki's purpose or topic." }, "theme": { diff --git a/src/firehose/handlers.ts b/src/firehose/handlers.ts index 26da928..946a016 100644 --- a/src/firehose/handlers.ts +++ b/src/firehose/handlers.ts @@ -4,6 +4,7 @@ import { COLLECTIONS, normalizeRole } from "../lib/collections.ts"; import { isValidLanguageTag } from "../lib/languages.ts"; import { LIMITS } from "../lib/limits.ts"; import { isSafeSlug, normalizeSlug } from "../lib/slug.ts"; +import { withinLimit } from "../lib/text-length.ts"; import { registerContributor } from "../registry/index.ts"; import { applyRevisionFromFirehose, @@ -85,7 +86,7 @@ function logDrop(atUri: string, reason: string): void { } function wikiIsInvalid(atUri: string, r: WikiRecord): boolean { - if (r.name.length > LIMITS.wiki.name) { + if (!withinLimit(r.name, LIMITS.wiki.name)) { logDrop(atUri, "wiki.name exceeds limit"); return true; } @@ -97,7 +98,7 @@ function wikiIsInvalid(atUri: string, r: WikiRecord): boolean { } if ( r.description !== undefined && - r.description.length > LIMITS.wiki.description + !withinLimit(r.description, LIMITS.wiki.description) ) { logDrop(atUri, "wiki.description exceeds limit"); return true; @@ -114,7 +115,7 @@ function noteIsInvalid(atUri: string, r: NoteRecord): boolean { logDrop(atUri, "note.slug is not a safe slug"); return true; } - if (r.title.length > LIMITS.note.title) { + if (!withinLimit(r.title, LIMITS.note.title)) { logDrop(atUri, "note.title exceeds limit"); return true; } @@ -130,7 +131,10 @@ function revisionIsInvalid(atUri: string, r: RevisionRecord): boolean { logDrop(atUri, "revision.diffFormat exceeds limit"); return true; } - if (r.message !== undefined && r.message.length > LIMITS.revision.message) { + if ( + r.message !== undefined && + !withinLimit(r.message, LIMITS.revision.message) + ) { logDrop(atUri, "revision.message exceeds limit"); return true; } @@ -148,10 +152,9 @@ function membershipIsInvalid(atUri: string, r: MembershipRecord): boolean { logDrop(atUri, "membership.memberDid is not a DID"); return true; } - if (r.role.length > LIMITS.membership.role) { - logDrop(atUri, "membership.role exceeds limit"); - return true; - } + // No role length check: normalizeRole maps anything unrecognised to + // "contributor", so an odd value is already handled rather than a reason to + // drop a membership grant. return false; } diff --git a/src/lib/limits.ts b/src/lib/limits.ts index 52b9cf2..c5843de 100644 --- a/src/lib/limits.ts +++ b/src/lib/limits.ts @@ -1,28 +1,26 @@ -// Mirrors lexicon maxLength/maxSize (lexicons/wiki.lichen.*.json); the mirror is -// enforced by tests/lib/limits-lexicon.test.ts. Some entries are only referenced -// there, because ingest applies a stricter predicate than the lexicon declares. +// Mirrors lexicon maxGraphemes/maxLength/maxSize (lexicons/wiki.lichen.*.json); +// the mirror is enforced by tests/lib/limits-lexicon.test.ts. +// +// User-facing strings carry both units the lexicon declares: `graphemes` is the +// limit a person experiences, `bytes` is wire safety. Checking only bytes would +// charge Japanese three times as much per character as English. export const LIMITS = { wiki: { - name: 256, - visibility: 32, + name: { graphemes: 256, bytes: 2560 }, language: 16, - description: 300, + description: { graphemes: 300, bytes: 3000 }, sidebar: 100_000, // appview-only, no lexicon constraint }, note: { - slug: 256, - title: 256, + slug: { graphemes: 256, bytes: 2560 }, + title: { graphemes: 256, bytes: 2560 }, }, revision: { - diff: 1_000_000, + diff: 1_000_000, // bytes; a machine-generated patch has no grapheme limit diffFormat: 32, - message: 1024, + message: { graphemes: 1024, bytes: 10240 }, blobCount: 32, }, - membership: { - memberDid: 2048, - role: 32, - }, image: { bytes: 2 * 1024 * 1024, }, diff --git a/src/lib/note-validation.ts b/src/lib/note-validation.ts index 366e41c..da81f62 100644 --- a/src/lib/note-validation.ts +++ b/src/lib/note-validation.ts @@ -3,6 +3,7 @@ import type { Messages } from "./i18n/index.ts"; import { fmt } from "./i18n/index.ts"; import { LIMITS } from "./limits.ts"; import { isSafeSlug, toSlug } from "./slug.ts"; +import { withinLimit } from "./text-length.ts"; export function validateNewNote( wikiAtUri: string, @@ -10,9 +11,11 @@ export function validateNewNote( msg: Messages, ): { noteSlug: string } | { error: string } { if (!title.trim()) return { error: msg.error.titleRequired }; - if (title.length > LIMITS.note.title) { + if (!withinLimit(title, LIMITS.note.title)) { return { - error: fmt(msg.error.titleTooLong, { max: String(LIMITS.note.title) }), + error: fmt(msg.error.titleTooLong, { + max: String(LIMITS.note.title.graphemes), + }), }; } const noteSlug = toSlug(title); diff --git a/src/lib/orchestrators/note.ts b/src/lib/orchestrators/note.ts index 0d0c8d7..5b6adfe 100644 --- a/src/lib/orchestrators/note.ts +++ b/src/lib/orchestrators/note.ts @@ -31,6 +31,7 @@ import { fmt, type Messages } from "../i18n/index.ts"; import { LIMITS } from "../limits.ts"; import { normalizeLF } from "../normalize.ts"; import { validateNewNote } from "../note-validation.ts"; +import { utf8Length, withinLimit } from "../text-length.ts"; import { withPdsError } from "./helpers.ts"; interface NoteFormFields { @@ -61,14 +62,16 @@ function validateRevisionFields( msg: Messages, ): void { // Cap content against the diff ceiling — the stored diff is rarely larger than the content. - if (content.length > LIMITS.revision.diff) { + if (utf8Length(content) > LIMITS.revision.diff) { throw new ValidationError( fmt(msg.error.contentTooLong, { max: String(LIMITS.revision.diff) }), ); } - if (message !== undefined && message.length > LIMITS.revision.message) { + if (message !== undefined && !withinLimit(message, LIMITS.revision.message)) { throw new ValidationError( - fmt(msg.error.messageTooLong, { max: String(LIMITS.revision.message) }), + fmt(msg.error.messageTooLong, { + max: String(LIMITS.revision.message.graphemes), + }), ); } } @@ -187,9 +190,11 @@ export async function editNoteAction( if (!fields.title.trim()) { throw new ValidationError(msg.error.titleRequired); } - if (fields.title.length > LIMITS.note.title) { + if (!withinLimit(fields.title, LIMITS.note.title)) { throw new ValidationError( - fmt(msg.error.titleTooLong, { max: String(LIMITS.note.title) }), + fmt(msg.error.titleTooLong, { + max: String(LIMITS.note.title.graphemes), + }), ); } diff --git a/src/lib/orchestrators/wiki.ts b/src/lib/orchestrators/wiki.ts index a71727d..5b2c266 100644 --- a/src/lib/orchestrators/wiki.ts +++ b/src/lib/orchestrators/wiki.ts @@ -31,6 +31,7 @@ import { fmt, type Messages, t } from "../i18n/index.ts"; import { isValidLanguageTag } from "../languages.ts"; import { LIMITS } from "../limits.ts"; import { isValidSlug, slugify } from "../slug.ts"; +import { truncateToLimit, withinLimit } from "../text-length.ts"; import { withPdsError } from "./helpers.ts"; export interface WikiFormFields { @@ -70,9 +71,11 @@ export async function createWikiCore( throw new ValidationError(msg.error.wikiNameRequired); } - if (fields.name.length > LIMITS.wiki.name) { + if (!withinLimit(fields.name, LIMITS.wiki.name)) { throw new ValidationError( - fmt(msg.error.wikiNameTooLong, { max: String(LIMITS.wiki.name) }), + fmt(msg.error.wikiNameTooLong, { + max: String(LIMITS.wiki.name.graphemes), + }), ); } @@ -113,9 +116,10 @@ export async function createWikiCore( let atUri = `at://${did}/wiki.lichen.wiki/${slug}`; const agent = ctx.session ? getAgent(ctx.session) : null; - const description = fields.description - .trim() - .slice(0, LIMITS.wiki.description); + const description = truncateToLimit( + fields.description.trim(), + LIMITS.wiki.description, + ); if (agent) { await withPdsError("create wiki", async () => { @@ -239,18 +243,21 @@ export async function editWikiAction( if (!fields.name.trim()) { throw new ValidationError(msg.error.wikiNameRequired); } - if (fields.name.length > LIMITS.wiki.name) { + if (!withinLimit(fields.name, LIMITS.wiki.name)) { throw new ValidationError( - fmt(msg.error.wikiNameTooLong, { max: String(LIMITS.wiki.name) }), + fmt(msg.error.wikiNameTooLong, { + max: String(LIMITS.wiki.name.graphemes), + }), ); } const did = ctx.did; if (!did) throw new ForbiddenError(); - const description = fields.description - .trim() - .slice(0, LIMITS.wiki.description); + const description = truncateToLimit( + fields.description.trim(), + LIMITS.wiki.description, + ); const agent = ctx.session ? getAgent(ctx.session) : null; if (agent) { diff --git a/src/lib/slug.ts b/src/lib/slug.ts index fdde9af..e018efa 100644 --- a/src/lib/slug.ts +++ b/src/lib/slug.ts @@ -1,9 +1,10 @@ import slugifyAscii from "@sindresorhus/slugify"; +import { LIMITS } from "./limits.ts"; +import { truncateToLimit, withinLimit } from "./text-length.ts"; export { default as slugify } from "@sindresorhus/slugify"; const SLUG_RE = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; -const SLUG_MAX = 128; // Letters, numbers and combining marks in any script, plus "-" and "_". Excludes // everything that gives a slug meaning in a URL or an HTML attribute: "/", "%", @@ -12,7 +13,7 @@ const UNSAFE_CHARS = /[^\p{L}\p{N}\p{M}_-]/u; /** The canonical form we mint: lowercase ASCII words joined by hyphens. */ export function isValidSlug(slug: string): boolean { - return SLUG_RE.test(slug) && slug.length >= 1 && slug.length <= SLUG_MAX; + return SLUG_RE.test(slug) && withinLimit(slug, LIMITS.note.slug); } /** @@ -23,7 +24,9 @@ export function isValidSlug(slug: string): boolean { */ export function isSafeSlug(slug: string): boolean { return ( - slug.length >= 1 && slug.length <= SLUG_MAX && !UNSAFE_CHARS.test(slug) + slug.length >= 1 && + withinLimit(slug, LIMITS.note.slug) && + !UNSAFE_CHARS.test(slug) ); } @@ -60,11 +63,12 @@ export function toSlug(text: string): string { const normalized = normalizeSlug(text); if (transliteratesCleanly(normalized)) return slugifyAscii(normalized); - return normalized + const unicode = normalized .replace(/[A-Z]/g, (c) => c.toLowerCase()) .replace(/\s+/gu, "-") .replace(new RegExp(UNSAFE_CHARS.source, "gu"), "") .replace(/-{2,}/g, "-") - .replace(/^-+|-+$/g, "") - .slice(0, SLUG_MAX); + .replace(/^-+|-+$/g, ""); + + return truncateToLimit(unicode, LIMITS.note.slug); } diff --git a/src/lib/text-length.ts b/src/lib/text-length.ts new file mode 100644 index 0000000..caff5b8 --- /dev/null +++ b/src/lib/text-length.ts @@ -0,0 +1,49 @@ +// Lexicon string limits come in two units, and neither is String.length: +// maxGraphemes counts what a reader would call a character, maxLength counts +// UTF-8 bytes. String.length is UTF-16 code units, which matches neither — it +// over-counts "é" in NFD and under-counts a 25-byte emoji by 14. + +const segmenter = new Intl.Segmenter("en", { granularity: "grapheme" }); +const encoder = new TextEncoder(); + +export function graphemeLength(text: string): number { + let n = 0; + for (const _ of segmenter.segment(text)) n++; + return n; +} + +export function utf8Length(text: string): number { + return encoder.encode(text).length; +} + +export interface TextLimit { + graphemes: number; + bytes: number; +} + +export function withinLimit(text: string, limit: TextLimit): boolean { + return ( + graphemeLength(text) <= limit.graphemes && utf8Length(text) <= limit.bytes + ); +} + +/** + * Truncate on grapheme boundaries, then trim further if the byte budget is + * still exceeded. Slicing by code unit would be enough for the grapheme cap but + * can split a surrogate pair or strip a combining mark from its base. + * + * Byte trimming almost never runs: maxLength is ~10x maxGraphemes, so only + * emoji-dense text reaches it. + */ +export function truncateToLimit(text: string, limit: TextLimit): string { + let graphemes = [...segmenter.segment(text)].map((s) => s.segment); + if (graphemes.length > limit.graphemes) { + graphemes = graphemes.slice(0, limit.graphemes); + } + let out = graphemes.join(""); + while (graphemes.length > 0 && utf8Length(out) > limit.bytes) { + graphemes.pop(); + out = graphemes.join(""); + } + return out; +} diff --git a/tests/firehose/hostile-records.test.ts b/tests/firehose/hostile-records.test.ts index 09e69e2..8185d62 100644 --- a/tests/firehose/hostile-records.test.ts +++ b/tests/firehose/hostile-records.test.ts @@ -60,7 +60,11 @@ describe("note.slug shape", () => { ["whitespace", "a b"], ["percent sign", "a%2fb"], ["angle brackets", "