Something went wrong. Try again.
This repository has no description
Something went wrong. Try again.
3.4 kB · 128 lines
TypeScript
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129// Taken from ATcute's richtext-segmenter// https://github.com/mary-ext/atcute/blob/trunk/packages/bluesky/richtext-segmenter/lib/index.ts// repoed b/c we need to import types from @atproto/api not @atcute/bskyimport { Main } from "@atproto/api/dist/client/types/app/bsky/richtext/facet";
type UnwrapArray<T> = T extends (infer V)[] ? V : never;
export type Facet = Main;export type FacetFeature = UnwrapArray<Facet["features"]>;
export interface RichtextSegment { text: string; features: FacetFeature[] | undefined;}
const segment = ( text: string, features: FacetFeature[] | undefined,): RichtextSegment => { return { text, features: text.length > 0 ? features : undefined };};
export const segmentize = ( text: string, facets: Facet[] | undefined,): RichtextSegment[] => { if (facets === undefined || facets.length === 0) { return [segment(text, undefined)]; }
const segments: RichtextSegment[] = []; const utf16Length = text.length; let utf16Cursor = 0; let utf8Cursor = 0;
const advanceCursor = (startUtf16: number, endUtf8: number): number => { let curs = startUtf16;
// Fast-path for entirely ASCII text const isLikelyAsciiText = text.charCodeAt(curs) < 0x80; if (isLikelyAsciiText) { curs += 1; utf8Cursor += 1;
// SIMD-like batch processing while (utf8Cursor + 8 <= endUtf8 && curs + 8 <= utf16Length) { const char1 = text.charCodeAt(curs); const char2 = text.charCodeAt(curs + 1); const char3 = text.charCodeAt(curs + 2); const char4 = text.charCodeAt(curs + 3); const char5 = text.charCodeAt(curs + 4); const char6 = text.charCodeAt(curs + 5); const char7 = text.charCodeAt(curs + 6); const char8 = text.charCodeAt(curs + 7);
if ( (char1 | char2 | char3 | char4 | char5 | char6 | char7 | char8) < 0x80 ) { curs += 8; utf8Cursor += 8; continue; }
break; } }
while (utf8Cursor < endUtf8 && curs < utf16Length) { const code = text.charCodeAt(curs);
if (code < 0x80) { curs += 1; utf8Cursor += 1; } else if (code < 0x800) { curs += 1; utf8Cursor += 2; } else if (code < 0xd800 || code > 0xdbff) { curs += 1; utf8Cursor += 3; } else { curs += 2; utf8Cursor += 4; } }
return curs; };
for (let idx = 0, len = facets.length; idx < len; idx++) { const facet = facets[idx];
const { byteStart, byteEnd } = facet.index; const features = facet.features;
if (byteStart > byteEnd || !features || features.length === 0) { continue; }
if (utf8Cursor < byteStart) { const nextUtf16Cursor = advanceCursor(utf16Cursor, byteStart); if (nextUtf16Cursor > utf16Cursor) { segments.push( segment(text.slice(utf16Cursor, nextUtf16Cursor), undefined), ); }
utf16Cursor = nextUtf16Cursor; }
{ const nextUtf16Cursor = advanceCursor(utf16Cursor, byteEnd); if (nextUtf16Cursor > utf16Cursor) { segments.push( segment(text.slice(utf16Cursor, nextUtf16Cursor), features), ); }
utf16Cursor = nextUtf16Cursor; } }
if (utf16Cursor < utf16Length) { segments.push(segment(text.slice(utf16Cursor), undefined)); }
return segments;};