From e418c8c4c2a4ad02400b8b4bc876fe7ea9f1c16e Mon Sep 17 00:00:00 2001 From: Pierre Le Fevre Date: Tue, 3 Mar 2026 17:17:03 +0100 Subject: [PATCH] Implement HTML5 tokenizer state machine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spec-compliant HTML5 tokenizer per WHATWG ยง13.2.5 with: - Full state machine: Data, TagOpen, EndTagOpen, TagName, attributes, comments, DOCTYPE (with PUBLIC/SYSTEM), character references - Named character references: 2125 entities with binary search lookup - Numeric character references: decimal, hex, Windows-1252 replacements - Proper token coalescing (adjacent Character tokens merged) - 34 unit tests, 6661/6807 html5lib tokenizer tests passing (97.7%) Co-Authored-By: Claude Opus 4.6 --- crates/html/src/entities.rs | 2162 ++++++++++++++++++++++++++++++++++ crates/html/src/lib.rs | 32 +- crates/html/src/tokenizer.rs | 2039 ++++++++++++++++++++++++++++++++ 3 files changed, 4229 insertions(+), 4 deletions(-) create mode 100644 crates/html/src/entities.rs create mode 100644 crates/html/src/tokenizer.rs diff --git a/crates/html/src/entities.rs b/crates/html/src/entities.rs new file mode 100644 index 0000000..4391740 --- /dev/null +++ b/crates/html/src/entities.rs @@ -0,0 +1,2162 @@ +//! HTML named character reference table. +//! +//! Complete list of 2125 named character references from the WHATWG HTML spec. +//! Uses a sorted array with binary search for O(log n) lookup. + +/// All 2125 named character references, sorted for binary search. +static ENTITIES: &[(&str, &str)] = &[ + ("AElig", "\u{00C6}"), + ("AMP", "&"), + ("Aacute", "\u{00C1}"), + ("Abreve", "\u{0102}"), + ("Acirc", "\u{00C2}"), + ("Acy", "\u{0410}"), + ("Afr", "\u{1D504}"), + ("Agrave", "\u{00C0}"), + ("Alpha", "\u{0391}"), + ("Amacr", "\u{0100}"), + ("And", "\u{2A53}"), + ("Aogon", "\u{0104}"), + ("Aopf", "\u{1D538}"), + ("ApplyFunction", "\u{2061}"), + ("Aring", "\u{00C5}"), + ("Ascr", "\u{1D49C}"), + ("Assign", "\u{2254}"), + ("Atilde", "\u{00C3}"), + ("Auml", "\u{00C4}"), + ("Backslash", "\u{2216}"), + ("Barv", "\u{2AE7}"), + ("Barwed", "\u{2306}"), + ("Bcy", "\u{0411}"), + ("Because", "\u{2235}"), + ("Bernoullis", "\u{212C}"), + ("Beta", "\u{0392}"), + ("Bfr", "\u{1D505}"), + ("Bopf", "\u{1D539}"), + ("Breve", "\u{02D8}"), + ("Bscr", "\u{212C}"), + ("Bumpeq", "\u{224E}"), + ("CHcy", "\u{0427}"), + ("COPY", "\u{00A9}"), + ("Cacute", "\u{0106}"), + ("Cap", "\u{22D2}"), + ("CapitalDifferentialD", "\u{2145}"), + ("Cayleys", "\u{212D}"), + ("Ccaron", "\u{010C}"), + ("Ccedil", "\u{00C7}"), + ("Ccirc", "\u{0108}"), + ("Cconint", "\u{2230}"), + ("Cdot", "\u{010A}"), + ("Cedilla", "\u{00B8}"), + ("CenterDot", "\u{00B7}"), + ("Cfr", "\u{212D}"), + ("Chi", "\u{03A7}"), + ("CircleDot", "\u{2299}"), + ("CircleMinus", "\u{2296}"), + ("CirclePlus", "\u{2295}"), + ("CircleTimes", "\u{2297}"), + ("ClockwiseContourIntegral", "\u{2232}"), + ("CloseCurlyDoubleQuote", "\u{201D}"), + ("CloseCurlyQuote", "\u{2019}"), + ("Colon", "\u{2237}"), + ("Colone", "\u{2A74}"), + ("Congruent", "\u{2261}"), + ("Conint", "\u{222F}"), + ("ContourIntegral", "\u{222E}"), + ("Copf", "\u{2102}"), + ("Coproduct", "\u{2210}"), + ("CounterClockwiseContourIntegral", "\u{2233}"), + ("Cross", "\u{2A2F}"), + ("Cscr", "\u{1D49E}"), + ("Cup", "\u{22D3}"), + ("CupCap", "\u{224D}"), + ("DD", "\u{2145}"), + ("DDotrahd", "\u{2911}"), + ("DJcy", "\u{0402}"), + ("DScy", "\u{0405}"), + ("DZcy", "\u{040F}"), + ("Dagger", "\u{2021}"), + ("Darr", "\u{21A1}"), + ("Dashv", "\u{2AE4}"), + ("Dcaron", "\u{010E}"), + ("Dcy", "\u{0414}"), + ("Del", "\u{2207}"), + ("Delta", "\u{0394}"), + ("Dfr", "\u{1D507}"), + ("DiacriticalAcute", "\u{00B4}"), + ("DiacriticalDot", "\u{02D9}"), + ("DiacriticalDoubleAcute", "\u{02DD}"), + ("DiacriticalGrave", "`"), + ("DiacriticalTilde", "\u{02DC}"), + ("Diamond", "\u{22C4}"), + ("DifferentialD", "\u{2146}"), + ("Dopf", "\u{1D53B}"), + ("Dot", "\u{00A8}"), + ("DotDot", "\u{20DC}"), + ("DotEqual", "\u{2250}"), + ("DoubleContourIntegral", "\u{222F}"), + ("DoubleDot", "\u{00A8}"), + ("DoubleDownArrow", "\u{21D3}"), + ("DoubleLeftArrow", "\u{21D0}"), + ("DoubleLeftRightArrow", "\u{21D4}"), + ("DoubleLeftTee", "\u{2AE4}"), + ("DoubleLongLeftArrow", "\u{27F8}"), + ("DoubleLongLeftRightArrow", "\u{27FA}"), + ("DoubleLongRightArrow", "\u{27F9}"), + ("DoubleRightArrow", "\u{21D2}"), + ("DoubleRightTee", "\u{22A8}"), + ("DoubleUpArrow", "\u{21D1}"), + ("DoubleUpDownArrow", "\u{21D5}"), + ("DoubleVerticalBar", "\u{2225}"), + ("DownArrow", "\u{2193}"), + ("DownArrowBar", "\u{2913}"), + ("DownArrowUpArrow", "\u{21F5}"), + ("DownBreve", "\u{0311}"), + ("DownLeftRightVector", "\u{2950}"), + ("DownLeftTeeVector", "\u{295E}"), + ("DownLeftVector", "\u{21BD}"), + ("DownLeftVectorBar", "\u{2956}"), + ("DownRightTeeVector", "\u{295F}"), + ("DownRightVector", "\u{21C1}"), + ("DownRightVectorBar", "\u{2957}"), + ("DownTee", "\u{22A4}"), + ("DownTeeArrow", "\u{21A7}"), + ("Downarrow", "\u{21D3}"), + ("Dscr", "\u{1D49F}"), + ("Dstrok", "\u{0110}"), + ("ENG", "\u{014A}"), + ("ETH", "\u{00D0}"), + ("Eacute", "\u{00C9}"), + ("Ecaron", "\u{011A}"), + ("Ecirc", "\u{00CA}"), + ("Ecy", "\u{042D}"), + ("Edot", "\u{0116}"), + ("Efr", "\u{1D508}"), + ("Egrave", "\u{00C8}"), + ("Element", "\u{2208}"), + ("Emacr", "\u{0112}"), + ("EmptySmallSquare", "\u{25FB}"), + ("EmptyVerySmallSquare", "\u{25AB}"), + ("Eogon", "\u{0118}"), + ("Eopf", "\u{1D53C}"), + ("Epsilon", "\u{0395}"), + ("Equal", "\u{2A75}"), + ("EqualTilde", "\u{2242}"), + ("Equilibrium", "\u{21CC}"), + ("Escr", "\u{2130}"), + ("Esim", "\u{2A73}"), + ("Eta", "\u{0397}"), + ("Euml", "\u{00CB}"), + ("Exists", "\u{2203}"), + ("ExponentialE", "\u{2147}"), + ("Fcy", "\u{0424}"), + ("Ffr", "\u{1D509}"), + ("FilledSmallSquare", "\u{25FC}"), + ("FilledVerySmallSquare", "\u{25AA}"), + ("Fopf", "\u{1D53D}"), + ("ForAll", "\u{2200}"), + ("Fouriertrf", "\u{2131}"), + ("Fscr", "\u{2131}"), + ("GJcy", "\u{0403}"), + ("GT", ">"), + ("Gamma", "\u{0393}"), + ("Gammad", "\u{03DC}"), + ("Gbreve", "\u{011E}"), + ("Gcedil", "\u{0122}"), + ("Gcirc", "\u{011C}"), + ("Gcy", "\u{0413}"), + ("Gdot", "\u{0120}"), + ("Gfr", "\u{1D50A}"), + ("Gg", "\u{22D9}"), + ("Gopf", "\u{1D53E}"), + ("GreaterEqual", "\u{2265}"), + ("GreaterEqualLess", "\u{22DB}"), + ("GreaterFullEqual", "\u{2267}"), + ("GreaterGreater", "\u{2AA2}"), + ("GreaterLess", "\u{2277}"), + ("GreaterSlantEqual", "\u{2A7E}"), + ("GreaterTilde", "\u{2273}"), + ("Gscr", "\u{1D4A2}"), + ("Gt", "\u{226B}"), + ("HARDcy", "\u{042A}"), + ("Hacek", "\u{02C7}"), + ("Hat", "^"), + ("Hcirc", "\u{0124}"), + ("Hfr", "\u{210C}"), + ("HilbertSpace", "\u{210B}"), + ("Hopf", "\u{210D}"), + ("HorizontalLine", "\u{2500}"), + ("Hscr", "\u{210B}"), + ("Hstrok", "\u{0126}"), + ("HumpDownHump", "\u{224E}"), + ("HumpEqual", "\u{224F}"), + ("IEcy", "\u{0415}"), + ("IJlig", "\u{0132}"), + ("IOcy", "\u{0401}"), + ("Iacute", "\u{00CD}"), + ("Icirc", "\u{00CE}"), + ("Icy", "\u{0418}"), + ("Idot", "\u{0130}"), + ("Ifr", "\u{2111}"), + ("Igrave", "\u{00CC}"), + ("Im", "\u{2111}"), + ("Imacr", "\u{012A}"), + ("ImaginaryI", "\u{2148}"), + ("Implies", "\u{21D2}"), + ("Int", "\u{222C}"), + ("Integral", "\u{222B}"), + ("Intersection", "\u{22C2}"), + ("InvisibleComma", "\u{2063}"), + ("InvisibleTimes", "\u{2062}"), + ("Iogon", "\u{012E}"), + ("Iopf", "\u{1D540}"), + ("Iota", "\u{0399}"), + ("Iscr", "\u{2110}"), + ("Itilde", "\u{0128}"), + ("Iukcy", "\u{0406}"), + ("Iuml", "\u{00CF}"), + ("Jcirc", "\u{0134}"), + ("Jcy", "\u{0419}"), + ("Jfr", "\u{1D50D}"), + ("Jopf", "\u{1D541}"), + ("Jscr", "\u{1D4A5}"), + ("Jsercy", "\u{0408}"), + ("Jukcy", "\u{0404}"), + ("KHcy", "\u{0425}"), + ("KJcy", "\u{040C}"), + ("Kappa", "\u{039A}"), + ("Kcedil", "\u{0136}"), + ("Kcy", "\u{041A}"), + ("Kfr", "\u{1D50E}"), + ("Kopf", "\u{1D542}"), + ("Kscr", "\u{1D4A6}"), + ("LJcy", "\u{0409}"), + ("LT", "<"), + ("Lacute", "\u{0139}"), + ("Lambda", "\u{039B}"), + ("Lang", "\u{27EA}"), + ("Laplacetrf", "\u{2112}"), + ("Larr", "\u{219E}"), + ("Lcaron", "\u{013D}"), + ("Lcedil", "\u{013B}"), + ("Lcy", "\u{041B}"), + ("LeftAngleBracket", "\u{27E8}"), + ("LeftArrow", "\u{2190}"), + ("LeftArrowBar", "\u{21E4}"), + ("LeftArrowRightArrow", "\u{21C6}"), + ("LeftCeiling", "\u{2308}"), + ("LeftDoubleBracket", "\u{27E6}"), + ("LeftDownTeeVector", "\u{2961}"), + ("LeftDownVector", "\u{21C3}"), + ("LeftDownVectorBar", "\u{2959}"), + ("LeftFloor", "\u{230A}"), + ("LeftRightArrow", "\u{2194}"), + ("LeftRightVector", "\u{294E}"), + ("LeftTee", "\u{22A3}"), + ("LeftTeeArrow", "\u{21A4}"), + ("LeftTeeVector", "\u{295A}"), + ("LeftTriangle", "\u{22B2}"), + ("LeftTriangleBar", "\u{29CF}"), + ("LeftTriangleEqual", "\u{22B4}"), + ("LeftUpDownVector", "\u{2951}"), + ("LeftUpTeeVector", "\u{2960}"), + ("LeftUpVector", "\u{21BF}"), + ("LeftUpVectorBar", "\u{2958}"), + ("LeftVector", "\u{21BC}"), + ("LeftVectorBar", "\u{2952}"), + ("Leftarrow", "\u{21D0}"), + ("Leftrightarrow", "\u{21D4}"), + ("LessEqualGreater", "\u{22DA}"), + ("LessFullEqual", "\u{2266}"), + ("LessGreater", "\u{2276}"), + ("LessLess", "\u{2AA1}"), + ("LessSlantEqual", "\u{2A7D}"), + ("LessTilde", "\u{2272}"), + ("Lfr", "\u{1D50F}"), + ("Ll", "\u{22D8}"), + ("Lleftarrow", "\u{21DA}"), + ("Lmidot", "\u{013F}"), + ("LongLeftArrow", "\u{27F5}"), + ("LongLeftRightArrow", "\u{27F7}"), + ("LongRightArrow", "\u{27F6}"), + ("Longleftarrow", "\u{27F8}"), + ("Longleftrightarrow", "\u{27FA}"), + ("Longrightarrow", "\u{27F9}"), + ("Lopf", "\u{1D543}"), + ("LowerLeftArrow", "\u{2199}"), + ("LowerRightArrow", "\u{2198}"), + ("Lscr", "\u{2112}"), + ("Lsh", "\u{21B0}"), + ("Lstrok", "\u{0141}"), + ("Lt", "\u{226A}"), + ("Map", "\u{2905}"), + ("Mcy", "\u{041C}"), + ("MediumSpace", "\u{205F}"), + ("Mellintrf", "\u{2133}"), + ("Mfr", "\u{1D510}"), + ("MinusPlus", "\u{2213}"), + ("Mopf", "\u{1D544}"), + ("Mscr", "\u{2133}"), + ("Mu", "\u{039C}"), + ("NJcy", "\u{040A}"), + ("Nacute", "\u{0143}"), + ("Ncaron", "\u{0147}"), + ("Ncedil", "\u{0145}"), + ("Ncy", "\u{041D}"), + ("NegativeMediumSpace", "\u{200B}"), + ("NegativeThickSpace", "\u{200B}"), + ("NegativeThinSpace", "\u{200B}"), + ("NegativeVeryThinSpace", "\u{200B}"), + ("NestedGreaterGreater", "\u{226B}"), + ("NestedLessLess", "\u{226A}"), + ("NewLine", "\u{000A}"), + ("Nfr", "\u{1D511}"), + ("NoBreak", "\u{2060}"), + ("NonBreakingSpace", "\u{00A0}"), + ("Nopf", "\u{2115}"), + ("Not", "\u{2AEC}"), + ("NotCongruent", "\u{2262}"), + ("NotCupCap", "\u{226D}"), + ("NotDoubleVerticalBar", "\u{2226}"), + ("NotElement", "\u{2209}"), + ("NotEqual", "\u{2260}"), + ("NotEqualTilde", "\u{2242}\u{0338}"), + ("NotExists", "\u{2204}"), + ("NotGreater", "\u{226F}"), + ("NotGreaterEqual", "\u{2271}"), + ("NotGreaterFullEqual", "\u{2267}\u{0338}"), + ("NotGreaterGreater", "\u{226B}\u{0338}"), + ("NotGreaterLess", "\u{2279}"), + ("NotGreaterSlantEqual", "\u{2A7E}\u{0338}"), + ("NotGreaterTilde", "\u{2275}"), + ("NotHumpDownHump", "\u{224E}\u{0338}"), + ("NotHumpEqual", "\u{224F}\u{0338}"), + ("NotLeftTriangle", "\u{22EA}"), + ("NotLeftTriangleBar", "\u{29CF}\u{0338}"), + ("NotLeftTriangleEqual", "\u{22EC}"), + ("NotLess", "\u{226E}"), + ("NotLessEqual", "\u{2270}"), + ("NotLessGreater", "\u{2278}"), + ("NotLessLess", "\u{226A}\u{0338}"), + ("NotLessSlantEqual", "\u{2A7D}\u{0338}"), + ("NotLessTilde", "\u{2274}"), + ("NotNestedGreaterGreater", "\u{2AA2}\u{0338}"), + ("NotNestedLessLess", "\u{2AA1}\u{0338}"), + ("NotPrecedes", "\u{2280}"), + ("NotPrecedesEqual", "\u{2AAF}\u{0338}"), + ("NotPrecedesSlantEqual", "\u{22E0}"), + ("NotReverseElement", "\u{220C}"), + ("NotRightTriangle", "\u{22EB}"), + ("NotRightTriangleBar", "\u{29D0}\u{0338}"), + ("NotRightTriangleEqual", "\u{22ED}"), + ("NotSquareSubset", "\u{228F}\u{0338}"), + ("NotSquareSubsetEqual", "\u{22E2}"), + ("NotSquareSuperset", "\u{2290}\u{0338}"), + ("NotSquareSupersetEqual", "\u{22E3}"), + ("NotSubset", "\u{2282}\u{20D2}"), + ("NotSubsetEqual", "\u{2288}"), + ("NotSucceeds", "\u{2281}"), + ("NotSucceedsEqual", "\u{2AB0}\u{0338}"), + ("NotSucceedsSlantEqual", "\u{22E1}"), + ("NotSucceedsTilde", "\u{227F}\u{0338}"), + ("NotSuperset", "\u{2283}\u{20D2}"), + ("NotSupersetEqual", "\u{2289}"), + ("NotTilde", "\u{2241}"), + ("NotTildeEqual", "\u{2244}"), + ("NotTildeFullEqual", "\u{2247}"), + ("NotTildeTilde", "\u{2249}"), + ("NotVerticalBar", "\u{2224}"), + ("Nscr", "\u{1D4A9}"), + ("Ntilde", "\u{00D1}"), + ("Nu", "\u{039D}"), + ("OElig", "\u{0152}"), + ("Oacute", "\u{00D3}"), + ("Ocirc", "\u{00D4}"), + ("Ocy", "\u{041E}"), + ("Odblac", "\u{0150}"), + ("Ofr", "\u{1D512}"), + ("Ograve", "\u{00D2}"), + ("Omacr", "\u{014C}"), + ("Omega", "\u{03A9}"), + ("Omicron", "\u{039F}"), + ("Oopf", "\u{1D546}"), + ("OpenCurlyDoubleQuote", "\u{201C}"), + ("OpenCurlyQuote", "\u{2018}"), + ("Or", "\u{2A54}"), + ("Oscr", "\u{1D4AA}"), + ("Oslash", "\u{00D8}"), + ("Otilde", "\u{00D5}"), + ("Otimes", "\u{2A37}"), + ("Ouml", "\u{00D6}"), + ("OverBar", "\u{203E}"), + ("OverBrace", "\u{23DE}"), + ("OverBracket", "\u{23B4}"), + ("OverParenthesis", "\u{23DC}"), + ("PartialD", "\u{2202}"), + ("Pcy", "\u{041F}"), + ("Pfr", "\u{1D513}"), + ("Phi", "\u{03A6}"), + ("Pi", "\u{03A0}"), + ("PlusMinus", "\u{00B1}"), + ("Poincareplane", "\u{210C}"), + ("Popf", "\u{2119}"), + ("Pr", "\u{2ABB}"), + ("Precedes", "\u{227A}"), + ("PrecedesEqual", "\u{2AAF}"), + ("PrecedesSlantEqual", "\u{227C}"), + ("PrecedesTilde", "\u{227E}"), + ("Prime", "\u{2033}"), + ("Product", "\u{220F}"), + ("Proportion", "\u{2237}"), + ("Proportional", "\u{221D}"), + ("Pscr", "\u{1D4AB}"), + ("Psi", "\u{03A8}"), + ("QUOT", "\u{0022}"), + ("Qfr", "\u{1D514}"), + ("Qopf", "\u{211A}"), + ("Qscr", "\u{1D4AC}"), + ("RBarr", "\u{2910}"), + ("REG", "\u{00AE}"), + ("Racute", "\u{0154}"), + ("Rang", "\u{27EB}"), + ("Rarr", "\u{21A0}"), + ("Rarrtl", "\u{2916}"), + ("Rcaron", "\u{0158}"), + ("Rcedil", "\u{0156}"), + ("Rcy", "\u{0420}"), + ("Re", "\u{211C}"), + ("ReverseElement", "\u{220B}"), + ("ReverseEquilibrium", "\u{21CB}"), + ("ReverseUpEquilibrium", "\u{296F}"), + ("Rfr", "\u{211C}"), + ("Rho", "\u{03A1}"), + ("RightAngleBracket", "\u{27E9}"), + ("RightArrow", "\u{2192}"), + ("RightArrowBar", "\u{21E5}"), + ("RightArrowLeftArrow", "\u{21C4}"), + ("RightCeiling", "\u{2309}"), + ("RightDoubleBracket", "\u{27E7}"), + ("RightDownTeeVector", "\u{295D}"), + ("RightDownVector", "\u{21C2}"), + ("RightDownVectorBar", "\u{2955}"), + ("RightFloor", "\u{230B}"), + ("RightTee", "\u{22A2}"), + ("RightTeeArrow", "\u{21A6}"), + ("RightTeeVector", "\u{295B}"), + ("RightTriangle", "\u{22B3}"), + ("RightTriangleBar", "\u{29D0}"), + ("RightTriangleEqual", "\u{22B5}"), + ("RightUpDownVector", "\u{294F}"), + ("RightUpTeeVector", "\u{295C}"), + ("RightUpVector", "\u{21BE}"), + ("RightUpVectorBar", "\u{2954}"), + ("RightVector", "\u{21C0}"), + ("RightVectorBar", "\u{2953}"), + ("Rightarrow", "\u{21D2}"), + ("Ropf", "\u{211D}"), + ("RoundImplies", "\u{2970}"), + ("Rrightarrow", "\u{21DB}"), + ("Rscr", "\u{211B}"), + ("Rsh", "\u{21B1}"), + ("RuleDelayed", "\u{29F4}"), + ("SHCHcy", "\u{0429}"), + ("SHcy", "\u{0428}"), + ("SOFTcy", "\u{042C}"), + ("Sacute", "\u{015A}"), + ("Sc", "\u{2ABC}"), + ("Scaron", "\u{0160}"), + ("Scedil", "\u{015E}"), + ("Scirc", "\u{015C}"), + ("Scy", "\u{0421}"), + ("Sfr", "\u{1D516}"), + ("ShortDownArrow", "\u{2193}"), + ("ShortLeftArrow", "\u{2190}"), + ("ShortRightArrow", "\u{2192}"), + ("ShortUpArrow", "\u{2191}"), + ("Sigma", "\u{03A3}"), + ("SmallCircle", "\u{2218}"), + ("Sopf", "\u{1D54A}"), + ("Sqrt", "\u{221A}"), + ("Square", "\u{25A1}"), + ("SquareIntersection", "\u{2293}"), + ("SquareSubset", "\u{228F}"), + ("SquareSubsetEqual", "\u{2291}"), + ("SquareSuperset", "\u{2290}"), + ("SquareSupersetEqual", "\u{2292}"), + ("SquareUnion", "\u{2294}"), + ("Sscr", "\u{1D4AE}"), + ("Star", "\u{22C6}"), + ("Sub", "\u{22D0}"), + ("Subset", "\u{22D0}"), + ("SubsetEqual", "\u{2286}"), + ("Succeeds", "\u{227B}"), + ("SucceedsEqual", "\u{2AB0}"), + ("SucceedsSlantEqual", "\u{227D}"), + ("SucceedsTilde", "\u{227F}"), + ("SuchThat", "\u{220B}"), + ("Sum", "\u{2211}"), + ("Sup", "\u{22D1}"), + ("Superset", "\u{2283}"), + ("SupersetEqual", "\u{2287}"), + ("Supset", "\u{22D1}"), + ("THORN", "\u{00DE}"), + ("TRADE", "\u{2122}"), + ("TSHcy", "\u{040B}"), + ("TScy", "\u{0426}"), + ("Tab", "\u{0009}"), + ("Tau", "\u{03A4}"), + ("Tcaron", "\u{0164}"), + ("Tcedil", "\u{0162}"), + ("Tcy", "\u{0422}"), + ("Tfr", "\u{1D517}"), + ("Therefore", "\u{2234}"), + ("Theta", "\u{0398}"), + ("ThickSpace", "\u{205F}\u{200A}"), + ("ThinSpace", "\u{2009}"), + ("Tilde", "\u{223C}"), + ("TildeEqual", "\u{2243}"), + ("TildeFullEqual", "\u{2245}"), + ("TildeTilde", "\u{2248}"), + ("Topf", "\u{1D54B}"), + ("TripleDot", "\u{20DB}"), + ("Tscr", "\u{1D4AF}"), + ("Tstrok", "\u{0166}"), + ("Uacute", "\u{00DA}"), + ("Uarr", "\u{219F}"), + ("Uarrocir", "\u{2949}"), + ("Ubrcy", "\u{040E}"), + ("Ubreve", "\u{016C}"), + ("Ucirc", "\u{00DB}"), + ("Ucy", "\u{0423}"), + ("Udblac", "\u{0170}"), + ("Ufr", "\u{1D518}"), + ("Ugrave", "\u{00D9}"), + ("Umacr", "\u{016A}"), + ("UnderBar", "_"), + ("UnderBrace", "\u{23DF}"), + ("UnderBracket", "\u{23B5}"), + ("UnderParenthesis", "\u{23DD}"), + ("Union", "\u{22C3}"), + ("UnionPlus", "\u{228E}"), + ("Uogon", "\u{0172}"), + ("Uopf", "\u{1D54C}"), + ("UpArrow", "\u{2191}"), + ("UpArrowBar", "\u{2912}"), + ("UpArrowDownArrow", "\u{21C5}"), + ("UpDownArrow", "\u{2195}"), + ("UpEquilibrium", "\u{296E}"), + ("UpTee", "\u{22A5}"), + ("UpTeeArrow", "\u{21A5}"), + ("Uparrow", "\u{21D1}"), + ("Updownarrow", "\u{21D5}"), + ("UpperLeftArrow", "\u{2196}"), + ("UpperRightArrow", "\u{2197}"), + ("Upsi", "\u{03D2}"), + ("Upsilon", "\u{03A5}"), + ("Uring", "\u{016E}"), + ("Uscr", "\u{1D4B0}"), + ("Utilde", "\u{0168}"), + ("Uuml", "\u{00DC}"), + ("VDash", "\u{22AB}"), + ("Vbar", "\u{2AEB}"), + ("Vcy", "\u{0412}"), + ("Vdash", "\u{22A9}"), + ("Vdashl", "\u{2AE6}"), + ("Vee", "\u{22C1}"), + ("Verbar", "\u{2016}"), + ("Vert", "\u{2016}"), + ("VerticalBar", "\u{2223}"), + ("VerticalLine", "|"), + ("VerticalSeparator", "\u{2758}"), + ("VerticalTilde", "\u{2240}"), + ("VeryThinSpace", "\u{200A}"), + ("Vfr", "\u{1D519}"), + ("Vopf", "\u{1D54D}"), + ("Vscr", "\u{1D4B1}"), + ("Vvdash", "\u{22AA}"), + ("Wcirc", "\u{0174}"), + ("Wedge", "\u{22C0}"), + ("Wfr", "\u{1D51A}"), + ("Wopf", "\u{1D54E}"), + ("Wscr", "\u{1D4B2}"), + ("Xfr", "\u{1D51B}"), + ("Xi", "\u{039E}"), + ("Xopf", "\u{1D54F}"), + ("Xscr", "\u{1D4B3}"), + ("YAcy", "\u{042F}"), + ("YIcy", "\u{0407}"), + ("YUcy", "\u{042E}"), + ("Yacute", "\u{00DD}"), + ("Ycirc", "\u{0176}"), + ("Ycy", "\u{042B}"), + ("Yfr", "\u{1D51C}"), + ("Yopf", "\u{1D550}"), + ("Yscr", "\u{1D4B4}"), + ("Yuml", "\u{0178}"), + ("ZHcy", "\u{0416}"), + ("Zacute", "\u{0179}"), + ("Zcaron", "\u{017D}"), + ("Zcy", "\u{0417}"), + ("Zdot", "\u{017B}"), + ("ZeroWidthSpace", "\u{200B}"), + ("Zeta", "\u{0396}"), + ("Zfr", "\u{2128}"), + ("Zopf", "\u{2124}"), + ("Zscr", "\u{1D4B5}"), + ("aacute", "\u{00E1}"), + ("abreve", "\u{0103}"), + ("ac", "\u{223E}"), + ("acE", "\u{223E}\u{0333}"), + ("acd", "\u{223F}"), + ("acirc", "\u{00E2}"), + ("acute", "\u{00B4}"), + ("acy", "\u{0430}"), + ("aelig", "\u{00E6}"), + ("af", "\u{2061}"), + ("afr", "\u{1D51E}"), + ("agrave", "\u{00E0}"), + ("alefsym", "\u{2135}"), + ("aleph", "\u{2135}"), + ("alpha", "\u{03B1}"), + ("amacr", "\u{0101}"), + ("amalg", "\u{2A3F}"), + ("amp", "&"), + ("and", "\u{2227}"), + ("andand", "\u{2A55}"), + ("andd", "\u{2A5C}"), + ("andslope", "\u{2A58}"), + ("andv", "\u{2A5A}"), + ("ang", "\u{2220}"), + ("ange", "\u{29A4}"), + ("angle", "\u{2220}"), + ("angmsd", "\u{2221}"), + ("angmsdaa", "\u{29A8}"), + ("angmsdab", "\u{29A9}"), + ("angmsdac", "\u{29AA}"), + ("angmsdad", "\u{29AB}"), + ("angmsdae", "\u{29AC}"), + ("angmsdaf", "\u{29AD}"), + ("angmsdag", "\u{29AE}"), + ("angmsdah", "\u{29AF}"), + ("angrt", "\u{221F}"), + ("angrtvb", "\u{22BE}"), + ("angrtvbd", "\u{299D}"), + ("angsph", "\u{2222}"), + ("angst", "\u{00C5}"), + ("angzarr", "\u{237C}"), + ("aogon", "\u{0105}"), + ("aopf", "\u{1D552}"), + ("ap", "\u{2248}"), + ("apE", "\u{2A70}"), + ("apacir", "\u{2A6F}"), + ("ape", "\u{224A}"), + ("apid", "\u{224B}"), + ("apos", "'"), + ("approx", "\u{2248}"), + ("approxeq", "\u{224A}"), + ("aring", "\u{00E5}"), + ("ascr", "\u{1D4B6}"), + ("ast", "*"), + ("asymp", "\u{2248}"), + ("asympeq", "\u{224D}"), + ("atilde", "\u{00E3}"), + ("auml", "\u{00E4}"), + ("awconint", "\u{2233}"), + ("awint", "\u{2A11}"), + ("bNot", "\u{2AED}"), + ("backcong", "\u{224C}"), + ("backepsilon", "\u{03F6}"), + ("backprime", "\u{2035}"), + ("backsim", "\u{223D}"), + ("backsimeq", "\u{22CD}"), + ("barvee", "\u{22BD}"), + ("barwed", "\u{2305}"), + ("barwedge", "\u{2305}"), + ("bbrk", "\u{23B5}"), + ("bbrktbrk", "\u{23B6}"), + ("bcong", "\u{224C}"), + ("bcy", "\u{0431}"), + ("bdquo", "\u{201E}"), + ("becaus", "\u{2235}"), + ("because", "\u{2235}"), + ("bemptyv", "\u{29B0}"), + ("bepsi", "\u{03F6}"), + ("bernou", "\u{212C}"), + ("beta", "\u{03B2}"), + ("beth", "\u{2136}"), + ("between", "\u{226C}"), + ("bfr", "\u{1D51F}"), + ("bigcap", "\u{22C2}"), + ("bigcirc", "\u{25EF}"), + ("bigcup", "\u{22C3}"), + ("bigodot", "\u{2A00}"), + ("bigoplus", "\u{2A01}"), + ("bigotimes", "\u{2A02}"), + ("bigsqcup", "\u{2A06}"), + ("bigstar", "\u{2605}"), + ("bigtriangledown", "\u{25BD}"), + ("bigtriangleup", "\u{25B3}"), + ("biguplus", "\u{2A04}"), + ("bigvee", "\u{22C1}"), + ("bigwedge", "\u{22C0}"), + ("bkarow", "\u{290D}"), + ("blacklozenge", "\u{29EB}"), + ("blacksquare", "\u{25AA}"), + ("blacktriangle", "\u{25B4}"), + ("blacktriangledown", "\u{25BE}"), + ("blacktriangleleft", "\u{25C2}"), + ("blacktriangleright", "\u{25B8}"), + ("blank", "\u{2423}"), + ("blk12", "\u{2592}"), + ("blk14", "\u{2591}"), + ("blk34", "\u{2593}"), + ("block", "\u{2588}"), + ("bne", "=\u{20E5}"), + ("bnequiv", "\u{2261}\u{20E5}"), + ("bnot", "\u{2310}"), + ("bopf", "\u{1D553}"), + ("bot", "\u{22A5}"), + ("bottom", "\u{22A5}"), + ("bowtie", "\u{22C8}"), + ("boxDL", "\u{2557}"), + ("boxDR", "\u{2554}"), + ("boxDl", "\u{2556}"), + ("boxDr", "\u{2553}"), + ("boxH", "\u{2550}"), + ("boxHD", "\u{2566}"), + ("boxHU", "\u{2569}"), + ("boxHd", "\u{2564}"), + ("boxHu", "\u{2567}"), + ("boxUL", "\u{255D}"), + ("boxUR", "\u{255A}"), + ("boxUl", "\u{255C}"), + ("boxUr", "\u{2559}"), + ("boxV", "\u{2551}"), + ("boxVH", "\u{256C}"), + ("boxVL", "\u{2563}"), + ("boxVR", "\u{2560}"), + ("boxVh", "\u{256B}"), + ("boxVl", "\u{2562}"), + ("boxVr", "\u{255F}"), + ("boxbox", "\u{29C9}"), + ("boxdL", "\u{2555}"), + ("boxdR", "\u{2552}"), + ("boxdl", "\u{2510}"), + ("boxdr", "\u{250C}"), + ("boxh", "\u{2500}"), + ("boxhD", "\u{2565}"), + ("boxhU", "\u{2568}"), + ("boxhd", "\u{252C}"), + ("boxhu", "\u{2534}"), + ("boxminus", "\u{229F}"), + ("boxplus", "\u{229E}"), + ("boxtimes", "\u{22A0}"), + ("boxuL", "\u{255B}"), + ("boxuR", "\u{2558}"), + ("boxul", "\u{2518}"), + ("boxur", "\u{2514}"), + ("boxv", "\u{2502}"), + ("boxvH", "\u{256A}"), + ("boxvL", "\u{2561}"), + ("boxvR", "\u{255E}"), + ("boxvh", "\u{253C}"), + ("boxvl", "\u{2524}"), + ("boxvr", "\u{251C}"), + ("bprime", "\u{2035}"), + ("breve", "\u{02D8}"), + ("brvbar", "\u{00A6}"), + ("bscr", "\u{1D4B7}"), + ("bsemi", "\u{204F}"), + ("bsim", "\u{223D}"), + ("bsime", "\u{22CD}"), + ("bsol", "\u{005C}"), + ("bsolb", "\u{29C5}"), + ("bsolhsub", "\u{27C8}"), + ("bull", "\u{2022}"), + ("bullet", "\u{2022}"), + ("bump", "\u{224E}"), + ("bumpE", "\u{2AAE}"), + ("bumpe", "\u{224F}"), + ("bumpeq", "\u{224F}"), + ("cacute", "\u{0107}"), + ("cap", "\u{2229}"), + ("capand", "\u{2A44}"), + ("capbrcup", "\u{2A49}"), + ("capcap", "\u{2A4B}"), + ("capcup", "\u{2A47}"), + ("capdot", "\u{2A40}"), + ("caps", "\u{2229}\u{FE00}"), + ("caret", "\u{2041}"), + ("caron", "\u{02C7}"), + ("ccaps", "\u{2A4D}"), + ("ccaron", "\u{010D}"), + ("ccedil", "\u{00E7}"), + ("ccirc", "\u{0109}"), + ("ccups", "\u{2A4C}"), + ("ccupssm", "\u{2A50}"), + ("cdot", "\u{010B}"), + ("cedil", "\u{00B8}"), + ("cemptyv", "\u{29B2}"), + ("cent", "\u{00A2}"), + ("centerdot", "\u{00B7}"), + ("cfr", "\u{1D520}"), + ("chcy", "\u{0447}"), + ("check", "\u{2713}"), + ("checkmark", "\u{2713}"), + ("chi", "\u{03C7}"), + ("cir", "\u{25CB}"), + ("cirE", "\u{29C3}"), + ("circ", "\u{02C6}"), + ("circeq", "\u{2257}"), + ("circlearrowleft", "\u{21BA}"), + ("circlearrowright", "\u{21BB}"), + ("circledR", "\u{00AE}"), + ("circledS", "\u{24C8}"), + ("circledast", "\u{229B}"), + ("circledcirc", "\u{229A}"), + ("circleddash", "\u{229D}"), + ("cire", "\u{2257}"), + ("cirfnint", "\u{2A10}"), + ("cirmid", "\u{2AEF}"), + ("cirscir", "\u{29C2}"), + ("clubs", "\u{2663}"), + ("clubsuit", "\u{2663}"), + ("colon", ":"), + ("colone", "\u{2254}"), + ("coloneq", "\u{2254}"), + ("comma", ","), + ("commat", "@"), + ("comp", "\u{2201}"), + ("compfn", "\u{2218}"), + ("complement", "\u{2201}"), + ("complexes", "\u{2102}"), + ("cong", "\u{2245}"), + ("congdot", "\u{2A6D}"), + ("conint", "\u{222E}"), + ("copf", "\u{1D554}"), + ("coprod", "\u{2210}"), + ("copy", "\u{00A9}"), + ("copysr", "\u{2117}"), + ("crarr", "\u{21B5}"), + ("cross", "\u{2717}"), + ("cscr", "\u{1D4B8}"), + ("csub", "\u{2ACF}"), + ("csube", "\u{2AD1}"), + ("csup", "\u{2AD0}"), + ("csupe", "\u{2AD2}"), + ("ctdot", "\u{22EF}"), + ("cudarrl", "\u{2938}"), + ("cudarrr", "\u{2935}"), + ("cuepr", "\u{22DE}"), + ("cuesc", "\u{22DF}"), + ("cularr", "\u{21B6}"), + ("cularrp", "\u{293D}"), + ("cup", "\u{222A}"), + ("cupbrcap", "\u{2A48}"), + ("cupcap", "\u{2A46}"), + ("cupcup", "\u{2A4A}"), + ("cupdot", "\u{228D}"), + ("cupor", "\u{2A45}"), + ("cups", "\u{222A}\u{FE00}"), + ("curarr", "\u{21B7}"), + ("curarrm", "\u{293C}"), + ("curlyeqprec", "\u{22DE}"), + ("curlyeqsucc", "\u{22DF}"), + ("curlyvee", "\u{22CE}"), + ("curlywedge", "\u{22CF}"), + ("curren", "\u{00A4}"), + ("curvearrowleft", "\u{21B6}"), + ("curvearrowright", "\u{21B7}"), + ("cuvee", "\u{22CE}"), + ("cuwed", "\u{22CF}"), + ("cwconint", "\u{2232}"), + ("cwint", "\u{2231}"), + ("cylcty", "\u{232D}"), + ("dArr", "\u{21D3}"), + ("dHar", "\u{2965}"), + ("dagger", "\u{2020}"), + ("daleth", "\u{2138}"), + ("darr", "\u{2193}"), + ("dash", "\u{2010}"), + ("dashv", "\u{22A3}"), + ("dbkarow", "\u{290F}"), + ("dblac", "\u{02DD}"), + ("dcaron", "\u{010F}"), + ("dcy", "\u{0434}"), + ("dd", "\u{2146}"), + ("ddagger", "\u{2021}"), + ("ddarr", "\u{21CA}"), + ("ddotseq", "\u{2A77}"), + ("deg", "\u{00B0}"), + ("delta", "\u{03B4}"), + ("demptyv", "\u{29B1}"), + ("dfisht", "\u{297F}"), + ("dfr", "\u{1D521}"), + ("dharl", "\u{21C3}"), + ("dharr", "\u{21C2}"), + ("diam", "\u{22C4}"), + ("diamond", "\u{22C4}"), + ("diamondsuit", "\u{2666}"), + ("diams", "\u{2666}"), + ("die", "\u{00A8}"), + ("digamma", "\u{03DD}"), + ("disin", "\u{22F2}"), + ("div", "\u{00F7}"), + ("divide", "\u{00F7}"), + ("divideontimes", "\u{22C7}"), + ("divonx", "\u{22C7}"), + ("djcy", "\u{0452}"), + ("dlcorn", "\u{231E}"), + ("dlcrop", "\u{230D}"), + ("dollar", "$"), + ("dopf", "\u{1D555}"), + ("dot", "\u{02D9}"), + ("doteq", "\u{2250}"), + ("doteqdot", "\u{2251}"), + ("dotminus", "\u{2238}"), + ("dotplus", "\u{2214}"), + ("dotsquare", "\u{22A1}"), + ("doublebarwedge", "\u{2306}"), + ("downarrow", "\u{2193}"), + ("downdownarrows", "\u{21CA}"), + ("downharpoonleft", "\u{21C3}"), + ("downharpoonright", "\u{21C2}"), + ("drbkarow", "\u{2910}"), + ("drcorn", "\u{231F}"), + ("drcrop", "\u{230C}"), + ("dscr", "\u{1D4B9}"), + ("dscy", "\u{0455}"), + ("dsol", "\u{29F6}"), + ("dstrok", "\u{0111}"), + ("dtdot", "\u{22F1}"), + ("dtri", "\u{25BF}"), + ("dtrif", "\u{25BE}"), + ("duarr", "\u{21F5}"), + ("duhar", "\u{296F}"), + ("dwangle", "\u{29A6}"), + ("dzcy", "\u{045F}"), + ("dzigrarr", "\u{27FF}"), + ("eDDot", "\u{2A77}"), + ("eDot", "\u{2251}"), + ("eacute", "\u{00E9}"), + ("easter", "\u{2A6E}"), + ("ecaron", "\u{011B}"), + ("ecir", "\u{2256}"), + ("ecirc", "\u{00EA}"), + ("ecolon", "\u{2255}"), + ("ecy", "\u{044D}"), + ("edot", "\u{0117}"), + ("ee", "\u{2147}"), + ("efDot", "\u{2252}"), + ("efr", "\u{1D522}"), + ("eg", "\u{2A9A}"), + ("egrave", "\u{00E8}"), + ("egs", "\u{2A96}"), + ("egsdot", "\u{2A98}"), + ("el", "\u{2A99}"), + ("elinters", "\u{23E7}"), + ("ell", "\u{2113}"), + ("els", "\u{2A95}"), + ("elsdot", "\u{2A97}"), + ("emacr", "\u{0113}"), + ("empty", "\u{2205}"), + ("emptyset", "\u{2205}"), + ("emptyv", "\u{2205}"), + ("emsp", "\u{2003}"), + ("emsp13", "\u{2004}"), + ("emsp14", "\u{2005}"), + ("eng", "\u{014B}"), + ("ensp", "\u{2002}"), + ("eogon", "\u{0119}"), + ("eopf", "\u{1D556}"), + ("epar", "\u{22D5}"), + ("eparsl", "\u{29E3}"), + ("eplus", "\u{2A71}"), + ("epsi", "\u{03B5}"), + ("epsilon", "\u{03B5}"), + ("epsiv", "\u{03F5}"), + ("eqcirc", "\u{2256}"), + ("eqcolon", "\u{2255}"), + ("eqsim", "\u{2242}"), + ("eqslantgtr", "\u{2A96}"), + ("eqslantless", "\u{2A95}"), + ("equals", "="), + ("equest", "\u{225F}"), + ("equiv", "\u{2261}"), + ("equivDD", "\u{2A78}"), + ("eqvparsl", "\u{29E5}"), + ("erDot", "\u{2253}"), + ("erarr", "\u{2971}"), + ("escr", "\u{212F}"), + ("esdot", "\u{2250}"), + ("esim", "\u{2242}"), + ("eta", "\u{03B7}"), + ("eth", "\u{00F0}"), + ("euml", "\u{00EB}"), + ("euro", "\u{20AC}"), + ("excl", "!"), + ("exist", "\u{2203}"), + ("expectation", "\u{2130}"), + ("exponentiale", "\u{2147}"), + ("fallingdotseq", "\u{2252}"), + ("fcy", "\u{0444}"), + ("female", "\u{2640}"), + ("ffilig", "\u{FB03}"), + ("fflig", "\u{FB00}"), + ("ffllig", "\u{FB04}"), + ("ffr", "\u{1D523}"), + ("filig", "\u{FB01}"), + ("fjlig", "fj"), + ("flat", "\u{266D}"), + ("fllig", "\u{FB02}"), + ("fltns", "\u{25B1}"), + ("fnof", "\u{0192}"), + ("fopf", "\u{1D557}"), + ("forall", "\u{2200}"), + ("fork", "\u{22D4}"), + ("forkv", "\u{2AD9}"), + ("fpartint", "\u{2A0D}"), + ("frac12", "\u{00BD}"), + ("frac13", "\u{2153}"), + ("frac14", "\u{00BC}"), + ("frac15", "\u{2155}"), + ("frac16", "\u{2159}"), + ("frac18", "\u{215B}"), + ("frac23", "\u{2154}"), + ("frac25", "\u{2156}"), + ("frac34", "\u{00BE}"), + ("frac35", "\u{2157}"), + ("frac38", "\u{215C}"), + ("frac45", "\u{2158}"), + ("frac56", "\u{215A}"), + ("frac58", "\u{215D}"), + ("frac78", "\u{215E}"), + ("frasl", "\u{2044}"), + ("frown", "\u{2322}"), + ("fscr", "\u{1D4BB}"), + ("gE", "\u{2267}"), + ("gEl", "\u{2A8C}"), + ("gacute", "\u{01F5}"), + ("gamma", "\u{03B3}"), + ("gammad", "\u{03DD}"), + ("gap", "\u{2A86}"), + ("gbreve", "\u{011F}"), + ("gcirc", "\u{011D}"), + ("gcy", "\u{0433}"), + ("gdot", "\u{0121}"), + ("ge", "\u{2265}"), + ("gel", "\u{22DB}"), + ("geq", "\u{2265}"), + ("geqq", "\u{2267}"), + ("geqslant", "\u{2A7E}"), + ("ges", "\u{2A7E}"), + ("gescc", "\u{2AA9}"), + ("gesdot", "\u{2A80}"), + ("gesdoto", "\u{2A82}"), + ("gesdotol", "\u{2A84}"), + ("gesl", "\u{22DB}\u{FE00}"), + ("gesles", "\u{2A94}"), + ("gfr", "\u{1D524}"), + ("gg", "\u{226B}"), + ("ggg", "\u{22D9}"), + ("gimel", "\u{2137}"), + ("gjcy", "\u{0453}"), + ("gl", "\u{2277}"), + ("glE", "\u{2A92}"), + ("gla", "\u{2AA5}"), + ("glj", "\u{2AA4}"), + ("gnE", "\u{2269}"), + ("gnap", "\u{2A8A}"), + ("gnapprox", "\u{2A8A}"), + ("gne", "\u{2A88}"), + ("gneq", "\u{2A88}"), + ("gneqq", "\u{2269}"), + ("gnsim", "\u{22E7}"), + ("gopf", "\u{1D558}"), + ("grave", "`"), + ("gscr", "\u{210A}"), + ("gsim", "\u{2273}"), + ("gsime", "\u{2A8E}"), + ("gsiml", "\u{2A90}"), + ("gt", ">"), + ("gtcc", "\u{2AA7}"), + ("gtcir", "\u{2A7A}"), + ("gtdot", "\u{22D7}"), + ("gtlPar", "\u{2995}"), + ("gtquest", "\u{2A7C}"), + ("gtrapprox", "\u{2A86}"), + ("gtrarr", "\u{2978}"), + ("gtrdot", "\u{22D7}"), + ("gtreqless", "\u{22DB}"), + ("gtreqqless", "\u{2A8C}"), + ("gtrless", "\u{2277}"), + ("gtrsim", "\u{2273}"), + ("gvertneqq", "\u{2269}\u{FE00}"), + ("gvnE", "\u{2269}\u{FE00}"), + ("hArr", "\u{21D4}"), + ("hairsp", "\u{200A}"), + ("half", "\u{00BD}"), + ("hamilt", "\u{210B}"), + ("hardcy", "\u{044A}"), + ("harr", "\u{2194}"), + ("harrcir", "\u{2948}"), + ("harrw", "\u{21AD}"), + ("hbar", "\u{210F}"), + ("hcirc", "\u{0125}"), + ("hearts", "\u{2665}"), + ("heartsuit", "\u{2665}"), + ("hellip", "\u{2026}"), + ("hercon", "\u{22B9}"), + ("hfr", "\u{1D525}"), + ("hksearow", "\u{2925}"), + ("hkswarow", "\u{2926}"), + ("hoarr", "\u{21FF}"), + ("homtht", "\u{223B}"), + ("hookleftarrow", "\u{21A9}"), + ("hookrightarrow", "\u{21AA}"), + ("hopf", "\u{1D559}"), + ("horbar", "\u{2015}"), + ("hscr", "\u{1D4BD}"), + ("hslash", "\u{210F}"), + ("hstrok", "\u{0127}"), + ("hybull", "\u{2043}"), + ("hyphen", "\u{2010}"), + ("iacute", "\u{00ED}"), + ("ic", "\u{2063}"), + ("icirc", "\u{00EE}"), + ("icy", "\u{0438}"), + ("iecy", "\u{0435}"), + ("iexcl", "\u{00A1}"), + ("iff", "\u{21D4}"), + ("ifr", "\u{1D526}"), + ("igrave", "\u{00EC}"), + ("ii", "\u{2148}"), + ("iiiint", "\u{2A0C}"), + ("iiint", "\u{222D}"), + ("iinfin", "\u{29DC}"), + ("iiota", "\u{2129}"), + ("ijlig", "\u{0133}"), + ("imacr", "\u{012B}"), + ("image", "\u{2111}"), + ("imagline", "\u{2110}"), + ("imagpart", "\u{2111}"), + ("imath", "\u{0131}"), + ("imof", "\u{22B7}"), + ("imped", "\u{01B5}"), + ("in", "\u{2208}"), + ("incare", "\u{2105}"), + ("infin", "\u{221E}"), + ("infintie", "\u{29DD}"), + ("inodot", "\u{0131}"), + ("int", "\u{222B}"), + ("intcal", "\u{22BA}"), + ("integers", "\u{2124}"), + ("intercal", "\u{22BA}"), + ("intlarhk", "\u{2A17}"), + ("intprod", "\u{2A3C}"), + ("iocy", "\u{0451}"), + ("iogon", "\u{012F}"), + ("iopf", "\u{1D55A}"), + ("iota", "\u{03B9}"), + ("iprod", "\u{2A3C}"), + ("iquest", "\u{00BF}"), + ("iscr", "\u{1D4BE}"), + ("isin", "\u{2208}"), + ("isinE", "\u{22F9}"), + ("isindot", "\u{22F5}"), + ("isins", "\u{22F4}"), + ("isinsv", "\u{22F3}"), + ("isinv", "\u{2208}"), + ("it", "\u{2062}"), + ("itilde", "\u{0129}"), + ("iukcy", "\u{0456}"), + ("iuml", "\u{00EF}"), + ("jcirc", "\u{0135}"), + ("jcy", "\u{0439}"), + ("jfr", "\u{1D527}"), + ("jmath", "\u{0237}"), + ("jopf", "\u{1D55B}"), + ("jscr", "\u{1D4BF}"), + ("jsercy", "\u{0458}"), + ("jukcy", "\u{0454}"), + ("kappa", "\u{03BA}"), + ("kappav", "\u{03F0}"), + ("kcedil", "\u{0137}"), + ("kcy", "\u{043A}"), + ("kfr", "\u{1D528}"), + ("kgreen", "\u{0138}"), + ("khcy", "\u{0445}"), + ("kjcy", "\u{045C}"), + ("kopf", "\u{1D55C}"), + ("kscr", "\u{1D4C0}"), + ("lAarr", "\u{21DA}"), + ("lArr", "\u{21D0}"), + ("lAtail", "\u{291B}"), + ("lBarr", "\u{290E}"), + ("lE", "\u{2266}"), + ("lEg", "\u{2A8B}"), + ("lHar", "\u{2962}"), + ("lacute", "\u{013A}"), + ("laemptyv", "\u{29B4}"), + ("lagran", "\u{2112}"), + ("lambda", "\u{03BB}"), + ("lang", "\u{27E8}"), + ("langd", "\u{2991}"), + ("langle", "\u{27E8}"), + ("lap", "\u{2A85}"), + ("laquo", "\u{00AB}"), + ("larr", "\u{2190}"), + ("larrb", "\u{21E4}"), + ("larrbfs", "\u{291F}"), + ("larrfs", "\u{291D}"), + ("larrhk", "\u{21A9}"), + ("larrlp", "\u{21AB}"), + ("larrpl", "\u{2939}"), + ("larrsim", "\u{2973}"), + ("larrtl", "\u{21A2}"), + ("lat", "\u{2AAB}"), + ("latail", "\u{2919}"), + ("late", "\u{2AAD}"), + ("lates", "\u{2AAD}\u{FE00}"), + ("lbarr", "\u{290C}"), + ("lbbrk", "\u{2772}"), + ("lbrace", "{"), + ("lbrack", "["), + ("lbrke", "\u{298B}"), + ("lbrksld", "\u{298F}"), + ("lbrkslu", "\u{298D}"), + ("lcaron", "\u{013E}"), + ("lcedil", "\u{013C}"), + ("lceil", "\u{2308}"), + ("lcub", "{"), + ("lcy", "\u{043B}"), + ("ldca", "\u{2936}"), + ("ldquo", "\u{201C}"), + ("ldquor", "\u{201E}"), + ("ldrdhar", "\u{2967}"), + ("ldrushar", "\u{294B}"), + ("ldsh", "\u{21B2}"), + ("le", "\u{2264}"), + ("leftarrow", "\u{2190}"), + ("leftarrowtail", "\u{21A2}"), + ("leftharpoondown", "\u{21BD}"), + ("leftharpoonup", "\u{21BC}"), + ("leftleftarrows", "\u{21C7}"), + ("leftrightarrow", "\u{2194}"), + ("leftrightarrows", "\u{21C6}"), + ("leftrightharpoons", "\u{21CB}"), + ("leftrightsquigarrow", "\u{21AD}"), + ("leftthreetimes", "\u{22CB}"), + ("leg", "\u{22DA}"), + ("leq", "\u{2264}"), + ("leqq", "\u{2266}"), + ("leqslant", "\u{2A7D}"), + ("les", "\u{2A7D}"), + ("lescc", "\u{2AA8}"), + ("lesdot", "\u{2A7F}"), + ("lesdoto", "\u{2A81}"), + ("lesdotor", "\u{2A83}"), + ("lesg", "\u{22DA}\u{FE00}"), + ("lesges", "\u{2A93}"), + ("lessapprox", "\u{2A85}"), + ("lessdot", "\u{22D6}"), + ("lesseqgtr", "\u{22DA}"), + ("lesseqqgtr", "\u{2A8B}"), + ("lessgtr", "\u{2276}"), + ("lesssim", "\u{2272}"), + ("lfisht", "\u{297C}"), + ("lfloor", "\u{230A}"), + ("lfr", "\u{1D529}"), + ("lg", "\u{2276}"), + ("lgE", "\u{2A91}"), + ("lhard", "\u{21BD}"), + ("lharu", "\u{21BC}"), + ("lharul", "\u{296A}"), + ("lhblk", "\u{2584}"), + ("ljcy", "\u{0459}"), + ("ll", "\u{226A}"), + ("llarr", "\u{21C7}"), + ("llcorner", "\u{231E}"), + ("llhard", "\u{296B}"), + ("lltri", "\u{25FA}"), + ("lmidot", "\u{0140}"), + ("lmoust", "\u{23B0}"), + ("lmoustache", "\u{23B0}"), + ("lnE", "\u{2268}"), + ("lnap", "\u{2A89}"), + ("lnapprox", "\u{2A89}"), + ("lne", "\u{2A87}"), + ("lneq", "\u{2A87}"), + ("lneqq", "\u{2268}"), + ("lnsim", "\u{22E6}"), + ("loang", "\u{27EC}"), + ("loarr", "\u{21FD}"), + ("lobrk", "\u{27E6}"), + ("longleftarrow", "\u{27F5}"), + ("longleftrightarrow", "\u{27F7}"), + ("longmapsto", "\u{27FC}"), + ("longrightarrow", "\u{27F6}"), + ("looparrowleft", "\u{21AB}"), + ("looparrowright", "\u{21AC}"), + ("lopar", "\u{2985}"), + ("lopf", "\u{1D55D}"), + ("loplus", "\u{2A2D}"), + ("lotimes", "\u{2A34}"), + ("lowast", "\u{2217}"), + ("lowbar", "_"), + ("loz", "\u{25CA}"), + ("lozenge", "\u{25CA}"), + ("lozf", "\u{29EB}"), + ("lpar", "("), + ("lparlt", "\u{2993}"), + ("lrarr", "\u{21C6}"), + ("lrcorner", "\u{231F}"), + ("lrhar", "\u{21CB}"), + ("lrhard", "\u{296D}"), + ("lrm", "\u{200E}"), + ("lrtri", "\u{22BF}"), + ("lsaquo", "\u{2039}"), + ("lscr", "\u{1D4C1}"), + ("lsh", "\u{21B0}"), + ("lsim", "\u{2272}"), + ("lsime", "\u{2A8D}"), + ("lsimg", "\u{2A8F}"), + ("lsqb", "["), + ("lsquo", "\u{2018}"), + ("lsquor", "\u{201A}"), + ("lstrok", "\u{0142}"), + ("lt", "<"), + ("ltcc", "\u{2AA6}"), + ("ltcir", "\u{2A79}"), + ("ltdot", "\u{22D6}"), + ("lthree", "\u{22CB}"), + ("ltimes", "\u{22C9}"), + ("ltlarr", "\u{2976}"), + ("ltquest", "\u{2A7B}"), + ("ltrPar", "\u{2996}"), + ("ltri", "\u{25C3}"), + ("ltrie", "\u{22B4}"), + ("ltrif", "\u{25C2}"), + ("lurdshar", "\u{294A}"), + ("luruhar", "\u{2966}"), + ("lvertneqq", "\u{2268}\u{FE00}"), + ("lvnE", "\u{2268}\u{FE00}"), + ("mDDot", "\u{223A}"), + ("macr", "\u{00AF}"), + ("male", "\u{2642}"), + ("malt", "\u{2720}"), + ("maltese", "\u{2720}"), + ("map", "\u{21A6}"), + ("mapsto", "\u{21A6}"), + ("mapstodown", "\u{21A7}"), + ("mapstoleft", "\u{21A4}"), + ("mapstoup", "\u{21A5}"), + ("marker", "\u{25AE}"), + ("mcomma", "\u{2A29}"), + ("mcy", "\u{043C}"), + ("mdash", "\u{2014}"), + ("measuredangle", "\u{2221}"), + ("mfr", "\u{1D52A}"), + ("mho", "\u{2127}"), + ("micro", "\u{00B5}"), + ("mid", "\u{2223}"), + ("midast", "*"), + ("midcir", "\u{2AF0}"), + ("middot", "\u{00B7}"), + ("minus", "\u{2212}"), + ("minusb", "\u{229F}"), + ("minusd", "\u{2238}"), + ("minusdu", "\u{2A2A}"), + ("mlcp", "\u{2ADB}"), + ("mldr", "\u{2026}"), + ("mnplus", "\u{2213}"), + ("models", "\u{22A7}"), + ("mopf", "\u{1D55E}"), + ("mp", "\u{2213}"), + ("mscr", "\u{1D4C2}"), + ("mstpos", "\u{223E}"), + ("mu", "\u{03BC}"), + ("multimap", "\u{22B8}"), + ("mumap", "\u{22B8}"), + ("nGg", "\u{22D9}\u{0338}"), + ("nGt", "\u{226B}\u{20D2}"), + ("nGtv", "\u{226B}\u{0338}"), + ("nLeftarrow", "\u{21CD}"), + ("nLeftrightarrow", "\u{21CE}"), + ("nLl", "\u{22D8}\u{0338}"), + ("nLt", "\u{226A}\u{20D2}"), + ("nLtv", "\u{226A}\u{0338}"), + ("nRightarrow", "\u{21CF}"), + ("nVDash", "\u{22AF}"), + ("nVdash", "\u{22AE}"), + ("nabla", "\u{2207}"), + ("nacute", "\u{0144}"), + ("nang", "\u{2220}\u{20D2}"), + ("nap", "\u{2249}"), + ("napE", "\u{2A70}\u{0338}"), + ("napid", "\u{224B}\u{0338}"), + ("napos", "\u{0149}"), + ("napprox", "\u{2249}"), + ("natur", "\u{266E}"), + ("natural", "\u{266E}"), + ("naturals", "\u{2115}"), + ("nbsp", "\u{00A0}"), + ("nbump", "\u{224E}\u{0338}"), + ("nbumpe", "\u{224F}\u{0338}"), + ("ncap", "\u{2A43}"), + ("ncaron", "\u{0148}"), + ("ncedil", "\u{0146}"), + ("ncong", "\u{2247}"), + ("ncongdot", "\u{2A6D}\u{0338}"), + ("ncup", "\u{2A42}"), + ("ncy", "\u{043D}"), + ("ndash", "\u{2013}"), + ("ne", "\u{2260}"), + ("neArr", "\u{21D7}"), + ("nearhk", "\u{2924}"), + ("nearr", "\u{2197}"), + ("nearrow", "\u{2197}"), + ("nedot", "\u{2250}\u{0338}"), + ("nequiv", "\u{2262}"), + ("nesear", "\u{2928}"), + ("nesim", "\u{2242}\u{0338}"), + ("nexist", "\u{2204}"), + ("nexists", "\u{2204}"), + ("nfr", "\u{1D52B}"), + ("ngE", "\u{2267}\u{0338}"), + ("nge", "\u{2271}"), + ("ngeq", "\u{2271}"), + ("ngeqq", "\u{2267}\u{0338}"), + ("ngeqslant", "\u{2A7E}\u{0338}"), + ("nges", "\u{2A7E}\u{0338}"), + ("ngsim", "\u{2275}"), + ("ngt", "\u{226F}"), + ("ngtr", "\u{226F}"), + ("nhArr", "\u{21CE}"), + ("nharr", "\u{21AE}"), + ("nhpar", "\u{2AF2}"), + ("ni", "\u{220B}"), + ("nis", "\u{22FC}"), + ("nisd", "\u{22FA}"), + ("niv", "\u{220B}"), + ("njcy", "\u{045A}"), + ("nlArr", "\u{21CD}"), + ("nlE", "\u{2266}\u{0338}"), + ("nlarr", "\u{219A}"), + ("nldr", "\u{2025}"), + ("nle", "\u{2270}"), + ("nleftarrow", "\u{219A}"), + ("nleftrightarrow", "\u{21AE}"), + ("nleq", "\u{2270}"), + ("nleqq", "\u{2266}\u{0338}"), + ("nleqslant", "\u{2A7D}\u{0338}"), + ("nles", "\u{2A7D}\u{0338}"), + ("nless", "\u{226E}"), + ("nlsim", "\u{2274}"), + ("nlt", "\u{226E}"), + ("nltri", "\u{22EA}"), + ("nltrie", "\u{22EC}"), + ("nmid", "\u{2224}"), + ("nopf", "\u{1D55F}"), + ("not", "\u{00AC}"), + ("notin", "\u{2209}"), + ("notinE", "\u{22F9}\u{0338}"), + ("notindot", "\u{22F5}\u{0338}"), + ("notinva", "\u{2209}"), + ("notinvb", "\u{22F7}"), + ("notinvc", "\u{22F6}"), + ("notni", "\u{220C}"), + ("notniva", "\u{220C}"), + ("notnivb", "\u{22FE}"), + ("notnivc", "\u{22FD}"), + ("npar", "\u{2226}"), + ("nparallel", "\u{2226}"), + ("nparsl", "\u{2AFD}\u{20E5}"), + ("npart", "\u{2202}\u{0338}"), + ("npolint", "\u{2A14}"), + ("npr", "\u{2280}"), + ("nprcue", "\u{22E0}"), + ("npre", "\u{2AAF}\u{0338}"), + ("nprec", "\u{2280}"), + ("npreceq", "\u{2AAF}\u{0338}"), + ("nrArr", "\u{21CF}"), + ("nrarr", "\u{219B}"), + ("nrarrc", "\u{2933}\u{0338}"), + ("nrarrw", "\u{219D}\u{0338}"), + ("nrightarrow", "\u{219B}"), + ("nrtri", "\u{22EB}"), + ("nrtrie", "\u{22ED}"), + ("nsc", "\u{2281}"), + ("nsccue", "\u{22E1}"), + ("nsce", "\u{2AB0}\u{0338}"), + ("nscr", "\u{1D4C3}"), + ("nshortmid", "\u{2224}"), + ("nshortparallel", "\u{2226}"), + ("nsim", "\u{2241}"), + ("nsime", "\u{2244}"), + ("nsimeq", "\u{2244}"), + ("nsmid", "\u{2224}"), + ("nspar", "\u{2226}"), + ("nsqsube", "\u{22E2}"), + ("nsqsupe", "\u{22E3}"), + ("nsub", "\u{2284}"), + ("nsubE", "\u{2AC5}\u{0338}"), + ("nsube", "\u{2288}"), + ("nsubset", "\u{2282}\u{20D2}"), + ("nsubseteq", "\u{2288}"), + ("nsubseteqq", "\u{2AC5}\u{0338}"), + ("nsucc", "\u{2281}"), + ("nsucceq", "\u{2AB0}\u{0338}"), + ("nsup", "\u{2285}"), + ("nsupE", "\u{2AC6}\u{0338}"), + ("nsupe", "\u{2289}"), + ("nsupset", "\u{2283}\u{20D2}"), + ("nsupseteq", "\u{2289}"), + ("nsupseteqq", "\u{2AC6}\u{0338}"), + ("ntgl", "\u{2279}"), + ("ntilde", "\u{00F1}"), + ("ntlg", "\u{2278}"), + ("ntriangleleft", "\u{22EA}"), + ("ntrianglelefteq", "\u{22EC}"), + ("ntriangleright", "\u{22EB}"), + ("ntrianglerighteq", "\u{22ED}"), + ("nu", "\u{03BD}"), + ("num", "#"), + ("numero", "\u{2116}"), + ("numsp", "\u{2007}"), + ("nvDash", "\u{22AD}"), + ("nvHarr", "\u{2904}"), + ("nvap", "\u{224D}\u{20D2}"), + ("nvdash", "\u{22AC}"), + ("nvge", "\u{2265}\u{20D2}"), + ("nvgt", ">\u{20D2}"), + ("nvinfin", "\u{29DE}"), + ("nvlArr", "\u{2902}"), + ("nvle", "\u{2264}\u{20D2}"), + ("nvlt", "<\u{20D2}"), + ("nvltrie", "\u{22B4}\u{20D2}"), + ("nvrArr", "\u{2903}"), + ("nvrtrie", "\u{22B5}\u{20D2}"), + ("nvsim", "\u{223C}\u{20D2}"), + ("nwArr", "\u{21D6}"), + ("nwarhk", "\u{2923}"), + ("nwarr", "\u{2196}"), + ("nwarrow", "\u{2196}"), + ("nwnear", "\u{2927}"), + ("oS", "\u{24C8}"), + ("oacute", "\u{00F3}"), + ("oast", "\u{229B}"), + ("ocir", "\u{229A}"), + ("ocirc", "\u{00F4}"), + ("ocy", "\u{043E}"), + ("odash", "\u{229D}"), + ("odblac", "\u{0151}"), + ("odiv", "\u{2A38}"), + ("odot", "\u{2299}"), + ("odsold", "\u{29BC}"), + ("oelig", "\u{0153}"), + ("ofcir", "\u{29BF}"), + ("ofr", "\u{1D52C}"), + ("ogon", "\u{02DB}"), + ("ograve", "\u{00F2}"), + ("ogt", "\u{29C1}"), + ("ohbar", "\u{29B5}"), + ("ohm", "\u{03A9}"), + ("oint", "\u{222E}"), + ("olarr", "\u{21BA}"), + ("olcir", "\u{29BE}"), + ("olcross", "\u{29BB}"), + ("oline", "\u{203E}"), + ("olt", "\u{29C0}"), + ("omacr", "\u{014D}"), + ("omega", "\u{03C9}"), + ("omicron", "\u{03BF}"), + ("omid", "\u{29B6}"), + ("ominus", "\u{2296}"), + ("oopf", "\u{1D560}"), + ("opar", "\u{29B7}"), + ("operp", "\u{29B9}"), + ("oplus", "\u{2295}"), + ("or", "\u{2228}"), + ("orarr", "\u{21BB}"), + ("ord", "\u{2A5D}"), + ("order", "\u{2134}"), + ("orderof", "\u{2134}"), + ("ordf", "\u{00AA}"), + ("ordm", "\u{00BA}"), + ("origof", "\u{22B6}"), + ("oror", "\u{2A56}"), + ("orslope", "\u{2A57}"), + ("orv", "\u{2A5B}"), + ("oscr", "\u{2134}"), + ("oslash", "\u{00F8}"), + ("osol", "\u{2298}"), + ("otilde", "\u{00F5}"), + ("otimes", "\u{2297}"), + ("otimesas", "\u{2A36}"), + ("ouml", "\u{00F6}"), + ("ovbar", "\u{233D}"), + ("par", "\u{2225}"), + ("para", "\u{00B6}"), + ("parallel", "\u{2225}"), + ("parsim", "\u{2AF3}"), + ("parsl", "\u{2AFD}"), + ("part", "\u{2202}"), + ("pcy", "\u{043F}"), + ("percnt", "%"), + ("period", "."), + ("permil", "\u{2030}"), + ("perp", "\u{22A5}"), + ("pertenk", "\u{2031}"), + ("pfr", "\u{1D52D}"), + ("phi", "\u{03C6}"), + ("phiv", "\u{03D5}"), + ("phmmat", "\u{2133}"), + ("phone", "\u{260E}"), + ("pi", "\u{03C0}"), + ("pitchfork", "\u{22D4}"), + ("piv", "\u{03D6}"), + ("planck", "\u{210F}"), + ("planckh", "\u{210E}"), + ("plankv", "\u{210F}"), + ("plus", "+"), + ("plusacir", "\u{2A23}"), + ("plusb", "\u{229E}"), + ("pluscir", "\u{2A22}"), + ("plusdo", "\u{2214}"), + ("plusdu", "\u{2A25}"), + ("pluse", "\u{2A72}"), + ("plusmn", "\u{00B1}"), + ("plussim", "\u{2A26}"), + ("plustwo", "\u{2A27}"), + ("pm", "\u{00B1}"), + ("pointint", "\u{2A15}"), + ("popf", "\u{1D561}"), + ("pound", "\u{00A3}"), + ("pr", "\u{227A}"), + ("prE", "\u{2AB3}"), + ("prap", "\u{2AB7}"), + ("prcue", "\u{227C}"), + ("pre", "\u{2AAF}"), + ("prec", "\u{227A}"), + ("precapprox", "\u{2AB7}"), + ("preccurlyeq", "\u{227C}"), + ("preceq", "\u{2AAF}"), + ("precnapprox", "\u{2AB9}"), + ("precneqq", "\u{2AB5}"), + ("precnsim", "\u{22E8}"), + ("precsim", "\u{227E}"), + ("prime", "\u{2032}"), + ("primes", "\u{2119}"), + ("prnE", "\u{2AB5}"), + ("prnap", "\u{2AB9}"), + ("prnsim", "\u{22E8}"), + ("prod", "\u{220F}"), + ("profalar", "\u{232E}"), + ("profline", "\u{2312}"), + ("profsurf", "\u{2313}"), + ("prop", "\u{221D}"), + ("propto", "\u{221D}"), + ("prsim", "\u{227E}"), + ("prurel", "\u{22B0}"), + ("pscr", "\u{1D4C5}"), + ("psi", "\u{03C8}"), + ("puncsp", "\u{2008}"), + ("qfr", "\u{1D52E}"), + ("qint", "\u{2A0C}"), + ("qopf", "\u{1D562}"), + ("qprime", "\u{2057}"), + ("qscr", "\u{1D4C6}"), + ("quaternions", "\u{210D}"), + ("quatint", "\u{2A16}"), + ("quest", "?"), + ("questeq", "\u{225F}"), + ("quot", "\u{0022}"), + ("rAarr", "\u{21DB}"), + ("rArr", "\u{21D2}"), + ("rAtail", "\u{291C}"), + ("rBarr", "\u{290F}"), + ("rHar", "\u{2964}"), + ("race", "\u{223D}\u{0331}"), + ("racute", "\u{0155}"), + ("radic", "\u{221A}"), + ("raemptyv", "\u{29B3}"), + ("rang", "\u{27E9}"), + ("rangd", "\u{2992}"), + ("range", "\u{29A5}"), + ("rangle", "\u{27E9}"), + ("raquo", "\u{00BB}"), + ("rarr", "\u{2192}"), + ("rarrap", "\u{2975}"), + ("rarrb", "\u{21E5}"), + ("rarrbfs", "\u{2920}"), + ("rarrc", "\u{2933}"), + ("rarrfs", "\u{291E}"), + ("rarrhk", "\u{21AA}"), + ("rarrlp", "\u{21AC}"), + ("rarrpl", "\u{2945}"), + ("rarrsim", "\u{2974}"), + ("rarrtl", "\u{21A3}"), + ("rarrw", "\u{219D}"), + ("ratail", "\u{291A}"), + ("ratio", "\u{2236}"), + ("rationals", "\u{211A}"), + ("rbarr", "\u{290D}"), + ("rbbrk", "\u{2773}"), + ("rbrace", "}"), + ("rbrack", "]"), + ("rbrke", "\u{298C}"), + ("rbrksld", "\u{298E}"), + ("rbrkslu", "\u{2990}"), + ("rcaron", "\u{0159}"), + ("rcedil", "\u{0157}"), + ("rceil", "\u{2309}"), + ("rcub", "}"), + ("rcy", "\u{0440}"), + ("rdca", "\u{2937}"), + ("rdldhar", "\u{2969}"), + ("rdquo", "\u{201D}"), + ("rdquor", "\u{201D}"), + ("rdsh", "\u{21B3}"), + ("real", "\u{211C}"), + ("realine", "\u{211B}"), + ("realpart", "\u{211C}"), + ("reals", "\u{211D}"), + ("rect", "\u{25AD}"), + ("reg", "\u{00AE}"), + ("rfisht", "\u{297D}"), + ("rfloor", "\u{230B}"), + ("rfr", "\u{1D52F}"), + ("rhard", "\u{21C1}"), + ("rharu", "\u{21C0}"), + ("rharul", "\u{296C}"), + ("rho", "\u{03C1}"), + ("rhov", "\u{03F1}"), + ("rightarrow", "\u{2192}"), + ("rightarrowtail", "\u{21A3}"), + ("rightharpoondown", "\u{21C1}"), + ("rightharpoonup", "\u{21C0}"), + ("rightleftarrows", "\u{21C4}"), + ("rightleftharpoons", "\u{21CC}"), + ("rightrightarrows", "\u{21C9}"), + ("rightsquigarrow", "\u{219D}"), + ("rightthreetimes", "\u{22CC}"), + ("ring", "\u{02DA}"), + ("risingdotseq", "\u{2253}"), + ("rlarr", "\u{21C4}"), + ("rlhar", "\u{21CC}"), + ("rlm", "\u{200F}"), + ("rmoust", "\u{23B1}"), + ("rmoustache", "\u{23B1}"), + ("rnmid", "\u{2AEE}"), + ("roang", "\u{27ED}"), + ("roarr", "\u{21FE}"), + ("robrk", "\u{27E7}"), + ("ropar", "\u{2986}"), + ("ropf", "\u{1D563}"), + ("roplus", "\u{2A2E}"), + ("rotimes", "\u{2A35}"), + ("rpar", ")"), + ("rpargt", "\u{2994}"), + ("rppolint", "\u{2A12}"), + ("rrarr", "\u{21C9}"), + ("rsaquo", "\u{203A}"), + ("rscr", "\u{1D4C7}"), + ("rsh", "\u{21B1}"), + ("rsqb", "]"), + ("rsquo", "\u{2019}"), + ("rsquor", "\u{2019}"), + ("rthree", "\u{22CC}"), + ("rtimes", "\u{22CA}"), + ("rtri", "\u{25B9}"), + ("rtrie", "\u{22B5}"), + ("rtrif", "\u{25B8}"), + ("rtriltri", "\u{29CE}"), + ("ruluhar", "\u{2968}"), + ("rx", "\u{211E}"), + ("sacute", "\u{015B}"), + ("sbquo", "\u{201A}"), + ("sc", "\u{227B}"), + ("scE", "\u{2AB4}"), + ("scap", "\u{2AB8}"), + ("scaron", "\u{0161}"), + ("sccue", "\u{227D}"), + ("sce", "\u{2AB0}"), + ("scedil", "\u{015F}"), + ("scirc", "\u{015D}"), + ("scnE", "\u{2AB6}"), + ("scnap", "\u{2ABA}"), + ("scnsim", "\u{22E9}"), + ("scpolint", "\u{2A13}"), + ("scsim", "\u{227F}"), + ("scy", "\u{0441}"), + ("sdot", "\u{22C5}"), + ("sdotb", "\u{22A1}"), + ("sdote", "\u{2A66}"), + ("seArr", "\u{21D8}"), + ("searhk", "\u{2925}"), + ("searr", "\u{2198}"), + ("searrow", "\u{2198}"), + ("sect", "\u{00A7}"), + ("semi", ";"), + ("seswar", "\u{2929}"), + ("setminus", "\u{2216}"), + ("setmn", "\u{2216}"), + ("sext", "\u{2736}"), + ("sfr", "\u{1D530}"), + ("sfrown", "\u{2322}"), + ("sharp", "\u{266F}"), + ("shchcy", "\u{0449}"), + ("shcy", "\u{0448}"), + ("shortmid", "\u{2223}"), + ("shortparallel", "\u{2225}"), + ("shy", "\u{00AD}"), + ("sigma", "\u{03C3}"), + ("sigmaf", "\u{03C2}"), + ("sigmav", "\u{03C2}"), + ("sim", "\u{223C}"), + ("simdot", "\u{2A6A}"), + ("sime", "\u{2243}"), + ("simeq", "\u{2243}"), + ("simg", "\u{2A9E}"), + ("simgE", "\u{2AA0}"), + ("siml", "\u{2A9D}"), + ("simlE", "\u{2A9F}"), + ("simne", "\u{2246}"), + ("simplus", "\u{2A24}"), + ("simrarr", "\u{2972}"), + ("slarr", "\u{2190}"), + ("smallsetminus", "\u{2216}"), + ("smashp", "\u{2A33}"), + ("smeparsl", "\u{29E4}"), + ("smid", "\u{2223}"), + ("smile", "\u{2323}"), + ("smt", "\u{2AAA}"), + ("smte", "\u{2AAC}"), + ("smtes", "\u{2AAC}\u{FE00}"), + ("softcy", "\u{044C}"), + ("sol", "/"), + ("solb", "\u{29C4}"), + ("solbar", "\u{233F}"), + ("sopf", "\u{1D564}"), + ("spades", "\u{2660}"), + ("spadesuit", "\u{2660}"), + ("spar", "\u{2225}"), + ("sqcap", "\u{2293}"), + ("sqcaps", "\u{2293}\u{FE00}"), + ("sqcup", "\u{2294}"), + ("sqcups", "\u{2294}\u{FE00}"), + ("sqsub", "\u{228F}"), + ("sqsube", "\u{2291}"), + ("sqsubset", "\u{228F}"), + ("sqsubseteq", "\u{2291}"), + ("sqsup", "\u{2290}"), + ("sqsupe", "\u{2292}"), + ("sqsupset", "\u{2290}"), + ("sqsupseteq", "\u{2292}"), + ("squ", "\u{25A1}"), + ("square", "\u{25A1}"), + ("squarf", "\u{25AA}"), + ("squf", "\u{25AA}"), + ("srarr", "\u{2192}"), + ("sscr", "\u{1D4C8}"), + ("ssetmn", "\u{2216}"), + ("ssmile", "\u{2323}"), + ("sstarf", "\u{22C6}"), + ("star", "\u{2606}"), + ("starf", "\u{2605}"), + ("straightepsilon", "\u{03F5}"), + ("straightphi", "\u{03D5}"), + ("strns", "\u{00AF}"), + ("sub", "\u{2282}"), + ("subE", "\u{2AC5}"), + ("subdot", "\u{2ABD}"), + ("sube", "\u{2286}"), + ("subedot", "\u{2AC3}"), + ("submult", "\u{2AC1}"), + ("subnE", "\u{2ACB}"), + ("subne", "\u{228A}"), + ("subplus", "\u{2ABF}"), + ("subrarr", "\u{2979}"), + ("subset", "\u{2282}"), + ("subseteq", "\u{2286}"), + ("subseteqq", "\u{2AC5}"), + ("subsetneq", "\u{228A}"), + ("subsetneqq", "\u{2ACB}"), + ("subsim", "\u{2AC7}"), + ("subsub", "\u{2AD5}"), + ("subsup", "\u{2AD3}"), + ("succ", "\u{227B}"), + ("succapprox", "\u{2AB8}"), + ("succcurlyeq", "\u{227D}"), + ("succeq", "\u{2AB0}"), + ("succnapprox", "\u{2ABA}"), + ("succneqq", "\u{2AB6}"), + ("succnsim", "\u{22E9}"), + ("succsim", "\u{227F}"), + ("sum", "\u{2211}"), + ("sung", "\u{266A}"), + ("sup", "\u{2283}"), + ("sup1", "\u{00B9}"), + ("sup2", "\u{00B2}"), + ("sup3", "\u{00B3}"), + ("supE", "\u{2AC6}"), + ("supdot", "\u{2ABE}"), + ("supdsub", "\u{2AD8}"), + ("supe", "\u{2287}"), + ("supedot", "\u{2AC4}"), + ("suphsol", "\u{27C9}"), + ("suphsub", "\u{2AD7}"), + ("suplarr", "\u{297B}"), + ("supmult", "\u{2AC2}"), + ("supnE", "\u{2ACC}"), + ("supne", "\u{228B}"), + ("supplus", "\u{2AC0}"), + ("supset", "\u{2283}"), + ("supseteq", "\u{2287}"), + ("supseteqq", "\u{2AC6}"), + ("supsetneq", "\u{228B}"), + ("supsetneqq", "\u{2ACC}"), + ("supsim", "\u{2AC8}"), + ("supsub", "\u{2AD4}"), + ("supsup", "\u{2AD6}"), + ("swArr", "\u{21D9}"), + ("swarhk", "\u{2926}"), + ("swarr", "\u{2199}"), + ("swarrow", "\u{2199}"), + ("swnwar", "\u{292A}"), + ("szlig", "\u{00DF}"), + ("target", "\u{2316}"), + ("tau", "\u{03C4}"), + ("tbrk", "\u{23B4}"), + ("tcaron", "\u{0165}"), + ("tcedil", "\u{0163}"), + ("tcy", "\u{0442}"), + ("tdot", "\u{20DB}"), + ("telrec", "\u{2315}"), + ("tfr", "\u{1D531}"), + ("there4", "\u{2234}"), + ("therefore", "\u{2234}"), + ("theta", "\u{03B8}"), + ("thetasym", "\u{03D1}"), + ("thetav", "\u{03D1}"), + ("thickapprox", "\u{2248}"), + ("thicksim", "\u{223C}"), + ("thinsp", "\u{2009}"), + ("thkap", "\u{2248}"), + ("thksim", "\u{223C}"), + ("thorn", "\u{00FE}"), + ("tilde", "\u{02DC}"), + ("times", "\u{00D7}"), + ("timesb", "\u{22A0}"), + ("timesbar", "\u{2A31}"), + ("timesd", "\u{2A30}"), + ("tint", "\u{222D}"), + ("toea", "\u{2928}"), + ("top", "\u{22A4}"), + ("topbot", "\u{2336}"), + ("topcir", "\u{2AF1}"), + ("topf", "\u{1D565}"), + ("topfork", "\u{2ADA}"), + ("tosa", "\u{2929}"), + ("tprime", "\u{2034}"), + ("trade", "\u{2122}"), + ("triangle", "\u{25B5}"), + ("triangledown", "\u{25BF}"), + ("triangleleft", "\u{25C3}"), + ("trianglelefteq", "\u{22B4}"), + ("triangleq", "\u{225C}"), + ("triangleright", "\u{25B9}"), + ("trianglerighteq", "\u{22B5}"), + ("tridot", "\u{25EC}"), + ("trie", "\u{225C}"), + ("triminus", "\u{2A3A}"), + ("triplus", "\u{2A39}"), + ("trisb", "\u{29CD}"), + ("tritime", "\u{2A3B}"), + ("trpezium", "\u{23E2}"), + ("tscr", "\u{1D4C9}"), + ("tscy", "\u{0446}"), + ("tshcy", "\u{045B}"), + ("tstrok", "\u{0167}"), + ("twixt", "\u{226C}"), + ("twoheadleftarrow", "\u{219E}"), + ("twoheadrightarrow", "\u{21A0}"), + ("uArr", "\u{21D1}"), + ("uHar", "\u{2963}"), + ("uacute", "\u{00FA}"), + ("uarr", "\u{2191}"), + ("ubrcy", "\u{045E}"), + ("ubreve", "\u{016D}"), + ("ucirc", "\u{00FB}"), + ("ucy", "\u{0443}"), + ("udarr", "\u{21C5}"), + ("udblac", "\u{0171}"), + ("udhar", "\u{296E}"), + ("ufisht", "\u{297E}"), + ("ufr", "\u{1D532}"), + ("ugrave", "\u{00F9}"), + ("uharl", "\u{21BF}"), + ("uharr", "\u{21BE}"), + ("uhblk", "\u{2580}"), + ("ulcorn", "\u{231C}"), + ("ulcorner", "\u{231C}"), + ("ulcrop", "\u{230F}"), + ("ultri", "\u{25F8}"), + ("umacr", "\u{016B}"), + ("uml", "\u{00A8}"), + ("uogon", "\u{0173}"), + ("uopf", "\u{1D566}"), + ("uparrow", "\u{2191}"), + ("updownarrow", "\u{2195}"), + ("upharpoonleft", "\u{21BF}"), + ("upharpoonright", "\u{21BE}"), + ("uplus", "\u{228E}"), + ("upsi", "\u{03C5}"), + ("upsih", "\u{03D2}"), + ("upsilon", "\u{03C5}"), + ("upuparrows", "\u{21C8}"), + ("urcorn", "\u{231D}"), + ("urcorner", "\u{231D}"), + ("urcrop", "\u{230E}"), + ("uring", "\u{016F}"), + ("urtri", "\u{25F9}"), + ("uscr", "\u{1D4CA}"), + ("utdot", "\u{22F0}"), + ("utilde", "\u{0169}"), + ("utri", "\u{25B5}"), + ("utrif", "\u{25B4}"), + ("uuarr", "\u{21C8}"), + ("uuml", "\u{00FC}"), + ("uwangle", "\u{29A7}"), + ("vArr", "\u{21D5}"), + ("vBar", "\u{2AE8}"), + ("vBarv", "\u{2AE9}"), + ("vDash", "\u{22A8}"), + ("vangrt", "\u{299C}"), + ("varepsilon", "\u{03F5}"), + ("varkappa", "\u{03F0}"), + ("varnothing", "\u{2205}"), + ("varphi", "\u{03D5}"), + ("varpi", "\u{03D6}"), + ("varpropto", "\u{221D}"), + ("varr", "\u{2195}"), + ("varrho", "\u{03F1}"), + ("varsigma", "\u{03C2}"), + ("varsubsetneq", "\u{228A}\u{FE00}"), + ("varsubsetneqq", "\u{2ACB}\u{FE00}"), + ("varsupsetneq", "\u{228B}\u{FE00}"), + ("varsupsetneqq", "\u{2ACC}\u{FE00}"), + ("vartheta", "\u{03D1}"), + ("vartriangleleft", "\u{22B2}"), + ("vartriangleright", "\u{22B3}"), + ("vcy", "\u{0432}"), + ("vdash", "\u{22A2}"), + ("vee", "\u{2228}"), + ("veebar", "\u{22BB}"), + ("veeeq", "\u{225A}"), + ("vellip", "\u{22EE}"), + ("verbar", "|"), + ("vert", "|"), + ("vfr", "\u{1D533}"), + ("vltri", "\u{22B2}"), + ("vnsub", "\u{2282}\u{20D2}"), + ("vnsup", "\u{2283}\u{20D2}"), + ("vopf", "\u{1D567}"), + ("vprop", "\u{221D}"), + ("vrtri", "\u{22B3}"), + ("vscr", "\u{1D4CB}"), + ("vsubnE", "\u{2ACB}\u{FE00}"), + ("vsubne", "\u{228A}\u{FE00}"), + ("vsupnE", "\u{2ACC}\u{FE00}"), + ("vsupne", "\u{228B}\u{FE00}"), + ("vzigzag", "\u{299A}"), + ("wcirc", "\u{0175}"), + ("wedbar", "\u{2A5F}"), + ("wedge", "\u{2227}"), + ("wedgeq", "\u{2259}"), + ("weierp", "\u{2118}"), + ("wfr", "\u{1D534}"), + ("wopf", "\u{1D568}"), + ("wp", "\u{2118}"), + ("wr", "\u{2240}"), + ("wreath", "\u{2240}"), + ("wscr", "\u{1D4CC}"), + ("xcap", "\u{22C2}"), + ("xcirc", "\u{25EF}"), + ("xcup", "\u{22C3}"), + ("xdtri", "\u{25BD}"), + ("xfr", "\u{1D535}"), + ("xhArr", "\u{27FA}"), + ("xharr", "\u{27F7}"), + ("xi", "\u{03BE}"), + ("xlArr", "\u{27F8}"), + ("xlarr", "\u{27F5}"), + ("xmap", "\u{27FC}"), + ("xnis", "\u{22FB}"), + ("xodot", "\u{2A00}"), + ("xopf", "\u{1D569}"), + ("xoplus", "\u{2A01}"), + ("xotime", "\u{2A02}"), + ("xrArr", "\u{27F9}"), + ("xrarr", "\u{27F6}"), + ("xscr", "\u{1D4CD}"), + ("xsqcup", "\u{2A06}"), + ("xuplus", "\u{2A04}"), + ("xutri", "\u{25B3}"), + ("xvee", "\u{22C1}"), + ("xwedge", "\u{22C0}"), + ("yacute", "\u{00FD}"), + ("yacy", "\u{044F}"), + ("ycirc", "\u{0177}"), + ("ycy", "\u{044B}"), + ("yen", "\u{00A5}"), + ("yfr", "\u{1D536}"), + ("yicy", "\u{0457}"), + ("yopf", "\u{1D56A}"), + ("yscr", "\u{1D4CE}"), + ("yucy", "\u{044E}"), + ("yuml", "\u{00FF}"), + ("zacute", "\u{017A}"), + ("zcaron", "\u{017E}"), + ("zcy", "\u{0437}"), + ("zdot", "\u{017C}"), + ("zeetrf", "\u{2128}"), + ("zeta", "\u{03B6}"), + ("zfr", "\u{1D537}"), + ("zhcy", "\u{0436}"), + ("zigrarr", "\u{21DD}"), + ("zopf", "\u{1D56B}"), + ("zscr", "\u{1D4CF}"), + ("zwj", "\u{200D}"), + ("zwnj", "\u{200C}"), +]; + +/// The 106 legacy named entities recognized without a trailing semicolon. +static LEGACY_ENTITIES: &[&str] = &[ + "AElig", "AMP", "Aacute", "Acirc", "Agrave", "Aring", "Atilde", "Auml", "COPY", "Ccedil", + "ETH", "Eacute", "Ecirc", "Egrave", "Euml", "GT", "Iacute", "Icirc", "Igrave", "Iuml", "LT", + "Ntilde", "Oacute", "Ocirc", "Ograve", "Oslash", "Otilde", "Ouml", "QUOT", "REG", "THORN", + "Uacute", "Ucirc", "Ugrave", "Uuml", "Yacute", "aacute", "acirc", "acute", "aelig", "agrave", + "amp", "aring", "atilde", "auml", "brvbar", "ccedil", "cedil", "cent", "copy", "curren", "deg", + "divide", "eacute", "ecirc", "egrave", "eth", "euml", "frac12", "frac14", "frac34", "gt", + "iacute", "icirc", "iexcl", "igrave", "iquest", "iuml", "laquo", "lt", "macr", "micro", + "middot", "nbsp", "not", "ntilde", "oacute", "ocirc", "ograve", "ordf", "ordm", "oslash", + "otilde", "ouml", "para", "plusmn", "pound", "quot", "raquo", "reg", "sect", "shy", "sup1", + "sup2", "sup3", "szlig", "thorn", "times", "uacute", "ucirc", "ugrave", "uml", "uuml", + "yacute", "yen", "yuml", +]; + +/// Look up a named character reference by name (without & prefix or ; suffix). +/// Returns the replacement string if found. +pub fn lookup_entity(name: &str) -> Option<&'static str> { + ENTITIES + .binary_search_by_key(&name, |&(n, _)| n) + .ok() + .map(|i| ENTITIES[i].1) +} + +/// Check if an entity name is a "legacy" entity recognized without semicolon. +pub fn is_legacy_entity(name: &str) -> bool { + LEGACY_ENTITIES.binary_search(&name).is_ok() +} diff --git a/crates/html/src/lib.rs b/crates/html/src/lib.rs index e9ce9ae..42fdc55 100644 --- a/crates/html/src/lib.rs +++ b/crates/html/src/lib.rs @@ -1,4 +1,11 @@ //! HTML5 tokenizer and tree builder. +//! +//! Implements the WHATWG HTML5 tokenizer state machine (ยง13.2.5). + +mod entities; +mod tokenizer; + +pub use tokenizer::Tokenizer; /// A token emitted by the HTML tokenizer. #[derive(Debug, Clone, PartialEq)] @@ -28,8 +35,25 @@ pub enum Token { /// Tokenize an HTML input string into a sequence of tokens. /// -/// This is a stub that returns an empty `Vec`. The real implementation -/// will be a spec-compliant HTML5 tokenizer state machine. -pub fn tokenize(_input: &str) -> Vec { - Vec::new() +/// Runs the HTML5 tokenizer state machine and returns all emitted tokens +/// (excluding Eof). Adjacent Character tokens are coalesced. +pub fn tokenize(input: &str) -> Vec { + let mut tok = Tokenizer::new(input); + let mut tokens = Vec::new(); + loop { + let token = tok.next_token(); + match token { + Token::Eof => break, + Token::Character(ref s) => { + // Coalesce adjacent character tokens. + if let Some(Token::Character(ref mut prev)) = tokens.last_mut() { + prev.push_str(s); + } else { + tokens.push(token); + } + } + _ => tokens.push(token), + } + } + tokens } diff --git a/crates/html/src/tokenizer.rs b/crates/html/src/tokenizer.rs new file mode 100644 index 0000000..378c9c6 --- /dev/null +++ b/crates/html/src/tokenizer.rs @@ -0,0 +1,2039 @@ +//! HTML5 tokenizer state machine per WHATWG spec ยง13.2.5. + +use crate::entities; +use crate::Token; + +#[derive(Debug, Clone, Copy, PartialEq)] +enum State { + Data, + TagOpen, + EndTagOpen, + TagName, + BeforeAttributeName, + AttributeName, + AfterAttributeName, + BeforeAttributeValue, + AttributeValueDoubleQuoted, + AttributeValueSingleQuoted, + AttributeValueUnquoted, + AfterAttributeValueQuoted, + SelfClosingStartTag, + BogusComment, + MarkupDeclarationOpen, + CommentStart, + CommentStartDash, + Comment, + CommentLessThanSign, + CommentLessThanSignBang, + CommentLessThanSignBangDash, + CommentLessThanSignBangDashDash, + CommentEndDash, + CommentEnd, + CommentEndBang, + Doctype, + BeforeDoctypeName, + DoctypeName, + AfterDoctypeName, + AfterDoctypePublicKeyword, + BeforeDoctypePublicIdentifier, + DoctypePublicIdentifierDoubleQuoted, + DoctypePublicIdentifierSingleQuoted, + AfterDoctypePublicIdentifier, + BetweenDoctypePublicAndSystemIdentifiers, + AfterDoctypeSystemKeyword, + BeforeDoctypeSystemIdentifier, + DoctypeSystemIdentifierDoubleQuoted, + DoctypeSystemIdentifierSingleQuoted, + AfterDoctypeSystemIdentifier, + BogusDoctype, + CharacterReference, + NumericCharacterReference, + HexCharacterReferenceStart, + DecCharacterReferenceStart, + HexCharacterReference, + DecCharacterReference, + NumericCharacterReferenceEnd, + NamedCharacterReference, +} + +/// HTML5 tokenizer state machine. +pub struct Tokenizer { + input: Vec, + pos: usize, + state: State, + return_state: State, + pending: Vec, + /// Current tag being built. + tag_name: String, + tag_self_closing: bool, + tag_is_end: bool, + tag_attributes: Vec<(String, String)>, + current_attr_name: String, + current_attr_value: String, + /// Current comment or doctype being built. + comment_data: String, + doctype_name: Option, + doctype_public_id: Option, + doctype_system_id: Option, + doctype_force_quirks: bool, + /// Character reference accumulator. + char_ref_code: u32, + temp_buf: String, +} + +impl Tokenizer { + /// Create a new tokenizer for the given input. + pub fn new(input: &str) -> Self { + Tokenizer { + input: input.chars().collect(), + pos: 0, + state: State::Data, + return_state: State::Data, + pending: Vec::new(), + tag_name: String::new(), + tag_self_closing: false, + tag_is_end: false, + tag_attributes: Vec::new(), + current_attr_name: String::new(), + current_attr_value: String::new(), + comment_data: String::new(), + doctype_name: None, + doctype_public_id: None, + doctype_system_id: None, + doctype_force_quirks: false, + char_ref_code: 0, + temp_buf: String::new(), + } + } + + /// Return the next token from the input. + pub fn next_token(&mut self) -> Token { + loop { + if let Some(token) = self.pending.pop() { + return token; + } + self.step(); + } + } + + fn next_char(&mut self) -> Option { + if self.pos < self.input.len() { + let ch = self.input[self.pos]; + self.pos += 1; + Some(ch) + } else { + None + } + } + + fn peek_char(&self) -> Option { + if self.pos < self.input.len() { + Some(self.input[self.pos]) + } else { + None + } + } + + fn reconsume(&mut self) { + if self.pos > 0 { + self.pos -= 1; + } + } + + fn emit(&mut self, token: Token) { + // We use a Vec as a stack, so push to front by inserting at 0. + self.pending.insert(0, token); + } + + fn emit_current_tag(&mut self) { + // Finalize the current attribute if there is one. + self.finish_attribute(); + + if self.tag_is_end { + self.emit(Token::EndTag { + name: self.tag_name.clone(), + }); + } else { + self.emit(Token::StartTag { + name: self.tag_name.clone(), + attributes: self.tag_attributes.clone(), + self_closing: self.tag_self_closing, + }); + } + } + + fn emit_current_comment(&mut self) { + self.emit(Token::Comment(self.comment_data.clone())); + } + + fn emit_current_doctype(&mut self) { + self.emit(Token::Doctype { + name: self.doctype_name.clone(), + public_id: self.doctype_public_id.clone(), + system_id: self.doctype_system_id.clone(), + force_quirks: self.doctype_force_quirks, + }); + } + + fn emit_char(&mut self, ch: char) { + self.emit(Token::Character(ch.to_string())); + } + + fn emit_eof(&mut self) { + self.emit(Token::Eof); + } + + fn start_new_tag(&mut self, is_end: bool) { + self.tag_name.clear(); + self.tag_self_closing = false; + self.tag_is_end = is_end; + self.tag_attributes.clear(); + self.current_attr_name.clear(); + self.current_attr_value.clear(); + } + + fn start_new_attribute(&mut self) { + self.finish_attribute(); + self.current_attr_name.clear(); + self.current_attr_value.clear(); + } + + fn finish_attribute(&mut self) { + if !self.current_attr_name.is_empty() { + // Per spec: if duplicate attribute name, ignore the later one. + let name = self.current_attr_name.clone(); + if !self.tag_attributes.iter().any(|(n, _)| n == &name) { + self.tag_attributes + .push((name, self.current_attr_value.clone())); + } + self.current_attr_name.clear(); + self.current_attr_value.clear(); + } + } + + /// Flush character reference code to the return state. + fn flush_char_ref(&mut self, s: &str) { + match self.return_state { + State::AttributeValueDoubleQuoted + | State::AttributeValueSingleQuoted + | State::AttributeValueUnquoted => { + self.current_attr_value.push_str(s); + } + _ => { + for ch in s.chars() { + self.emit_char(ch); + } + } + } + } + + fn step(&mut self) { + match self.state { + State::Data => self.state_data(), + State::TagOpen => self.state_tag_open(), + State::EndTagOpen => self.state_end_tag_open(), + State::TagName => self.state_tag_name(), + State::BeforeAttributeName => self.state_before_attribute_name(), + State::AttributeName => self.state_attribute_name(), + State::AfterAttributeName => self.state_after_attribute_name(), + State::BeforeAttributeValue => self.state_before_attribute_value(), + State::AttributeValueDoubleQuoted => self.state_attribute_value_double_quoted(), + State::AttributeValueSingleQuoted => self.state_attribute_value_single_quoted(), + State::AttributeValueUnquoted => self.state_attribute_value_unquoted(), + State::AfterAttributeValueQuoted => self.state_after_attribute_value_quoted(), + State::SelfClosingStartTag => self.state_self_closing_start_tag(), + State::BogusComment => self.state_bogus_comment(), + State::MarkupDeclarationOpen => self.state_markup_declaration_open(), + State::CommentStart => self.state_comment_start(), + State::CommentStartDash => self.state_comment_start_dash(), + State::Comment => self.state_comment(), + State::CommentLessThanSign => self.state_comment_less_than_sign(), + State::CommentLessThanSignBang => self.state_comment_less_than_sign_bang(), + State::CommentLessThanSignBangDash => self.state_comment_less_than_sign_bang_dash(), + State::CommentLessThanSignBangDashDash => { + self.state_comment_less_than_sign_bang_dash_dash() + } + State::CommentEndDash => self.state_comment_end_dash(), + State::CommentEnd => self.state_comment_end(), + State::CommentEndBang => self.state_comment_end_bang(), + State::Doctype => self.state_doctype(), + State::BeforeDoctypeName => self.state_before_doctype_name(), + State::DoctypeName => self.state_doctype_name(), + State::AfterDoctypeName => self.state_after_doctype_name(), + State::AfterDoctypePublicKeyword => self.state_after_doctype_public_keyword(), + State::BeforeDoctypePublicIdentifier => self.state_before_doctype_public_identifier(), + State::DoctypePublicIdentifierDoubleQuoted => { + self.state_doctype_public_identifier_double_quoted() + } + State::DoctypePublicIdentifierSingleQuoted => { + self.state_doctype_public_identifier_single_quoted() + } + State::AfterDoctypePublicIdentifier => self.state_after_doctype_public_identifier(), + State::BetweenDoctypePublicAndSystemIdentifiers => { + self.state_between_doctype_public_and_system_identifiers() + } + State::AfterDoctypeSystemKeyword => self.state_after_doctype_system_keyword(), + State::BeforeDoctypeSystemIdentifier => self.state_before_doctype_system_identifier(), + State::DoctypeSystemIdentifierDoubleQuoted => { + self.state_doctype_system_identifier_double_quoted() + } + State::DoctypeSystemIdentifierSingleQuoted => { + self.state_doctype_system_identifier_single_quoted() + } + State::AfterDoctypeSystemIdentifier => self.state_after_doctype_system_identifier(), + State::BogusDoctype => self.state_bogus_doctype(), + State::CharacterReference => self.state_character_reference(), + State::NumericCharacterReference => self.state_numeric_character_reference(), + State::HexCharacterReferenceStart => self.state_hex_character_reference_start(), + State::DecCharacterReferenceStart => self.state_dec_character_reference_start(), + State::HexCharacterReference => self.state_hex_character_reference(), + State::DecCharacterReference => self.state_dec_character_reference(), + State::NumericCharacterReferenceEnd => self.state_numeric_character_reference_end(), + State::NamedCharacterReference => self.state_named_character_reference(), + } + } + + // --- State implementations --- + + fn state_data(&mut self) { + match self.next_char() { + Some('&') => { + self.return_state = State::Data; + self.state = State::CharacterReference; + } + Some('<') => { + self.state = State::TagOpen; + } + Some('\0') => { + // Parse error. Emit replacement character. + self.emit_char('\u{FFFD}'); + } + None => { + self.emit_eof(); + } + Some(c) => { + self.emit_char(c); + } + } + } + + fn state_tag_open(&mut self) { + match self.next_char() { + Some('!') => { + self.state = State::MarkupDeclarationOpen; + } + Some('/') => { + self.state = State::EndTagOpen; + } + Some(c) if c.is_ascii_alphabetic() => { + self.start_new_tag(false); + self.reconsume(); + self.state = State::TagName; + } + Some('?') => { + // Parse error. Create a comment token. + self.comment_data.clear(); + self.reconsume(); + self.state = State::BogusComment; + } + None => { + // Parse error. Emit '<' and EOF. + self.emit_char('<'); + self.emit_eof(); + } + Some(_) => { + // Parse error. Emit '<' and reconsume. + self.emit_char('<'); + self.reconsume(); + self.state = State::Data; + } + } + } + + fn state_end_tag_open(&mut self) { + match self.next_char() { + Some(c) if c.is_ascii_alphabetic() => { + self.start_new_tag(true); + self.reconsume(); + self.state = State::TagName; + } + Some('>') => { + // Parse error. Switch to data state. + self.state = State::Data; + } + None => { + self.emit_char('<'); + self.emit_char('/'); + self.emit_eof(); + } + Some(_) => { + // Parse error. Create a comment. + self.comment_data.clear(); + self.reconsume(); + self.state = State::BogusComment; + } + } + } + + fn state_tag_name(&mut self) { + match self.next_char() { + Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { + self.state = State::BeforeAttributeName; + } + Some('/') => { + self.state = State::SelfClosingStartTag; + } + Some('>') => { + self.state = State::Data; + self.emit_current_tag(); + } + Some(c) if c.is_ascii_uppercase() => { + self.tag_name.push(c.to_ascii_lowercase()); + } + Some('\0') => { + self.tag_name.push('\u{FFFD}'); + } + None => { + self.emit_eof(); + } + Some(c) => { + self.tag_name.push(c); + } + } + } + + fn state_before_attribute_name(&mut self) { + match self.next_char() { + Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { + // Ignore whitespace. + } + Some('/') | Some('>') => { + self.reconsume(); + self.state = State::AfterAttributeName; + } + None => { + // EOF: go to AfterAttributeName without reconsuming. + self.state = State::AfterAttributeName; + } + Some('=') => { + // Parse error. Start a new attribute with '=' as name. + self.start_new_attribute(); + self.current_attr_name.push('='); + self.state = State::AttributeName; + } + Some(_) => { + self.start_new_attribute(); + self.reconsume(); + self.state = State::AttributeName; + } + } + } + + fn state_attribute_name(&mut self) { + match self.next_char() { + Some('\t') | Some('\n') | Some('\x0C') | Some(' ') | Some('/') | Some('>') => { + self.reconsume(); + self.state = State::AfterAttributeName; + } + None => { + self.state = State::AfterAttributeName; + } + Some('=') => { + self.state = State::BeforeAttributeValue; + } + Some(c) if c.is_ascii_uppercase() => { + self.current_attr_name.push(c.to_ascii_lowercase()); + } + Some('\0') => { + self.current_attr_name.push('\u{FFFD}'); + } + Some(c) => { + self.current_attr_name.push(c); + } + } + } + + fn state_after_attribute_name(&mut self) { + match self.next_char() { + Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { + // Ignore. + } + Some('/') => { + self.state = State::SelfClosingStartTag; + } + Some('=') => { + self.state = State::BeforeAttributeValue; + } + Some('>') => { + self.state = State::Data; + self.emit_current_tag(); + } + None => { + self.emit_eof(); + } + Some(_) => { + self.start_new_attribute(); + self.reconsume(); + self.state = State::AttributeName; + } + } + } + + fn state_before_attribute_value(&mut self) { + match self.next_char() { + Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { + // Ignore. + } + Some('"') => { + self.state = State::AttributeValueDoubleQuoted; + } + Some('\'') => { + self.state = State::AttributeValueSingleQuoted; + } + Some('>') => { + // Parse error. Emit tag with missing value. + self.state = State::Data; + self.emit_current_tag(); + } + _ => { + self.reconsume(); + self.state = State::AttributeValueUnquoted; + } + } + } + + fn state_attribute_value_double_quoted(&mut self) { + match self.next_char() { + Some('"') => { + self.state = State::AfterAttributeValueQuoted; + } + Some('&') => { + self.return_state = State::AttributeValueDoubleQuoted; + self.state = State::CharacterReference; + } + Some('\0') => { + self.current_attr_value.push('\u{FFFD}'); + } + None => { + self.emit_eof(); + } + Some(c) => { + self.current_attr_value.push(c); + } + } + } + + fn state_attribute_value_single_quoted(&mut self) { + match self.next_char() { + Some('\'') => { + self.state = State::AfterAttributeValueQuoted; + } + Some('&') => { + self.return_state = State::AttributeValueSingleQuoted; + self.state = State::CharacterReference; + } + Some('\0') => { + self.current_attr_value.push('\u{FFFD}'); + } + None => { + self.emit_eof(); + } + Some(c) => { + self.current_attr_value.push(c); + } + } + } + + fn state_attribute_value_unquoted(&mut self) { + match self.next_char() { + Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { + self.state = State::BeforeAttributeName; + } + Some('&') => { + self.return_state = State::AttributeValueUnquoted; + self.state = State::CharacterReference; + } + Some('>') => { + self.state = State::Data; + self.emit_current_tag(); + } + Some('\0') => { + self.current_attr_value.push('\u{FFFD}'); + } + None => { + self.emit_eof(); + } + Some(c) => { + self.current_attr_value.push(c); + } + } + } + + fn state_after_attribute_value_quoted(&mut self) { + match self.next_char() { + Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { + self.state = State::BeforeAttributeName; + } + Some('/') => { + self.state = State::SelfClosingStartTag; + } + Some('>') => { + self.state = State::Data; + self.emit_current_tag(); + } + None => { + self.emit_eof(); + } + Some(_) => { + // Parse error. Reconsume in before attribute name. + self.reconsume(); + self.state = State::BeforeAttributeName; + } + } + } + + fn state_self_closing_start_tag(&mut self) { + match self.next_char() { + Some('>') => { + self.tag_self_closing = true; + self.state = State::Data; + self.emit_current_tag(); + } + None => { + self.emit_eof(); + } + Some(_) => { + // Parse error. Reconsume in before attribute name. + self.reconsume(); + self.state = State::BeforeAttributeName; + } + } + } + + fn state_bogus_comment(&mut self) { + match self.next_char() { + Some('>') => { + self.state = State::Data; + self.emit_current_comment(); + } + None => { + self.emit_current_comment(); + self.emit_eof(); + } + Some('\0') => { + self.comment_data.push('\u{FFFD}'); + } + Some(c) => { + self.comment_data.push(c); + } + } + } + + fn state_markup_declaration_open(&mut self) { + // Check for `--`, `DOCTYPE`, or `[CDATA[` + if self.starts_with("--") { + self.pos += 2; + self.comment_data.clear(); + self.state = State::CommentStart; + } else if self.starts_with_case_insensitive("DOCTYPE") { + self.pos += 7; + self.state = State::Doctype; + } else if self.starts_with("[CDATA[") { + // Per spec, if not in foreign content, parse error โ†’ bogus comment. + self.pos += 7; + self.comment_data.clear(); + self.comment_data.push_str("[CDATA["); + self.state = State::BogusComment; + } else { + // Parse error. Bogus comment. + self.comment_data.clear(); + self.state = State::BogusComment; + } + } + + fn state_comment_start(&mut self) { + match self.next_char() { + Some('-') => { + self.state = State::CommentStartDash; + } + Some('>') => { + // Parse error. Emit empty comment. + self.state = State::Data; + self.emit_current_comment(); + } + _ => { + self.reconsume(); + self.state = State::Comment; + } + } + } + + fn state_comment_start_dash(&mut self) { + match self.next_char() { + Some('-') => { + self.state = State::CommentEnd; + } + Some('>') => { + // Parse error. + self.state = State::Data; + self.emit_current_comment(); + } + None => { + self.emit_current_comment(); + self.emit_eof(); + } + Some(_) => { + self.comment_data.push('-'); + self.reconsume(); + self.state = State::Comment; + } + } + } + + fn state_comment(&mut self) { + match self.next_char() { + Some('<') => { + self.comment_data.push('<'); + self.state = State::CommentLessThanSign; + } + Some('-') => { + self.state = State::CommentEndDash; + } + Some('\0') => { + self.comment_data.push('\u{FFFD}'); + } + None => { + self.emit_current_comment(); + self.emit_eof(); + } + Some(c) => { + self.comment_data.push(c); + } + } + } + + fn state_comment_less_than_sign(&mut self) { + match self.next_char() { + Some('!') => { + self.comment_data.push('!'); + self.state = State::CommentLessThanSignBang; + } + Some('<') => { + self.comment_data.push('<'); + } + None => { + // Don't reconsume on EOF โ€” pos didn't advance, so reconsuming + // would back up to '<' and loop forever between here and Comment. + self.state = State::Comment; + } + Some(_) => { + self.reconsume(); + self.state = State::Comment; + } + } + } + + fn state_comment_less_than_sign_bang(&mut self) { + match self.next_char() { + Some('-') => { + self.state = State::CommentLessThanSignBangDash; + } + _ => { + self.reconsume(); + self.state = State::Comment; + } + } + } + + fn state_comment_less_than_sign_bang_dash(&mut self) { + match self.next_char() { + Some('-') => { + self.state = State::CommentLessThanSignBangDashDash; + } + _ => { + self.reconsume(); + self.state = State::CommentEndDash; + } + } + } + + fn state_comment_less_than_sign_bang_dash_dash(&mut self) { + match self.next_char() { + Some('>') | None => { + self.reconsume(); + self.state = State::CommentEnd; + } + Some(_) => { + // Parse error. + self.reconsume(); + self.state = State::CommentEnd; + } + } + } + + fn state_comment_end_dash(&mut self) { + match self.next_char() { + Some('-') => { + self.state = State::CommentEnd; + } + None => { + self.emit_current_comment(); + self.emit_eof(); + } + Some(_) => { + self.comment_data.push('-'); + self.reconsume(); + self.state = State::Comment; + } + } + } + + fn state_comment_end(&mut self) { + match self.next_char() { + Some('>') => { + self.state = State::Data; + self.emit_current_comment(); + } + Some('!') => { + self.state = State::CommentEndBang; + } + Some('-') => { + self.comment_data.push('-'); + } + None => { + self.emit_current_comment(); + self.emit_eof(); + } + Some(_) => { + self.comment_data.push('-'); + self.comment_data.push('-'); + self.reconsume(); + self.state = State::Comment; + } + } + } + + fn state_comment_end_bang(&mut self) { + match self.next_char() { + Some('-') => { + self.comment_data.push('-'); + self.comment_data.push('-'); + self.comment_data.push('!'); + self.state = State::CommentEndDash; + } + Some('>') => { + self.state = State::Data; + self.emit_current_comment(); + } + None => { + self.emit_current_comment(); + self.emit_eof(); + } + Some(_) => { + self.comment_data.push('-'); + self.comment_data.push('-'); + self.comment_data.push('!'); + self.reconsume(); + self.state = State::Comment; + } + } + } + + fn state_doctype(&mut self) { + match self.next_char() { + Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { + self.state = State::BeforeDoctypeName; + } + Some('>') => { + self.reconsume(); + self.state = State::BeforeDoctypeName; + } + None => { + self.doctype_name = None; + self.doctype_public_id = None; + self.doctype_system_id = None; + self.doctype_force_quirks = true; + self.emit_current_doctype(); + self.emit_eof(); + } + Some(_) => { + // Parse error. Missing whitespace before DOCTYPE name. + self.reconsume(); + self.state = State::BeforeDoctypeName; + } + } + } + + fn state_before_doctype_name(&mut self) { + match self.next_char() { + Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { + // Ignore whitespace. + } + Some(c) if c.is_ascii_uppercase() => { + self.doctype_name = Some(c.to_ascii_lowercase().to_string()); + self.doctype_public_id = None; + self.doctype_system_id = None; + self.doctype_force_quirks = false; + self.state = State::DoctypeName; + } + Some('\0') => { + self.doctype_name = Some("\u{FFFD}".to_string()); + self.doctype_public_id = None; + self.doctype_system_id = None; + self.doctype_force_quirks = false; + self.state = State::DoctypeName; + } + Some('>') => { + // Parse error. Force quirks. + self.doctype_name = None; + self.doctype_public_id = None; + self.doctype_system_id = None; + self.doctype_force_quirks = true; + self.state = State::Data; + self.emit_current_doctype(); + } + None => { + self.doctype_name = None; + self.doctype_public_id = None; + self.doctype_system_id = None; + self.doctype_force_quirks = true; + self.emit_current_doctype(); + self.emit_eof(); + } + Some(c) => { + self.doctype_name = Some(c.to_string()); + self.doctype_public_id = None; + self.doctype_system_id = None; + self.doctype_force_quirks = false; + self.state = State::DoctypeName; + } + } + } + + fn state_doctype_name(&mut self) { + match self.next_char() { + Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { + self.state = State::AfterDoctypeName; + } + Some('>') => { + self.state = State::Data; + self.emit_current_doctype(); + } + Some(c) if c.is_ascii_uppercase() => { + if let Some(ref mut name) = self.doctype_name { + name.push(c.to_ascii_lowercase()); + } + } + Some('\0') => { + if let Some(ref mut name) = self.doctype_name { + name.push('\u{FFFD}'); + } + } + None => { + self.doctype_force_quirks = true; + self.emit_current_doctype(); + self.emit_eof(); + } + Some(c) => { + if let Some(ref mut name) = self.doctype_name { + name.push(c); + } + } + } + } + + fn state_after_doctype_name(&mut self) { + match self.next_char() { + Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { + // Ignore. + } + Some('>') => { + self.state = State::Data; + self.emit_current_doctype(); + } + None => { + self.doctype_force_quirks = true; + self.emit_current_doctype(); + self.emit_eof(); + } + Some(_) => { + // Check for PUBLIC or SYSTEM keyword. + self.reconsume(); + if self.starts_with_case_insensitive("PUBLIC") { + self.pos += 6; + self.state = State::AfterDoctypePublicKeyword; + } else if self.starts_with_case_insensitive("SYSTEM") { + self.pos += 6; + self.state = State::AfterDoctypeSystemKeyword; + } else { + // Parse error. + self.doctype_force_quirks = true; + self.next_char(); // consume the reconsumed char + self.state = State::BogusDoctype; + } + } + } + } + + fn state_after_doctype_public_keyword(&mut self) { + match self.next_char() { + Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { + self.state = State::BeforeDoctypePublicIdentifier; + } + Some('"') => { + // Parse error. Missing whitespace. + self.doctype_public_id = Some(String::new()); + self.state = State::DoctypePublicIdentifierDoubleQuoted; + } + Some('\'') => { + self.doctype_public_id = Some(String::new()); + self.state = State::DoctypePublicIdentifierSingleQuoted; + } + Some('>') => { + self.doctype_force_quirks = true; + self.state = State::Data; + self.emit_current_doctype(); + } + None => { + self.doctype_force_quirks = true; + self.emit_current_doctype(); + self.emit_eof(); + } + Some(_) => { + self.doctype_force_quirks = true; + self.reconsume(); + self.state = State::BogusDoctype; + } + } + } + + fn state_before_doctype_public_identifier(&mut self) { + match self.next_char() { + Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { + // Ignore. + } + Some('"') => { + self.doctype_public_id = Some(String::new()); + self.state = State::DoctypePublicIdentifierDoubleQuoted; + } + Some('\'') => { + self.doctype_public_id = Some(String::new()); + self.state = State::DoctypePublicIdentifierSingleQuoted; + } + Some('>') => { + self.doctype_force_quirks = true; + self.state = State::Data; + self.emit_current_doctype(); + } + None => { + self.doctype_force_quirks = true; + self.emit_current_doctype(); + self.emit_eof(); + } + Some(_) => { + self.doctype_force_quirks = true; + self.reconsume(); + self.state = State::BogusDoctype; + } + } + } + + fn state_doctype_public_identifier_double_quoted(&mut self) { + match self.next_char() { + Some('"') => { + self.state = State::AfterDoctypePublicIdentifier; + } + Some('\0') => { + if let Some(ref mut id) = self.doctype_public_id { + id.push('\u{FFFD}'); + } + } + Some('>') => { + self.doctype_force_quirks = true; + self.state = State::Data; + self.emit_current_doctype(); + } + None => { + self.doctype_force_quirks = true; + self.emit_current_doctype(); + self.emit_eof(); + } + Some(c) => { + if let Some(ref mut id) = self.doctype_public_id { + id.push(c); + } + } + } + } + + fn state_doctype_public_identifier_single_quoted(&mut self) { + match self.next_char() { + Some('\'') => { + self.state = State::AfterDoctypePublicIdentifier; + } + Some('\0') => { + if let Some(ref mut id) = self.doctype_public_id { + id.push('\u{FFFD}'); + } + } + Some('>') => { + self.doctype_force_quirks = true; + self.state = State::Data; + self.emit_current_doctype(); + } + None => { + self.doctype_force_quirks = true; + self.emit_current_doctype(); + self.emit_eof(); + } + Some(c) => { + if let Some(ref mut id) = self.doctype_public_id { + id.push(c); + } + } + } + } + + fn state_after_doctype_public_identifier(&mut self) { + match self.next_char() { + Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { + self.state = State::BetweenDoctypePublicAndSystemIdentifiers; + } + Some('>') => { + self.state = State::Data; + self.emit_current_doctype(); + } + Some('"') => { + // Parse error. Missing whitespace. + self.doctype_system_id = Some(String::new()); + self.state = State::DoctypeSystemIdentifierDoubleQuoted; + } + Some('\'') => { + self.doctype_system_id = Some(String::new()); + self.state = State::DoctypeSystemIdentifierSingleQuoted; + } + None => { + self.doctype_force_quirks = true; + self.emit_current_doctype(); + self.emit_eof(); + } + Some(_) => { + self.doctype_force_quirks = true; + self.reconsume(); + self.state = State::BogusDoctype; + } + } + } + + fn state_between_doctype_public_and_system_identifiers(&mut self) { + match self.next_char() { + Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { + // Ignore. + } + Some('>') => { + self.state = State::Data; + self.emit_current_doctype(); + } + Some('"') => { + self.doctype_system_id = Some(String::new()); + self.state = State::DoctypeSystemIdentifierDoubleQuoted; + } + Some('\'') => { + self.doctype_system_id = Some(String::new()); + self.state = State::DoctypeSystemIdentifierSingleQuoted; + } + None => { + self.doctype_force_quirks = true; + self.emit_current_doctype(); + self.emit_eof(); + } + Some(_) => { + self.doctype_force_quirks = true; + self.reconsume(); + self.state = State::BogusDoctype; + } + } + } + + fn state_after_doctype_system_keyword(&mut self) { + match self.next_char() { + Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { + self.state = State::BeforeDoctypeSystemIdentifier; + } + Some('"') => { + self.doctype_system_id = Some(String::new()); + self.state = State::DoctypeSystemIdentifierDoubleQuoted; + } + Some('\'') => { + self.doctype_system_id = Some(String::new()); + self.state = State::DoctypeSystemIdentifierSingleQuoted; + } + Some('>') => { + self.doctype_force_quirks = true; + self.state = State::Data; + self.emit_current_doctype(); + } + None => { + self.doctype_force_quirks = true; + self.emit_current_doctype(); + self.emit_eof(); + } + Some(_) => { + self.doctype_force_quirks = true; + self.reconsume(); + self.state = State::BogusDoctype; + } + } + } + + fn state_before_doctype_system_identifier(&mut self) { + match self.next_char() { + Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { + // Ignore. + } + Some('"') => { + self.doctype_system_id = Some(String::new()); + self.state = State::DoctypeSystemIdentifierDoubleQuoted; + } + Some('\'') => { + self.doctype_system_id = Some(String::new()); + self.state = State::DoctypeSystemIdentifierSingleQuoted; + } + Some('>') => { + self.doctype_force_quirks = true; + self.state = State::Data; + self.emit_current_doctype(); + } + None => { + self.doctype_force_quirks = true; + self.emit_current_doctype(); + self.emit_eof(); + } + Some(_) => { + self.doctype_force_quirks = true; + self.reconsume(); + self.state = State::BogusDoctype; + } + } + } + + fn state_doctype_system_identifier_double_quoted(&mut self) { + match self.next_char() { + Some('"') => { + self.state = State::AfterDoctypeSystemIdentifier; + } + Some('\0') => { + if let Some(ref mut id) = self.doctype_system_id { + id.push('\u{FFFD}'); + } + } + Some('>') => { + self.doctype_force_quirks = true; + self.state = State::Data; + self.emit_current_doctype(); + } + None => { + self.doctype_force_quirks = true; + self.emit_current_doctype(); + self.emit_eof(); + } + Some(c) => { + if let Some(ref mut id) = self.doctype_system_id { + id.push(c); + } + } + } + } + + fn state_doctype_system_identifier_single_quoted(&mut self) { + match self.next_char() { + Some('\'') => { + self.state = State::AfterDoctypeSystemIdentifier; + } + Some('\0') => { + if let Some(ref mut id) = self.doctype_system_id { + id.push('\u{FFFD}'); + } + } + Some('>') => { + self.doctype_force_quirks = true; + self.state = State::Data; + self.emit_current_doctype(); + } + None => { + self.doctype_force_quirks = true; + self.emit_current_doctype(); + self.emit_eof(); + } + Some(c) => { + if let Some(ref mut id) = self.doctype_system_id { + id.push(c); + } + } + } + } + + fn state_after_doctype_system_identifier(&mut self) { + match self.next_char() { + Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { + // Ignore. + } + Some('>') => { + self.state = State::Data; + self.emit_current_doctype(); + } + None => { + self.doctype_force_quirks = true; + self.emit_current_doctype(); + self.emit_eof(); + } + Some(_) => { + // Parse error, but do NOT set force_quirks. + self.reconsume(); + self.state = State::BogusDoctype; + } + } + } + + fn state_bogus_doctype(&mut self) { + match self.next_char() { + Some('>') => { + self.state = State::Data; + self.emit_current_doctype(); + } + Some('\0') => { + // Parse error. Ignore. + } + None => { + self.emit_current_doctype(); + self.emit_eof(); + } + Some(_) => { + // Ignore. + } + } + } + + // --- Character reference states --- + + fn state_character_reference(&mut self) { + self.temp_buf.clear(); + self.temp_buf.push('&'); + + match self.peek_char() { + Some(c) if c.is_ascii_alphanumeric() => { + self.state = State::NamedCharacterReference; + } + Some('#') => { + self.temp_buf.push('#'); + self.next_char(); + self.state = State::NumericCharacterReference; + } + _ => { + // Not a character reference. Flush '&' to return state. + self.flush_char_ref("&"); + self.state = self.return_state; + } + } + } + + fn state_named_character_reference(&mut self) { + // Collect alphanumeric characters to form the entity name. + // Per spec, entity names can also contain digits after the first char. + let mut name = String::new(); + let start_pos = self.pos; + + while let Some(c) = self.peek_char() { + if c.is_ascii_alphanumeric() { + name.push(c); + self.pos += 1; + } else { + break; + } + } + + // Try to find a match, trying longest match first. + // First check if the full name + semicolon matches. + let has_trailing_semi = self.peek_char() == Some(';'); + + let mut matched_value: Option<&str> = None; + let mut matched_len = 0; + + // Try the full name first (with semicolon if present). + if has_trailing_semi { + if let Some(val) = entities::lookup_entity(&name) { + matched_value = Some(val); + matched_len = name.len(); + } + } + + // If no match with full name, try progressively shorter prefixes. + if matched_value.is_none() { + for i in (1..=name.len()).rev() { + let candidate = &name[..i]; + if let Some(val) = entities::lookup_entity(candidate) { + // Without semicolon, only legacy entities are recognized. + if entities::is_legacy_entity(candidate) { + matched_value = Some(val); + matched_len = i; + break; + } + } + } + } + + // Also try the full name without semicolon for legacy entities. + if matched_value.is_none() && !has_trailing_semi { + if let Some(val) = entities::lookup_entity(&name) { + if entities::is_legacy_entity(&name) { + matched_value = Some(val); + matched_len = name.len(); + } + } + } + + if let Some(value) = matched_value { + // Rewind to just after the matched portion. + self.pos = start_pos + matched_len; + + // Check for semicolon after the matched portion. + let has_semi = self.peek_char() == Some(';'); + if has_semi { + self.pos += 1; + } + + // Per spec: if consumed as part of an attribute and the character + // after the match is `=` or alphanumeric, and no semicolon, + // flush the original text instead. + let in_attribute = matches!( + self.return_state, + State::AttributeValueDoubleQuoted + | State::AttributeValueSingleQuoted + | State::AttributeValueUnquoted + ); + + if !has_semi && in_attribute { + if let Some(next) = self.peek_char() { + if next == '=' || next.is_ascii_alphanumeric() { + // Not a reference. Flush original text. + let mut original = "&".to_string(); + original.push_str(&name[..matched_len]); + self.flush_char_ref(&original); + self.state = self.return_state; + return; + } + } + } + + self.flush_char_ref(value); + self.state = self.return_state; + } else { + // No match. Rewind and flush '&' + all collected chars. + self.pos = start_pos; + self.flush_char_ref("&"); + for _ in 0..name.len() { + let c = self.next_char().unwrap(); + let s = c.to_string(); + self.flush_char_ref(&s); + } + self.state = self.return_state; + } + } + + fn state_numeric_character_reference(&mut self) { + self.char_ref_code = 0; + match self.peek_char() { + Some('x') | Some('X') => { + self.temp_buf.push(self.peek_char().unwrap()); + self.next_char(); + self.state = State::HexCharacterReferenceStart; + } + _ => { + self.state = State::DecCharacterReferenceStart; + } + } + } + + fn state_hex_character_reference_start(&mut self) { + match self.peek_char() { + Some(c) if c.is_ascii_hexdigit() => { + self.state = State::HexCharacterReference; + } + _ => { + // Parse error. Flush temp_buf. + let buf = self.temp_buf.clone(); + self.flush_char_ref(&buf); + self.state = self.return_state; + } + } + } + + fn state_dec_character_reference_start(&mut self) { + match self.peek_char() { + Some(c) if c.is_ascii_digit() => { + self.state = State::DecCharacterReference; + } + _ => { + let buf = self.temp_buf.clone(); + self.flush_char_ref(&buf); + self.state = self.return_state; + } + } + } + + fn state_hex_character_reference(&mut self) { + match self.next_char() { + Some(c) if c.is_ascii_hexdigit() => { + // Cap at a value that's clearly out of range but won't overflow. + self.char_ref_code = self + .char_ref_code + .saturating_mul(16) + .saturating_add(c.to_digit(16).unwrap()); + if self.char_ref_code > 0x10FFFF { + self.char_ref_code = 0x110000; + } + } + Some(';') => { + self.state = State::NumericCharacterReferenceEnd; + } + None => { + // EOF: missing semicolon parse error. Don't reconsume. + self.state = State::NumericCharacterReferenceEnd; + } + Some(_) => { + // Parse error: missing semicolon. + self.reconsume(); + self.state = State::NumericCharacterReferenceEnd; + } + } + } + + fn state_dec_character_reference(&mut self) { + match self.next_char() { + Some(c) if c.is_ascii_digit() => { + self.char_ref_code = self + .char_ref_code + .saturating_mul(10) + .saturating_add(c.to_digit(10).unwrap()); + if self.char_ref_code > 0x10FFFF { + self.char_ref_code = 0x110000; + } + } + Some(';') => { + self.state = State::NumericCharacterReferenceEnd; + } + None => { + // EOF: missing semicolon parse error. Don't reconsume. + self.state = State::NumericCharacterReferenceEnd; + } + Some(_) => { + self.reconsume(); + self.state = State::NumericCharacterReferenceEnd; + } + } + } + + fn state_numeric_character_reference_end(&mut self) { + let code = self.char_ref_code; + let ch = match code { + 0 => '\u{FFFD}', + // Surrogate range. + 0xD800..=0xDFFF => '\u{FFFD}', + // Out of Unicode range. + c if c > 0x10FFFF => '\u{FFFD}', + // Windows-1252 replacement table for 0x80..0x9F. + 0x80 => '\u{20AC}', + 0x82 => '\u{201A}', + 0x83 => '\u{0192}', + 0x84 => '\u{201E}', + 0x85 => '\u{2026}', + 0x86 => '\u{2020}', + 0x87 => '\u{2021}', + 0x88 => '\u{02C6}', + 0x89 => '\u{2030}', + 0x8A => '\u{0160}', + 0x8B => '\u{2039}', + 0x8C => '\u{0152}', + 0x8E => '\u{017D}', + 0x91 => '\u{2018}', + 0x92 => '\u{2019}', + 0x93 => '\u{201C}', + 0x94 => '\u{201D}', + 0x95 => '\u{2022}', + 0x96 => '\u{2013}', + 0x97 => '\u{2014}', + 0x98 => '\u{02DC}', + 0x99 => '\u{2122}', + 0x9A => '\u{0161}', + 0x9B => '\u{203A}', + 0x9C => '\u{0153}', + 0x9E => '\u{017E}', + 0x9F => '\u{0178}', + c => char::from_u32(c).unwrap_or('\u{FFFD}'), + }; + + let s = ch.to_string(); + self.flush_char_ref(&s); + self.state = self.return_state; + } + + // --- Helpers --- + + fn starts_with(&self, s: &str) -> bool { + let bytes: Vec = s.chars().collect(); + if self.pos + bytes.len() > self.input.len() { + return false; + } + for (i, &c) in bytes.iter().enumerate() { + if self.input[self.pos + i] != c { + return false; + } + } + true + } + + fn starts_with_case_insensitive(&self, s: &str) -> bool { + let bytes: Vec = s.chars().collect(); + if self.pos + bytes.len() > self.input.len() { + return false; + } + for (i, &c) in bytes.iter().enumerate() { + if !self.input[self.pos + i].eq_ignore_ascii_case(&c) { + return false; + } + } + true + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tokenize; + + #[test] + fn empty_input() { + let tokens = tokenize(""); + assert!(tokens.is_empty()); + } + + #[test] + fn plain_text() { + let tokens = tokenize("Hello, world!"); + assert_eq!(tokens, vec![Token::Character("Hello, world!".to_string())]); + } + + #[test] + fn simple_element() { + let tokens = tokenize("

Hello

"); + assert_eq!( + tokens, + vec![ + Token::StartTag { + name: "p".to_string(), + attributes: vec![], + self_closing: false, + }, + Token::Character("Hello".to_string()), + Token::EndTag { + name: "p".to_string(), + }, + ] + ); + } + + #[test] + fn self_closing_tag() { + let tokens = tokenize("
"); + assert_eq!( + tokens, + vec![Token::StartTag { + name: "br".to_string(), + attributes: vec![], + self_closing: true, + }] + ); + } + + #[test] + fn self_closing_img() { + let tokens = tokenize(""); + assert_eq!( + tokens, + vec![Token::StartTag { + name: "img".to_string(), + attributes: vec![], + self_closing: true, + }] + ); + } + + #[test] + fn tag_with_attributes() { + let tokens = tokenize(r#""#); + assert_eq!( + tokens, + vec![Token::StartTag { + name: "a".to_string(), + attributes: vec![ + ("href".to_string(), "url".to_string()), + ("class".to_string(), "link".to_string()), + ], + self_closing: false, + }] + ); + } + + #[test] + fn tag_with_single_quoted_attributes() { + let tokens = tokenize("
"); + assert_eq!( + tokens, + vec![Token::StartTag { + name: "div".to_string(), + attributes: vec![("id".to_string(), "main".to_string())], + self_closing: false, + }] + ); + } + + #[test] + fn tag_with_unquoted_attribute() { + let tokens = tokenize(""); + assert_eq!( + tokens, + vec![Token::StartTag { + name: "input".to_string(), + attributes: vec![("type".to_string(), "text".to_string())], + self_closing: false, + }] + ); + } + + #[test] + fn comment() { + let tokens = tokenize(""); + assert_eq!(tokens, vec![Token::Comment(" comment ".to_string())]); + } + + #[test] + fn empty_comment() { + let tokens = tokenize(""); + assert_eq!(tokens, vec![Token::Comment("".to_string())]); + } + + #[test] + fn doctype_html() { + let tokens = tokenize(""); + assert_eq!( + tokens, + vec![Token::Doctype { + name: Some("html".to_string()), + public_id: None, + system_id: None, + force_quirks: false, + }] + ); + } + + #[test] + fn doctype_case_insensitive() { + let tokens = tokenize(""); + assert_eq!( + tokens, + vec![Token::Doctype { + name: Some("html".to_string()), + public_id: None, + system_id: None, + force_quirks: false, + }] + ); + } + + #[test] + fn char_ref_named() { + let tokens = tokenize("&<>""); + assert_eq!(tokens, vec![Token::Character("&<>\"".to_string())]); + } + + #[test] + fn char_ref_numeric_decimal() { + let tokens = tokenize("A"); + assert_eq!(tokens, vec![Token::Character("A".to_string())]); + } + + #[test] + fn char_ref_numeric_hex() { + let tokens = tokenize("A"); + assert_eq!(tokens, vec![Token::Character("A".to_string())]); + } + + #[test] + fn char_ref_numeric_hex_uppercase() { + let tokens = tokenize("A"); + assert_eq!(tokens, vec![Token::Character("A".to_string())]); + } + + #[test] + fn full_html_document() { + let tokens = + tokenize("Test

Hello

"); + assert_eq!( + tokens, + vec![ + Token::StartTag { + name: "html".to_string(), + attributes: vec![], + self_closing: false, + }, + Token::StartTag { + name: "head".to_string(), + attributes: vec![], + self_closing: false, + }, + Token::StartTag { + name: "title".to_string(), + attributes: vec![], + self_closing: false, + }, + Token::Character("Test".to_string()), + Token::EndTag { + name: "title".to_string(), + }, + Token::EndTag { + name: "head".to_string(), + }, + Token::StartTag { + name: "body".to_string(), + attributes: vec![], + self_closing: false, + }, + Token::StartTag { + name: "p".to_string(), + attributes: vec![], + self_closing: false, + }, + Token::Character("Hello".to_string()), + Token::EndTag { + name: "p".to_string(), + }, + Token::EndTag { + name: "body".to_string(), + }, + Token::EndTag { + name: "html".to_string(), + }, + ] + ); + } + + #[test] + fn uppercase_tag_names_lowercased() { + let tokens = tokenize("
"); + assert_eq!( + tokens, + vec![ + Token::StartTag { + name: "div".to_string(), + attributes: vec![], + self_closing: false, + }, + Token::EndTag { + name: "div".to_string(), + }, + ] + ); + } + + #[test] + fn uppercase_attribute_names_lowercased() { + let tokens = tokenize(r#"
"#); + assert_eq!( + tokens, + vec![Token::StartTag { + name: "div".to_string(), + attributes: vec![("class".to_string(), "x".to_string())], + self_closing: false, + }] + ); + } + + #[test] + fn duplicate_attributes_first_wins() { + let tokens = tokenize(r#"
"#); + assert_eq!( + tokens, + vec![Token::StartTag { + name: "div".to_string(), + attributes: vec![("class".to_string(), "a".to_string())], + self_closing: false, + }] + ); + } + + #[test] + fn char_ref_in_attribute() { + let tokens = tokenize(r#""#); + assert_eq!( + tokens, + vec![Token::StartTag { + name: "a".to_string(), + attributes: vec![("href".to_string(), "?a=1&b=2".to_string())], + self_closing: false, + }] + ); + } + + #[test] + fn multiple_attributes() { + let tokens = tokenize(r#""#); + assert_eq!( + tokens, + vec![Token::StartTag { + name: "input".to_string(), + attributes: vec![ + ("type".to_string(), "text".to_string()), + ("name".to_string(), "foo".to_string()), + ("value".to_string(), "bar".to_string()), + ], + self_closing: false, + }] + ); + } + + #[test] + fn boolean_attribute() { + let tokens = tokenize(""); + assert_eq!( + tokens, + vec![Token::StartTag { + name: "input".to_string(), + attributes: vec![("disabled".to_string(), "".to_string())], + self_closing: false, + }] + ); + } + + #[test] + fn mixed_content() { + let tokens = tokenize("Hello World"); + assert_eq!( + tokens, + vec![ + Token::Character("Hello ".to_string()), + Token::Comment(" comment ".to_string()), + Token::Character(" World".to_string()), + ] + ); + } + + #[test] + fn doctype_with_public_id() { + let tokens = tokenize( + r#""#, + ); + assert_eq!( + tokens, + vec![Token::Doctype { + name: Some("html".to_string()), + public_id: Some("-//W3C//DTD XHTML 1.0 Strict//EN".to_string()), + system_id: Some("http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd".to_string()), + force_quirks: false, + }] + ); + } + + #[test] + fn null_in_text() { + let tokens = tokenize("a\0b"); + assert_eq!(tokens, vec![Token::Character("a\u{FFFD}b".to_string())]); + } + + #[test] + fn windows_1252_numeric_refs() { + // € should map to Euro sign. + let tokens = tokenize("€"); + assert_eq!(tokens, vec![Token::Character("\u{20AC}".to_string())]); + } + + #[test] + fn attribute_with_empty_value() { + let tokens = tokenize(r#"
"#); + assert_eq!( + tokens, + vec![Token::StartTag { + name: "div".to_string(), + attributes: vec![("class".to_string(), "".to_string())], + self_closing: false, + }] + ); + } + + #[test] + fn adjacent_tags() { + let tokens = tokenize(""); + assert_eq!( + tokens, + vec![ + Token::StartTag { + name: "b".to_string(), + attributes: vec![], + self_closing: false, + }, + Token::EndTag { + name: "b".to_string(), + }, + Token::StartTag { + name: "i".to_string(), + attributes: vec![], + self_closing: false, + }, + Token::EndTag { + name: "i".to_string(), + }, + ] + ); + } + + #[test] + fn newlines_in_text() { + let tokens = tokenize("line1\nline2\nline3"); + assert_eq!( + tokens, + vec![Token::Character("line1\nline2\nline3".to_string())] + ); + } + + #[test] + fn self_closing_with_attribute() { + let tokens = tokenize(r#""#); + assert_eq!( + tokens, + vec![Token::StartTag { + name: "img".to_string(), + attributes: vec![("src".to_string(), "test.png".to_string())], + self_closing: true, + }] + ); + } + + #[test] + fn less_than_in_text_not_tag() { + // A bare '<' not followed by a letter should be emitted as text. + let tokens = tokenize("1 < 2"); + assert_eq!(tokens, vec![Token::Character("1 < 2".to_string())]); + } + + #[test] + fn ampersand_not_entity() { + let tokens = tokenize("a & b"); + assert_eq!(tokens, vec![Token::Character("a & b".to_string())]); + } + + #[test] + fn cdata_in_html_becomes_comment() { + let tokens = tokenize(""); + // In HTML (non-foreign) context, CDATA is a parse error โ†’ bogus comment. + assert_eq!(tokens, vec![Token::Comment("[CDATA[hello]]".to_string())]); + } +} -- 2.51.2