Something went wrong. Try again.
we (web engine): Experimental web browser project to understand the limits of Claude
Something went wrong. Try again.
Rust
at url-parser
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040//! HTML5 tokenizer state machine per WHATWG spec §13.2.5.
use crate::entities;use crate::Token;
#[derive(Debug, Clone, Copy, PartialEq)]enum State { Data, TagOpen, EndTagOpen, TagName, BeforeAttributeName, AttributeName, AfterAttributeName, BeforeAttributeValue, AttributeValueDoubleQuoted, AttributeValueSingleQuoted, AttributeValueUnquoted, AfterAttributeValueQuoted, SelfClosingStartTag, BogusComment, MarkupDeclarationOpen, CommentStart, CommentStartDash, Comment, CommentLessThanSign, CommentLessThanSignBang, CommentLessThanSignBangDash, CommentLessThanSignBangDashDash, CommentEndDash, CommentEnd, CommentEndBang, Doctype, BeforeDoctypeName, DoctypeName, AfterDoctypeName, AfterDoctypePublicKeyword, BeforeDoctypePublicIdentifier, DoctypePublicIdentifierDoubleQuoted, DoctypePublicIdentifierSingleQuoted, AfterDoctypePublicIdentifier, BetweenDoctypePublicAndSystemIdentifiers, AfterDoctypeSystemKeyword, BeforeDoctypeSystemIdentifier, DoctypeSystemIdentifierDoubleQuoted, DoctypeSystemIdentifierSingleQuoted, AfterDoctypeSystemIdentifier, BogusDoctype, CharacterReference, NumericCharacterReference, HexCharacterReferenceStart, DecCharacterReferenceStart, HexCharacterReference, DecCharacterReference, NumericCharacterReferenceEnd, NamedCharacterReference,}
/// HTML5 tokenizer state machine.pub struct Tokenizer { input: Vec<char>, pos: usize, state: State, return_state: State, pending: Vec<Token>, /// Current tag being built. tag_name: String, tag_self_closing: bool, tag_is_end: bool, tag_attributes: Vec<(String, String)>, current_attr_name: String, current_attr_value: String, /// Current comment or doctype being built. comment_data: String, doctype_name: Option<String>, doctype_public_id: Option<String>, doctype_system_id: Option<String>, doctype_force_quirks: bool, /// Character reference accumulator. char_ref_code: u32, temp_buf: String,}
impl Tokenizer { /// Create a new tokenizer for the given input. pub fn new(input: &str) -> Self { Tokenizer { input: input.chars().collect(), pos: 0, state: State::Data, return_state: State::Data, pending: Vec::new(), tag_name: String::new(), tag_self_closing: false, tag_is_end: false, tag_attributes: Vec::new(), current_attr_name: String::new(), current_attr_value: String::new(), comment_data: String::new(), doctype_name: None, doctype_public_id: None, doctype_system_id: None, doctype_force_quirks: false, char_ref_code: 0, temp_buf: String::new(), } }
/// Return the next token from the input. pub fn next_token(&mut self) -> Token { loop { if let Some(token) = self.pending.pop() { return token; } self.step(); } }
fn next_char(&mut self) -> Option<char> { if self.pos < self.input.len() { let ch = self.input[self.pos]; self.pos += 1; Some(ch) } else { None } }
fn peek_char(&self) -> Option<char> { if self.pos < self.input.len() { Some(self.input[self.pos]) } else { None } }
fn reconsume(&mut self) { if self.pos > 0 { self.pos -= 1; } }
fn emit(&mut self, token: Token) { // We use a Vec as a stack, so push to front by inserting at 0. self.pending.insert(0, token); }
fn emit_current_tag(&mut self) { // Finalize the current attribute if there is one. self.finish_attribute();
if self.tag_is_end { self.emit(Token::EndTag { name: self.tag_name.clone(), }); } else { self.emit(Token::StartTag { name: self.tag_name.clone(), attributes: self.tag_attributes.clone(), self_closing: self.tag_self_closing, }); } }
fn emit_current_comment(&mut self) { self.emit(Token::Comment(self.comment_data.clone())); }
fn emit_current_doctype(&mut self) { self.emit(Token::Doctype { name: self.doctype_name.clone(), public_id: self.doctype_public_id.clone(), system_id: self.doctype_system_id.clone(), force_quirks: self.doctype_force_quirks, }); }
fn emit_char(&mut self, ch: char) { self.emit(Token::Character(ch.to_string())); }
fn emit_eof(&mut self) { self.emit(Token::Eof); }
fn start_new_tag(&mut self, is_end: bool) { self.tag_name.clear(); self.tag_self_closing = false; self.tag_is_end = is_end; self.tag_attributes.clear(); self.current_attr_name.clear(); self.current_attr_value.clear(); }
fn start_new_attribute(&mut self) { self.finish_attribute(); self.current_attr_name.clear(); self.current_attr_value.clear(); }
fn finish_attribute(&mut self) { if !self.current_attr_name.is_empty() { // Per spec: if duplicate attribute name, ignore the later one. let name = self.current_attr_name.clone(); if !self.tag_attributes.iter().any(|(n, _)| n == &name) { self.tag_attributes .push((name, self.current_attr_value.clone())); } self.current_attr_name.clear(); self.current_attr_value.clear(); } }
/// Flush character reference code to the return state. fn flush_char_ref(&mut self, s: &str) { match self.return_state { State::AttributeValueDoubleQuoted | State::AttributeValueSingleQuoted | State::AttributeValueUnquoted => { self.current_attr_value.push_str(s); } _ => { for ch in s.chars() { self.emit_char(ch); } } } }
fn step(&mut self) { match self.state { State::Data => self.state_data(), State::TagOpen => self.state_tag_open(), State::EndTagOpen => self.state_end_tag_open(), State::TagName => self.state_tag_name(), State::BeforeAttributeName => self.state_before_attribute_name(), State::AttributeName => self.state_attribute_name(), State::AfterAttributeName => self.state_after_attribute_name(), State::BeforeAttributeValue => self.state_before_attribute_value(), State::AttributeValueDoubleQuoted => self.state_attribute_value_double_quoted(), State::AttributeValueSingleQuoted => self.state_attribute_value_single_quoted(), State::AttributeValueUnquoted => self.state_attribute_value_unquoted(), State::AfterAttributeValueQuoted => self.state_after_attribute_value_quoted(), State::SelfClosingStartTag => self.state_self_closing_start_tag(), State::BogusComment => self.state_bogus_comment(), State::MarkupDeclarationOpen => self.state_markup_declaration_open(), State::CommentStart => self.state_comment_start(), State::CommentStartDash => self.state_comment_start_dash(), State::Comment => self.state_comment(), State::CommentLessThanSign => self.state_comment_less_than_sign(), State::CommentLessThanSignBang => self.state_comment_less_than_sign_bang(), State::CommentLessThanSignBangDash => self.state_comment_less_than_sign_bang_dash(), State::CommentLessThanSignBangDashDash => { self.state_comment_less_than_sign_bang_dash_dash() } State::CommentEndDash => self.state_comment_end_dash(), State::CommentEnd => self.state_comment_end(), State::CommentEndBang => self.state_comment_end_bang(), State::Doctype => self.state_doctype(), State::BeforeDoctypeName => self.state_before_doctype_name(), State::DoctypeName => self.state_doctype_name(), State::AfterDoctypeName => self.state_after_doctype_name(), State::AfterDoctypePublicKeyword => self.state_after_doctype_public_keyword(), State::BeforeDoctypePublicIdentifier => self.state_before_doctype_public_identifier(), State::DoctypePublicIdentifierDoubleQuoted => { self.state_doctype_public_identifier_double_quoted() } State::DoctypePublicIdentifierSingleQuoted => { self.state_doctype_public_identifier_single_quoted() } State::AfterDoctypePublicIdentifier => self.state_after_doctype_public_identifier(), State::BetweenDoctypePublicAndSystemIdentifiers => { self.state_between_doctype_public_and_system_identifiers() } State::AfterDoctypeSystemKeyword => self.state_after_doctype_system_keyword(), State::BeforeDoctypeSystemIdentifier => self.state_before_doctype_system_identifier(), State::DoctypeSystemIdentifierDoubleQuoted => { self.state_doctype_system_identifier_double_quoted() } State::DoctypeSystemIdentifierSingleQuoted => { self.state_doctype_system_identifier_single_quoted() } State::AfterDoctypeSystemIdentifier => self.state_after_doctype_system_identifier(), State::BogusDoctype => self.state_bogus_doctype(), State::CharacterReference => self.state_character_reference(), State::NumericCharacterReference => self.state_numeric_character_reference(), State::HexCharacterReferenceStart => self.state_hex_character_reference_start(), State::DecCharacterReferenceStart => self.state_dec_character_reference_start(), State::HexCharacterReference => self.state_hex_character_reference(), State::DecCharacterReference => self.state_dec_character_reference(), State::NumericCharacterReferenceEnd => self.state_numeric_character_reference_end(), State::NamedCharacterReference => self.state_named_character_reference(), } }
// --- State implementations ---
fn state_data(&mut self) { match self.next_char() { Some('&') => { self.return_state = State::Data; self.state = State::CharacterReference; } Some('<') => { self.state = State::TagOpen; } Some('\0') => { // Parse error. Emit replacement character. self.emit_char('\u{FFFD}'); } None => { self.emit_eof(); } Some(c) => { self.emit_char(c); } } }
fn state_tag_open(&mut self) { match self.next_char() { Some('!') => { self.state = State::MarkupDeclarationOpen; } Some('/') => { self.state = State::EndTagOpen; } Some(c) if c.is_ascii_alphabetic() => { self.start_new_tag(false); self.reconsume(); self.state = State::TagName; } Some('?') => { // Parse error. Create a comment token. self.comment_data.clear(); self.reconsume(); self.state = State::BogusComment; } None => { // Parse error. Emit '<' and EOF. self.emit_char('<'); self.emit_eof(); } Some(_) => { // Parse error. Emit '<' and reconsume. self.emit_char('<'); self.reconsume(); self.state = State::Data; } } }
fn state_end_tag_open(&mut self) { match self.next_char() { Some(c) if c.is_ascii_alphabetic() => { self.start_new_tag(true); self.reconsume(); self.state = State::TagName; } Some('>') => { // Parse error. Switch to data state. self.state = State::Data; } None => { self.emit_char('<'); self.emit_char('/'); self.emit_eof(); } Some(_) => { // Parse error. Create a comment. self.comment_data.clear(); self.reconsume(); self.state = State::BogusComment; } } }
fn state_tag_name(&mut self) { match self.next_char() { Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { self.state = State::BeforeAttributeName; } Some('/') => { self.state = State::SelfClosingStartTag; } Some('>') => { self.state = State::Data; self.emit_current_tag(); } Some(c) if c.is_ascii_uppercase() => { self.tag_name.push(c.to_ascii_lowercase()); } Some('\0') => { self.tag_name.push('\u{FFFD}'); } None => { self.emit_eof(); } Some(c) => { self.tag_name.push(c); } } }
fn state_before_attribute_name(&mut self) { match self.next_char() { Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { // Ignore whitespace. } Some('/') | Some('>') => { self.reconsume(); self.state = State::AfterAttributeName; } None => { // EOF: go to AfterAttributeName without reconsuming. self.state = State::AfterAttributeName; } Some('=') => { // Parse error. Start a new attribute with '=' as name. self.start_new_attribute(); self.current_attr_name.push('='); self.state = State::AttributeName; } Some(_) => { self.start_new_attribute(); self.reconsume(); self.state = State::AttributeName; } } }
fn state_attribute_name(&mut self) { match self.next_char() { Some('\t') | Some('\n') | Some('\x0C') | Some(' ') | Some('/') | Some('>') => { self.reconsume(); self.state = State::AfterAttributeName; } None => { self.state = State::AfterAttributeName; } Some('=') => { self.state = State::BeforeAttributeValue; } Some(c) if c.is_ascii_uppercase() => { self.current_attr_name.push(c.to_ascii_lowercase()); } Some('\0') => { self.current_attr_name.push('\u{FFFD}'); } Some(c) => { self.current_attr_name.push(c); } } }
fn state_after_attribute_name(&mut self) { match self.next_char() { Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { // Ignore. } Some('/') => { self.state = State::SelfClosingStartTag; } Some('=') => { self.state = State::BeforeAttributeValue; } Some('>') => { self.state = State::Data; self.emit_current_tag(); } None => { self.emit_eof(); } Some(_) => { self.start_new_attribute(); self.reconsume(); self.state = State::AttributeName; } } }
fn state_before_attribute_value(&mut self) { match self.next_char() { Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { // Ignore. } Some('"') => { self.state = State::AttributeValueDoubleQuoted; } Some('\'') => { self.state = State::AttributeValueSingleQuoted; } Some('>') => { // Parse error. Emit tag with missing value. self.state = State::Data; self.emit_current_tag(); } _ => { self.reconsume(); self.state = State::AttributeValueUnquoted; } } }
fn state_attribute_value_double_quoted(&mut self) { match self.next_char() { Some('"') => { self.state = State::AfterAttributeValueQuoted; } Some('&') => { self.return_state = State::AttributeValueDoubleQuoted; self.state = State::CharacterReference; } Some('\0') => { self.current_attr_value.push('\u{FFFD}'); } None => { self.emit_eof(); } Some(c) => { self.current_attr_value.push(c); } } }
fn state_attribute_value_single_quoted(&mut self) { match self.next_char() { Some('\'') => { self.state = State::AfterAttributeValueQuoted; } Some('&') => { self.return_state = State::AttributeValueSingleQuoted; self.state = State::CharacterReference; } Some('\0') => { self.current_attr_value.push('\u{FFFD}'); } None => { self.emit_eof(); } Some(c) => { self.current_attr_value.push(c); } } }
fn state_attribute_value_unquoted(&mut self) { match self.next_char() { Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { self.state = State::BeforeAttributeName; } Some('&') => { self.return_state = State::AttributeValueUnquoted; self.state = State::CharacterReference; } Some('>') => { self.state = State::Data; self.emit_current_tag(); } Some('\0') => { self.current_attr_value.push('\u{FFFD}'); } None => { self.emit_eof(); } Some(c) => { self.current_attr_value.push(c); } } }
fn state_after_attribute_value_quoted(&mut self) { match self.next_char() { Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { self.state = State::BeforeAttributeName; } Some('/') => { self.state = State::SelfClosingStartTag; } Some('>') => { self.state = State::Data; self.emit_current_tag(); } None => { self.emit_eof(); } Some(_) => { // Parse error. Reconsume in before attribute name. self.reconsume(); self.state = State::BeforeAttributeName; } } }
fn state_self_closing_start_tag(&mut self) { match self.next_char() { Some('>') => { self.tag_self_closing = true; self.state = State::Data; self.emit_current_tag(); } None => { self.emit_eof(); } Some(_) => { // Parse error. Reconsume in before attribute name. self.reconsume(); self.state = State::BeforeAttributeName; } } }
fn state_bogus_comment(&mut self) { match self.next_char() { Some('>') => { self.state = State::Data; self.emit_current_comment(); } None => { self.emit_current_comment(); self.emit_eof(); } Some('\0') => { self.comment_data.push('\u{FFFD}'); } Some(c) => { self.comment_data.push(c); } } }
fn state_markup_declaration_open(&mut self) { // Check for `--`, `DOCTYPE`, or `[CDATA[` if self.starts_with("--") { self.pos += 2; self.comment_data.clear(); self.state = State::CommentStart; } else if self.starts_with_case_insensitive("DOCTYPE") { self.pos += 7; self.state = State::Doctype; } else if self.starts_with("[CDATA[") { // Per spec, if not in foreign content, parse error → bogus comment. self.pos += 7; self.comment_data.clear(); self.comment_data.push_str("[CDATA["); self.state = State::BogusComment; } else { // Parse error. Bogus comment. self.comment_data.clear(); self.state = State::BogusComment; } }
fn state_comment_start(&mut self) { match self.next_char() { Some('-') => { self.state = State::CommentStartDash; } Some('>') => { // Parse error. Emit empty comment. self.state = State::Data; self.emit_current_comment(); } _ => { self.reconsume(); self.state = State::Comment; } } }
fn state_comment_start_dash(&mut self) { match self.next_char() { Some('-') => { self.state = State::CommentEnd; } Some('>') => { // Parse error. self.state = State::Data; self.emit_current_comment(); } None => { self.emit_current_comment(); self.emit_eof(); } Some(_) => { self.comment_data.push('-'); self.reconsume(); self.state = State::Comment; } } }
fn state_comment(&mut self) { match self.next_char() { Some('<') => { self.comment_data.push('<'); self.state = State::CommentLessThanSign; } Some('-') => { self.state = State::CommentEndDash; } Some('\0') => { self.comment_data.push('\u{FFFD}'); } None => { self.emit_current_comment(); self.emit_eof(); } Some(c) => { self.comment_data.push(c); } } }
fn state_comment_less_than_sign(&mut self) { match self.next_char() { Some('!') => { self.comment_data.push('!'); self.state = State::CommentLessThanSignBang; } Some('<') => { self.comment_data.push('<'); } None => { // Don't reconsume on EOF — pos didn't advance, so reconsuming // would back up to '<' and loop forever between here and Comment. self.state = State::Comment; } Some(_) => { self.reconsume(); self.state = State::Comment; } } }
fn state_comment_less_than_sign_bang(&mut self) { match self.next_char() { Some('-') => { self.state = State::CommentLessThanSignBangDash; } _ => { self.reconsume(); self.state = State::Comment; } } }
fn state_comment_less_than_sign_bang_dash(&mut self) { match self.next_char() { Some('-') => { self.state = State::CommentLessThanSignBangDashDash; } _ => { self.reconsume(); self.state = State::CommentEndDash; } } }
fn state_comment_less_than_sign_bang_dash_dash(&mut self) { match self.next_char() { Some('>') | None => { self.reconsume(); self.state = State::CommentEnd; } Some(_) => { // Parse error. self.reconsume(); self.state = State::CommentEnd; } } }
fn state_comment_end_dash(&mut self) { match self.next_char() { Some('-') => { self.state = State::CommentEnd; } None => { self.emit_current_comment(); self.emit_eof(); } Some(_) => { self.comment_data.push('-'); self.reconsume(); self.state = State::Comment; } } }
fn state_comment_end(&mut self) { match self.next_char() { Some('>') => { self.state = State::Data; self.emit_current_comment(); } Some('!') => { self.state = State::CommentEndBang; } Some('-') => { self.comment_data.push('-'); } None => { self.emit_current_comment(); self.emit_eof(); } Some(_) => { self.comment_data.push('-'); self.comment_data.push('-'); self.reconsume(); self.state = State::Comment; } } }
fn state_comment_end_bang(&mut self) { match self.next_char() { Some('-') => { self.comment_data.push('-'); self.comment_data.push('-'); self.comment_data.push('!'); self.state = State::CommentEndDash; } Some('>') => { self.state = State::Data; self.emit_current_comment(); } None => { self.emit_current_comment(); self.emit_eof(); } Some(_) => { self.comment_data.push('-'); self.comment_data.push('-'); self.comment_data.push('!'); self.reconsume(); self.state = State::Comment; } } }
fn state_doctype(&mut self) { match self.next_char() { Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { self.state = State::BeforeDoctypeName; } Some('>') => { self.reconsume(); self.state = State::BeforeDoctypeName; } None => { self.doctype_name = None; self.doctype_public_id = None; self.doctype_system_id = None; self.doctype_force_quirks = true; self.emit_current_doctype(); self.emit_eof(); } Some(_) => { // Parse error. Missing whitespace before DOCTYPE name. self.reconsume(); self.state = State::BeforeDoctypeName; } } }
fn state_before_doctype_name(&mut self) { match self.next_char() { Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { // Ignore whitespace. } Some(c) if c.is_ascii_uppercase() => { self.doctype_name = Some(c.to_ascii_lowercase().to_string()); self.doctype_public_id = None; self.doctype_system_id = None; self.doctype_force_quirks = false; self.state = State::DoctypeName; } Some('\0') => { self.doctype_name = Some("\u{FFFD}".to_string()); self.doctype_public_id = None; self.doctype_system_id = None; self.doctype_force_quirks = false; self.state = State::DoctypeName; } Some('>') => { // Parse error. Force quirks. self.doctype_name = None; self.doctype_public_id = None; self.doctype_system_id = None; self.doctype_force_quirks = true; self.state = State::Data; self.emit_current_doctype(); } None => { self.doctype_name = None; self.doctype_public_id = None; self.doctype_system_id = None; self.doctype_force_quirks = true; self.emit_current_doctype(); self.emit_eof(); } Some(c) => { self.doctype_name = Some(c.to_string()); self.doctype_public_id = None; self.doctype_system_id = None; self.doctype_force_quirks = false; self.state = State::DoctypeName; } } }
fn state_doctype_name(&mut self) { match self.next_char() { Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { self.state = State::AfterDoctypeName; } Some('>') => { self.state = State::Data; self.emit_current_doctype(); } Some(c) if c.is_ascii_uppercase() => { if let Some(ref mut name) = self.doctype_name { name.push(c.to_ascii_lowercase()); } } Some('\0') => { if let Some(ref mut name) = self.doctype_name { name.push('\u{FFFD}'); } } None => { self.doctype_force_quirks = true; self.emit_current_doctype(); self.emit_eof(); } Some(c) => { if let Some(ref mut name) = self.doctype_name { name.push(c); } } } }
fn state_after_doctype_name(&mut self) { match self.next_char() { Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { // Ignore. } Some('>') => { self.state = State::Data; self.emit_current_doctype(); } None => { self.doctype_force_quirks = true; self.emit_current_doctype(); self.emit_eof(); } Some(_) => { // Check for PUBLIC or SYSTEM keyword. self.reconsume(); if self.starts_with_case_insensitive("PUBLIC") { self.pos += 6; self.state = State::AfterDoctypePublicKeyword; } else if self.starts_with_case_insensitive("SYSTEM") { self.pos += 6; self.state = State::AfterDoctypeSystemKeyword; } else { // Parse error. self.doctype_force_quirks = true; self.next_char(); // consume the reconsumed char self.state = State::BogusDoctype; } } } }
fn state_after_doctype_public_keyword(&mut self) { match self.next_char() { Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { self.state = State::BeforeDoctypePublicIdentifier; } Some('"') => { // Parse error. Missing whitespace. self.doctype_public_id = Some(String::new()); self.state = State::DoctypePublicIdentifierDoubleQuoted; } Some('\'') => { self.doctype_public_id = Some(String::new()); self.state = State::DoctypePublicIdentifierSingleQuoted; } Some('>') => { self.doctype_force_quirks = true; self.state = State::Data; self.emit_current_doctype(); } None => { self.doctype_force_quirks = true; self.emit_current_doctype(); self.emit_eof(); } Some(_) => { self.doctype_force_quirks = true; self.reconsume(); self.state = State::BogusDoctype; } } }
fn state_before_doctype_public_identifier(&mut self) { match self.next_char() { Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { // Ignore. } Some('"') => { self.doctype_public_id = Some(String::new()); self.state = State::DoctypePublicIdentifierDoubleQuoted; } Some('\'') => { self.doctype_public_id = Some(String::new()); self.state = State::DoctypePublicIdentifierSingleQuoted; } Some('>') => { self.doctype_force_quirks = true; self.state = State::Data; self.emit_current_doctype(); } None => { self.doctype_force_quirks = true; self.emit_current_doctype(); self.emit_eof(); } Some(_) => { self.doctype_force_quirks = true; self.reconsume(); self.state = State::BogusDoctype; } } }
fn state_doctype_public_identifier_double_quoted(&mut self) { match self.next_char() { Some('"') => { self.state = State::AfterDoctypePublicIdentifier; } Some('\0') => { if let Some(ref mut id) = self.doctype_public_id { id.push('\u{FFFD}'); } } Some('>') => { self.doctype_force_quirks = true; self.state = State::Data; self.emit_current_doctype(); } None => { self.doctype_force_quirks = true; self.emit_current_doctype(); self.emit_eof(); } Some(c) => { if let Some(ref mut id) = self.doctype_public_id { id.push(c); } } } }
fn state_doctype_public_identifier_single_quoted(&mut self) { match self.next_char() { Some('\'') => { self.state = State::AfterDoctypePublicIdentifier; } Some('\0') => { if let Some(ref mut id) = self.doctype_public_id { id.push('\u{FFFD}'); } } Some('>') => { self.doctype_force_quirks = true; self.state = State::Data; self.emit_current_doctype(); } None => { self.doctype_force_quirks = true; self.emit_current_doctype(); self.emit_eof(); } Some(c) => { if let Some(ref mut id) = self.doctype_public_id { id.push(c); } } } }
fn state_after_doctype_public_identifier(&mut self) { match self.next_char() { Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { self.state = State::BetweenDoctypePublicAndSystemIdentifiers; } Some('>') => { self.state = State::Data; self.emit_current_doctype(); } Some('"') => { // Parse error. Missing whitespace. self.doctype_system_id = Some(String::new()); self.state = State::DoctypeSystemIdentifierDoubleQuoted; } Some('\'') => { self.doctype_system_id = Some(String::new()); self.state = State::DoctypeSystemIdentifierSingleQuoted; } None => { self.doctype_force_quirks = true; self.emit_current_doctype(); self.emit_eof(); } Some(_) => { self.doctype_force_quirks = true; self.reconsume(); self.state = State::BogusDoctype; } } }
fn state_between_doctype_public_and_system_identifiers(&mut self) { match self.next_char() { Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { // Ignore. } Some('>') => { self.state = State::Data; self.emit_current_doctype(); } Some('"') => { self.doctype_system_id = Some(String::new()); self.state = State::DoctypeSystemIdentifierDoubleQuoted; } Some('\'') => { self.doctype_system_id = Some(String::new()); self.state = State::DoctypeSystemIdentifierSingleQuoted; } None => { self.doctype_force_quirks = true; self.emit_current_doctype(); self.emit_eof(); } Some(_) => { self.doctype_force_quirks = true; self.reconsume(); self.state = State::BogusDoctype; } } }
fn state_after_doctype_system_keyword(&mut self) { match self.next_char() { Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { self.state = State::BeforeDoctypeSystemIdentifier; } Some('"') => { self.doctype_system_id = Some(String::new()); self.state = State::DoctypeSystemIdentifierDoubleQuoted; } Some('\'') => { self.doctype_system_id = Some(String::new()); self.state = State::DoctypeSystemIdentifierSingleQuoted; } Some('>') => { self.doctype_force_quirks = true; self.state = State::Data; self.emit_current_doctype(); } None => { self.doctype_force_quirks = true; self.emit_current_doctype(); self.emit_eof(); } Some(_) => { self.doctype_force_quirks = true; self.reconsume(); self.state = State::BogusDoctype; } } }
fn state_before_doctype_system_identifier(&mut self) { match self.next_char() { Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { // Ignore. } Some('"') => { self.doctype_system_id = Some(String::new()); self.state = State::DoctypeSystemIdentifierDoubleQuoted; } Some('\'') => { self.doctype_system_id = Some(String::new()); self.state = State::DoctypeSystemIdentifierSingleQuoted; } Some('>') => { self.doctype_force_quirks = true; self.state = State::Data; self.emit_current_doctype(); } None => { self.doctype_force_quirks = true; self.emit_current_doctype(); self.emit_eof(); } Some(_) => { self.doctype_force_quirks = true; self.reconsume(); self.state = State::BogusDoctype; } } }
fn state_doctype_system_identifier_double_quoted(&mut self) { match self.next_char() { Some('"') => { self.state = State::AfterDoctypeSystemIdentifier; } Some('\0') => { if let Some(ref mut id) = self.doctype_system_id { id.push('\u{FFFD}'); } } Some('>') => { self.doctype_force_quirks = true; self.state = State::Data; self.emit_current_doctype(); } None => { self.doctype_force_quirks = true; self.emit_current_doctype(); self.emit_eof(); } Some(c) => { if let Some(ref mut id) = self.doctype_system_id { id.push(c); } } } }
fn state_doctype_system_identifier_single_quoted(&mut self) { match self.next_char() { Some('\'') => { self.state = State::AfterDoctypeSystemIdentifier; } Some('\0') => { if let Some(ref mut id) = self.doctype_system_id { id.push('\u{FFFD}'); } } Some('>') => { self.doctype_force_quirks = true; self.state = State::Data; self.emit_current_doctype(); } None => { self.doctype_force_quirks = true; self.emit_current_doctype(); self.emit_eof(); } Some(c) => { if let Some(ref mut id) = self.doctype_system_id { id.push(c); } } } }
fn state_after_doctype_system_identifier(&mut self) { match self.next_char() { Some('\t') | Some('\n') | Some('\x0C') | Some(' ') => { // Ignore. } Some('>') => { self.state = State::Data; self.emit_current_doctype(); } None => { self.doctype_force_quirks = true; self.emit_current_doctype(); self.emit_eof(); } Some(_) => { // Parse error, but do NOT set force_quirks. self.reconsume(); self.state = State::BogusDoctype; } } }
fn state_bogus_doctype(&mut self) { match self.next_char() { Some('>') => { self.state = State::Data; self.emit_current_doctype(); } Some('\0') => { // Parse error. Ignore. } None => { self.emit_current_doctype(); self.emit_eof(); } Some(_) => { // Ignore. } } }
// --- Character reference states ---
fn state_character_reference(&mut self) { self.temp_buf.clear(); self.temp_buf.push('&');
match self.peek_char() { Some(c) if c.is_ascii_alphanumeric() => { self.state = State::NamedCharacterReference; } Some('#') => { self.temp_buf.push('#'); self.next_char(); self.state = State::NumericCharacterReference; } _ => { // Not a character reference. Flush '&' to return state. self.flush_char_ref("&"); self.state = self.return_state; } } }
fn state_named_character_reference(&mut self) { // Collect alphanumeric characters to form the entity name. // Per spec, entity names can also contain digits after the first char. let mut name = String::new(); let start_pos = self.pos;
while let Some(c) = self.peek_char() { if c.is_ascii_alphanumeric() { name.push(c); self.pos += 1; } else { break; } }
// Try to find a match, trying longest match first. // First check if the full name + semicolon matches. let has_trailing_semi = self.peek_char() == Some(';');
let mut matched_value: Option<&str> = None; let mut matched_len = 0;
// Try the full name first (with semicolon if present). if has_trailing_semi { if let Some(val) = entities::lookup_entity(&name) { matched_value = Some(val); matched_len = name.len(); } }
// If no match with full name, try progressively shorter prefixes. if matched_value.is_none() { for i in (1..=name.len()).rev() { let candidate = &name[..i]; if let Some(val) = entities::lookup_entity(candidate) { // Without semicolon, only legacy entities are recognized. if entities::is_legacy_entity(candidate) { matched_value = Some(val); matched_len = i; break; } } } }
// Also try the full name without semicolon for legacy entities. if matched_value.is_none() && !has_trailing_semi { if let Some(val) = entities::lookup_entity(&name) { if entities::is_legacy_entity(&name) { matched_value = Some(val); matched_len = name.len(); } } }
if let Some(value) = matched_value { // Rewind to just after the matched portion. self.pos = start_pos + matched_len;
// Check for semicolon after the matched portion. let has_semi = self.peek_char() == Some(';'); if has_semi { self.pos += 1; }
// Per spec: if consumed as part of an attribute and the character // after the match is `=` or alphanumeric, and no semicolon, // flush the original text instead. let in_attribute = matches!( self.return_state, State::AttributeValueDoubleQuoted | State::AttributeValueSingleQuoted | State::AttributeValueUnquoted );
if !has_semi && in_attribute { if let Some(next) = self.peek_char() { if next == '=' || next.is_ascii_alphanumeric() { // Not a reference. Flush original text. let mut original = "&".to_string(); original.push_str(&name[..matched_len]); self.flush_char_ref(&original); self.state = self.return_state; return; } } }
self.flush_char_ref(value); self.state = self.return_state; } else { // No match. Rewind and flush '&' + all collected chars. self.pos = start_pos; self.flush_char_ref("&"); for _ in 0..name.len() { let c = self.next_char().unwrap(); let s = c.to_string(); self.flush_char_ref(&s); } self.state = self.return_state; } }
fn state_numeric_character_reference(&mut self) { self.char_ref_code = 0; match self.peek_char() { Some('x') | Some('X') => { self.temp_buf.push(self.peek_char().unwrap()); self.next_char(); self.state = State::HexCharacterReferenceStart; } _ => { self.state = State::DecCharacterReferenceStart; } } }
fn state_hex_character_reference_start(&mut self) { match self.peek_char() { Some(c) if c.is_ascii_hexdigit() => { self.state = State::HexCharacterReference; } _ => { // Parse error. Flush temp_buf. let buf = self.temp_buf.clone(); self.flush_char_ref(&buf); self.state = self.return_state; } } }
fn state_dec_character_reference_start(&mut self) { match self.peek_char() { Some(c) if c.is_ascii_digit() => { self.state = State::DecCharacterReference; } _ => { let buf = self.temp_buf.clone(); self.flush_char_ref(&buf); self.state = self.return_state; } } }
fn state_hex_character_reference(&mut self) { match self.next_char() { Some(c) if c.is_ascii_hexdigit() => { // Cap at a value that's clearly out of range but won't overflow. self.char_ref_code = self .char_ref_code .saturating_mul(16) .saturating_add(c.to_digit(16).unwrap()); if self.char_ref_code > 0x10FFFF { self.char_ref_code = 0x110000; } } Some(';') => { self.state = State::NumericCharacterReferenceEnd; } None => { // EOF: missing semicolon parse error. Don't reconsume. self.state = State::NumericCharacterReferenceEnd; } Some(_) => { // Parse error: missing semicolon. self.reconsume(); self.state = State::NumericCharacterReferenceEnd; } } }
fn state_dec_character_reference(&mut self) { match self.next_char() { Some(c) if c.is_ascii_digit() => { self.char_ref_code = self .char_ref_code .saturating_mul(10) .saturating_add(c.to_digit(10).unwrap()); if self.char_ref_code > 0x10FFFF { self.char_ref_code = 0x110000; } } Some(';') => { self.state = State::NumericCharacterReferenceEnd; } None => { // EOF: missing semicolon parse error. Don't reconsume. self.state = State::NumericCharacterReferenceEnd; } Some(_) => { self.reconsume(); self.state = State::NumericCharacterReferenceEnd; } } }
fn state_numeric_character_reference_end(&mut self) { let code = self.char_ref_code; let ch = match code { 0 => '\u{FFFD}', // Surrogate range. 0xD800..=0xDFFF => '\u{FFFD}', // Out of Unicode range. c if c > 0x10FFFF => '\u{FFFD}', // Windows-1252 replacement table for 0x80..0x9F. 0x80 => '\u{20AC}', 0x82 => '\u{201A}', 0x83 => '\u{0192}', 0x84 => '\u{201E}', 0x85 => '\u{2026}', 0x86 => '\u{2020}', 0x87 => '\u{2021}', 0x88 => '\u{02C6}', 0x89 => '\u{2030}', 0x8A => '\u{0160}', 0x8B => '\u{2039}', 0x8C => '\u{0152}', 0x8E => '\u{017D}', 0x91 => '\u{2018}', 0x92 => '\u{2019}', 0x93 => '\u{201C}', 0x94 => '\u{201D}', 0x95 => '\u{2022}', 0x96 => '\u{2013}', 0x97 => '\u{2014}', 0x98 => '\u{02DC}', 0x99 => '\u{2122}', 0x9A => '\u{0161}', 0x9B => '\u{203A}', 0x9C => '\u{0153}', 0x9E => '\u{017E}', 0x9F => '\u{0178}', c => char::from_u32(c).unwrap_or('\u{FFFD}'), };
let s = ch.to_string(); self.flush_char_ref(&s); self.state = self.return_state; }
// --- Helpers ---
fn starts_with(&self, s: &str) -> bool { let bytes: Vec<char> = s.chars().collect(); if self.pos + bytes.len() > self.input.len() { return false; } for (i, &c) in bytes.iter().enumerate() { if self.input[self.pos + i] != c { return false; } } true }
fn starts_with_case_insensitive(&self, s: &str) -> bool { let bytes: Vec<char> = s.chars().collect(); if self.pos + bytes.len() > self.input.len() { return false; } for (i, &c) in bytes.iter().enumerate() { if !self.input[self.pos + i].eq_ignore_ascii_case(&c) { return false; } } true }}
#[cfg(test)]mod tests { use super::*; use crate::tokenize;
#[test] fn empty_input() { let tokens = tokenize(""); assert!(tokens.is_empty()); }
#[test] fn plain_text() { let tokens = tokenize("Hello, world!"); assert_eq!(tokens, vec![Token::Character("Hello, world!".to_string())]); }
#[test] fn simple_element() { let tokens = tokenize("<p>Hello</p>"); assert_eq!( tokens, vec![ Token::StartTag { name: "p".to_string(), attributes: vec![], self_closing: false, }, Token::Character("Hello".to_string()), Token::EndTag { name: "p".to_string(), }, ] ); }
#[test] fn self_closing_tag() { let tokens = tokenize("<br/>"); assert_eq!( tokens, vec![Token::StartTag { name: "br".to_string(), attributes: vec![], self_closing: true, }] ); }
#[test] fn self_closing_img() { let tokens = tokenize("<img/>"); assert_eq!( tokens, vec![Token::StartTag { name: "img".to_string(), attributes: vec![], self_closing: true, }] ); }
#[test] fn tag_with_attributes() { let tokens = tokenize(r#"<a href="url" class="link">"#); assert_eq!( tokens, vec![Token::StartTag { name: "a".to_string(), attributes: vec![ ("href".to_string(), "url".to_string()), ("class".to_string(), "link".to_string()), ], self_closing: false, }] ); }
#[test] fn tag_with_single_quoted_attributes() { let tokens = tokenize("<div id='main'>"); assert_eq!( tokens, vec![Token::StartTag { name: "div".to_string(), attributes: vec![("id".to_string(), "main".to_string())], self_closing: false, }] ); }
#[test] fn tag_with_unquoted_attribute() { let tokens = tokenize("<input type=text>"); assert_eq!( tokens, vec![Token::StartTag { name: "input".to_string(), attributes: vec![("type".to_string(), "text".to_string())], self_closing: false, }] ); }
#[test] fn comment() { let tokens = tokenize("<!-- comment -->"); assert_eq!(tokens, vec![Token::Comment(" comment ".to_string())]); }
#[test] fn empty_comment() { let tokens = tokenize("<!---->"); assert_eq!(tokens, vec![Token::Comment("".to_string())]); }
#[test] fn doctype_html() { let tokens = tokenize("<!DOCTYPE html>"); assert_eq!( tokens, vec![Token::Doctype { name: Some("html".to_string()), public_id: None, system_id: None, force_quirks: false, }] ); }
#[test] fn doctype_case_insensitive() { let tokens = tokenize("<!doctype html>"); assert_eq!( tokens, vec![Token::Doctype { name: Some("html".to_string()), public_id: None, system_id: None, force_quirks: false, }] ); }
#[test] fn char_ref_named() { let tokens = tokenize("&<>""); assert_eq!(tokens, vec![Token::Character("&<>\"".to_string())]); }
#[test] fn char_ref_numeric_decimal() { let tokens = tokenize("A"); assert_eq!(tokens, vec![Token::Character("A".to_string())]); }
#[test] fn char_ref_numeric_hex() { let tokens = tokenize("A"); assert_eq!(tokens, vec![Token::Character("A".to_string())]); }
#[test] fn char_ref_numeric_hex_uppercase() { let tokens = tokenize("A"); assert_eq!(tokens, vec![Token::Character("A".to_string())]); }
#[test] fn full_html_document() { let tokens = tokenize("<html><head><title>Test</title></head><body><p>Hello</p></body></html>"); assert_eq!( tokens, vec![ Token::StartTag { name: "html".to_string(), attributes: vec![], self_closing: false, }, Token::StartTag { name: "head".to_string(), attributes: vec![], self_closing: false, }, Token::StartTag { name: "title".to_string(), attributes: vec![], self_closing: false, }, Token::Character("Test".to_string()), Token::EndTag { name: "title".to_string(), }, Token::EndTag { name: "head".to_string(), }, Token::StartTag { name: "body".to_string(), attributes: vec![], self_closing: false, }, Token::StartTag { name: "p".to_string(), attributes: vec![], self_closing: false, }, Token::Character("Hello".to_string()), Token::EndTag { name: "p".to_string(), }, Token::EndTag { name: "body".to_string(), }, Token::EndTag { name: "html".to_string(), }, ] ); }
#[test] fn uppercase_tag_names_lowercased() { let tokens = tokenize("<DIV></DIV>"); assert_eq!( tokens, vec![ Token::StartTag { name: "div".to_string(), attributes: vec![], self_closing: false, }, Token::EndTag { name: "div".to_string(), }, ] ); }
#[test] fn uppercase_attribute_names_lowercased() { let tokens = tokenize(r#"<div CLASS="x">"#); assert_eq!( tokens, vec![Token::StartTag { name: "div".to_string(), attributes: vec![("class".to_string(), "x".to_string())], self_closing: false, }] ); }
#[test] fn duplicate_attributes_first_wins() { let tokens = tokenize(r#"<div class="a" class="b">"#); assert_eq!( tokens, vec![Token::StartTag { name: "div".to_string(), attributes: vec![("class".to_string(), "a".to_string())], self_closing: false, }] ); }
#[test] fn char_ref_in_attribute() { let tokens = tokenize(r#"<a href="?a=1&b=2">"#); assert_eq!( tokens, vec![Token::StartTag { name: "a".to_string(), attributes: vec![("href".to_string(), "?a=1&b=2".to_string())], self_closing: false, }] ); }
#[test] fn multiple_attributes() { let tokens = tokenize(r#"<input type="text" name="foo" value="bar">"#); assert_eq!( tokens, vec![Token::StartTag { name: "input".to_string(), attributes: vec![ ("type".to_string(), "text".to_string()), ("name".to_string(), "foo".to_string()), ("value".to_string(), "bar".to_string()), ], self_closing: false, }] ); }
#[test] fn boolean_attribute() { let tokens = tokenize("<input disabled>"); assert_eq!( tokens, vec![Token::StartTag { name: "input".to_string(), attributes: vec![("disabled".to_string(), "".to_string())], self_closing: false, }] ); }
#[test] fn mixed_content() { let tokens = tokenize("Hello <!-- comment --> World"); assert_eq!( tokens, vec![ Token::Character("Hello ".to_string()), Token::Comment(" comment ".to_string()), Token::Character(" World".to_string()), ] ); }
#[test] fn doctype_with_public_id() { let tokens = tokenize( r#"<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">"#, ); assert_eq!( tokens, vec![Token::Doctype { name: Some("html".to_string()), public_id: Some("-//W3C//DTD XHTML 1.0 Strict//EN".to_string()), system_id: Some("http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd".to_string()), force_quirks: false, }] ); }
#[test] fn null_in_text() { let tokens = tokenize("a\0b"); assert_eq!(tokens, vec![Token::Character("a\u{FFFD}b".to_string())]); }
#[test] fn windows_1252_numeric_refs() { // € should map to Euro sign. let tokens = tokenize("€"); assert_eq!(tokens, vec![Token::Character("\u{20AC}".to_string())]); }
#[test] fn attribute_with_empty_value() { let tokens = tokenize(r#"<div class="">"#); assert_eq!( tokens, vec![Token::StartTag { name: "div".to_string(), attributes: vec![("class".to_string(), "".to_string())], self_closing: false, }] ); }
#[test] fn adjacent_tags() { let tokens = tokenize("<b></b><i></i>"); assert_eq!( tokens, vec![ Token::StartTag { name: "b".to_string(), attributes: vec![], self_closing: false, }, Token::EndTag { name: "b".to_string(), }, Token::StartTag { name: "i".to_string(), attributes: vec![], self_closing: false, }, Token::EndTag { name: "i".to_string(), }, ] ); }
#[test] fn newlines_in_text() { let tokens = tokenize("line1\nline2\nline3"); assert_eq!( tokens, vec![Token::Character("line1\nline2\nline3".to_string())] ); }
#[test] fn self_closing_with_attribute() { let tokens = tokenize(r#"<img src="test.png"/>"#); assert_eq!( tokens, vec![Token::StartTag { name: "img".to_string(), attributes: vec![("src".to_string(), "test.png".to_string())], self_closing: true, }] ); }
#[test] fn less_than_in_text_not_tag() { // A bare '<' not followed by a letter should be emitted as text. let tokens = tokenize("1 < 2"); assert_eq!(tokens, vec![Token::Character("1 < 2".to_string())]); }
#[test] fn ampersand_not_entity() { let tokens = tokenize("a & b"); assert_eq!(tokens, vec![Token::Character("a & b".to_string())]); }
#[test] fn cdata_in_html_becomes_comment() { let tokens = tokenize("<![CDATA[hello]]>"); // In HTML (non-foreign) context, CDATA is a parse error → bogus comment. assert_eq!(tokens, vec![Token::Comment("[CDATA[hello]]".to_string())]); }}