Something went wrong. Try again.
we (web engine): Experimental web browser project to understand the limits of Claude
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263//! HTML5 tokenizer and tree builder.//!//! Implements the WHATWG HTML5 tokenizer state machine (§13.2.5)//! and a simplified tree builder for constructing DOM trees from tokens.
mod entities;mod tokenizer;mod tree_builder;
pub use tokenizer::Tokenizer;pub use tree_builder::{parse_html, TreeBuilder};
/// A token emitted by the HTML tokenizer.#[derive(Debug, Clone, PartialEq)]pub enum Token { /// `<!DOCTYPE name public_id system_id>` Doctype { name: Option<String>, public_id: Option<String>, system_id: Option<String>, force_quirks: bool, }, /// `<tag attr="val">` StartTag { name: String, attributes: Vec<(String, String)>, self_closing: bool, }, /// `</tag>` EndTag { name: String }, /// Character data (may be coalesced). Character(String), /// `<!-- comment -->` Comment(String), /// End of file. Eof,}
/// Tokenize an HTML input string into a sequence of tokens.////// Runs the HTML5 tokenizer state machine and returns all emitted tokens/// (excluding Eof). Adjacent Character tokens are coalesced.pub fn tokenize(input: &str) -> Vec<Token> { let mut tok = Tokenizer::new(input); let mut tokens = Vec::new(); loop { let token = tok.next_token(); match token { Token::Eof => break, Token::Character(ref s) => { // Coalesce adjacent character tokens. if let Some(Token::Character(ref mut prev)) = tokens.last_mut() { prev.push_str(s); } else { tokens.push(token); } } _ => tokens.push(token), } } tokens}