Something went wrong. Try again.
we (web engine): Experimental web browser project to understand the limits of Claude
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835//! Resource loader: fetch URLs and decode their content.//!//! Brings together `net` (HTTP client), `encoding` (charset detection and decoding),//! `url` (URL parsing and resolution), and `image` (image decoding) into a single//! `ResourceLoader` that the browser uses to load web pages and subresources.
use std::fmt;
use we_encoding::sniff::sniff_encoding;use we_encoding::Encoding;use we_net::client::{ClientError, HttpClient};use we_net::http::ContentType;use we_url::data_url::{is_data_url, parse_data_url};use we_url::Url;
// ---------------------------------------------------------------------------// Error type// ---------------------------------------------------------------------------
/// Errors that can occur during resource loading.#[derive(Debug)]pub enum LoadError { /// URL parsing failed. InvalidUrl(String), /// Network or HTTP error from the underlying client. Network(ClientError), /// HTTP response indicated an error status. HttpStatus { status: u16, reason: String }, /// Encoding or decoding error. Encoding(String),}
impl fmt::Display for LoadError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::InvalidUrl(s) => write!(f, "invalid URL: {s}"), Self::Network(e) => write!(f, "network error: {e}"), Self::HttpStatus { status, reason } => { write!(f, "HTTP {status} {reason}") } Self::Encoding(s) => write!(f, "encoding error: {s}"), } }}
impl From<ClientError> for LoadError { fn from(e: ClientError) -> Self { Self::Network(e) }}
// ---------------------------------------------------------------------------// Resource types// ---------------------------------------------------------------------------
/// A loaded resource with its decoded content and metadata.#[derive(Debug)]pub enum Resource { /// An HTML document. Html { text: String, base_url: Url, encoding: Encoding, }, /// A CSS stylesheet. Css { text: String, url: Url }, /// A decoded image. Image { data: Vec<u8>, mime_type: String, url: Url, }, /// Any other resource type (binary). Other { data: Vec<u8>, mime_type: String, url: Url, },}
// ---------------------------------------------------------------------------// ResourceLoader// ---------------------------------------------------------------------------
/// Loads resources over HTTP/HTTPS with encoding detection and content-type handling.pub struct ResourceLoader { client: HttpClient,}
impl ResourceLoader { /// Create a new resource loader with default settings. pub fn new() -> Self { Self { client: HttpClient::new(), } }
/// Fetch a resource at the given URL. /// /// Determines the resource type from the HTTP Content-Type header, decodes /// text resources using the appropriate character encoding (per WHATWG spec), /// and returns the result as a typed `Resource`. /// /// Handles `data:` and `about:` URLs locally without network access. pub fn fetch(&mut self, url: &Url) -> Result<Resource, LoadError> { // Handle data: URLs without network fetch. if url.scheme() == "data" { return fetch_data_url(&url.serialize()); }
// Handle about: URLs without network fetch. if url.scheme() == "about" { return fetch_about_url(url); }
let response = self.client.get(url)?;
// Check for HTTP error status codes if response.status_code >= 400 { return Err(LoadError::HttpStatus { status: response.status_code, reason: response.reason.clone(), }); }
let content_type = response.content_type(); let mime = content_type .as_ref() .map(|ct| ct.mime_type.as_str()) .unwrap_or("application/octet-stream");
match classify_mime(mime) { MimeClass::Html => { let (text, encoding) = decode_text_resource(&response.body, content_type.as_ref(), true); Ok(Resource::Html { text, base_url: url.clone(), encoding, }) } MimeClass::Css => { let (text, _encoding) = decode_text_resource(&response.body, content_type.as_ref(), false); Ok(Resource::Css { text, url: url.clone(), }) } MimeClass::Image => Ok(Resource::Image { data: response.body, mime_type: mime.to_string(), url: url.clone(), }), MimeClass::Other => { // Check if it's a text type we should decode if mime.starts_with("text/") { let (text, _encoding) = decode_text_resource(&response.body, content_type.as_ref(), false); Ok(Resource::Other { data: text.into_bytes(), mime_type: mime.to_string(), url: url.clone(), }) } else { Ok(Resource::Other { data: response.body, mime_type: mime.to_string(), url: url.clone(), }) } } } }
/// Fetch a URL string, resolving it against an optional base URL. /// /// Handles `data:` and `about:` URLs locally without network access. pub fn fetch_url(&mut self, url_str: &str, base: Option<&Url>) -> Result<Resource, LoadError> { // Handle data URLs directly — no network fetch needed. if is_data_url(url_str) { return fetch_data_url(url_str); }
// Handle about: URLs without network fetch. if url_str.starts_with("about:") { let url = Url::parse(url_str).map_err(|_| LoadError::InvalidUrl(url_str.to_string()))?; return fetch_about_url(&url); }
let url = match base { Some(base_url) => Url::parse_with_base(url_str, base_url) .or_else(|_| Url::parse(url_str)) .map_err(|_| LoadError::InvalidUrl(url_str.to_string()))?, None => Url::parse(url_str).map_err(|_| LoadError::InvalidUrl(url_str.to_string()))?, }; self.fetch(&url) }}
impl Default for ResourceLoader { fn default() -> Self { Self::new() }}
// ---------------------------------------------------------------------------// MIME classification// ---------------------------------------------------------------------------
enum MimeClass { Html, Css, Image, Other,}
fn classify_mime(mime: &str) -> MimeClass { match mime { "text/html" | "application/xhtml+xml" => MimeClass::Html, "text/css" => MimeClass::Css, "image/png" | "image/jpeg" | "image/gif" | "image/webp" | "image/svg+xml" => { MimeClass::Image } _ => MimeClass::Other, }}
// ---------------------------------------------------------------------------// Text decoding// ---------------------------------------------------------------------------
/// Decode a text resource's bytes to a String using WHATWG encoding sniffing.////// For HTML resources, uses BOM > HTTP charset > meta prescan > default./// For non-HTML text resources, uses BOM > HTTP charset > default (UTF-8).fn decode_text_resource( bytes: &[u8], content_type: Option<&ContentType>, is_html: bool,) -> (String, Encoding) { let http_ct_value = content_type.map(|ct| { // Reconstruct a Content-Type header value for the sniffing function match &ct.charset { Some(charset) => format!("{}; charset={}", ct.mime_type, charset), None => ct.mime_type.clone(), } });
if is_html { // Full WHATWG sniffing: BOM > HTTP > meta prescan > default (Windows-1252) let (encoding, _source) = sniff_encoding(bytes, http_ct_value.as_deref()); let text = decode_with_bom_handling(bytes, encoding); (text, encoding) } else { // Non-HTML: BOM > HTTP charset > default (UTF-8) let (bom_enc, after_bom) = we_encoding::bom_sniff(bytes); if let Some(enc) = bom_enc { let text = we_encoding::decode(after_bom, enc); return (text, enc); }
// Try HTTP charset if let Some(charset) = content_type.and_then(|ct| ct.charset.as_deref()) { if let Some(enc) = we_encoding::lookup(charset) { let text = we_encoding::decode(bytes, enc); return (text, enc); } }
// Default to UTF-8 for non-HTML text let text = we_encoding::decode(bytes, Encoding::Utf8); (text, Encoding::Utf8) }}
/// Decode bytes with BOM handling — strip BOM bytes before decoding.fn decode_with_bom_handling(bytes: &[u8], encoding: Encoding) -> String { let (bom_enc, after_bom) = we_encoding::bom_sniff(bytes); if bom_enc.is_some() { // BOM was present — decode the bytes after the BOM we_encoding::decode(after_bom, encoding) } else { we_encoding::decode(bytes, encoding) }}
// ---------------------------------------------------------------------------// Data URL handling// ---------------------------------------------------------------------------
/// Fetch a data URL, decoding its payload and returning the appropriate Resource type.fn fetch_data_url(url_str: &str) -> Result<Resource, LoadError> { let parsed = parse_data_url(url_str) .map_err(|e| LoadError::InvalidUrl(format!("data URL error: {e}")))?;
let mime = &parsed.mime_type;
// Create a synthetic Url for the resource metadata. let url = Url::parse(url_str).map_err(|_| LoadError::InvalidUrl(url_str.to_string()))?;
match classify_mime(mime) { MimeClass::Html => { let encoding = charset_to_encoding(parsed.charset.as_deref()); let text = we_encoding::decode(&parsed.data, encoding); Ok(Resource::Html { text, base_url: url, encoding, }) } MimeClass::Css => { let encoding = charset_to_encoding(parsed.charset.as_deref()); let text = we_encoding::decode(&parsed.data, encoding); Ok(Resource::Css { text, url }) } MimeClass::Image => Ok(Resource::Image { data: parsed.data, mime_type: mime.to_string(), url, }), MimeClass::Other => { if mime.starts_with("text/") { let encoding = charset_to_encoding(parsed.charset.as_deref()); let text = we_encoding::decode(&parsed.data, encoding); Ok(Resource::Other { data: text.into_bytes(), mime_type: mime.to_string(), url, }) } else { Ok(Resource::Other { data: parsed.data, mime_type: mime.to_string(), url, }) } } }}
// ---------------------------------------------------------------------------// about: URL handling// ---------------------------------------------------------------------------
/// The minimal HTML document for about:blank.pub const ABOUT_BLANK_HTML: &str = "<!DOCTYPE html><html><head></head><body></body></html>";
/// Fetch an about: URL, returning the appropriate resource.////// Currently only `about:blank` is supported, which returns an empty HTML/// document with UTF-8 encoding.fn fetch_about_url(url: &Url) -> Result<Resource, LoadError> { match url.path().as_str() { "blank" => Ok(Resource::Html { text: ABOUT_BLANK_HTML.to_string(), base_url: url.clone(), encoding: Encoding::Utf8, }), other => Err(LoadError::InvalidUrl(format!( "unsupported about: URL: about:{other}" ))), }}
/// Map a charset name to an Encoding, defaulting to UTF-8.fn charset_to_encoding(charset: Option<&str>) -> Encoding { charset .and_then(we_encoding::lookup) .unwrap_or(Encoding::Utf8)}
// ---------------------------------------------------------------------------// Tests// ---------------------------------------------------------------------------
#[cfg(test)]mod tests { use super::*;
// ----------------------------------------------------------------------- // LoadError Display // -----------------------------------------------------------------------
#[test] fn load_error_display_invalid_url() { let e = LoadError::InvalidUrl("bad://url".to_string()); assert_eq!(e.to_string(), "invalid URL: bad://url"); }
#[test] fn load_error_display_http_status() { let e = LoadError::HttpStatus { status: 404, reason: "Not Found".to_string(), }; assert_eq!(e.to_string(), "HTTP 404 Not Found"); }
#[test] fn load_error_display_encoding() { let e = LoadError::Encoding("bad charset".to_string()); assert_eq!(e.to_string(), "encoding error: bad charset"); }
// ----------------------------------------------------------------------- // MIME classification // -----------------------------------------------------------------------
#[test] fn classify_text_html() { assert!(matches!(classify_mime("text/html"), MimeClass::Html)); }
#[test] fn classify_xhtml() { assert!(matches!( classify_mime("application/xhtml+xml"), MimeClass::Html )); }
#[test] fn classify_text_css() { assert!(matches!(classify_mime("text/css"), MimeClass::Css)); }
#[test] fn classify_image_png() { assert!(matches!(classify_mime("image/png"), MimeClass::Image)); }
#[test] fn classify_image_jpeg() { assert!(matches!(classify_mime("image/jpeg"), MimeClass::Image)); }
#[test] fn classify_image_gif() { assert!(matches!(classify_mime("image/gif"), MimeClass::Image)); }
#[test] fn classify_application_json() { assert!(matches!( classify_mime("application/json"), MimeClass::Other )); }
#[test] fn classify_text_plain() { assert!(matches!(classify_mime("text/plain"), MimeClass::Other)); }
#[test] fn classify_octet_stream() { assert!(matches!( classify_mime("application/octet-stream"), MimeClass::Other )); }
// ----------------------------------------------------------------------- // Text decoding — HTML // -----------------------------------------------------------------------
#[test] fn decode_html_utf8_bom() { let bytes = b"\xEF\xBB\xBF<html>Hello</html>"; let (text, enc) = decode_text_resource(bytes, None, true); assert_eq!(enc, Encoding::Utf8); assert_eq!(text, "<html>Hello</html>"); }
#[test] fn decode_html_utf8_from_http_charset() { let ct = ContentType { mime_type: "text/html".to_string(), charset: Some("utf-8".to_string()), }; let bytes = b"<html>Hello</html>"; let (text, enc) = decode_text_resource(bytes, Some(&ct), true); assert_eq!(enc, Encoding::Utf8); assert_eq!(text, "<html>Hello</html>"); }
#[test] fn decode_html_meta_charset() { let html = b"<meta charset=\"utf-8\"><html>Hello</html>"; let (text, enc) = decode_text_resource(html, None, true); assert_eq!(enc, Encoding::Utf8); assert!(text.contains("Hello")); }
#[test] fn decode_html_default_windows_1252() { let bytes = b"<html>Hello</html>"; let (text, enc) = decode_text_resource(bytes, None, true); assert_eq!(enc, Encoding::Windows1252); assert!(text.contains("Hello")); }
#[test] fn decode_html_windows_1252_special_chars() { // \x93 and \x94 are left/right double quotation marks in Windows-1252 let bytes = b"<html>\x93Hello\x94</html>"; let (text, enc) = decode_text_resource(bytes, None, true); assert_eq!(enc, Encoding::Windows1252); assert!(text.contains('\u{201C}')); // left double quote assert!(text.contains('\u{201D}')); // right double quote }
#[test] fn decode_html_bom_beats_http_charset() { let ct = ContentType { mime_type: "text/html".to_string(), charset: Some("windows-1252".to_string()), }; let mut bytes = vec![0xEF, 0xBB, 0xBF]; bytes.extend_from_slice(b"<html>Hello</html>"); let (text, enc) = decode_text_resource(&bytes, Some(&ct), true); assert_eq!(enc, Encoding::Utf8); assert_eq!(text, "<html>Hello</html>"); }
// ----------------------------------------------------------------------- // Text decoding — non-HTML (CSS, etc.) // -----------------------------------------------------------------------
#[test] fn decode_css_utf8_default() { let bytes = b"body { color: red; }"; let (text, enc) = decode_text_resource(bytes, None, false); assert_eq!(enc, Encoding::Utf8); assert_eq!(text, "body { color: red; }"); }
#[test] fn decode_css_bom_utf8() { let bytes = b"\xEF\xBB\xBFbody { color: red; }"; let (text, enc) = decode_text_resource(bytes, None, false); assert_eq!(enc, Encoding::Utf8); assert_eq!(text, "body { color: red; }"); }
#[test] fn decode_css_http_charset() { let ct = ContentType { mime_type: "text/css".to_string(), charset: Some("utf-8".to_string()), }; let bytes = b"body { color: red; }"; let (text, enc) = decode_text_resource(bytes, Some(&ct), false); assert_eq!(enc, Encoding::Utf8); assert_eq!(text, "body { color: red; }"); }
// ----------------------------------------------------------------------- // BOM handling // -----------------------------------------------------------------------
#[test] fn decode_with_bom_strips_utf8_bom() { let bytes = b"\xEF\xBB\xBFHello"; let text = decode_with_bom_handling(bytes, Encoding::Utf8); assert_eq!(text, "Hello"); }
#[test] fn decode_without_bom_passes_through() { let bytes = b"Hello"; let text = decode_with_bom_handling(bytes, Encoding::Utf8); assert_eq!(text, "Hello"); }
#[test] fn decode_with_utf16le_bom() { let bytes = b"\xFF\xFEH\x00e\x00l\x00l\x00o\x00"; let text = decode_with_bom_handling(bytes, Encoding::Utf16Le); assert_eq!(text, "Hello"); }
#[test] fn decode_with_utf16be_bom() { let bytes = b"\xFE\xFF\x00H\x00e\x00l\x00l\x00o"; let text = decode_with_bom_handling(bytes, Encoding::Utf16Be); assert_eq!(text, "Hello"); }
// ----------------------------------------------------------------------- // ResourceLoader construction // -----------------------------------------------------------------------
#[test] fn resource_loader_new() { let _loader = ResourceLoader::new(); }
#[test] fn resource_loader_default() { let _loader = ResourceLoader::default(); }
// ----------------------------------------------------------------------- // URL resolution // -----------------------------------------------------------------------
#[test] fn fetch_url_invalid_url_error() { let mut loader = ResourceLoader::new(); let result = loader.fetch_url("not a url at all", None); assert!(matches!(result, Err(LoadError::InvalidUrl(_)))); }
#[test] fn fetch_url_relative_without_base_errors() { let mut loader = ResourceLoader::new(); let result = loader.fetch_url("/relative/path", None); assert!(matches!(result, Err(LoadError::InvalidUrl(_)))); }
#[test] fn fetch_url_relative_with_base_resolves() { let mut loader = ResourceLoader::new(); let base = Url::parse("http://example.com/page").unwrap(); // This will fail since we can't actually connect in tests, // but the URL resolution itself should work (it won't be InvalidUrl). let result = loader.fetch_url("/style.css", Some(&base)); assert!(result.is_err()); // The error should NOT be InvalidUrl — the URL resolved successfully. assert!(!matches!(result, Err(LoadError::InvalidUrl(_)))); }
// ----------------------------------------------------------------------- // Data URL loading // -----------------------------------------------------------------------
#[test] fn data_url_plain_text() { let mut loader = ResourceLoader::new(); let result = loader.fetch_url("data:text/plain,Hello%20World", None); assert!(result.is_ok()); match result.unwrap() { Resource::Other { data, mime_type, .. } => { assert_eq!(mime_type, "text/plain"); assert_eq!(String::from_utf8(data).unwrap(), "Hello World"); } other => panic!("expected Other, got {:?}", other), } }
#[test] fn data_url_html() { let mut loader = ResourceLoader::new(); let result = loader.fetch_url("data:text/html,<h1>Hello</h1>", None); assert!(result.is_ok()); match result.unwrap() { Resource::Html { text, .. } => { assert_eq!(text, "<h1>Hello</h1>"); } other => panic!("expected Html, got {:?}", other), } }
#[test] fn data_url_css() { let mut loader = ResourceLoader::new(); let result = loader.fetch_url("data:text/css,body{color:red}", None); assert!(result.is_ok()); match result.unwrap() { Resource::Css { text, .. } => { assert_eq!(text, "body{color:red}"); } other => panic!("expected Css, got {:?}", other), } }
#[test] fn data_url_image() { let mut loader = ResourceLoader::new(); let result = loader.fetch_url("data:image/png;base64,/wCq", None); assert!(result.is_ok()); match result.unwrap() { Resource::Image { data, mime_type, .. } => { assert_eq!(mime_type, "image/png"); assert_eq!(data, vec![0xFF, 0x00, 0xAA]); } other => panic!("expected Image, got {:?}", other), } }
#[test] fn data_url_base64() { let mut loader = ResourceLoader::new(); let result = loader.fetch_url("data:text/plain;base64,SGVsbG8=", None); assert!(result.is_ok()); match result.unwrap() { Resource::Other { data, .. } => { assert_eq!(String::from_utf8(data).unwrap(), "Hello"); } other => panic!("expected Other, got {:?}", other), } }
#[test] fn data_url_empty() { let mut loader = ResourceLoader::new(); let result = loader.fetch_url("data:,", None); assert!(result.is_ok()); }
#[test] fn data_url_via_fetch_method() { let mut loader = ResourceLoader::new(); let url = Url::parse("data:text/plain,Hello").unwrap(); let result = loader.fetch(&url); assert!(result.is_ok()); match result.unwrap() { Resource::Other { data, .. } => { assert_eq!(String::from_utf8(data).unwrap(), "Hello"); } other => panic!("expected Other, got {:?}", other), } }
#[test] fn data_url_invalid() { let mut loader = ResourceLoader::new(); let result = loader.fetch_url("data:text/plain", None); assert!(matches!(result, Err(LoadError::InvalidUrl(_)))); }
#[test] fn data_url_binary() { let mut loader = ResourceLoader::new(); let result = loader.fetch_url("data:application/octet-stream;base64,/wCq", None); assert!(result.is_ok()); match result.unwrap() { Resource::Other { data, mime_type, .. } => { assert_eq!(mime_type, "application/octet-stream"); assert_eq!(data, vec![0xFF, 0x00, 0xAA]); } other => panic!("expected Other, got {:?}", other), } }
// ----------------------------------------------------------------------- // about: URL loading // -----------------------------------------------------------------------
#[test] fn about_blank_via_fetch_url() { let mut loader = ResourceLoader::new(); let result = loader.fetch_url("about:blank", None); assert!(result.is_ok()); match result.unwrap() { Resource::Html { text, encoding, base_url, .. } => { assert_eq!(text, ABOUT_BLANK_HTML); assert_eq!(encoding, Encoding::Utf8); assert_eq!(base_url.scheme(), "about"); } other => panic!("expected Html, got {:?}", other), } }
#[test] fn about_blank_via_fetch() { let mut loader = ResourceLoader::new(); let url = Url::parse("about:blank").unwrap(); let result = loader.fetch(&url); assert!(result.is_ok()); match result.unwrap() { Resource::Html { text, encoding, base_url, .. } => { assert_eq!(text, ABOUT_BLANK_HTML); assert_eq!(encoding, Encoding::Utf8); assert_eq!(base_url.scheme(), "about"); } other => panic!("expected Html, got {:?}", other), } }
#[test] fn about_blank_dom_structure() { let doc = we_html::parse_html(ABOUT_BLANK_HTML);
// Find the <html> element under the document root. let html = doc .children(doc.root()) .find(|&n| doc.tag_name(n) == Some("html")); assert!(html.is_some(), "document should have an <html> element"); let html = html.unwrap();
// The DOM should have html > head + body structure. let children: Vec<_> = doc .children(html) .filter(|&n| doc.tag_name(n).is_some()) .collect(); assert_eq!(children.len(), 2); assert_eq!(doc.tag_name(children[0]).unwrap(), "head"); assert_eq!(doc.tag_name(children[1]).unwrap(), "body");
// Body should have no child elements. let body_children: Vec<_> = doc .children(children[1]) .filter(|&n| doc.tag_name(n).is_some()) .collect(); assert!(body_children.is_empty()); }
#[test] fn about_unsupported_url() { let mut loader = ResourceLoader::new(); let result = loader.fetch_url("about:invalid", None); assert!(matches!(result, Err(LoadError::InvalidUrl(_)))); }}