diff --git a/crates/encoding/src/error.rs b/crates/encoding/src/error.rs new file mode 100644 index 0000000..acd4fff --- /dev/null +++ b/crates/encoding/src/error.rs @@ -0,0 +1,70 @@ +//! Encoding error types. + +use std::fmt; + +/// Errors that can occur during encoding/decoding operations. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum EncodingError { + /// An invalid byte sequence was encountered in fatal (strict) mode. + InvalidSequence { + encoding: &'static str, + position: usize, + }, + /// The requested encoding label is not recognized. + UnknownLabel(String), + /// The encoding does not support the encode operation. + /// Per WHATWG spec, UTF-16BE/LE are decode-only. + EncodeNotSupported { encoding: &'static str }, +} + +impl fmt::Display for EncodingError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InvalidSequence { encoding, position } => { + write!( + f, + "invalid byte sequence in {encoding} at position {position}" + ) + } + Self::UnknownLabel(label) => { + write!(f, "unknown encoding label: {label}") + } + Self::EncodeNotSupported { encoding } => { + write!(f, "encode not supported for {encoding}") + } + } + } +} + +pub type Result = std::result::Result; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn display_invalid_sequence() { + let err = EncodingError::InvalidSequence { + encoding: "UTF-8", + position: 5, + }; + assert_eq!( + err.to_string(), + "invalid byte sequence in UTF-8 at position 5" + ); + } + + #[test] + fn display_unknown_label() { + let err = EncodingError::UnknownLabel("bogus".to_string()); + assert_eq!(err.to_string(), "unknown encoding label: bogus"); + } + + #[test] + fn display_encode_not_supported() { + let err = EncodingError::EncodeNotSupported { + encoding: "UTF-16LE", + }; + assert_eq!(err.to_string(), "encode not supported for UTF-16LE"); + } +} diff --git a/crates/encoding/src/lib.rs b/crates/encoding/src/lib.rs index e82d244..bb8188c 100644 --- a/crates/encoding/src/lib.rs +++ b/crates/encoding/src/lib.rs @@ -1 +1,341 @@ -//! WHATWG Encoding Standard — all encodings, pure Rust. +//! WHATWG Encoding Standard — UTF-8 and UTF-16 codecs, pure Rust. + +pub mod error; +mod utf16; +mod utf8; + +use error::{EncodingError, Result}; +use utf8::ErrorMode; + +// --------------------------------------------------------------------------- +// Encoding enum +// --------------------------------------------------------------------------- + +/// Supported text encodings per WHATWG Encoding Standard. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Encoding { + Utf8, + Utf16Be, + Utf16Le, +} + +impl Encoding { + /// Canonical name per WHATWG spec. + pub fn name(&self) -> &'static str { + match self { + Self::Utf8 => "UTF-8", + Self::Utf16Be => "UTF-16BE", + Self::Utf16Le => "UTF-16LE", + } + } +} + +// --------------------------------------------------------------------------- +// Label lookup (WHATWG Encoding Standard §4.2) +// --------------------------------------------------------------------------- + +/// WHATWG encoding label mappings. +/// Labels are stored in lowercase; lookup normalizes input to lowercase. +const ENCODING_LABELS: &[(&str, Encoding)] = &[ + // UTF-8 labels + ("unicode-1-1-utf-8", Encoding::Utf8), + ("unicode11utf8", Encoding::Utf8), + ("unicode20utf8", Encoding::Utf8), + ("utf-8", Encoding::Utf8), + ("utf8", Encoding::Utf8), + ("x-unicode20utf8", Encoding::Utf8), + // UTF-16BE labels + ("unicodefffe", Encoding::Utf16Be), + ("utf-16be", Encoding::Utf16Be), + // UTF-16LE labels + ("csunicode", Encoding::Utf16Le), + ("iso-10646-ucs-2", Encoding::Utf16Le), + ("ucs-2", Encoding::Utf16Le), + ("unicode", Encoding::Utf16Le), + ("unicodefeff", Encoding::Utf16Le), + ("utf-16", Encoding::Utf16Le), + ("utf-16le", Encoding::Utf16Le), +]; + +/// Look up an encoding by its WHATWG label. +/// +/// Strips leading/trailing ASCII whitespace and compares case-insensitively, +/// per the WHATWG Encoding Standard. +pub fn lookup(label: &str) -> Option { + let trimmed = trim_ascii_whitespace(label); + if trimmed.is_empty() { + return None; + } + for &(name, enc) in ENCODING_LABELS { + if ascii_eq_ignore_case(trimmed, name) { + return Some(enc); + } + } + None +} + +/// Sniff BOM from the start of a byte slice. +/// +/// Returns the detected encoding (if any) and the remaining bytes after the BOM. +pub fn bom_sniff(bytes: &[u8]) -> (Option, &[u8]) { + if bytes.len() >= 3 && bytes[0] == 0xEF && bytes[1] == 0xBB && bytes[2] == 0xBF { + (Some(Encoding::Utf8), &bytes[3..]) + } else if bytes.len() >= 2 && bytes[0] == 0xFE && bytes[1] == 0xFF { + (Some(Encoding::Utf16Be), &bytes[2..]) + } else if bytes.len() >= 2 && bytes[0] == 0xFF && bytes[1] == 0xFE { + (Some(Encoding::Utf16Le), &bytes[2..]) + } else { + (None, bytes) + } +} + +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +/// Decode bytes to a `String` using the given encoding. +/// +/// Invalid sequences are replaced with U+FFFD (replacement mode per WHATWG spec). +pub fn decode(bytes: &[u8], encoding: Encoding) -> String { + // Replacement mode never fails + match encoding { + Encoding::Utf8 => utf8::decode_utf8(bytes, ErrorMode::Replacement).unwrap(), + Encoding::Utf16Le => utf16::decode_utf16le(bytes, ErrorMode::Replacement).unwrap(), + Encoding::Utf16Be => utf16::decode_utf16be(bytes, ErrorMode::Replacement).unwrap(), + } +} + +/// Decode bytes to a `String`, returning an error on any invalid sequence. +/// +/// Fatal mode per WHATWG spec — returns `Err` on the first invalid byte sequence. +pub fn decode_strict(bytes: &[u8], encoding: Encoding) -> Result { + match encoding { + Encoding::Utf8 => utf8::decode_utf8(bytes, ErrorMode::Fatal), + Encoding::Utf16Le => utf16::decode_utf16le(bytes, ErrorMode::Fatal), + Encoding::Utf16Be => utf16::decode_utf16be(bytes, ErrorMode::Fatal), + } +} + +/// Encode a string to bytes using the given encoding. +/// +/// Only UTF-8 encoding is supported for encode. Per WHATWG spec, UTF-16 +/// encodings are decode-only. +pub fn encode(text: &str, encoding: Encoding) -> Result> { + match encoding { + Encoding::Utf8 => Ok(utf8::encode_utf8(text)), + Encoding::Utf16Be => Err(EncodingError::EncodeNotSupported { + encoding: "UTF-16BE", + }), + Encoding::Utf16Le => Err(EncodingError::EncodeNotSupported { + encoding: "UTF-16LE", + }), + } +} + +// --------------------------------------------------------------------------- +// Internal helpers +// --------------------------------------------------------------------------- + +/// ASCII whitespace per WHATWG spec: TAB, LF, FF, CR, SPACE. +fn trim_ascii_whitespace(s: &str) -> &str { + let bytes = s.as_bytes(); + let start = bytes + .iter() + .position(|&b| !is_ascii_whitespace(b)) + .unwrap_or(bytes.len()); + let end = bytes + .iter() + .rposition(|&b| !is_ascii_whitespace(b)) + .map(|p| p + 1) + .unwrap_or(0); + if start >= end { + return ""; + } + &s[start..end] +} + +fn is_ascii_whitespace(b: u8) -> bool { + matches!(b, 0x09 | 0x0A | 0x0C | 0x0D | 0x20) +} + +fn ascii_eq_ignore_case(a: &str, b: &str) -> bool { + a.eq_ignore_ascii_case(b) +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + // -- Encoding enum -- + + #[test] + fn encoding_names() { + assert_eq!(Encoding::Utf8.name(), "UTF-8"); + assert_eq!(Encoding::Utf16Be.name(), "UTF-16BE"); + assert_eq!(Encoding::Utf16Le.name(), "UTF-16LE"); + } + + // -- Label lookup -- + + #[test] + fn lookup_utf8_labels() { + assert_eq!(lookup("utf-8"), Some(Encoding::Utf8)); + assert_eq!(lookup("UTF-8"), Some(Encoding::Utf8)); + assert_eq!(lookup("utf8"), Some(Encoding::Utf8)); + assert_eq!(lookup("Utf8"), Some(Encoding::Utf8)); + assert_eq!(lookup("unicode-1-1-utf-8"), Some(Encoding::Utf8)); + assert_eq!(lookup("x-unicode20utf8"), Some(Encoding::Utf8)); + } + + #[test] + fn lookup_utf16_labels() { + assert_eq!(lookup("utf-16be"), Some(Encoding::Utf16Be)); + assert_eq!(lookup("UTF-16BE"), Some(Encoding::Utf16Be)); + assert_eq!(lookup("unicodefffe"), Some(Encoding::Utf16Be)); + assert_eq!(lookup("utf-16le"), Some(Encoding::Utf16Le)); + assert_eq!(lookup("utf-16"), Some(Encoding::Utf16Le)); + assert_eq!(lookup("unicode"), Some(Encoding::Utf16Le)); + assert_eq!(lookup("ucs-2"), Some(Encoding::Utf16Le)); + assert_eq!(lookup("iso-10646-ucs-2"), Some(Encoding::Utf16Le)); + } + + #[test] + fn lookup_with_whitespace() { + assert_eq!(lookup(" utf-8 "), Some(Encoding::Utf8)); + assert_eq!(lookup("\tutf-8\n"), Some(Encoding::Utf8)); + assert_eq!(lookup("\r\nutf-16le\r\n"), Some(Encoding::Utf16Le)); + } + + #[test] + fn lookup_unknown() { + assert_eq!(lookup("latin1"), None); + assert_eq!(lookup(""), None); + assert_eq!(lookup(" "), None); + assert_eq!(lookup("utf-99"), None); + } + + // -- BOM sniffing -- + + #[test] + fn bom_utf8() { + let (enc, rest) = bom_sniff(&[0xEF, 0xBB, 0xBF, 0x41]); + assert_eq!(enc, Some(Encoding::Utf8)); + assert_eq!(rest, &[0x41]); + } + + #[test] + fn bom_utf16be() { + let (enc, rest) = bom_sniff(&[0xFE, 0xFF, 0x00, 0x41]); + assert_eq!(enc, Some(Encoding::Utf16Be)); + assert_eq!(rest, &[0x00, 0x41]); + } + + #[test] + fn bom_utf16le() { + let (enc, rest) = bom_sniff(&[0xFF, 0xFE, 0x41, 0x00]); + assert_eq!(enc, Some(Encoding::Utf16Le)); + assert_eq!(rest, &[0x41, 0x00]); + } + + #[test] + fn bom_none() { + let data = [0x41, 0x42, 0x43]; + let (enc, rest) = bom_sniff(&data); + assert_eq!(enc, None); + assert_eq!(rest, &data); + } + + #[test] + fn bom_empty() { + let (enc, rest) = bom_sniff(&[]); + assert_eq!(enc, None); + assert_eq!(rest, &[] as &[u8]); + } + + #[test] + fn bom_short() { + let (enc, rest) = bom_sniff(&[0xEF, 0xBB]); + assert_eq!(enc, None); + assert_eq!(rest, &[0xEF, 0xBB]); + } + + // -- Top-level decode -- + + #[test] + fn decode_utf8_basic() { + assert_eq!(decode(b"Hello", Encoding::Utf8), "Hello"); + } + + #[test] + fn decode_utf8_invalid_replaces() { + assert_eq!(decode(&[0xFF], Encoding::Utf8), "\u{FFFD}"); + } + + #[test] + fn decode_utf16le_basic() { + assert_eq!(decode(&[0x41, 0x00], Encoding::Utf16Le), "A"); + } + + #[test] + fn decode_utf16be_basic() { + assert_eq!(decode(&[0x00, 0x41], Encoding::Utf16Be), "A"); + } + + // -- Top-level decode_strict -- + + #[test] + fn decode_strict_valid() { + assert_eq!(decode_strict(b"Hello", Encoding::Utf8).unwrap(), "Hello"); + } + + #[test] + fn decode_strict_invalid() { + assert!(decode_strict(&[0xFF], Encoding::Utf8).is_err()); + } + + // -- Top-level encode -- + + #[test] + fn encode_utf8_basic() { + assert_eq!(encode("Hello", Encoding::Utf8).unwrap(), b"Hello"); + } + + #[test] + fn encode_utf16_not_supported() { + assert!(matches!( + encode("Hello", Encoding::Utf16Le), + Err(EncodingError::EncodeNotSupported { + encoding: "UTF-16LE" + }) + )); + assert!(matches!( + encode("Hello", Encoding::Utf16Be), + Err(EncodingError::EncodeNotSupported { + encoding: "UTF-16BE" + }) + )); + } + + // -- Trim helpers -- + + #[test] + fn trim_ascii_whitespace_basic() { + assert_eq!(trim_ascii_whitespace(" hello "), "hello"); + assert_eq!(trim_ascii_whitespace("hello"), "hello"); + assert_eq!(trim_ascii_whitespace(""), ""); + assert_eq!(trim_ascii_whitespace(" "), ""); + assert_eq!(trim_ascii_whitespace("\t\nhello\r\n"), "hello"); + } + + #[test] + fn ascii_eq_ignore_case_basic() { + assert!(ascii_eq_ignore_case("utf-8", "UTF-8")); + assert!(ascii_eq_ignore_case("Utf-8", "utf-8")); + assert!(!ascii_eq_ignore_case("utf-8", "utf-9")); + assert!(!ascii_eq_ignore_case("utf-8", "utf-8x")); + } +} diff --git a/crates/encoding/src/utf16.rs b/crates/encoding/src/utf16.rs new file mode 100644 index 0000000..9616b07 --- /dev/null +++ b/crates/encoding/src/utf16.rs @@ -0,0 +1,390 @@ +//! UTF-16 decoder per WHATWG Encoding Standard. + +use crate::error::{EncodingError, Result}; +use crate::utf8::ErrorMode; + +/// Decode a byte slice as UTF-16LE. +pub(crate) fn decode_utf16le(bytes: &[u8], mode: ErrorMode) -> Result { + decode_utf16(bytes, false, mode) +} + +/// Decode a byte slice as UTF-16BE. +pub(crate) fn decode_utf16be(bytes: &[u8], mode: ErrorMode) -> Result { + decode_utf16(bytes, true, mode) +} + +/// Shared UTF-16 decoder (WHATWG Encoding Standard §14.2). +fn decode_utf16(bytes: &[u8], big_endian: bool, mode: ErrorMode) -> Result { + let mut output = String::with_capacity(bytes.len() / 2); + let mut i = 0; + let mut lead_surrogate: Option = None; + let mut bom_checked = false; + + while i + 1 < bytes.len() { + let code_unit = if big_endian { + ((bytes[i] as u16) << 8) | (bytes[i + 1] as u16) + } else { + ((bytes[i + 1] as u16) << 8) | (bytes[i] as u16) + }; + i += 2; + + // BOM handling: strip BOM matching our endianness at the start + if !bom_checked { + bom_checked = true; + if code_unit == 0xFEFF { + // BOM matches our endianness — consume it + continue; + } + // 0xFFFE is NOT treated as a BOM — fall through to normal processing + } + + if is_lead_surrogate(code_unit) { + // If we already have an unpaired lead, emit error for it + if let Some(_prev) = lead_surrogate { + if mode == ErrorMode::Fatal { + return Err(EncodingError::InvalidSequence { + encoding: encoding_name(big_endian), + position: i - 4, // position of the previous unpaired lead + }); + } + output.push('\u{FFFD}'); + } + lead_surrogate = Some(code_unit); + } else if is_trail_surrogate(code_unit) { + if let Some(lead) = lead_surrogate.take() { + // Valid surrogate pair — compute supplementary code point + let cp = 0x10000 + ((lead as u32 - 0xD800) << 10) + (code_unit as u32 - 0xDC00); + let ch = char::from_u32(cp).unwrap_or('\u{FFFD}'); + output.push(ch); + } else { + // Trail surrogate without lead + if mode == ErrorMode::Fatal { + return Err(EncodingError::InvalidSequence { + encoding: encoding_name(big_endian), + position: i - 2, + }); + } + output.push('\u{FFFD}'); + } + } else { + // Regular BMP character + if let Some(_lead) = lead_surrogate.take() { + // Unpaired lead surrogate before this code unit + if mode == ErrorMode::Fatal { + return Err(EncodingError::InvalidSequence { + encoding: encoding_name(big_endian), + position: i - 4, + }); + } + output.push('\u{FFFD}'); + } + let ch = char::from_u32(code_unit as u32).unwrap_or('\u{FFFD}'); + output.push(ch); + } + } + + // Handle trailing single byte (odd byte count) + if i < bytes.len() { + // Flush any pending lead surrogate first + if lead_surrogate.take().is_some() { + if mode == ErrorMode::Fatal { + return Err(EncodingError::InvalidSequence { + encoding: encoding_name(big_endian), + position: i - 2, + }); + } + output.push('\u{FFFD}'); + } + if mode == ErrorMode::Fatal { + return Err(EncodingError::InvalidSequence { + encoding: encoding_name(big_endian), + position: i, + }); + } + output.push('\u{FFFD}'); + } else if lead_surrogate.is_some() { + // Unpaired lead surrogate at end of input + if mode == ErrorMode::Fatal { + return Err(EncodingError::InvalidSequence { + encoding: encoding_name(big_endian), + position: i - 2, + }); + } + output.push('\u{FFFD}'); + } + + Ok(output) +} + +fn is_lead_surrogate(cu: u16) -> bool { + (0xD800..=0xDBFF).contains(&cu) +} + +fn is_trail_surrogate(cu: u16) -> bool { + (0xDC00..=0xDFFF).contains(&cu) +} + +fn encoding_name(big_endian: bool) -> &'static str { + if big_endian { + "UTF-16BE" + } else { + "UTF-16LE" + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + fn le(bytes: &[u8]) -> String { + decode_utf16le(bytes, ErrorMode::Replacement).unwrap() + } + + fn be(bytes: &[u8]) -> String { + decode_utf16be(bytes, ErrorMode::Replacement).unwrap() + } + + // -- Basic ASCII -- + + #[test] + fn le_ascii() { + assert_eq!(le(&[0x41, 0x00]), "A"); + } + + #[test] + fn be_ascii() { + assert_eq!(be(&[0x00, 0x41]), "A"); + } + + #[test] + fn le_hello() { + assert_eq!(le(&[0x48, 0x00, 0x69, 0x00]), "Hi"); + } + + #[test] + fn be_hello() { + assert_eq!(be(&[0x00, 0x48, 0x00, 0x69]), "Hi"); + } + + // -- BMP characters -- + + #[test] + fn le_bmp() { + // U+00E9 (e with acute) = 0xE9 0x00 in LE + assert_eq!(le(&[0xE9, 0x00]), "\u{00E9}"); + } + + #[test] + fn be_bmp() { + // U+00E9 in BE = 0x00 0xE9 + assert_eq!(be(&[0x00, 0xE9]), "\u{00E9}"); + } + + #[test] + fn le_cjk() { + // U+4E16 = 0x16 0x4E in LE + assert_eq!(le(&[0x16, 0x4E]), "\u{4E16}"); + } + + // -- Surrogate pairs -- + + #[test] + fn le_surrogate_pair() { + // U+1F600 = D83D DE00 in UTF-16 + // LE: 3D D8 00 DE + assert_eq!(le(&[0x3D, 0xD8, 0x00, 0xDE]), "\u{1F600}"); + } + + #[test] + fn be_surrogate_pair() { + // U+1F600 = D83D DE00 in UTF-16 + // BE: D8 3D DE 00 + assert_eq!(be(&[0xD8, 0x3D, 0xDE, 0x00]), "\u{1F600}"); + } + + #[test] + fn le_supplementary_u10000() { + // U+10000 = D800 DC00 + // LE: 00 D8 00 DC + assert_eq!(le(&[0x00, 0xD8, 0x00, 0xDC]), "\u{10000}"); + } + + #[test] + fn le_supplementary_u10ffff() { + // U+10FFFF = DBFF DFFF + // LE: FF DB FF DF + assert_eq!(le(&[0xFF, 0xDB, 0xFF, 0xDF]), "\u{10FFFF}"); + } + + // -- Unpaired surrogates -- + + #[test] + fn le_unpaired_lead() { + // Lead surrogate D800 followed by non-surrogate 0041 + // LE: 00 D8 41 00 + assert_eq!(le(&[0x00, 0xD8, 0x41, 0x00]), "\u{FFFD}A"); + } + + #[test] + fn le_unpaired_trail() { + // Trail surrogate DC00 without lead + // LE: 00 DC + assert_eq!(le(&[0x00, 0xDC]), "\u{FFFD}"); + } + + #[test] + fn le_lead_at_end() { + // Lead surrogate at end of input + assert_eq!(le(&[0x00, 0xD8]), "\u{FFFD}"); + } + + #[test] + fn le_two_leads_in_a_row() { + // Two lead surrogates: D800 D801 — first is unpaired, second is unpaired at end + // LE: 00 D8 01 D8 + assert_eq!(le(&[0x00, 0xD8, 0x01, 0xD8]), "\u{FFFD}\u{FFFD}"); + } + + // -- BOM handling -- + + #[test] + fn le_bom_stripped() { + // UTF-16LE BOM: FF FE + assert_eq!(le(&[0xFF, 0xFE, 0x41, 0x00]), "A"); + } + + #[test] + fn be_bom_stripped() { + // UTF-16BE BOM: FE FF + assert_eq!(be(&[0xFE, 0xFF, 0x00, 0x41]), "A"); + } + + #[test] + fn le_wrong_bom_not_stripped() { + // FE FF is NOT the LE BOM — it's U+FEFF (ZWNBSP) + assert_eq!(le(&[0xFE, 0xFF]), "\u{FFFE}"); + } + + #[test] + fn be_wrong_bom_not_stripped() { + // FF FE is NOT the BE BOM — it's U+FFFE + assert_eq!(be(&[0xFF, 0xFE]), "\u{FFFE}"); + } + + #[test] + fn le_bom_only() { + assert_eq!(le(&[0xFF, 0xFE]), ""); + } + + #[test] + fn be_bom_only() { + assert_eq!(be(&[0xFE, 0xFF]), ""); + } + + // -- Odd byte count -- + + #[test] + fn le_odd_byte() { + assert_eq!(le(&[0x41, 0x00, 0x42]), "A\u{FFFD}"); + } + + #[test] + fn be_odd_byte() { + assert_eq!(be(&[0x00, 0x41, 0x42]), "A\u{FFFD}"); + } + + #[test] + fn single_byte() { + assert_eq!(le(&[0x41]), "\u{FFFD}"); + } + + // -- Empty input -- + + #[test] + fn empty_le() { + assert_eq!(le(&[]), ""); + } + + #[test] + fn empty_be() { + assert_eq!(be(&[]), ""); + } + + // -- Fatal mode -- + + #[test] + fn fatal_valid_le() { + assert_eq!( + decode_utf16le(&[0x41, 0x00], ErrorMode::Fatal).unwrap(), + "A" + ); + } + + #[test] + fn fatal_unpaired_lead_le() { + let err = decode_utf16le(&[0x00, 0xD8, 0x41, 0x00], ErrorMode::Fatal).unwrap_err(); + assert!(matches!( + err, + EncodingError::InvalidSequence { + encoding: "UTF-16LE", + .. + } + )); + } + + #[test] + fn fatal_unpaired_trail_le() { + let err = decode_utf16le(&[0x00, 0xDC], ErrorMode::Fatal).unwrap_err(); + assert!(matches!( + err, + EncodingError::InvalidSequence { + encoding: "UTF-16LE", + .. + } + )); + } + + #[test] + fn fatal_odd_byte_le() { + let err = decode_utf16le(&[0x41, 0x00, 0x42], ErrorMode::Fatal).unwrap_err(); + assert!(matches!( + err, + EncodingError::InvalidSequence { + encoding: "UTF-16LE", + .. + } + )); + } + + // -- Mixed content -- + + #[test] + fn le_mixed_bmp_and_supplementary() { + // "A" + U+1F600 + "B" + // LE: 41 00 | 3D D8 00 DE | 42 00 + assert_eq!( + le(&[0x41, 0x00, 0x3D, 0xD8, 0x00, 0xDE, 0x42, 0x00]), + "A\u{1F600}B" + ); + } + + #[test] + fn be_mixed_bmp_and_supplementary() { + // "A" + U+1F600 + "B" + // BE: 00 41 | D8 3D DE 00 | 00 42 + assert_eq!( + be(&[0x00, 0x41, 0xD8, 0x3D, 0xDE, 0x00, 0x00, 0x42]), + "A\u{1F600}B" + ); + } + + #[test] + fn le_null_character() { + // U+0000 = 00 00 in LE + assert_eq!(le(&[0x00, 0x00]), "\0"); + } +} diff --git a/crates/encoding/src/utf8.rs b/crates/encoding/src/utf8.rs new file mode 100644 index 0000000..03bc27b --- /dev/null +++ b/crates/encoding/src/utf8.rs @@ -0,0 +1,486 @@ +//! UTF-8 decoder and encoder per WHATWG Encoding Standard. + +use crate::error::{EncodingError, Result}; + +/// Error handling mode. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ErrorMode { + Replacement, + Fatal, +} + +/// Decode a byte slice as UTF-8. +/// +/// In replacement mode, invalid sequences are replaced with U+FFFD. +/// In fatal mode, the first invalid sequence causes an error. +pub(crate) fn decode_utf8(bytes: &[u8], mode: ErrorMode) -> Result { + // Strip UTF-8 BOM if present + let bytes = if bytes.len() >= 3 && bytes[0] == 0xEF && bytes[1] == 0xBB && bytes[2] == 0xBF { + &bytes[3..] + } else { + bytes + }; + + let mut output = String::with_capacity(bytes.len()); + let mut decoder = Utf8Decoder::new(); + let mut i = 0; + + while i < bytes.len() { + match decoder.process_byte(bytes[i]) { + DecoderResult::CodePoint(ch) => { + output.push(ch); + i += 1; + } + DecoderResult::Error(error_pos) => { + if mode == ErrorMode::Fatal { + return Err(EncodingError::InvalidSequence { + encoding: "UTF-8", + position: error_pos, + }); + } + output.push('\u{FFFD}'); + i += 1; + } + DecoderResult::ErrorPrepend(error_pos) => { + if mode == ErrorMode::Fatal { + return Err(EncodingError::InvalidSequence { + encoding: "UTF-8", + position: error_pos, + }); + } + output.push('\u{FFFD}'); + // Do NOT advance i — re-process this byte + } + DecoderResult::Continue => { + i += 1; + } + } + } + + // Handle incomplete sequence at end of input + if decoder.bytes_needed > 0 { + if mode == ErrorMode::Fatal { + return Err(EncodingError::InvalidSequence { + encoding: "UTF-8", + position: bytes.len().saturating_sub(decoder.bytes_seen as usize), + }); + } + output.push('\u{FFFD}'); + } + + Ok(output) +} + +/// Encode a string as UTF-8 bytes. +/// +/// Since Rust strings are already valid UTF-8, this is a straightforward copy. +pub(crate) fn encode_utf8(text: &str) -> Vec { + text.as_bytes().to_vec() +} + +// --------------------------------------------------------------------------- +// Streaming UTF-8 decoder (WHATWG Encoding Standard §8.1.1) +// --------------------------------------------------------------------------- + +enum DecoderResult { + /// A valid code point was decoded. + CodePoint(char), + /// An error occurred at the given byte position; advance to next byte. + Error(usize), + /// An error occurred at the given byte position; re-process current byte. + ErrorPrepend(usize), + /// More bytes needed; continue feeding. + Continue, +} + +struct Utf8Decoder { + code_point: u32, + bytes_seen: u8, + bytes_needed: u8, + lower_boundary: u8, + upper_boundary: u8, + /// Position of the start of the current multi-byte sequence. + sequence_start: usize, + /// Total bytes processed so far. + position: usize, +} + +impl Utf8Decoder { + fn new() -> Self { + Self { + code_point: 0, + bytes_seen: 0, + bytes_needed: 0, + lower_boundary: 0x80, + upper_boundary: 0xBF, + sequence_start: 0, + position: 0, + } + } + + fn process_byte(&mut self, byte: u8) -> DecoderResult { + let pos = self.position; + self.position += 1; + + if self.bytes_needed == 0 { + match byte { + 0x00..=0x7F => DecoderResult::CodePoint(byte as char), + 0xC2..=0xDF => { + self.bytes_needed = 1; + self.code_point = (byte & 0x1F) as u32; + self.sequence_start = pos; + DecoderResult::Continue + } + 0xE0 => { + self.bytes_needed = 2; + self.lower_boundary = 0xA0; + self.code_point = (byte & 0x0F) as u32; + self.sequence_start = pos; + DecoderResult::Continue + } + 0xE1..=0xEC | 0xEE..=0xEF => { + self.bytes_needed = 2; + self.code_point = (byte & 0x0F) as u32; + self.sequence_start = pos; + DecoderResult::Continue + } + 0xED => { + self.bytes_needed = 2; + self.upper_boundary = 0x9F; + self.code_point = (byte & 0x0F) as u32; + self.sequence_start = pos; + DecoderResult::Continue + } + 0xF0 => { + self.bytes_needed = 3; + self.lower_boundary = 0x90; + self.code_point = (byte & 0x07) as u32; + self.sequence_start = pos; + DecoderResult::Continue + } + 0xF1..=0xF3 => { + self.bytes_needed = 3; + self.code_point = (byte & 0x07) as u32; + self.sequence_start = pos; + DecoderResult::Continue + } + 0xF4 => { + self.bytes_needed = 3; + self.upper_boundary = 0x8F; + self.code_point = (byte & 0x07) as u32; + self.sequence_start = pos; + DecoderResult::Continue + } + _ => { + // 0x80..=0xC1, 0xF5..=0xFF: invalid lead byte + DecoderResult::Error(pos) + } + } + } else { + // Expecting continuation byte + if byte < self.lower_boundary || byte > self.upper_boundary { + // Invalid continuation — reset and prepend byte + let err_pos = self.sequence_start; + self.reset(); + self.position -= 1; // will be re-processed + return DecoderResult::ErrorPrepend(err_pos); + } + + // Valid continuation byte + self.lower_boundary = 0x80; + self.upper_boundary = 0xBF; + self.code_point = (self.code_point << 6) | (byte & 0x3F) as u32; + self.bytes_seen += 1; + + if self.bytes_seen == self.bytes_needed { + let cp = self.code_point; + self.reset(); + // The WHATWG state machine guarantees valid scalar values here, + // but use fallback for defense-in-depth. + let ch = char::from_u32(cp).unwrap_or('\u{FFFD}'); + DecoderResult::CodePoint(ch) + } else { + DecoderResult::Continue + } + } + } + + fn reset(&mut self) { + self.code_point = 0; + self.bytes_seen = 0; + self.bytes_needed = 0; + self.lower_boundary = 0x80; + self.upper_boundary = 0xBF; + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + fn decode_replace(bytes: &[u8]) -> String { + decode_utf8(bytes, ErrorMode::Replacement).unwrap() + } + + fn decode_fatal(bytes: &[u8]) -> Result { + decode_utf8(bytes, ErrorMode::Fatal) + } + + // -- Basic ASCII -- + + #[test] + fn ascii_roundtrip() { + assert_eq!(decode_replace(b"Hello, world!"), "Hello, world!"); + } + + #[test] + fn empty_input() { + assert_eq!(decode_replace(b""), ""); + } + + #[test] + fn null_byte() { + assert_eq!(decode_replace(&[0x00]), "\0"); + } + + // -- Multi-byte sequences -- + + #[test] + fn two_byte_sequence() { + // U+00E9 (e with acute) = 0xC3 0xA9 + assert_eq!(decode_replace(&[0xC3, 0xA9]), "\u{00E9}"); + } + + #[test] + fn three_byte_sequence() { + // U+4E16 (CJK character) = 0xE4 0xB8 0x96 + assert_eq!(decode_replace(&[0xE4, 0xB8, 0x96]), "\u{4E16}"); + } + + #[test] + fn four_byte_sequence() { + // U+1F600 (grinning face) = 0xF0 0x9F 0x98 0x80 + assert_eq!(decode_replace(&[0xF0, 0x9F, 0x98, 0x80]), "\u{1F600}"); + } + + #[test] + fn mixed_ascii_and_multibyte() { + // "Caf\u{00E9}" = [0x43, 0x61, 0x66, 0xC3, 0xA9] + assert_eq!( + decode_replace(&[0x43, 0x61, 0x66, 0xC3, 0xA9]), + "Caf\u{00E9}" + ); + } + + // -- BOM handling -- + + #[test] + fn bom_stripped() { + // UTF-8 BOM + "A" + assert_eq!(decode_replace(&[0xEF, 0xBB, 0xBF, 0x41]), "A"); + } + + #[test] + fn bom_only() { + assert_eq!(decode_replace(&[0xEF, 0xBB, 0xBF]), ""); + } + + // -- Invalid sequences (replacement mode) -- + + #[test] + fn invalid_byte_ff() { + assert_eq!(decode_replace(&[0xFF]), "\u{FFFD}"); + } + + #[test] + fn invalid_byte_fe() { + assert_eq!(decode_replace(&[0xFE]), "\u{FFFD}"); + } + + #[test] + fn invalid_continuation_byte_standalone() { + // 0x80 without a lead byte + assert_eq!(decode_replace(&[0x80]), "\u{FFFD}"); + } + + #[test] + fn overlong_two_byte() { + // 0xC0 0xAF is an overlong encoding of U+002F ('/') + // 0xC0 is always invalid (lead byte rejected), 0xAF is a continuation + // byte without a lead (also invalid) — both produce U+FFFD + assert_eq!(decode_replace(&[0xC0, 0xAF]), "\u{FFFD}\u{FFFD}"); + } + + #[test] + fn truncated_two_byte() { + // 0xC3 without continuation + assert_eq!(decode_replace(&[0xC3]), "\u{FFFD}"); + } + + #[test] + fn truncated_three_byte() { + // 0xE4 0xB8 without third byte + assert_eq!(decode_replace(&[0xE4, 0xB8]), "\u{FFFD}"); + } + + #[test] + fn truncated_four_byte() { + // 0xF0 0x9F 0x98 without fourth byte + assert_eq!(decode_replace(&[0xF0, 0x9F, 0x98]), "\u{FFFD}"); + } + + #[test] + fn surrogate_half_rejected() { + // U+D800 would encode as 0xED 0xA0 0x80, but surrogates are invalid in UTF-8 + // 0xED with upper_boundary 0x9F rejects 0xA0 + assert_eq!( + decode_replace(&[0xED, 0xA0, 0x80]), + "\u{FFFD}\u{FFFD}\u{FFFD}" + ); + } + + #[test] + fn invalid_continuation_mid_sequence() { + // 0xE4 expects continuation, but 0x41 is ASCII — error + prepend + assert_eq!(decode_replace(&[0xE4, 0x41]), "\u{FFFD}A"); + } + + #[test] + fn invalid_between_valid() { + // Valid 'A', invalid 0xFF, valid 'B' + assert_eq!(decode_replace(&[0x41, 0xFF, 0x42]), "A\u{FFFD}B"); + } + + #[test] + fn multiple_errors_in_a_row() { + assert_eq!( + decode_replace(&[0xFE, 0xFF, 0xFE]), + "\u{FFFD}\u{FFFD}\u{FFFD}" + ); + } + + // -- Fatal mode -- + + #[test] + fn fatal_valid() { + assert_eq!(decode_fatal(b"Hello").unwrap(), "Hello"); + } + + #[test] + fn fatal_invalid() { + let err = decode_fatal(&[0x41, 0xFF]).unwrap_err(); + assert!(matches!( + err, + EncodingError::InvalidSequence { + encoding: "UTF-8", + position: 1 + } + )); + } + + #[test] + fn fatal_truncated() { + let err = decode_fatal(&[0xC3]).unwrap_err(); + assert!(matches!( + err, + EncodingError::InvalidSequence { + encoding: "UTF-8", + .. + } + )); + } + + // -- Encoder -- + + #[test] + fn encode_ascii() { + assert_eq!(encode_utf8("Hello"), b"Hello"); + } + + #[test] + fn encode_multibyte() { + assert_eq!(encode_utf8("\u{00E9}"), &[0xC3, 0xA9]); + } + + #[test] + fn encode_emoji() { + assert_eq!(encode_utf8("\u{1F600}"), &[0xF0, 0x9F, 0x98, 0x80]); + } + + #[test] + fn encode_empty() { + assert_eq!(encode_utf8(""), b""); + } + + #[test] + fn roundtrip() { + let original = "Hello \u{4E16}\u{754C} \u{1F600}"; + let encoded = encode_utf8(original); + let decoded = decode_replace(&encoded); + assert_eq!(decoded, original); + } + + // -- Edge cases -- + + #[test] + fn max_two_byte() { + // U+07FF = 0xDF 0xBF + assert_eq!(decode_replace(&[0xDF, 0xBF]), "\u{07FF}"); + } + + #[test] + fn min_three_byte() { + // U+0800 = 0xE0 0xA0 0x80 + assert_eq!(decode_replace(&[0xE0, 0xA0, 0x80]), "\u{0800}"); + } + + #[test] + fn max_three_byte() { + // U+FFFF = 0xEF 0xBF 0xBF + assert_eq!(decode_replace(&[0xEF, 0xBF, 0xBF]), "\u{FFFF}"); + } + + #[test] + fn min_four_byte() { + // U+10000 = 0xF0 0x90 0x80 0x80 + assert_eq!(decode_replace(&[0xF0, 0x90, 0x80, 0x80]), "\u{10000}"); + } + + #[test] + fn max_unicode() { + // U+10FFFF = 0xF4 0x8F 0xBF 0xBF + assert_eq!(decode_replace(&[0xF4, 0x8F, 0xBF, 0xBF]), "\u{10FFFF}"); + } + + #[test] + fn above_max_unicode_rejected() { + // 0xF4 0x90 would start U+110000, which is above max + // 0xF4 has upper_boundary = 0x8F, so 0x90 is rejected + assert_eq!( + decode_replace(&[0xF4, 0x90, 0x80, 0x80]), + "\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}" + ); + } + + #[test] + fn overlong_three_byte_rejected() { + // 0xE0 requires lower_boundary = 0xA0, so 0xE0 0x80 0x80 is rejected + assert_eq!( + decode_replace(&[0xE0, 0x80, 0x80]), + "\u{FFFD}\u{FFFD}\u{FFFD}" + ); + } + + #[test] + fn overlong_four_byte_rejected() { + // 0xF0 requires lower_boundary = 0x90, so 0xF0 0x80 0x80 0x80 is rejected + assert_eq!( + decode_replace(&[0xF0, 0x80, 0x80, 0x80]), + "\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}" + ); + } +}