diff --git a/crates/core/src/ir.rs b/crates/core/src/ir.rs index 2b03abb..e0860e2 100644 --- a/crates/core/src/ir.rs +++ b/crates/core/src/ir.rs @@ -28,6 +28,7 @@ pub enum ScalarRepresentation { #[derive(Debug, Clone, PartialEq, Eq)] pub enum HeapRepresentation { String, + BitArray, Tuple, List, Record, @@ -283,6 +284,15 @@ pub enum ExpressionKind { Branch(Branch), Tuple(Vec), List(Vec), + BitArray(BitArrayLiteral), + BitArrayConcat { + left: Box, + right: Box, + }, + BitStringDeconstruct { + bit_array: Box, + segments: Vec, + }, Record(RecordValue), Constructor(ConstructorValue), FieldAccess { @@ -319,6 +329,51 @@ pub enum ExpressionKind { Failure(FailurePath), } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BitArrayLiteral { + pub segments: Vec, + pub bit_len: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BitArraySegment { + pub value: u64, + pub bit_size: u32, + pub type_: BitSegmentType, + pub options: Vec, + pub span: Span, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BitStringPatternSegment { + pub binding: Option, + pub bit_size: Option, + pub type_: BitSegmentType, + pub options: Vec, + pub span: Span, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum BitSegmentType { + Integer, + Float, + Binary, + Utf8, + Utf16, + Utf32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum BitSegmentOption { + Size(u32), + Unit(u32), + Signed, + Unsigned, + BigEndian, + LittleEndian, + NativeEndian, +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct DirectCall { pub function: String, @@ -489,6 +544,7 @@ pub enum IrPattern { name: String, arguments: Vec, }, + BitString(Vec), } #[derive(Debug, Clone, PartialEq, Eq)] @@ -799,6 +855,11 @@ impl Lowerer { }, }) } + AstExpression::Raw(raw) if raw.kind == "bit_string" => Some(Expression { + type_: Type::BitArray, + span: raw.span, + kind: ExpressionKind::BitArray(bit_array_literal(raw)), + }), AstExpression::Raw(raw) if raw.kind == "record" => { let type_ = self.typed_expression_type(raw.span).unwrap_or(Type::Nil); Some(Expression { @@ -982,10 +1043,11 @@ impl Lowerer { context.bind(alias.alias.text.clone(), local.id); Some(IrPattern::Alias { pattern: Box::new(inner), local: local.id }) } - Pattern::BitString(raw) => { - self.unsupported_pattern(raw.span, "bit string pattern"); - None - } + Pattern::BitString(raw) => Some(IrPattern::BitString(bit_string_pattern_segments( + context, + raw, + subject_type, + ))), Pattern::Raw(raw) => { self.diagnostics.push( Diagnostic::new( @@ -1023,19 +1085,13 @@ impl Lowerer { | IrPattern::Literal(_) | IrPattern::Tuple(_) | IrPattern::List { .. } - | IrPattern::Constructor { .. } => { + | IrPattern::Constructor { .. } + | IrPattern::BitString(_) => { let _ = context; } } } - fn unsupported_pattern(&mut self, span: Span, kind: &str) { - self.diagnostics.push( - Diagnostic::new(DiagnosticCode::LoweringError, format!("{kind} cannot be lowered")) - .with_label(Label::primary(span, "unsupported pattern here")), - ); - } - fn typed_expression_type(&self, span: Span) -> Option { self.expression_types.get(&span).cloned() } @@ -1157,6 +1213,18 @@ fn collect_successful_bindings( ); } } + IrPattern::BitString(segments) => { + let subject = binding_subject(&path); + for (index, segment) in segments.iter().enumerate() { + if let Some(local) = segment.binding { + bindings.push(SuccessfulBinding { + local, + path: BindingPath::ListElement { subject, index }, + span: context.local(local).span, + }); + } + } + } IrPattern::Discard | IrPattern::Literal(_) => {} } } @@ -1172,6 +1240,107 @@ fn binding_subject(path: &BindingPath) -> usize { } } +fn bit_array_literal(raw: &ast::RawSyntax) -> BitArrayLiteral { + let segments = bit_array_segments(raw); + let bit_len = segments.iter().map(|segment| segment.bit_size).sum(); + BitArrayLiteral { segments, bit_len } +} + +fn bit_array_segments(raw: &ast::RawSyntax) -> Vec { + let Some(inner) = raw + .source + .trim() + .strip_prefix("<<") + .and_then(|source| source.strip_suffix(">>")) + else { + return Vec::new(); + }; + inner + .split(',') + .filter_map(|segment| bit_array_segment(segment.trim(), raw.span)) + .collect() +} + +fn bit_array_segment(source: &str, span: Span) -> Option { + if source.is_empty() { + return None; + } + let (value, options) = source.split_once(':').unwrap_or((source, "")); + let value = value.trim().parse::().ok()?; + let options = bit_segment_options(options); + let bit_size = options + .iter() + .find_map(|option| match option { + BitSegmentOption::Size(size) => Some(*size), + _ => None, + }) + .unwrap_or(8); + Some(BitArraySegment { value, bit_size, type_: BitSegmentType::Integer, options, span }) +} + +fn bit_segment_options(source: &str) -> Vec { + source + .split('-') + .filter_map(|option| { + let option = option.trim(); + if option.is_empty() { + return None; + } + match option { + "signed" => Some(BitSegmentOption::Signed), + "unsigned" => Some(BitSegmentOption::Unsigned), + "big" => Some(BitSegmentOption::BigEndian), + "little" => Some(BitSegmentOption::LittleEndian), + "native" => Some(BitSegmentOption::NativeEndian), + "float" => None, + "binary" | "bytes" | "bits" | "bit_string" | "utf8" | "utf16" | "utf32" => None, + _ if let Some(size) = option.strip_prefix("size(").and_then(|value| value.strip_suffix(')')) => { + size.parse().ok().map(BitSegmentOption::Size) + } + _ if let Some(unit) = option.strip_prefix("unit(").and_then(|value| value.strip_suffix(')')) => { + unit.parse().ok().map(BitSegmentOption::Unit) + } + _ => option.parse().ok().map(BitSegmentOption::Size), + } + }) + .collect() +} + +fn bit_string_pattern_segments( + context: &mut FunctionContext, raw: &ast::RawSyntax, subject_type: &Type, +) -> Vec { + let Some(inner) = raw + .source + .trim() + .strip_prefix("<<") + .and_then(|source| source.strip_suffix(">>")) + else { + return Vec::new(); + }; + inner + .split(',') + .map(|segment| bit_string_pattern_segment(context, segment.trim(), raw.span, subject_type)) + .collect() +} + +fn bit_string_pattern_segment( + context: &mut FunctionContext, source: &str, span: Span, subject_type: &Type, +) -> BitStringPatternSegment { + let (value, options) = source.split_once(':').unwrap_or((source, "")); + let options = bit_segment_options(options); + let bit_size = options.iter().find_map(|option| match option { + BitSegmentOption::Size(size) => Some(*size), + _ => None, + }); + let binding = value.trim().chars().next().filter(|char| char.is_lowercase()).map(|_| { + let name = ast::Name { span, text: value.trim().into() }; + let local = context.allocate(&name, subject_type.clone()); + context.bind(name.text, local.id); + local.id + }); + BitStringPatternSegment { binding, bit_size, type_: BitSegmentType::Integer, options, span } +} + fn type_for_literal(kind: LiteralKind) -> Type { match kind { LiteralKind::Int => Type::Int, @@ -1209,6 +1378,7 @@ impl From<&Type> for RepresentationType { Type::Bool => Self::Scalar(ScalarRepresentation::I32), Type::Nil => Self::Scalar(ScalarRepresentation::Unit), Type::String => Self::HeapManaged(HeapRepresentation::String), + Type::BitArray => Self::HeapManaged(HeapRepresentation::BitArray), Type::Tuple(_) => Self::HeapManaged(HeapRepresentation::Tuple), Type::List(_) => Self::HeapManaged(HeapRepresentation::List), Type::Record { .. } => Self::HeapManaged(HeapRepresentation::Record), @@ -1491,6 +1661,17 @@ mod tests { )); } + #[test] + fn lowers_bit_arrays_to_managed_value_forms() { + let module = lower_source("fn bits() { <<1, 2:size(4), 3>> }"); + let bits = &module.functions[0]; + + assert!(matches!( + bits.body.result.kind, + ExpressionKind::BitArray(BitArrayLiteral { bit_len: 20, .. }) + )); + } + #[test] fn represents_runtime_managed_value_types() { assert_eq!( @@ -1505,6 +1686,10 @@ mod tests { RepresentationType::from(&Type::String), RepresentationType::HeapManaged(HeapRepresentation::String) ); + assert_eq!( + RepresentationType::from(&Type::BitArray), + RepresentationType::HeapManaged(HeapRepresentation::BitArray) + ); } #[test] diff --git a/crates/core/src/resolve.rs b/crates/core/src/resolve.rs index a226a0c..ffe71d6 100644 --- a/crates/core/src/resolve.rs +++ b/crates/core/src/resolve.rs @@ -187,7 +187,7 @@ impl Resolver { } fn collect_prelude(&mut self, scope: ScopeId) { - for name in ["Int", "Float", "String", "Bool", "Nil", "List", "Result"] { + for name in ["Int", "Float", "String", "BitArray", "Bool", "Nil", "List", "Result"] { let name = ast::Name { span: self.module.span, text: name.into() }; self.define(scope, &name, Namespace::Type, SymbolKind::Prelude); } diff --git a/crates/core/src/runtime.rs b/crates/core/src/runtime.rs index 8db196e..15186c1 100644 --- a/crates/core/src/runtime.rs +++ b/crates/core/src/runtime.rs @@ -16,6 +16,10 @@ impl Layout { self.header_size + self.align_to(byte_len) } + pub fn bit_array_size(self, bit_len: u32) -> u32 { + self.header_size + self.align_to(bit_array_payload_len(bit_len)) + } + pub fn list_cons_size(self, value_size: u32) -> u32 { self.align_to(self.header_size + value_size + self.word_size) } @@ -56,6 +60,7 @@ pub enum ObjectTag { Record, Custom, Closure, + BitArray, } impl From for u32 { @@ -67,6 +72,7 @@ impl From for u32 { ObjectTag::Record => 4, ObjectTag::Custom => 5, ObjectTag::Closure => 6, + ObjectTag::BitArray => 7, } } } @@ -82,6 +88,7 @@ impl TryFrom for ObjectTag { 4 => Ok(ObjectTag::Record), 5 => Ok(ObjectTag::Custom), 6 => Ok(ObjectTag::Closure), + 7 => Ok(ObjectTag::BitArray), _ => Err(()), } } @@ -110,6 +117,66 @@ pub fn string_object(config: RuntimeConfig, offset: u32, string: &str) -> Static StaticObject { offset, bytes } } +pub fn bit_array_object(config: RuntimeConfig, offset: u32, data: &[u8], bit_len: u32) -> StaticObject { + assert!(bit_len <= data.len() as u32 * 8, "bit length exceeds payload bytes"); + let size = config.layout.bit_array_size(bit_len); + let payload_len = bit_array_payload_len(bit_len) as usize; + let mut bytes = Vec::with_capacity(size as usize); + bytes.extend_from_slice(&u32::from(ObjectTag::BitArray).to_le_bytes()); + bytes.extend_from_slice(&bit_len.to_le_bytes()); + bytes.extend_from_slice(&data[..payload_len]); + bytes.resize(size as usize, 0); + StaticObject { offset, bytes } +} + +pub fn bit_array_payload_len(bit_len: u32) -> u32 { + bit_len.div_ceil(8) +} + +pub fn bit_array_get_bit(data: &[u8], bit_len: u32, index: u32) -> Option { + if index >= bit_len { + return None; + } + let byte = data.get((index / 8) as usize)?; + let shift = 7 - index % 8; + Some((byte >> shift) & 1) +} + +pub fn bit_array_slice(data: &[u8], source_bit_len: u32, start: u32, bit_len: u32) -> Option> { + if start.checked_add(bit_len)? > source_bit_len { + return None; + } + let mut output = vec![0; bit_array_payload_len(bit_len) as usize]; + for offset in 0..bit_len { + let bit = bit_array_get_bit(data, source_bit_len, start + offset)?; + bit_array_set_bit(&mut output, offset, bit); + } + Some(output) +} + +pub fn bit_array_append(left: &[u8], left_bit_len: u32, right: &[u8], right_bit_len: u32) -> Vec { + let bit_len = left_bit_len + right_bit_len; + let mut output = vec![0; bit_array_payload_len(bit_len) as usize]; + for index in 0..left_bit_len { + let bit = bit_array_get_bit(left, left_bit_len, index).expect("left bit in range"); + bit_array_set_bit(&mut output, index, bit); + } + for index in 0..right_bit_len { + let bit = bit_array_get_bit(right, right_bit_len, index).expect("right bit in range"); + bit_array_set_bit(&mut output, left_bit_len + index, bit); + } + output +} + +fn bit_array_set_bit(data: &mut [u8], index: u32, bit: u8) { + if bit & 1 == 0 { + return; + } + let byte = &mut data[(index / 8) as usize]; + let shift = 7 - index % 8; + *byte |= 1 << shift; +} + #[cfg(test)] mod tests { use super::*; @@ -117,7 +184,9 @@ mod tests { #[test] fn converts_object_tags_to_and_from_u32() { assert_eq!(u32::from(ObjectTag::String), 1); + assert_eq!(u32::from(ObjectTag::BitArray), 7); assert_eq!(ObjectTag::try_from(6), Ok(ObjectTag::Closure)); + assert_eq!(ObjectTag::try_from(7), Ok(ObjectTag::BitArray)); assert_eq!(ObjectTag::try_from(99), Err(())); } @@ -139,10 +208,44 @@ mod tests { assert_eq!(object.bytes.len() as u32, 16); } + #[test] + fn encodes_static_bit_arrays_with_header_and_padding() { + let object = bit_array_object( + RuntimeConfig::DEFAULT, + RuntimeConfig::DEFAULT.static_data_start, + &[0b1010_0101, 0b1100_0000], + 10, + ); + + assert_eq!(object.offset, 1024); + assert_eq!( + ObjectTag::try_from(u32::from_le_bytes(object.bytes[0..4].try_into().unwrap())), + Ok(ObjectTag::BitArray) + ); + assert_eq!(u32::from_le_bytes(object.bytes[4..8].try_into().unwrap()), 10); + assert_eq!(&object.bytes[8..10], &[0b1010_0101, 0b1100_0000]); + assert_eq!(object.bytes.len() as u32, 16); + } + + #[test] + fn slices_and_appends_bit_arrays() { + let data = [0b1010_0101, 0b1100_0000]; + + assert_eq!(bit_array_get_bit(&data, 10, 0), Some(1)); + assert_eq!(bit_array_get_bit(&data, 10, 1), Some(0)); + assert_eq!(bit_array_get_bit(&data, 10, 10), None); + assert_eq!(bit_array_slice(&data, 10, 2, 5), Some(vec![0b1001_0000])); + assert_eq!( + bit_array_append(&[0b1010_0000], 4, &[0b1100_0000], 4), + vec![0b1010_1100] + ); + } + #[test] fn computes_aligned_object_sizes() { let layout = Layout::DEFAULT; + assert_eq!(layout.bit_array_size(10), 16); assert_eq!(layout.list_cons_size(8), 24); assert_eq!(layout.tuple_size(3, 8), 32); assert_eq!(layout.record_size(2, 8), 24); diff --git a/crates/core/src/types.rs b/crates/core/src/types.rs index eda6a3a..1c198ed 100644 --- a/crates/core/src/types.rs +++ b/crates/core/src/types.rs @@ -14,6 +14,7 @@ pub enum Type { Int, Float, String, + BitArray, Bool, Nil, Tuple(Vec), @@ -357,6 +358,7 @@ impl TypeChecker { self.pop_scope(); type_ } + Expression::Raw(raw) if raw.kind == "bit_string" => Type::BitArray, Expression::Raw(raw) if raw.kind == "record" => { let Some(name) = raw.source.split(['(', ' ']).next() else { return None; @@ -447,10 +449,7 @@ impl TypeChecker { self.bind_pattern(&alias.pattern, type_); self.define(alias.alias.text.clone(), type_.clone()); } - Pattern::BitString(raw) => self.diagnostics.push( - Diagnostic::new(DiagnosticCode::TypeError, "bit string patterns cannot be typed yet") - .with_label(Label::primary(raw.span, "bit string pattern here")), - ), + Pattern::BitString(raw) => self.expect_same(&Type::BitArray, type_, raw.span), Pattern::Raw(raw) => self.diagnostics.push( Diagnostic::new(DiagnosticCode::TypeError, format!("unsupported pattern `{}`", raw.kind)) .with_label(Label::primary(raw.span, "unsupported pattern here")), @@ -758,6 +757,7 @@ impl Type { Type::Int => "Int".into(), Type::Float => "Float".into(), Type::String => "String".into(), + Type::BitArray => "BitArray".into(), Type::Bool => "Bool".into(), Type::Nil => "Nil".into(), Type::Tuple(items) => format!("#({})", items.iter().map(Type::display).collect::>().join(", ")), @@ -886,6 +886,7 @@ fn parse_type_source(source: &str) -> Option { "Int" => Some(Type::Int), "Float" => Some(Type::Float), "String" => Some(Type::String), + "BitArray" => Some(Type::BitArray), "Bool" => Some(Type::Bool), "Nil" => Some(Type::Nil), _ if source.starts_with("fn(") => parse_function_type(source), @@ -1138,6 +1139,7 @@ mod tests { #[test] fn parses_generic_tuple_list_and_custom_annotations() { + assert_eq!(Type::from_source("BitArray"), Some(Type::BitArray)); assert_eq!(Type::from_source("List(Int)"), Some(Type::List(Box::new(Type::Int)))); assert_eq!( Type::from_source("#(String, Int)"), diff --git a/crates/core/src/wasm.rs b/crates/core/src/wasm.rs index e0fbd09..fa152b7 100644 --- a/crates/core/src/wasm.rs +++ b/crates/core/src/wasm.rs @@ -161,11 +161,17 @@ impl Emitter { writeln!(self.functions, " call ${}", call.function).expect("write WAT"); } ExpressionKind::Branch(branch) => self.branch(branch, &expression.type_, expression.span), + ExpressionKind::BitArray(bit_array) => { + let pointer = self.static_bit_array(bit_array); + writeln!(self.functions, " i32.const {pointer}").expect("write WAT"); + } ExpressionKind::IndirectCall(_) | ExpressionKind::FunctionValue(_) | ExpressionKind::AnonymousFunction(_) | ExpressionKind::Pipeline(_) | ExpressionKind::Use(_) + | ExpressionKind::BitArrayConcat { .. } + | ExpressionKind::BitStringDeconstruct { .. } | ExpressionKind::Tuple(_) | ExpressionKind::List(_) | ExpressionKind::Record(_) @@ -268,7 +274,8 @@ impl Emitter { | ir::IrPattern::Literal(_) | ir::IrPattern::Tuple(_) | ir::IrPattern::List { .. } - | ir::IrPattern::Constructor { .. } => {} + | ir::IrPattern::Constructor { .. } + | ir::IrPattern::BitString(_) => {} } } @@ -292,7 +299,10 @@ impl Emitter { ), } } - ir::IrPattern::Tuple(_) | ir::IrPattern::List { .. } | ir::IrPattern::Constructor { .. } => { + ir::IrPattern::Tuple(_) + | ir::IrPattern::List { .. } + | ir::IrPattern::Constructor { .. } + | ir::IrPattern::BitString(_) => { self.diagnostics.push( Diagnostic::new( DiagnosticCode::WasmError, @@ -333,6 +343,16 @@ impl Emitter { pointer } + fn static_bit_array(&mut self, bit_array: &ir::BitArrayLiteral) -> u32 { + self.uses_runtime = true; + let bytes = bit_array_bytes(bit_array); + let object = runtime::bit_array_object(self.config, self.next_static_offset, &bytes, bit_array.bit_len); + let pointer = object.offset; + self.next_static_offset = self.config.layout.align_to(object.offset + object.bytes.len() as u32); + self.data.push(object); + pointer + } + fn unsupported_expression(&mut self, expression: &ir::Expression) { self.diagnostics.push( Diagnostic::new( @@ -410,11 +430,29 @@ impl From for String { } } +fn bit_array_bytes(bit_array: &ir::BitArrayLiteral) -> Vec { + let mut bytes = vec![0; runtime::bit_array_payload_len(bit_array.bit_len) as usize]; + let mut offset = 0; + for segment in &bit_array.segments { + for bit_index in 0..segment.bit_size { + let source_shift = segment.bit_size - bit_index - 1; + let bit = if source_shift < u64::BITS { (segment.value >> source_shift) & 1 } else { 0 }; + if bit == 1 { + let byte = &mut bytes[(offset / 8) as usize]; + let target_shift = 7 - offset % 8; + *byte |= 1 << target_shift; + } + offset += 1; + } + } + bytes +} + fn wasm_type(type_: &Type) -> Option<&'static str> { match type_ { Type::Int => Some("i64"), Type::Float => Some("f64"), - Type::Bool | Type::String => Some("i32"), + Type::Bool | Type::String | Type::BitArray => Some("i32"), Type::Nil | Type::Tuple(_) | Type::List(_) @@ -587,6 +625,29 @@ mod tests { assert_eq!(keep.call(&mut store, 42).expect("call keep"), 42); } + #[test] + fn returns_bit_array_pointer_with_inspectable_memory_layout() { + let wasm = compile_wasm("pub fn bits() { <<1, 2, 3>> }"); + let engine = Engine::default(); + let module = Module::new(&engine, &wasm.bytes).expect("compile wasm module"); + let mut store = Store::new(&engine, ()); + let instance = Instance::new(&mut store, &module, &[]).expect("instantiate module"); + let bits = instance + .get_typed_func::<(), i32>(&mut store, "bits") + .expect("get bits export"); + let pointer = bits.call(&mut store, ()).expect("call bits") as usize; + let memory = instance.get_memory(&mut store, "memory").expect("memory export"); + let mut bytes = [0; 16]; + memory.read(&store, pointer, &mut bytes).expect("read bit array object"); + + assert_eq!( + ObjectTag::try_from(u32::from_le_bytes(bytes[0..4].try_into().unwrap())), + Ok(ObjectTag::BitArray) + ); + assert_eq!(u32::from_le_bytes(bytes[4..8].try_into().unwrap()), 24); + assert_eq!(&bytes[8..11], &[1, 2, 3]); + } + #[test] fn returns_string_pointer_with_inspectable_memory_layout() { let wasm = compile_wasm("pub fn greeting() { \"hello\" }"); diff --git a/docs/src/internal/tasks/08_wasm_backend_and_runtime.md b/docs/src/internal/tasks/08_wasm_backend_and_runtime.md index d9703de..461eda5 100644 --- a/docs/src/internal/tasks/08_wasm_backend_and_runtime.md +++ b/docs/src/internal/tasks/08_wasm_backend_and_runtime.md @@ -8,14 +8,14 @@ Emit WebAssembly for scalar and runtime-managed Gleam values. ### Bit arrays -- [ ] Add bit-array and bit-string segment types, including segment options and +- [x] Add bit-array and bit-string segment types, including segment options and bit-string patterns. -- [ ] Add IR forms for bit-array literals, segments, concatenation, and +- [x] Add IR forms for bit-array literals, segments, concatenation, and bit-string pattern deconstruction. -- [ ] Specify bit-array layout, segment packing, bit offsets, and byte +- [x] Specify bit-array layout, segment packing, bit offsets, and byte alignment rules. -- [ ] Implement bit-array allocation, append/slice helpers, and segment access. -- [ ] Add bit-array memory-layout tests in Wasmtime. +- [x] Implement bit-array allocation, append/slice helpers, and segment access. +- [x] Add bit-array memory-layout tests in Wasmtime. ### Runtime memory and values