From 9650543de59bab1d4374d2c57451c85bf7e8171a Mon Sep 17 00:00:00 2001 From: Corbin Crutchley Date: Fri, 24 Jul 2026 11:04:03 -0700 Subject: [PATCH] feat: automate static CPython runtime --- PLAN.md | 4 +- README.md | 8 + crates/aarch64/src/lib.rs | 603 ++++++++++++++++++++++++++------ guest-tests/cpython/README.md | 35 ++ guest-tests/cpython/build.sh | 140 ++++++++ guest-tests/cpython/config.site | 3 + guest-tests/cpython/verify.sh | 52 +++ 7 files changed, 744 insertions(+), 101 deletions(-) create mode 100644 guest-tests/cpython/README.md create mode 100755 guest-tests/cpython/build.sh create mode 100644 guest-tests/cpython/config.site create mode 100755 guest-tests/cpython/verify.sh diff --git a/PLAN.md b/PLAN.md index 439a80b..261cada 100644 --- a/PLAN.md +++ b/PLAN.md @@ -1,7 +1,7 @@ # AArch64 ELF-to-WebAssembly Browser Runtime ## Engineering Build Plan and Agent Handoff -**Status:** Phase 2 complete; controlled C/Rust/filesystem guests execute in native and browser Workers +**Status:** Phase 2 complete; Phase 3 filesystem services and native CPython checkpoint implemented **Primary implementation language:** Rust **Initial browser target:** Google Chrome **Guest architecture:** AArch64, little-endian, Linux userspace @@ -1857,7 +1857,7 @@ The Phase 2 control checkpoint is complete: the Worker accepts explicit C, Rust, Phase 2 is complete. `binarrow-memory-fs` supplies a bounded ephemeral `/tmp` regular-file store behind the project host filesystem trait, while `binarrow-linux-runtime` owns guest descriptors and the observed `openat`, `close`, `lseek`, `read`, and file `write` ABI. A freestanding C fixture creates, writes, seeks, reads, and closes `/tmp/roundtrip.txt` before printing the recovered bytes; native, CLI, and Chromium tests all run the identical ELF. ADR-0002 records the interpreter's interim browser memory design: project-owned sparse 64-bit guest mappings backed by bounded Wasm32 allocations, with Memory64 retained as a required feature gate for the future translator. Every Phase 2 deliverable and acceptance criterion now has an automated regression. Phase 3 begins with a mountable VFS, directory and metadata syscalls, OPFS persistence, terminal input, clock/random services, and the suspension model needed by CPython. -The first CPython checkpoint is complete in the native runtime. A static AArch64 CPython 3.12.13 image loads the packaged standard library, compiles `encodings`, executes `-c 'print(6 * 7)'`, prints `42`, and exits 0 under a reproducible CLI random seed. The observed path fixed negative `ADR`/`ADRP` offsets and BFXIL field preservation, and added `writev`, descriptor `fstat`, and standard-descriptor `fcntl` behavior. The source-only runtime image still reports the expected missing platform-dependent `lib-dynload` directory; packaging extension modules and adding this long-running command as an automated fixture are the next CPython tasks. +The first CPython checkpoint is complete and reproducible in the native runtime. `guest-tests/cpython/build.sh` checksum-pins CPython 3.12.13, isolates all build and Zig caches under the repository's ignored `.tmp` directory, cross-compiles a static AArch64 musl interpreter, links the configured dependency-free extension modules into the executable, and packages the standard library with its empty pre-dynamic-linking `lib-dynload` landmark. `guest-tests/cpython/verify.sh` checks the version and runs the bounded `-c 'print(6 * 7)'` regression with a deterministic CLI random seed. The freshly built broader module profile added focused semantics for NEON table lookup, unsigned variable shifts, extraction, and 64-bit-group reversal. Running this packaged runtime in Chromium is the next CPython acceptance task; shared native extensions remain deferred to Phase 7. Do not begin the full web IDE before item 30 passes. diff --git a/README.md b/README.md index d0fbf02..cd55077 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,14 @@ cargo run -p binarrow-cli -- run path/to/aarch64-static.elf [guest arguments...] The runnable instruction/syscall profile remains fixture-driven. The current static Rust program adds single-thread atomics and barriers, 128-bit vector moves/stores, byte popcount and reduction, multiplication/division, and signed shifts to the earlier libc instruction path. The runtime implements `openat`, `close`, `lseek`, file `read`/`write`, polling, deterministic process/signal setup, anonymous memory management, terminal output, and exit. Invalid guest arguments return Linux errno values; instruction, syscall, committed-memory, ephemeral-filesystem, open-file, and combined-output limits are enforced before host side effects. +The native CPython checkpoint is reproducible without committing its large +generated artifacts. `guest-tests/cpython/build.sh` creates a statically linked +AArch64 musl CPython 3.12.13 executable and standard-library image entirely +under `.tmp`; `guest-tests/cpython/verify.sh` runs the bounded version and +`print(6 * 7)` regressions. See +[`guest-tests/cpython/README.md`](guest-tests/cpython/README.md) for prerequisites +and packaging details. + The browser build generates its Memory64, JSPI, and P-code `.wasm` probes before starting Vite. Generated artifacts are not committed. ## Scope diff --git a/crates/aarch64/src/lib.rs b/crates/aarch64/src/lib.rs index 5f455ba..313ebd6 100644 --- a/crates/aarch64/src/lib.rs +++ b/crates/aarch64/src/lib.rs @@ -42,6 +42,10 @@ const NEON_MINIMUM_NUMBER_USER_OP: &str = "NEON_fminnm"; const NEON_FUSED_MULTIPLY_ADD_USER_OP: &str = "NEON_fmadd"; const NEON_ROUND_INTEGRAL_USER_OP: &str = "NEON_frint"; const NEON_UNSIGNED_MULTIPLY_LONG_USER_OP: &str = "NEON_umull"; +const NEON_TABLE_LOOKUP_USER_OP: &str = "a64_TBL"; +const NEON_UNSIGNED_SHIFT_LEFT_USER_OP: &str = "NEON_ushl"; +const NEON_EXTRACT_USER_OP: &str = "NEON_ext"; +const NEON_REVERSE_64_USER_OP: &str = "NEON_rev64"; const SLEIGH_SOURCES: [(&str, &str); 7] = [ ( @@ -737,6 +741,30 @@ enum Operation { right: Value, element_size: Value, }, + NeonTableLookup { + destination: Place, + original: Value, + table: Value, + indices: Value, + }, + NeonUnsignedShiftLeft { + destination: Place, + source: Value, + shifts: Value, + element_size: Value, + }, + NeonExtract { + destination: Place, + first: Value, + second: Value, + byte_offset: Value, + element_size: Value, + }, + NeonReverse64 { + destination: Place, + source: Value, + element_size: Value, + }, SupervisorCall { immediate: Value, }, @@ -1096,93 +1124,151 @@ fn lower_control_pcode( label, }), PcodeOp::PcodeLabel(label) => Some(Operation::Label(label)), - PcodeOp::PcodeOp(id) => match language.get_user_ops().nth(usize::from(id))? { - SUPERVISOR_USER_OP => Some(Operation::SupervisorCall { - immediate: lower_value(language, left)?, - }), - EXCLUSIVE_MONITOR_PASS_USER_OP => Some(Operation::ExclusiveMonitorPass { - destination: lower_place(language, output)?, - address: lower_value(language, left)?, - size: lower_value(language, right)?, - }), - EXCLUSIVE_MONITORS_STATUS_USER_OP => Some(Operation::ExclusiveMonitorsStatus { - destination: lower_place(language, output)?, - }), - DATA_MEMORY_BARRIER_USER_OP => Some(Operation::MemoryBarrier), - NEON_COUNT_USER_OP => Some(Operation::NeonCount { - destination: lower_place(language, output)?, - source: lower_value(language, left)?, - element_size: lower_value(language, right)?, - }), - NEON_ADD_ACROSS_USER_OP => Some(Operation::NeonAddAcross { - destination: lower_place(language, output)?, - source: lower_value(language, left)?, - element_size: lower_value(language, right)?, - }), - NEON_COMPARE_EQUAL_USER_OP => Some(Operation::NeonCompareEqual { - destination: lower_place(language, output)?, - left: lower_value(language, left)?, - right: lower_value(language, right)?, - element_size: arguments[0]?, - }), - NEON_UNSIGNED_MAXIMUM_ACROSS_USER_OP => Some(Operation::NeonUnsignedMaximumAcross { - destination: lower_place(language, output)?, - source: lower_value(language, left)?, - element_size: lower_value(language, right)?, - }), - NEON_BITWISE_SELECT_USER_OP => Some(Operation::NeonBitwiseSelect { - destination: lower_place(language, output)?, - selector: lower_value(language, left)?, - when_set: lower_value(language, right)?, - when_clear: arguments[0]?, - element_size: arguments[1]?, - }), - NEON_UNSIGNED_MINIMUM_ACROSS_USER_OP => Some(Operation::NeonUnsignedMinimumAcross { - destination: lower_place(language, output)?, - source: lower_value(language, left)?, - element_size: lower_value(language, right)?, - }), - NEON_COMPARE_LESS_THAN_USER_OP => Some(Operation::NeonCompareLessThan { - destination: lower_place(language, output)?, - left: lower_value(language, left)?, - right: lower_value(language, right)?, - element_size: arguments[0]?, - }), - NEON_BITWISE_INSERT_USER_OP => Some(Operation::NeonBitwiseInsert { - destination: lower_place(language, output)?, - original: lower_value(language, left)?, - source: lower_value(language, right)?, - mask: arguments[0]?, - element_size: arguments[1]?, - }), - NEON_MINIMUM_NUMBER_USER_OP if arguments[0].is_none() => { - Some(Operation::FloatMinimumNumber { + PcodeOp::PcodeOp(id) => { + let user_op = language.get_user_ops().nth(usize::from(id))?; + if matches!( + user_op, + NEON_TABLE_LOOKUP_USER_OP + | NEON_UNSIGNED_SHIFT_LEFT_USER_OP + | NEON_EXTRACT_USER_OP + | NEON_REVERSE_64_USER_OP + | NEON_MINIMUM_NUMBER_USER_OP + | NEON_FUSED_MULTIPLY_ADD_USER_OP + | NEON_ROUND_INTEGRAL_USER_OP + ) { + return lower_extended_user_pcode( + language, user_op, output, left, right, arguments, + ); + } + match user_op { + SUPERVISOR_USER_OP => Some(Operation::SupervisorCall { + immediate: lower_value(language, left)?, + }), + EXCLUSIVE_MONITOR_PASS_USER_OP => Some(Operation::ExclusiveMonitorPass { + destination: lower_place(language, output)?, + address: lower_value(language, left)?, + size: lower_value(language, right)?, + }), + EXCLUSIVE_MONITORS_STATUS_USER_OP => Some(Operation::ExclusiveMonitorsStatus { + destination: lower_place(language, output)?, + }), + DATA_MEMORY_BARRIER_USER_OP => Some(Operation::MemoryBarrier), + NEON_COUNT_USER_OP => Some(Operation::NeonCount { + destination: lower_place(language, output)?, + source: lower_value(language, left)?, + element_size: lower_value(language, right)?, + }), + NEON_ADD_ACROSS_USER_OP => Some(Operation::NeonAddAcross { + destination: lower_place(language, output)?, + source: lower_value(language, left)?, + element_size: lower_value(language, right)?, + }), + NEON_COMPARE_EQUAL_USER_OP => Some(Operation::NeonCompareEqual { destination: lower_place(language, output)?, left: lower_value(language, left)?, right: lower_value(language, right)?, - }) - } - NEON_FUSED_MULTIPLY_ADD_USER_OP if arguments[1].is_none() => { - Some(Operation::FloatFusedMultiplyAdd { + element_size: arguments[0]?, + }), + NEON_UNSIGNED_MAXIMUM_ACROSS_USER_OP => { + Some(Operation::NeonUnsignedMaximumAcross { + destination: lower_place(language, output)?, + source: lower_value(language, left)?, + element_size: lower_value(language, right)?, + }) + } + NEON_BITWISE_SELECT_USER_OP => Some(Operation::NeonBitwiseSelect { + destination: lower_place(language, output)?, + selector: lower_value(language, left)?, + when_set: lower_value(language, right)?, + when_clear: arguments[0]?, + element_size: arguments[1]?, + }), + NEON_UNSIGNED_MINIMUM_ACROSS_USER_OP => { + Some(Operation::NeonUnsignedMinimumAcross { + destination: lower_place(language, output)?, + source: lower_value(language, left)?, + element_size: lower_value(language, right)?, + }) + } + NEON_COMPARE_LESS_THAN_USER_OP => Some(Operation::NeonCompareLessThan { destination: lower_place(language, output)?, left: lower_value(language, left)?, right: lower_value(language, right)?, - addend: arguments[0]?, - }) + element_size: arguments[0]?, + }), + NEON_BITWISE_INSERT_USER_OP => Some(Operation::NeonBitwiseInsert { + destination: lower_place(language, output)?, + original: lower_value(language, left)?, + source: lower_value(language, right)?, + mask: arguments[0]?, + element_size: arguments[1]?, + }), + NEON_UNSIGNED_MULTIPLY_LONG_USER_OP => Some(Operation::NeonUnsignedMultiplyLong { + destination: lower_place(language, output)?, + left: lower_value(language, left)?, + right: lower_value(language, right)?, + element_size: arguments[0]?, + }), + _ => None, } - NEON_ROUND_INTEGRAL_USER_OP => Some(Operation::FloatRoundIntegral { + } + _ => None, + } +} + +fn lower_extended_user_pcode( + language: &SleighData, + user_op: &str, + output: VarNode, + left: PcodeValue, + right: PcodeValue, + arguments: [Option; 2], +) -> Option { + match user_op { + NEON_MINIMUM_NUMBER_USER_OP if arguments[0].is_none() => { + Some(Operation::FloatMinimumNumber { destination: lower_place(language, output)?, - source: lower_value(language, left)?, - mode: lower_value(language, right)?, - }), - NEON_UNSIGNED_MULTIPLY_LONG_USER_OP => Some(Operation::NeonUnsignedMultiplyLong { + left: lower_value(language, left)?, + right: lower_value(language, right)?, + }) + } + NEON_FUSED_MULTIPLY_ADD_USER_OP if arguments[1].is_none() => { + Some(Operation::FloatFusedMultiplyAdd { destination: lower_place(language, output)?, left: lower_value(language, left)?, right: lower_value(language, right)?, - element_size: arguments[0]?, - }), - _ => None, - }, + addend: arguments[0]?, + }) + } + NEON_ROUND_INTEGRAL_USER_OP => Some(Operation::FloatRoundIntegral { + destination: lower_place(language, output)?, + source: lower_value(language, left)?, + mode: lower_value(language, right)?, + }), + NEON_TABLE_LOOKUP_USER_OP if arguments[1].is_none() => Some(Operation::NeonTableLookup { + destination: lower_place(language, output)?, + original: lower_value(language, left)?, + table: lower_value(language, right)?, + indices: arguments[0]?, + }), + NEON_UNSIGNED_SHIFT_LEFT_USER_OP => Some(Operation::NeonUnsignedShiftLeft { + destination: lower_place(language, output)?, + source: lower_value(language, left)?, + shifts: lower_value(language, right)?, + element_size: arguments[0]?, + }), + NEON_EXTRACT_USER_OP => Some(Operation::NeonExtract { + destination: lower_place(language, output)?, + first: lower_value(language, left)?, + second: lower_value(language, right)?, + byte_offset: arguments[0]?, + element_size: arguments[1]?, + }), + NEON_REVERSE_64_USER_OP => Some(Operation::NeonReverse64 { + destination: lower_place(language, output)?, + source: lower_value(language, left)?, + element_size: lower_value(language, right)?, + }), _ => None, } } @@ -1332,7 +1418,11 @@ fn execute_operations( | Operation::NeonUnsignedMinimumAcross { .. } | Operation::NeonCompareLessThan { .. } | Operation::NeonBitwiseInsert { .. } - | Operation::NeonUnsignedMultiplyLong { .. } => { + | Operation::NeonUnsignedMultiplyLong { .. } + | Operation::NeonTableLookup { .. } + | Operation::NeonUnsignedShiftLeft { .. } + | Operation::NeonExtract { .. } + | Operation::NeonReverse64 { .. } => { execute_value_operation(state, &mut scratch, operation, pc, encoding)?; } Operation::Load { .. } | Operation::Store { .. } => { @@ -1451,7 +1541,11 @@ fn execute_value_operation( | Operation::NeonUnsignedMinimumAcross { .. } | Operation::NeonCompareLessThan { .. } | Operation::NeonBitwiseInsert { .. } - | Operation::NeonUnsignedMultiplyLong { .. }) => { + | Operation::NeonUnsignedMultiplyLong { .. } + | Operation::NeonTableLookup { .. } + | Operation::NeonUnsignedShiftLeft { .. } + | Operation::NeonExtract { .. } + | Operation::NeonReverse64 { .. }) => { execute_neon_value_operation(state, scratch, &operation, pc, encoding)? } Operation::Load { .. } @@ -1908,6 +2002,17 @@ fn execute_neon_value_operation( pc: GuestAddress, encoding: u32, ) -> Result<(Place, u128), Trap> { + if matches!( + operation, + Operation::NeonTableLookup { .. } + | Operation::NeonUnsignedShiftLeft { .. } + | Operation::NeonExtract { .. } + | Operation::NeonReverse64 { .. } + | Operation::NeonBitwiseSelect { .. } + | Operation::NeonBitwiseInsert { .. } + ) { + return execute_neon_permutation_operation(state, scratch, operation, pc, encoding); + } let unsupported = || Trap::UnsupportedInstruction { pc, encoding }; let read = |value| read_value(state, scratch, value).ok_or_else(unsupported); match *operation { @@ -1915,24 +2020,21 @@ fn execute_neon_value_operation( destination, source, element_size, - } => execute_neon_byte_reduction( - state, - scratch, - (destination, source, element_size), - true, - (pc, encoding), - ), - Operation::NeonAddAcross { + } + | Operation::NeonAddAcross { destination, source, element_size, - } => execute_neon_byte_reduction( - state, - scratch, - (destination, source, element_size), - false, - (pc, encoding), - ), + } => { + let count_bits = matches!(operation, Operation::NeonCount { .. }); + execute_neon_byte_reduction( + state, + scratch, + (destination, source, element_size), + count_bits, + (pc, encoding), + ) + } Operation::NeonCompareEqual { destination, left, @@ -1970,6 +2072,29 @@ fn execute_neon_value_operation( .ok_or_else(unsupported)?; Ok((destination, result)) } + Operation::NeonUnsignedMultiplyLong { + destination, + left, + right, + element_size, + } => execute_neon_unsigned_multiply_long( + state, + scratch, + (destination, left, right, element_size), + (pc, encoding), + ), + _ => Err(unsupported()), + } +} + +fn execute_neon_permutation_operation( + state: &Aarch64State, + scratch: &ScratchValues, + operation: &Operation, + pc: GuestAddress, + encoding: u32, +) -> Result<(Place, u128), Trap> { + match *operation { Operation::NeonBitwiseSelect { destination, selector, @@ -1994,19 +2119,192 @@ fn execute_neon_value_operation( (destination, original, source, mask, element_size), (pc, encoding), ), - Operation::NeonUnsignedMultiplyLong { + Operation::NeonTableLookup { destination, - left, - right, + original, + table, + indices, + } => execute_neon_table_lookup( + state, + scratch, + (destination, original, table, indices), + (pc, encoding), + ), + Operation::NeonUnsignedShiftLeft { + destination, + source, + shifts, element_size, - } => execute_neon_unsigned_multiply_long( + } => execute_neon_unsigned_shift_left( state, scratch, - (destination, left, right, element_size), + (destination, source, shifts, element_size), (pc, encoding), ), - _ => Err(unsupported()), + Operation::NeonExtract { + destination, + first, + second, + byte_offset, + element_size, + } => execute_neon_extract( + state, + scratch, + (destination, first, second, byte_offset, element_size), + (pc, encoding), + ), + Operation::NeonReverse64 { + destination, + source, + element_size, + } => execute_neon_reverse_64( + state, + scratch, + (destination, source, element_size), + (pc, encoding), + ), + _ => Err(Trap::UnsupportedInstruction { pc, encoding }), + } +} + +fn execute_neon_reverse_64( + state: &Aarch64State, + scratch: &ScratchValues, + operands: (Place, Value, Value), + location: (GuestAddress, u32), +) -> Result<(Place, u128), Trap> { + let (destination, source, element_size) = operands; + let (pc, encoding) = location; + let unsupported = || Trap::UnsupportedInstruction { pc, encoding }; + let read = |value| read_value(state, scratch, value).ok_or_else(unsupported); + let element_size = u8::try_from(read(element_size)?).map_err(|_| unsupported())?; + if destination.size != source.size + || element_size == 0 + || !8_u8.is_multiple_of(element_size) + || !source.size.is_multiple_of(8) + { + return Err(unsupported()); + } + let source = read(source)?; + let element_bits = u32::from(element_size) * 8; + let element_mask = value_mask(element_size).ok_or_else(unsupported)?; + let elements_per_group = 8 / element_size; + let mut result = 0_u128; + for group in 0..(destination.size / 8) { + for output_element in 0..elements_per_group { + let input_element = elements_per_group - 1 - output_element; + let input_shift = (u32::from(group) * 64) + (u32::from(input_element) * element_bits); + let output_shift = (u32::from(group) * 64) + (u32::from(output_element) * element_bits); + result |= ((source >> input_shift) & element_mask) << output_shift; + } + } + Ok((destination, result)) +} + +fn execute_neon_extract( + state: &Aarch64State, + scratch: &ScratchValues, + operands: (Place, Value, Value, Value, Value), + location: (GuestAddress, u32), +) -> Result<(Place, u128), Trap> { + let (destination, first, second, byte_offset, element_size) = operands; + let (pc, encoding) = location; + let unsupported = || Trap::UnsupportedInstruction { pc, encoding }; + let read = |value| read_value(state, scratch, value).ok_or_else(unsupported); + if read(element_size)? != 1 || destination.size != first.size || first.size != second.size { + return Err(unsupported()); + } + let byte_offset = u8::try_from(read(byte_offset)?).map_err(|_| unsupported())?; + if byte_offset >= destination.size { + return Err(unsupported()); + } + let source_size = first.size; + let first = read(first)?; + let second = read(second)?; + let mut result = 0_u128; + for destination_byte in 0..destination.size { + let source_byte = destination_byte + byte_offset; + let byte = if source_byte < source_size { + (first >> (u32::from(source_byte) * 8)) & 0xff + } else { + (second >> (u32::from(source_byte - source_size) * 8)) & 0xff + }; + result |= byte << (u32::from(destination_byte) * 8); + } + Ok((destination, result)) +} + +fn execute_neon_unsigned_shift_left( + state: &Aarch64State, + scratch: &ScratchValues, + operands: (Place, Value, Value, Value), + location: (GuestAddress, u32), +) -> Result<(Place, u128), Trap> { + let (destination, source, shifts, element_size) = operands; + let (pc, encoding) = location; + let unsupported = || Trap::UnsupportedInstruction { pc, encoding }; + let read = |value| read_value(state, scratch, value).ok_or_else(unsupported); + let element_size = u8::try_from(read(element_size)?).map_err(|_| unsupported())?; + if element_size == 0 + || destination.size != source.size + || source.size != shifts.size + || !source.size.is_multiple_of(element_size) + { + return Err(unsupported()); + } + let lane_bits = u32::from(element_size) * 8; + let lane_mask = value_mask(element_size).ok_or_else(unsupported)?; + let source = read(source)?; + let shifts = read(shifts)?; + let mut result = 0_u128; + for lane in 0..(destination.size / element_size) { + let lane_shift = u32::from(lane) * lane_bits; + let value = (source >> lane_shift) & lane_mask; + let shift_byte = u8::try_from((shifts >> lane_shift) & 0xff).map_err(|_| unsupported())?; + let shift = i8::from_ne_bytes([shift_byte]); + let shifted = if i32::from(shift) >= i32::try_from(lane_bits).unwrap() { + 0 + } else if shift >= 0 { + (value << u32::try_from(shift).unwrap()) & lane_mask + } else if i32::from(shift) <= -i32::try_from(lane_bits).unwrap() { + 0 + } else { + value >> u32::try_from(-i32::from(shift)).unwrap() + }; + result |= shifted << lane_shift; + } + Ok((destination, result)) +} + +fn execute_neon_table_lookup( + state: &Aarch64State, + scratch: &ScratchValues, + operands: (Place, Value, Value, Value), + location: (GuestAddress, u32), +) -> Result<(Place, u128), Trap> { + let (destination, original, table, indices) = operands; + let (pc, encoding) = location; + let unsupported = || Trap::UnsupportedInstruction { pc, encoding }; + let read = |value| read_value(state, scratch, value).ok_or_else(unsupported); + if destination.size != original.size || destination.size != indices.size || table.size != 16 { + return Err(unsupported()); + } + let table_size = table.size; + let original = read(original)?; + let table = read(table)?; + let indices = read(indices)?; + let mut result = 0_u128; + for byte_index in 0..destination.size { + let shift = u32::from(byte_index) * 8; + let index = u8::try_from((indices >> shift) & 0xff).map_err(|_| unsupported())?; + let byte = if index < table_size { + (table >> (u32::from(index) * 8)) & 0xff + } else { + (original >> shift) & 0xff + }; + result |= byte << shift; } + Ok((destination, result)) } fn execute_neon_byte_reduction( @@ -2368,6 +2666,10 @@ fn execute_memory_operation( | Operation::NeonCompareLessThan { .. } | Operation::NeonBitwiseInsert { .. } | Operation::NeonUnsignedMultiplyLong { .. } + | Operation::NeonTableLookup { .. } + | Operation::NeonUnsignedShiftLeft { .. } + | Operation::NeonExtract { .. } + | Operation::NeonReverse64 { .. } | Operation::SupervisorCall { .. } => Err(unsupported()), } } @@ -3047,6 +3349,109 @@ mod tests { assert_eq!(state.vector(0), Some(0xaa0f_aa0f_aa0f_aa0f)); } + #[test] + fn executes_cpython_neon_table_lookup_semantics() { + const CODE: &[u8] = &[ + 0x86, 0x00, 0x03, 0x4e, // tbl v6.16b, {v4.16b}, v3.16b + 0x87, 0x10, 0x03, 0x4e, // tbx v7.16b, {v4.16b}, v3.16b + ]; + let table = core::array::from_fn(|index| 0x10 + u8::try_from(index).unwrap()); + let indices = [0, 15, 16, 7, 2, 99, 3, 14, 8, 9, 10, 11, 12, 13, 1, 4]; + let expected_table: [u8; 16] = core::array::from_fn(|index| { + let table_index = indices[index]; + if table_index < 16 { + table[usize::from(table_index)] + } else { + 0 + } + }); + let expected_extension: [u8; 16] = core::array::from_fn(|index| { + let table_index = indices[index]; + if table_index < 16 { + table[usize::from(table_index)] + } else { + 0xaa + } + }); + let mut memory = executable_memory(CODE); + let mut state = Aarch64State::new(CODE_ADDRESS, GuestAddress::new(0x8000)); + state.set_vector(3, u128::from_le_bytes(indices)).unwrap(); + state.set_vector(4, u128::from_le_bytes(table)).unwrap(); + state + .set_vector(7, u128::from_le_bytes([0xaa; 16])) + .unwrap(); + let mut interpreter = Interpreter::new().unwrap(); + + interpreter.step(&mut state, &mut memory).unwrap(); + interpreter.step(&mut state, &mut memory).unwrap(); + + assert_eq!(state.vector(6), Some(u128::from_le_bytes(expected_table))); + assert_eq!( + state.vector(7), + Some(u128::from_le_bytes(expected_extension)) + ); + } + + #[test] + fn executes_cpython_neon_unsigned_variable_shift_semantics() { + const CODE: &[u8] = &[ + 0x00, 0x44, 0xa1, 0x2e, // ushl v0.2s, v0.2s, v1.2s + ]; + let mut memory = executable_memory(CODE); + let mut state = Aarch64State::new(CODE_ADDRESS, GuestAddress::new(0x8000)); + state + .set_vector(0, 7_u128 << 32 | u128::from(0x8000_0001_u32)) + .unwrap(); + state + .set_vector(1, u128::from(0x0000_00fe_u32) << 32 | 1) + .unwrap(); + let mut interpreter = Interpreter::new().unwrap(); + + interpreter.step(&mut state, &mut memory).unwrap(); + + assert_eq!(state.vector(0), Some(1_u128 << 32 | 2)); + } + + #[test] + fn executes_cpython_neon_extract_semantics() { + const CODE: &[u8] = &[ + 0x01, 0x40, 0x00, 0x6e, // ext v1.16b, v0.16b, v0.16b, #8 + ]; + let source = [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, + 0x0e, 0x0f, + ]; + let expected = [ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, + 0x06, 0x07, + ]; + let mut memory = executable_memory(CODE); + let mut state = Aarch64State::new(CODE_ADDRESS, GuestAddress::new(0x8000)); + state.set_vector(0, u128::from_le_bytes(source)).unwrap(); + let mut interpreter = Interpreter::new().unwrap(); + + interpreter.step(&mut state, &mut memory).unwrap(); + + assert_eq!(state.vector(1), Some(u128::from_le_bytes(expected))); + } + + #[test] + fn executes_cpython_neon_reverse_64_semantics() { + const CODE: &[u8] = &[ + 0x00, 0x08, 0xa0, 0x0e, // rev64 v0.2s, v0.2s + ]; + let mut memory = executable_memory(CODE); + let mut state = Aarch64State::new(CODE_ADDRESS, GuestAddress::new(0x8000)); + state + .set_vector(0, 0x1122_3344_u128 << 32 | 0xaabb_ccdd) + .unwrap(); + let mut interpreter = Interpreter::new().unwrap(); + + interpreter.step(&mut state, &mut memory).unwrap(); + + assert_eq!(state.vector(0), Some(0xaabb_ccdd_u128 << 32 | 0x1122_3344)); + } + #[test] fn executes_musl_time_conversion_vector_semantics() { const CODE: &[u8] = &[ diff --git a/guest-tests/cpython/README.md b/guest-tests/cpython/README.md new file mode 100644 index 0000000..cc24eca --- /dev/null +++ b/guest-tests/cpython/README.md @@ -0,0 +1,35 @@ +# CPython 3.12.13 checkpoint + +This fixture cross-compiles an ordinary, statically linked AArch64 musl +CPython and packages its standard library as a Binarrow filesystem image. The +generated runtime and image are intentionally kept in the repository's +gitignored `.tmp/cpython-3.12.13` directory rather than committed to Git. + +Prerequisites: + +- Zig 0.16.0 +- the repository Rust toolchain +- `curl`, `make`, `tar`, `shasum`, and a native C compiler + +Build the host bootstrap interpreter, target interpreter, and filesystem +image: + +```sh +guest-tests/cpython/build.sh +``` + +The download is pinned by SHA-256. Host package discovery is disabled for the +cross build so unavailable target libraries cannot leak in through the host's +`pkg-config`. Configured dependency-free extension modules are linked into the +executable because shared extensions remain out of scope until Phase 7. + +Run the bounded native regression: + +```sh +guest-tests/cpython/verify.sh +``` + +The verification checks `Python 3.12.13`, loads the packaged `encodings` +module, evaluates `print(6 * 7)`, checks for exactly `42`, and requires a clean +guest exit. The current interpreter can take several minutes to complete this +regression even when the native runner is built in release mode. diff --git a/guest-tests/cpython/build.sh b/guest-tests/cpython/build.sh new file mode 100755 index 0000000..736b1bb --- /dev/null +++ b/guest-tests/cpython/build.sh @@ -0,0 +1,140 @@ +#!/bin/sh +set -eu + +fixture_directory=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +workspace_directory=$(CDPATH= cd -- "$fixture_directory/../.." && pwd) +work_directory=$workspace_directory/.tmp/cpython-3.12.13 +archive=$work_directory/Python-3.12.13.tar.xz +source_directory=$work_directory/source +host_build_directory=$work_directory/build-host +target_build_directory=$work_directory/build-aarch64-static +cache_directory=$work_directory/zig-cache +runtime=$work_directory/python.aarch64.elf +filesystem_image=$work_directory/python-lib.bnfs +archive_url=https://www.python.org/ftp/python/3.12.13/Python-3.12.13.tar.xz +archive_sha256=c08bc65a81971c1dd5783182826503369466c7e67374d1646519adf05207b684 +jobs=${JOBS:-4} + +zig=$(command -v zig || true) +if [ -z "$zig" ]; then + echo "CPython fixture requires Zig 0.16.0" >&2 + exit 1 +fi +zig_version=$($zig version) +if [ "$zig_version" != "0.16.0" ]; then + echo "CPython fixture requires Zig 0.16.0; found $zig_version" >&2 + exit 1 +fi + +mkdir -p \ + "$workspace_directory/.tmp/rust-tmp" \ + "$work_directory" \ + "$cache_directory/local" \ + "$cache_directory/global" +export TMPDIR=$workspace_directory/.tmp/rust-tmp +export ZIG_LOCAL_CACHE_DIR=$cache_directory/local +export ZIG_GLOBAL_CACHE_DIR=$cache_directory/global +export PYTHONDONTWRITEBYTECODE=1 + +verify_archive() { + actual_sha256=$(shasum -a 256 "$1" | awk '{print $1}') + if [ "$actual_sha256" != "$archive_sha256" ]; then + echo "unexpected CPython archive checksum: $actual_sha256" >&2 + return 1 + fi +} + +if [ ! -f "$archive" ]; then + partial_archive=$archive.part + curl --fail --location --retry 3 --continue-at - \ + --output "$partial_archive" "$archive_url" + verify_archive "$partial_archive" + mv "$partial_archive" "$archive" +else + verify_archive "$archive" +fi + +if [ ! -x "$source_directory/configure" ]; then + mkdir -p "$source_directory" + tar -xJf "$archive" --strip-components=1 -C "$source_directory" +fi + +case $(uname -s) in + Darwin) host_executable=python.exe ;; + *) host_executable=python ;; +esac +build_triplet=$($source_directory/config.guess) + +mkdir -p "$host_build_directory" +if [ ! -f "$host_build_directory/Makefile" ]; then + ( + cd "$host_build_directory" + "$source_directory/configure" \ + --without-ensurepip \ + --disable-test-modules + ) +fi +( + cd "$host_build_directory" + make -j"$jobs" "$host_executable" +) + +mkdir -p "$target_build_directory" +if [ ! -f "$target_build_directory/Makefile" ]; then + ( + cd "$target_build_directory" + env \ + CONFIG_SITE="$fixture_directory/config.site" \ + MODULE_BUILDTYPE=static \ + PKG_CONFIG=false \ + CC="$zig cc -target aarch64-linux-musl" \ + CXX="$zig c++ -target aarch64-linux-musl" \ + AR="$zig ar" \ + RANLIB="$zig ranlib" \ + "$source_directory/configure" \ + --build="$build_triplet" \ + --host=aarch64-linux-musl \ + --with-build-python="$host_build_directory/$host_executable" \ + --disable-shared \ + --without-ensurepip \ + --disable-test-modules + ) +fi + +# Fold every dependency-free configured extension into the executable. This +# avoids requiring PT_INTERP or dlopen support before the dynamic-ELF phase. +if ! cmp -s \ + "$target_build_directory/Modules/Setup.stdlib" \ + "$target_build_directory/Modules/Setup.local" +then + cp "$target_build_directory/Modules/Setup.stdlib" \ + "$target_build_directory/Modules/Setup.local" +fi +( + cd "$target_build_directory" + make -j"$jobs" CFLAGS_NODIST=-Wno-error=date-time "$host_executable" +) + +cp "$target_build_directory/$host_executable" "$runtime" +chmod 0644 "$runtime" + +# Keep the packaged image source-only and independent of build-host cache +# history. These files are generated by CPython's own build-time imports. +find "$source_directory/Lib" -type f -name '*.pyc' -delete +find "$source_directory/Lib" -depth -type d -name __pycache__ -empty -delete + +# CPython uses this directory as an installation-layout landmark even though +# this pre-dynamic-linking image intentionally contains no shared extensions. +mkdir -p "$source_directory/Lib/lib-dynload" + +env TMPDIR="$TMPDIR" cargo build \ + --manifest-path "$workspace_directory/Cargo.toml" \ + --release \ + -p binarrow-cli +"$workspace_directory/target/release/binarrow" image pack \ + --guest-root /project/python/lib/python3.12 \ + "$source_directory/Lib" \ + "$filesystem_image" + +echo "CPython runtime: $runtime" +echo "CPython filesystem image: $filesystem_image" diff --git a/guest-tests/cpython/config.site b/guest-tests/cpython/config.site new file mode 100644 index 0000000..1f70b30 --- /dev/null +++ b/guest-tests/cpython/config.site @@ -0,0 +1,3 @@ +ac_cv_buggy_getaddrinfo=no +ac_cv_file__dev_ptmx=no +ac_cv_file__dev_ptc=no diff --git a/guest-tests/cpython/verify.sh b/guest-tests/cpython/verify.sh new file mode 100755 index 0000000..202b175 --- /dev/null +++ b/guest-tests/cpython/verify.sh @@ -0,0 +1,52 @@ +#!/bin/sh +set -eu + +fixture_directory=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +workspace_directory=$(CDPATH= cd -- "$fixture_directory/../.." && pwd) +work_directory=$workspace_directory/.tmp/cpython-3.12.13 +runtime=$work_directory/python.aarch64.elf +filesystem_image=$work_directory/python-lib.bnfs +runner=$workspace_directory/target/release/binarrow + +if [ ! -f "$runtime" ] || [ ! -f "$filesystem_image" ]; then + echo "build the CPython fixture first with guest-tests/cpython/build.sh" >&2 + exit 1 +fi + +mkdir -p "$workspace_directory/.tmp/rust-tmp" +export TMPDIR=$workspace_directory/.tmp/rust-tmp + +cargo build \ + --manifest-path "$workspace_directory/Cargo.toml" \ + --release \ + -p binarrow-cli + +version=$( + "$runner" run \ + --instruction-budget 100000000 \ + --filesystem-limit 67108864 \ + --filesystem-snapshot "$filesystem_image" \ + --random-seed 1 \ + --argv0 /project/python/bin/python \ + "$runtime" --version +) +if [ "$version" != "Python 3.12.13" ]; then + echo "unexpected CPython version output: $version" >&2 + exit 1 +fi + +output=$( + "$runner" run \ + --instruction-budget 100000000 \ + --filesystem-limit 67108864 \ + --filesystem-snapshot "$filesystem_image" \ + --random-seed 1 \ + --argv0 /project/python/bin/python \ + "$runtime" -S -E -c 'print(6 * 7)' +) +if [ "$output" != "42" ]; then + echo "unexpected CPython script output: $output" >&2 + exit 1 +fi + +echo "CPython 3.12.13 fixture passed" -- 2.51.2