From f2497d02afb2e5d2d038d59d14965ec83a161f4e Mon Sep 17 00:00:00 2001 From: Orual Date: Sun, 2 Aug 2026 18:07:11 -0400 Subject: [PATCH] PM-78: add local model resolver Epic: PM-86 Task: PM-78 --- Cargo.lock | 7 + crates/polymodel-ldraw-core/Cargo.toml | 1 + crates/polymodel-ldraw-core/src/parser.rs | 223 +++++++++----------- crates/polymodel-ldraw-core/src/resolver.rs | 0 crates/polymodel-ldraw-core/src/texmap.rs | 16 ++ crates/polymodel-ldraw-core/src/types.rs | 67 ++++++ 6 files changed, 189 insertions(+), 125 deletions(-) create mode 100644 crates/polymodel-ldraw-core/src/resolver.rs diff --git a/Cargo.lock b/Cargo.lock index ef05ce7..bd072da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6404,6 +6404,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "thiserror 2.0.18", + "typed-arena", ] [[package]] @@ -9010,6 +9011,12 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +[[package]] +name = "typed-arena" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" + [[package]] name = "typed-path" version = "0.12.3" diff --git a/crates/polymodel-ldraw-core/Cargo.toml b/crates/polymodel-ldraw-core/Cargo.toml index cebfb6d..1915f88 100644 --- a/crates/polymodel-ldraw-core/Cargo.toml +++ b/crates/polymodel-ldraw-core/Cargo.toml @@ -11,6 +11,7 @@ serde = { workspace = true } miette = { workspace = true } sha2 = "0.10" thiserror = { workspace = true } +typed-arena = "2" [dev-dependencies] serde_json = "1" diff --git a/crates/polymodel-ldraw-core/src/parser.rs b/crates/polymodel-ldraw-core/src/parser.rs index 82b9a26..d3582cb 100644 --- a/crates/polymodel-ldraw-core/src/parser.rs +++ b/crates/polymodel-ldraw-core/src/parser.rs @@ -4,16 +4,18 @@ use crate::geom::{Bounds3, parse_primitive, process_geometry}; use crate::model::{ ColourTable, ModelData, ParseResult, SemanticRecord, SyntaxRecord, parse_colour, }; -use crate::mpd::{OwnedVirtualFile, split_mpd}; +use crate::mpd::{OwnedVirtualFile, VirtualFile, split_mpd}; use crate::scanner::{LineType, ScannedLine, TokenKind, scan_lines}; use crate::texmap::{TexmapState, TextureDescriptor, texmap}; use crate::traversal::traverse; use crate::types::{ DEFAULT_COLOUR, DEFAULT_PROVENANCE, Diagnostic, DiagnosticCode, LdrawLimits, LimitCounters, - LimitKind, ParseError, ParseOptions, ParserProfile, SemanticArenas, add_diag, limit_error, + LimitKind, ParseError, ParseOptions, ParserProfile, Resolver, SemanticArenas, Span, add_diag, + limit_error, }; use std::borrow::Cow; use std::collections::BTreeSet; +use typed_arena::Arena; pub struct LdrawParser; impl Default for LdrawParser { @@ -26,8 +28,10 @@ impl LdrawParser { &self, bytes: &'src [u8], options: ParseOptions<'a>, + resolver: Option<&dyn Resolver>, ) -> Result, ParseError> { let mut counters = LimitCounters::default(); + let arena = Arena::>::new(); let input_len = u64::try_from(bytes.len()).map_err(|_| ParseError::Overflow("resource bytes"))?; counters @@ -78,135 +82,104 @@ impl LdrawParser { let mut colours = ColourTable::new(); let mut texture_ids = BTreeSet::new(); for file in &files { - let colours_before = colours.local_len(); - if options.cancellation.cancelled { - return Err(ParseError::Cancelled); - } - let path = NormalizedPath::new(&file.name)?; - let file_bytes = file - .lines + models.push(parse_model( + file, + &options, + &mut counters, + &mut diagnostics, + &mut arenas, + &mut colours, + &mut texture_ids, + )?); + } + + if let Some(resolver) = resolver { + let mut visited_names = models .iter() - .flat_map(|line| { - line.raw - .iter() - .copied() - .chain(line.ending.as_str().as_bytes().iter().copied()) - }) - .collect::>(); - let key = CacheKey::new(path.clone(), options.resolved_root, &file_bytes); - let mut model = ModelData { - file_id: file.id, - path, - key, - bfc: BfcFrame::default(), - colour: DEFAULT_COLOUR, - colours: colours.clone(), - steps: Vec::new(), - texmap_state: TexmapState::Inactive, - texmap_descriptor: None, - texmap_stack: Vec::new(), - texmap_next_span: None, - texmap_fallback_seen: false, - texmap_events: Vec::new(), - data_payload: false, - includes: Vec::new(), - triangles: 0, - quads: 0, - lines: 0, - conditional_lines: 0, - geometry: Vec::new(), - bounds: Bounds3::default(), - reflection: false, - }; - for line in &file.lines { - if line.blank { - continue; - } - if options.cancellation.cancelled { - return Err(ParseError::Cancelled); - } - if counters - .add(LimitKind::Commands, 1, &options.limits) - .is_err() - { - return Err(limit_error(LimitKind::Commands, Some(line.span))); - } - let before = diagnostics.len(); - let result = process_line( - &mut model, - line, - options.profile, - &mut diagnostics, - &mut counters, - &options.limits, - &mut colours, - &mut texture_ids, - ); - if let Err(error) = result { - let continue_diagnostic = match &error { - ParseError::Diagnostic(diagnostic) => { - options.profile != ParserProfile::Strict - && !matches!( - diagnostic.code, - DiagnosticCode::LineBytesLimit - | DiagnosticCode::FilesLimit - | DiagnosticCode::SemanticArenaReservationLimit - | DiagnosticCode::IncludeDepthLimit - | DiagnosticCode::CommandsLimit - | DiagnosticCode::InstancesLimit - | DiagnosticCode::TrianglesLimit - | DiagnosticCode::LinesLimit - | DiagnosticCode::TexturesLimit - | DiagnosticCode::FetchesLimit - | DiagnosticCode::DiagnosticsLimit - | DiagnosticCode::ResourceBytesLimit - ) - } - _ => false, + .map(|model| model.path.as_str().to_owned()) + .collect::>(); + let mut model_index = 0; + while model_index < models.len() { + let include_names = models[model_index] + .includes + .iter() + .map(|include| include.name.to_string()) + .collect::>(); + for include_name in include_names { + let normalized_name = include_name.replace('\\\\', "/"); + if models.iter().any(|model| { + model + .path + .as_str() + .eq_ignore_ascii_case(&normalized_name) + }) || !visited_names.insert(normalized_name.to_ascii_lowercase()) + { + continue; + } + counters + .add(LimitKind::Fetches, 1, &options.limits) + .map_err(|name| limit_error(name, None))?; + let Some(bytes) = resolver.resolve(&normalized_name) else { + continue; }; - if !continue_diagnostic { - return Err(error); + let part_bytes = arena.alloc(bytes.into_owned()).as_slice(); + counters + .add( + LimitKind::ResourceBytes, + u64::try_from(part_bytes.len()) + .map_err(|_| ParseError::Overflow("resource bytes"))?, + &options.limits, + ) + .map_err(|name| limit_error(name, None))?; + let resolved_scanned = scan_lines(part_bytes, &options.limits)?; + for line in &resolved_scanned { + counters + .add( + LimitKind::LineBytes, + u64::try_from(line.raw.len()) + .map_err(|_| ParseError::Overflow("line bytes"))?, + &options.limits, + ) + .map_err(|name| limit_error(name, Some(line.span)))?; } + let file = VirtualFile { + id: u32::try_from(files.len()) + .map_err(|_| ParseError::Overflow("virtual file id"))?, + name: Cow::Owned(normalized_name), + lines: resolved_scanned, + source_span: Span { + start: 0, + end: u32::try_from(part_bytes.len()) + .map_err(|_| ParseError::Overflow("source span"))?, + line: 1, + column: 1, + }, + }; + counters + .add(LimitKind::Files, 1, &options.limits) + .map_err(|name| limit_error(name, None))?; + files.push(file.clone()); + models.push(parse_model( + &file, + &options, + &mut counters, + &mut diagnostics, + &mut arenas, + &mut colours, + &mut texture_ids, + )?); } - let diagnostic_count = u64::try_from(diagnostics.len()) - .map_err(|_| ParseError::Overflow("diagnostic count conversion"))?; - if diagnostics.len() > before && diagnostic_count > options.limits.diagnostics { - return Err(limit_error(LimitKind::Diagnostics, Some(line.span))); - } + model_index += 1; } - let include_count = u64::try_from(model.includes.len()) - .map_err(|_| ParseError::Overflow("include count conversion"))?; - let colour_entries = colours - .local_len() - .checked_sub(colours_before) - .ok_or(ParseError::Overflow("colour entry delta"))?; - let colour_entries = u64::try_from(colour_entries) - .map_err(|_| ParseError::Overflow("colour entry conversion"))?; - let graph_entries = include_count - .checked_add(1) - .ok_or(ParseError::Overflow("graph entry estimate"))?; - let texture_events = u64::try_from(model.texmap_events.len()) - .map_err(|_| ParseError::Overflow("TEXMAP event conversion"))?; - let child_charge = 256u64 - .checked_add( - include_count - .checked_mul(128) - .ok_or(ParseError::Overflow("instance arena estimate"))?, - ) - .and_then(|charge| { - charge.checked_add(graph_entries.checked_mul(64).unwrap_or(u64::MAX)) - }) - .and_then(|charge| { - charge.checked_add(colour_entries.checked_mul(32).unwrap_or(u64::MAX)) - }) - .and_then(|charge| { - charge.checked_add(texture_events.checked_mul(48).unwrap_or(u64::MAX)) - }) - .ok_or(ParseError::Overflow("model arena estimate"))?; - arenas.child(&options, child_charge)?; - models.push(model); } - let (scene, summaries) = traverse(&models, &options, &mut counters, &mut diagnostics)?; + let (scene, summaries) = traverse( + &mut models, + &options, + &mut counters, + &mut diagnostics, + resolver, + &arena, + )?; let root = models.first(); let semantic = SemanticRecord { canonical_path: root.map(|m| Cow::Owned(m.path.to_string())), diff --git a/crates/polymodel-ldraw-core/src/resolver.rs b/crates/polymodel-ldraw-core/src/resolver.rs new file mode 100644 index 0000000..e69de29 diff --git a/crates/polymodel-ldraw-core/src/texmap.rs b/crates/polymodel-ldraw-core/src/texmap.rs index d266df4..01b5b0c 100644 --- a/crates/polymodel-ldraw-core/src/texmap.rs +++ b/crates/polymodel-ldraw-core/src/texmap.rs @@ -47,6 +47,22 @@ pub struct TextureDescriptor<'src> { #[serde(borrow)] pub glossmap: Option>, } +impl<'src> TextureDescriptor<'src> { + pub(crate) fn into_owned(self) -> TextureDescriptor<'static> { + TextureDescriptor { + mode: self.mode, + parameters: self + .parameters + .into_iter() + .map(|value| Cow::Owned(value.into_owned())) + .collect(), + pngfile: Cow::Owned(self.pngfile.into_owned()), + glossmap: self + .glossmap + .map(|value| Cow::Owned(value.into_owned())), + } + } +} #[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum TexmapKind { diff --git a/crates/polymodel-ldraw-core/src/types.rs b/crates/polymodel-ldraw-core/src/types.rs index b91e9e3..eb4a7bd 100644 --- a/crates/polymodel-ldraw-core/src/types.rs +++ b/crates/polymodel-ldraw-core/src/types.rs @@ -333,6 +333,73 @@ pub struct ParseOptions<'a> { pub fetch_count: u64, pub cancellation: CancellationPolicy, } + +pub trait Resolver { + fn resolve(&self, name: &str) -> Option>; +} + +/// Caller-owned storage for source buffers used during a parse. +/// +/// Each buffer has its own allocation, so adding another buffer never moves +/// the bytes already handed out by this bank. +pub struct ByteBank { + buffers: Vec>, +} + +impl ByteBank { + pub fn new() -> Self { + Self { buffers: Vec::new() } + } + + pub fn add(&mut self, bytes: Vec) -> &[u8] { + self.buffers.push(bytes); + self.buffers + .last() + .map(Vec::as_slice) + .expect("buffer was just added") + } + + pub(crate) fn len(&self) -> usize { + self.buffers.len() + } + + pub(crate) fn get(&self, index: usize) -> Option<&[u8]> { + self.buffers.get(index).map(Vec::as_slice) + } +} + +impl Default for ByteBank { + fn default() -> Self { + Self::new() + } +} + +pub struct LocalResolver { + base_dir: std::path::PathBuf, +} + +impl LocalResolver { + pub fn new>(base_dir: P) -> Self { + Self { base_dir: base_dir.into() } + } + + fn try_path(&self, name: &str, subdir: &str) -> Option> { + let path = self.base_dir.join(subdir).join(name); + match std::fs::read(&path) { + Ok(bytes) => Some(std::borrow::Cow::Owned(bytes)), + Err(_) => None, + } + } +} + +impl Resolver for LocalResolver { + fn resolve(&self, name: &str) -> Option> { + self.try_path(name, "parts") + .or_else(|| self.try_path(name, "p")) + .or_else(|| self.try_path(name, "parts/s")) + .or_else(|| self.try_path(name, "models")) + } +} impl<'a> ParseOptions<'a> { pub fn new(owner: ReservationOwner, ledger: &'a ReservationLedger) -> Self { Self { -- 2.51.2