diff --git a/mlf-cli/src/check.rs b/mlf-cli/src/check.rs index 4a432f9..85f3a59 100644 --- a/mlf-cli/src/check.rs +++ b/mlf-cli/src/check.rs @@ -1,4 +1,5 @@ -use crate::config::{find_project_root, ConfigError, MlfConfig}; +use crate::config::{find_project_root, get_mlf_cache_dir, ConfigError, MlfConfig}; +use crate::workspace_ext::workspace_with_std_and_cache; use miette::Diagnostic; use mlf_diagnostics::{ParseDiagnostic, ValidationDiagnostic}; use std::path::PathBuf; @@ -106,9 +107,21 @@ pub fn run_check(input_patterns: Vec) -> Result<(), CheckError> { } } - let mut workspace = mlf_lang::Workspace::with_std().map_err(|e| { + // Try to load cached lexicons from .mlf directory + let current_dir = std::env::current_dir() + .map_err(|e| CheckError::ReadFile { + path: ".".to_string(), + source: e, + })?; + + let mlf_cache_dir = find_project_root(¤t_dir) + .ok() + .map(|root| get_mlf_cache_dir(&root)); + + let mut workspace = workspace_with_std_and_cache(mlf_cache_dir.as_deref()).map_err(|e| { + eprintln!("Error loading workspace: {}", e); CheckError::ValidationErrors { - help: Some(format!("Failed to load standard library: {:?}", e)), + help: Some(format!("Failed to load workspace: {}", e)), } })?; diff --git a/mlf-cli/src/fetch.rs b/mlf-cli/src/fetch.rs index a82ca0b..c31beee 100644 --- a/mlf-cli/src/fetch.rs +++ b/mlf-cli/src/fetch.rs @@ -269,11 +269,14 @@ pub fn fetch_lexicon(nsid: &str, project_root: &std::path::Path) -> Result<(), F println!(" Processing: {}", record_nsid); - // Save JSON file + // Save JSON file with directory structure + // e.g., "place.stream.key" -> "place/stream/key.json" let json_str = serde_json::to_string_pretty(&record.value)?; - let json_path = mlf_dir - .join("lexicons/json") - .join(format!("{}.json", record_nsid)); + let mut json_path = mlf_dir.join("lexicons/json"); + for segment in record_nsid.split('.') { + json_path.push(segment); + } + json_path.set_extension("json"); // Create parent directories if let Some(parent) = json_path.parent() { @@ -287,10 +290,13 @@ pub fn fetch_lexicon(nsid: &str, project_root: &std::path::Path) -> Result<(), F let mlf_content = crate::generate::mlf::generate_mlf_from_json(&record.value) .map_err(|e| FetchError::ConversionError(format!("{:?}", e)))?; - // Save MLF file - let mlf_path = mlf_dir - .join("lexicons/mlf") - .join(format!("{}.mlf", record_nsid)); + // Save MLF file with directory structure + // e.g., "place.stream.key" -> "place/stream/key.mlf" + let mut mlf_path = mlf_dir.join("lexicons/mlf"); + for segment in record_nsid.split('.') { + mlf_path.push(segment); + } + mlf_path.set_extension("mlf"); // Create parent directories if let Some(parent) = mlf_path.parent() { diff --git a/mlf-cli/src/generate/mlf.rs b/mlf-cli/src/generate/mlf.rs index 8709fbf..774b050 100644 --- a/mlf-cli/src/generate/mlf.rs +++ b/mlf-cli/src/generate/mlf.rs @@ -214,7 +214,7 @@ pub fn generate_mlf_from_json(json: &Value) -> Result output.push('\n'); } "object" => { - let mlf = generate_def_type(name, def)?; + let mlf = generate_def_type(name, def, last_segment)?; output.push_str(&mlf); output.push('\n'); } @@ -302,13 +302,14 @@ fn generate_record(name: &str, def: &Value, last_segment: &str) -> Result Result Result Result Result Result< let is_required = required.contains(¶m_name.as_str()); let required_marker = if is_required { "!" } else { "" }; let param_type = generate_type(param_def).unwrap_or_else(|_| "unknown".to_string()); + let escaped_param_name = escape_name(param_name); - format!("{}{}: {}", param_name, required_marker, param_type) + format!("{}{}: {}", escaped_param_name, required_marker, param_type) }) .collect(); @@ -566,14 +570,22 @@ fn generate_token(name: &str, def: &Value) -> Result { } } - output.push_str(&format!("token {};\n", name)); + let escaped_name = escape_name(name); + output.push_str(&format!("token {};\n", escaped_name)); Ok(output) } -fn generate_def_type(name: &str, def: &Value) -> Result { +fn generate_def_type(name: &str, def: &Value, last_segment: &str) -> Result { let mut output = String::new(); - output.push_str(&format!("def type {} = ", name)); + // Use last segment of NSID for "main" definitions + let def_name = if name == "main" { + escape_name(last_segment) + } else { + escape_name(name) + }; + + output.push_str(&format!("def type {} = ", def_name)); let type_str = generate_type_with_indent(def, 0)?; output.push_str(&type_str); output.push_str(";\n"); @@ -620,9 +632,10 @@ fn generate_type_with_indent(type_def: &Value, indent_level: usize) -> Result Result { message: "Missing 'items' in array type".to_string(), } })?; - let item_type = generate_type(items)?; + + // Check if items have constraints + let items_obj = items.as_object(); + let has_item_constraints = items_obj.map_or(false, |obj| { + obj.contains_key("minLength") || + obj.contains_key("maxLength") || + obj.contains_key("minGraphemes") || + obj.contains_key("maxGraphemes") || + obj.contains_key("minimum") || + obj.contains_key("maximum") || + obj.contains_key("enum") || + obj.contains_key("knownValues") || + obj.contains_key("default") + }); + + let item_type = if has_item_constraints { + // If item has constraints, we need to wrap in parentheses to apply constraints before [] + // For now, just generate the base type without item constraints + // TODO: Consider generating a type alias for complex constrained items + items.get("type") + .and_then(|t| t.as_str()) + .unwrap_or("unknown") + .to_string() + } else { + generate_type(items)? + }; + let mut result = format!("{}[]", item_type); result = apply_constraints(result, type_def); Ok(result) @@ -735,9 +774,10 @@ fn generate_type(type_def: &Value) -> Result { let is_required = required.contains(&field_name.as_str()); let required_marker = if is_required { "!" } else { "" }; let field_type = generate_type(field_def)?; + let escaped_field_name = escape_name(field_name); output.push_str(&format!( " {}{}: {},\n", - field_name, required_marker, field_type + escaped_field_name, required_marker, field_type )); } @@ -767,8 +807,9 @@ fn generate_type(type_def: &Value) -> Result { } Some("ref") => { if let Some(ref_str) = type_def.get("ref").and_then(|v| v.as_str()) { - // Convert ref format from namespace#name to namespace.name - // Also strip leading # for local refs + // Convert refs: strip leading # and convert remaining # to . + // "#audio" -> "audio" (local ref, just the name) + // "com.example#foo" -> "com.example.foo" (external ref) let clean_ref = ref_str.trim_start_matches('#').replace('#', "."); Ok(clean_ref) } else { diff --git a/mlf-cli/src/workspace_ext.rs b/mlf-cli/src/workspace_ext.rs index 4d6ac78..c74ecef 100644 --- a/mlf-cli/src/workspace_ext.rs +++ b/mlf-cli/src/workspace_ext.rs @@ -18,7 +18,7 @@ pub fn load_mlf_directory(workspace: &mut Workspace, dir: &Path) -> Result<(), S .map_err(|e| format!("Failed to read {}: {}", file_path.display(), e))?; // Convert file path to namespace - // e.g., ".mlf/lexicons/mlf/stream.place.mlf" -> "stream.place" + // e.g., ".mlf/lexicons/mlf/place/stream/key.mlf" -> "place.stream" let namespace = extract_namespace_from_path(&file_path, dir)?; // Parse the lexicon @@ -63,7 +63,9 @@ fn find_mlf_files(dir: &Path) -> Result, String> { } /// Extract namespace from file path relative to base directory -/// e.g., base=".mlf/lexicons/mlf", path=".mlf/lexicons/mlf/stream.place.mlf" -> "stream.place" +/// The namespace includes the full path WITH the filename (minus .mlf extension) +/// e.g., base=".mlf/lexicons/mlf", path=".mlf/lexicons/mlf/place/stream/key.mlf" -> "place.stream.key" +/// This allows "place.stream.key" to resolve to a definition named "key" in namespace "place.stream.key" fn extract_namespace_from_path(path: &Path, base: &Path) -> Result { let relative = path .strip_prefix(base) @@ -80,7 +82,7 @@ fn extract_namespace_from_path(path: &Path, base: &Path) -> Result "stream.place.foo" + // e.g., "place/stream/key" -> "place.stream.key" let namespace = without_ext.replace(std::path::MAIN_SEPARATOR, "."); Ok(namespace) @@ -122,20 +124,29 @@ mod tests { use super::*; #[test] - fn test_extract_namespace() { + fn test_extract_namespace_nested() { let base = Path::new(".mlf/lexicons/mlf"); - let path = Path::new(".mlf/lexicons/mlf/stream.place.mlf"); + let path = Path::new(".mlf/lexicons/mlf/place/stream/key.mlf"); let namespace = extract_namespace_from_path(path, base).unwrap(); - assert_eq!(namespace, "stream.place"); + // Full path "place/stream/key" becomes "place.stream.key" + assert_eq!(namespace, "place.stream.key"); } #[test] - fn test_extract_namespace_nested() { + fn test_extract_namespace_deep() { let base = Path::new(".mlf/lexicons/mlf"); let path = Path::new(".mlf/lexicons/mlf/com/atproto/admin/defs.mlf"); let namespace = extract_namespace_from_path(path, base).unwrap(); - // On Unix: "com/atproto/admin/defs" -> "com.atproto.admin.defs" - // Note: This depends on the directory structure - assert!(namespace.contains("com")); + // Full path "com/atproto/admin/defs" becomes "com.atproto.admin.defs" + assert_eq!(namespace, "com.atproto.admin.defs"); + } + + #[test] + fn test_extract_namespace_root() { + let base = Path::new(".mlf/lexicons/mlf"); + let path = Path::new(".mlf/lexicons/mlf/simple.mlf"); + let namespace = extract_namespace_from_path(path, base).unwrap(); + // File "simple" becomes namespace "simple" + assert_eq!(namespace, "simple"); } } diff --git a/mlf-codegen/src/lib.rs b/mlf-codegen/src/lib.rs index 5446e8c..5aaea78 100644 --- a/mlf-codegen/src/lib.rs +++ b/mlf-codegen/src/lib.rs @@ -159,6 +159,7 @@ fn analyze_type_usage(lexicon: &Lexicon) -> HashMap { count_type_references(¶m.ty, &mut usage_counts); } match &query.returns { + ReturnType::None { .. } => {} ReturnType::Type(ty) => count_type_references(ty, &mut usage_counts), ReturnType::TypeWithErrors { success, .. } => { count_type_references(success, &mut usage_counts) @@ -170,6 +171,7 @@ fn analyze_type_usage(lexicon: &Lexicon) -> HashMap { count_type_references(¶m.ty, &mut usage_counts); } match &procedure.returns { + ReturnType::None { .. } => {} ReturnType::Type(ty) => count_type_references(ty, &mut usage_counts), ReturnType::TypeWithErrors { success, .. } => { count_type_references(success, &mut usage_counts) @@ -180,7 +182,9 @@ fn analyze_type_usage(lexicon: &Lexicon) -> HashMap { for param in &subscription.params { count_type_references(¶m.ty, &mut usage_counts); } - count_type_references(&subscription.messages, &mut usage_counts); + if let Some(messages) = &subscription.messages { + count_type_references(messages, &mut usage_counts); + } } Item::InlineType(inline_type) => { count_type_references(&inline_type.ty, &mut usage_counts); @@ -292,11 +296,12 @@ fn generate_query_json(query: &Query, usage_counts: &HashMap, wor }; let output = match &query.returns { + ReturnType::None { .. } => None, ReturnType::Type(ty) => { let mut output_obj = Map::new(); output_obj.insert("encoding".to_string(), json!("application/json")); output_obj.insert("schema".to_string(), generate_type_json(ty, usage_counts, workspace, current_namespace)); - Value::Object(output_obj) + Some(Value::Object(output_obj)) } ReturnType::TypeWithErrors { success, errors, .. } => { let mut error_defs = Map::new(); @@ -313,7 +318,7 @@ fn generate_query_json(query: &Query, usage_counts: &HashMap, wor output_obj.insert("encoding".to_string(), json!("application/json")); output_obj.insert("schema".to_string(), generate_type_json(success, usage_counts, workspace, current_namespace)); output_obj.insert("errors".to_string(), json!(error_defs)); - Value::Object(output_obj) + Some(Value::Object(output_obj)) } }; @@ -321,7 +326,9 @@ fn generate_query_json(query: &Query, usage_counts: &HashMap, wor query_obj.insert("type".to_string(), json!("query")); query_obj.insert("description".to_string(), json!(extract_docs(&query.docs))); query_obj.insert("parameters".to_string(), params); - query_obj.insert("output".to_string(), output); + if let Some(output_val) = output { + query_obj.insert("output".to_string(), output_val); + } Value::Object(query_obj) } @@ -358,11 +365,12 @@ fn generate_procedure_json(procedure: &Procedure, usage_counts: &HashMap None, ReturnType::Type(ty) => { let mut output_obj = Map::new(); output_obj.insert("encoding".to_string(), json!("application/json")); output_obj.insert("schema".to_string(), generate_type_json(ty, usage_counts, workspace, current_namespace)); - Value::Object(output_obj) + Some(Value::Object(output_obj)) } ReturnType::TypeWithErrors { success, errors, .. } => { let mut error_defs = Map::new(); @@ -379,7 +387,7 @@ fn generate_procedure_json(procedure: &Procedure, usage_counts: &HashMap, pub name: Ident, pub params: Vec, - pub messages: Type, // Union of message types + pub messages: Option, // Union of message types (optional) pub span: Span, } /// Return type for queries and procedures #[derive(Debug, Clone, PartialEq)] pub enum ReturnType { + /// No return type specified + None { span: Span }, /// Simple return type Type(Type), /// Return type with error handling @@ -184,6 +186,7 @@ pub enum ReturnType { impl ReturnType { pub fn span(&self) -> Span { match self { + ReturnType::None { span } => *span, ReturnType::Type(ty) => ty.span(), ReturnType::TypeWithErrors { span, .. } => *span, } diff --git a/mlf-lang/src/parser.rs b/mlf-lang/src/parser.rs index c825c6e..599b7a6 100644 --- a/mlf-lang/src/parser.rs +++ b/mlf-lang/src/parser.rs @@ -386,37 +386,44 @@ impl Parser { let params = self.parse_params()?; - self.expect(LexToken::RightParen)?; - self.expect(LexToken::Colon)?; + let right_paren_span = self.expect(LexToken::RightParen)?; - let output = self.parse_base_type()?; + // Check if there's a return type (colon present) + let returns = if matches!(self.current().token, LexToken::Colon) { + self.advance(); // consume colon - let returns = if matches!(self.current().token, LexToken::Pipe) { - self.advance(); - if matches!(self.current().token, LexToken::Error) { - self.advance(); - let errors = self.parse_errors()?; - let error_span = errors.last().map(|e| e.span).unwrap_or(output.span()); - ReturnType::TypeWithErrors { - success: output, - errors, - span: error_span, - } - } else { - let mut types = alloc::vec![output]; - types.push(self.parse_base_type()?); + let output = self.parse_base_type()?; - while matches!(self.current().token, LexToken::Pipe) { + if matches!(self.current().token, LexToken::Pipe) { + self.advance(); + if matches!(self.current().token, LexToken::Error) { self.advance(); + let errors = self.parse_errors()?; + let error_span = errors.last().map(|e| e.span).unwrap_or(output.span()); + ReturnType::TypeWithErrors { + success: output, + errors, + span: error_span, + } + } else { + let mut types = alloc::vec![output]; types.push(self.parse_base_type()?); - } - let span = Span::new(types[0].span().start, types.last().unwrap().span().end); - // Return type unions are open by default (no ! support in return types yet) - ReturnType::Type(Type::Union { types, closed: false, span }) + while matches!(self.current().token, LexToken::Pipe) { + self.advance(); + types.push(self.parse_base_type()?); + } + + let span = Span::new(types[0].span().start, types.last().unwrap().span().end); + // Return type unions are open by default (no ! support in return types yet) + ReturnType::Type(Type::Union { types, closed: false, span }) + } + } else { + ReturnType::Type(output) } } else { - ReturnType::Type(output) + // No return type specified + ReturnType::None { span: right_paren_span } }; let end = self.expect(LexToken::Semicolon)?; @@ -438,37 +445,44 @@ impl Parser { let params = self.parse_params()?; - self.expect(LexToken::RightParen)?; - self.expect(LexToken::Colon)?; + let right_paren_span = self.expect(LexToken::RightParen)?; - let output = self.parse_base_type()?; + // Check if there's a return type (colon present) + let returns = if matches!(self.current().token, LexToken::Colon) { + self.advance(); // consume colon - let returns = if matches!(self.current().token, LexToken::Pipe) { - self.advance(); - if matches!(self.current().token, LexToken::Error) { - self.advance(); - let errors = self.parse_errors()?; - let error_span = errors.last().map(|e| e.span).unwrap_or(output.span()); - ReturnType::TypeWithErrors { - success: output, - errors, - span: error_span, - } - } else { - let mut types = alloc::vec![output]; - types.push(self.parse_base_type()?); + let output = self.parse_base_type()?; - while matches!(self.current().token, LexToken::Pipe) { + if matches!(self.current().token, LexToken::Pipe) { + self.advance(); + if matches!(self.current().token, LexToken::Error) { self.advance(); + let errors = self.parse_errors()?; + let error_span = errors.last().map(|e| e.span).unwrap_or(output.span()); + ReturnType::TypeWithErrors { + success: output, + errors, + span: error_span, + } + } else { + let mut types = alloc::vec![output]; types.push(self.parse_base_type()?); - } - let span = Span::new(types[0].span().start, types.last().unwrap().span().end); - // Return type unions are open by default (no ! support in return types yet) - ReturnType::Type(Type::Union { types, closed: false, span }) + while matches!(self.current().token, LexToken::Pipe) { + self.advance(); + types.push(self.parse_base_type()?); + } + + let span = Span::new(types[0].span().start, types.last().unwrap().span().end); + // Return type unions are open by default (no ! support in return types yet) + ReturnType::Type(Type::Union { types, closed: false, span }) + } + } else { + ReturnType::Type(output) } } else { - ReturnType::Type(output) + // No return type specified + ReturnType::None { span: right_paren_span } }; let end = self.expect(LexToken::Semicolon)?; @@ -491,9 +505,14 @@ impl Parser { let params = self.parse_params()?; self.expect(LexToken::RightParen)?; - self.expect(LexToken::Colon)?; - let messages = self.parse_type()?; + // Check if there's a messages type (colon present) + let messages = if matches!(self.current().token, LexToken::Colon) { + self.advance(); // consume colon + Some(self.parse_type()?) + } else { + None + }; let end = self.expect(LexToken::Semicolon)?; @@ -1415,4 +1434,72 @@ mod tests { _ => panic!("Expected inline type"), } } + + #[test] + fn test_parse_query_without_return_type() { + let input = "query foo();"; + let result = parse_lexicon(input); + assert!(result.is_ok()); + let lexicon = result.unwrap(); + assert_eq!(lexicon.items.len(), 1); + match &lexicon.items[0] { + Item::Query(q) => { + assert_eq!(q.name.name, "foo"); + assert_eq!(q.params.len(), 0); + assert!(matches!(q.returns, ReturnType::None { .. })); + } + _ => panic!("Expected query"), + } + } + + #[test] + fn test_parse_query_with_params_no_return() { + let input = "query bar(id: string, count: integer);"; + let result = parse_lexicon(input); + assert!(result.is_ok()); + let lexicon = result.unwrap(); + assert_eq!(lexicon.items.len(), 1); + match &lexicon.items[0] { + Item::Query(q) => { + assert_eq!(q.name.name, "bar"); + assert_eq!(q.params.len(), 2); + assert!(matches!(q.returns, ReturnType::None { .. })); + } + _ => panic!("Expected query"), + } + } + + #[test] + fn test_parse_procedure_without_return_type() { + let input = "procedure baz();"; + let result = parse_lexicon(input); + assert!(result.is_ok()); + let lexicon = result.unwrap(); + assert_eq!(lexicon.items.len(), 1); + match &lexicon.items[0] { + Item::Procedure(p) => { + assert_eq!(p.name.name, "baz"); + assert_eq!(p.params.len(), 0); + assert!(matches!(p.returns, ReturnType::None { .. })); + } + _ => panic!("Expected procedure"), + } + } + + #[test] + fn test_parse_subscription_without_messages() { + let input = "subscription updates();"; + let result = parse_lexicon(input); + assert!(result.is_ok()); + let lexicon = result.unwrap(); + assert_eq!(lexicon.items.len(), 1); + match &lexicon.items[0] { + Item::Subscription(s) => { + assert_eq!(s.name.name, "updates"); + assert_eq!(s.params.len(), 0); + assert!(s.messages.is_none()); + } + _ => panic!("Expected subscription"), + } + } } diff --git a/mlf-lang/src/workspace.rs b/mlf-lang/src/workspace.rs index d23fe87..4b25724 100644 --- a/mlf-lang/src/workspace.rs +++ b/mlf-lang/src/workspace.rs @@ -963,6 +963,9 @@ impl Workspace { } match &query.returns { + ReturnType::None { .. } => { + // No return type to resolve + } ReturnType::Type(ty) => { if let Err(mut ty_errors) = self.resolve_type(namespace, ty) { errors.append(&mut ty_errors); @@ -992,6 +995,9 @@ impl Workspace { } match &procedure.returns { + ReturnType::None { .. } => { + // No return type to resolve + } ReturnType::Type(ty) => { if let Err(mut ty_errors) = self.resolve_type(namespace, ty) { errors.append(&mut ty_errors); @@ -1020,8 +1026,10 @@ impl Workspace { } } - if let Err(mut msg_errors) = self.resolve_type(namespace, &subscription.messages) { - errors.append(&mut msg_errors); + if let Some(messages) = &subscription.messages { + if let Err(mut msg_errors) = self.resolve_type(namespace, messages) { + errors.append(&mut msg_errors); + } } if errors.is_empty() {