From 6209a66d136d733b3f5824855486dbe38149fe9a Mon Sep 17 00:00:00 2001 From: Matt Stavola Date: Mon, 6 Oct 2025 02:00:16 -0400 Subject: [PATCH] Change in syntax and semantics --- .gitignore | 2 + Cargo.lock | 23 ++ SPEC.md | 389 ++++++++++++++++------------ mlf-cli/src/generate/lexicon.rs | 39 ++- mlf-codegen/Cargo.toml | 2 +- mlf-codegen/src/lib.rs | 298 +++++++++++++-------- mlf-diagnostics/src/lib.rs | 8 + mlf-lang/src/ast.rs | 72 +++-- mlf-lang/src/error.rs | 1 + mlf-lang/src/lexer.rs | 12 +- mlf-lang/src/parser.rs | 324 +++++++++++++++++------ mlf-lang/src/workspace.rs | 183 +++++++++++-- mlf-validation/src/lib.rs | 26 +- mlf-wasm/src/lib.rs | 35 ++- resources/prelude.mlf | 22 +- tree-sitter-mlf/grammar.js | 33 +-- tree-sitter-mlf/src/grammar.json | 51 +++- tree-sitter-mlf/src/node-types.json | 104 +++++--- website/content/docs/syntax.md | 303 ++++++++++++---------- website/sass/style.scss | 130 ++++++++-- website/static/js/app.js | 204 ++++++++++++--- website/syntaxes/mlf.sublime-syntax | 4 +- website/templates/playground.html | 9 +- 23 files changed, 1589 insertions(+), 685 deletions(-) diff --git a/.gitignore b/.gitignore index 5793325..25f4ac2 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,5 @@ Thumbs.db # Claude Code CLAUDE.md .claude/*.local.json + +output-testing diff --git a/Cargo.lock b/Cargo.lock index 4733d93..9755166 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -174,6 +174,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + [[package]] name = "errno" version = "0.3.14" @@ -202,12 +208,28 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "hashbrown" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" + [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "indexmap" +version = "2.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "is_ci" version = "1.2.0" @@ -540,6 +562,7 @@ version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" dependencies = [ + "indexmap", "itoa", "memchr", "ryu", diff --git a/SPEC.md b/SPEC.md index d99a7ec..439edad 100644 --- a/SPEC.md +++ b/SPEC.md @@ -24,10 +24,12 @@ MLF is a domain-specific language (DSL) for writing ATProto Lexicons with 100% f The `#` character is reserved for shebangs only and is not used elsewhere in the syntax. ### File Naming Convention -Files should follow the lexicon NSID: +The file path determines the lexicon NSID. Files should follow the lexicon NSID structure: - `app.bsky.feed.post.mlf` → Lexicon NSID: `app.bsky.feed.post` - `sh.tangled.repo.issue.mlf` → Lexicon NSID: `sh.tangled.repo.issue` +The lexicon NSID is derived solely from the filename, not from any internal namespace declarations. + ## Core Concepts ### NSIDs (Namespaced Identifiers) @@ -50,22 +52,64 @@ References to definitions can be: 1. **Local (same file)**: Just use the name ```mlf record myRecord { - field: myAlias, // References alias in same file + field: myType // References type in same file } - alias myAlias = { /* ... */ }; + def type myType = { /* ... */ } ``` 2. **Cross-file (different lexicon)**: Use full dotted path ```mlf record myRecord { - profile: app.bsky.actor.profile, // References app/bsky/actor/profile.mlf - author: com.example.user.author, // References com/example/user/author.mlf + profile: app.bsky.actor.profile // References app/bsky/actor/profile.mlf + author: com.example.user.author // References com/example/user/author.mlf } ``` **Note**: The `#` character is NOT used for references. All references use dotted notation. +### Syntax Rules + +#### Semicolons + +- **Records** do NOT have semicolons after the closing brace `}` +- All other definitions require semicolons: + - `use` statements end with `;` + - `token` definitions end with `;` + - `inline type` definitions end with `;` + - `def type` definitions end with `;` + - `query` definitions end with `;` + - `procedure` definitions end with `;` + - `subscription` definitions end with `;` + +#### Commas + +Commas are **required** between items, with **trailing commas allowed**: + +- **Record fields**: Commas required between fields, trailing comma allowed + ```mlf + record example { + field1: string, + field2: integer, // trailing comma allowed + } + ``` + +- **Constraints**: Commas required between constraint properties, trailing comma allowed + ```mlf + title: string constrained { + maxLength: 200, + minLength: 1, // trailing comma allowed + } + ``` + +- **Error definitions**: Commas required between errors, trailing comma allowed + ```mlf + query getThread(): thread | error { + NotFound, + BadRequest, // trailing comma allowed + } + ``` + ## Type System ### Primitive Types @@ -106,8 +150,8 @@ blob // Generic blob With constraints: ```mlf avatar: blob constrained { - accept: ["image/png", "image/jpeg"], - maxSize: 1000000, // bytes + accept: ["image/png", "image/jpeg"] + maxSize: 1000000 // bytes } ``` @@ -126,26 +170,36 @@ Records are the primary data structure, stored in repositories: ```mlf record post { text: string constrained { - maxLength: 300, - maxGraphemes: 300, - }, - createdAt: Datetime, - reply?: replyRef, // Optional field + maxLength: 300 + maxGraphemes: 300 + } + createdAt: Datetime + reply?: replyRef // Optional field } ``` -### Aliases +### Type Definitions + +MLF supports two kinds of type definitions: -Type aliases define reusable object shapes: +**Inline Types** - Expanded at the point of use, never appear in generated lexicon defs: ```mlf -alias replyRef = { - root: AtUri, - parent: AtUri, +inline type AtIdentifier = string constrained { + format "at-identifier" }; ``` -If used in multiple places, they will be hoisted to a def. If only used in a single place, they will be inlined. +**Def Types** - Become named definitions in the lexicon's defs block: + +```mlf +def type ReplyRef = { + root: AtUri + parent: AtUri +}; +``` + +Use `inline type` for type aliases that should be expanded inline (like primitive type wrappers). Use `def type` for types that should be referenced by name in the generated lexicon. ### Tokens @@ -161,11 +215,11 @@ token closed; record issue { state: string constrained { knownValues: [ - open, // References token defined above - closed, - ], - default: "open", - }, + open // References token defined above + closed + ] + default: "open" + } } ``` @@ -179,14 +233,14 @@ Queries are read-only HTTP endpoints (GET): /// Get a user profile query getProfile( /// The actor's DID or handle - actor: AtIdentifier, + actor: AtIdentifier /// Optional viewer context - viewer?: Did, + viewer?: Did ): profileView | error { /// Profile not found - ProfileNotFound, + ProfileNotFound /// Invalid request parameters - BadRequest, + BadRequest }; ``` @@ -197,14 +251,14 @@ Procedures are write operations (POST): ```mlf /// Create a new post procedure createPost( - text: string, - createdAt: Datetime, + text: string + createdAt: Datetime ): { - uri: AtUri, - cid: Cid, + uri: AtUri + cid: Cid } | error { /// Text exceeds maximum length - TextTooLong, + TextTooLong }; ``` @@ -216,32 +270,32 @@ Subscriptions are WebSocket-based event streams that emit messages over time. Th /// Subscribe to repository events subscription subscribeRepos( /// Optional cursor for resuming from a specific point - cursor?: integer, + cursor?: integer ): commit | identity | handle | migrate | tombstone | info; ``` -**Message definitions** for subscriptions are defined as aliases or records: +**Message definitions** for subscriptions are defined as def types or records: ```mlf /// Commit message emitted by subscribeRepos -alias commit = { - seq: integer, - rebase: boolean, - tooBig: boolean, - repo: Did, - commit: Cid, - rev: string, - since: string, - blocks: bytes, - ops: repoOp[], - blobs: Cid[], - time: Datetime, +def type commit = { + seq: integer + rebase: boolean + tooBig: boolean + repo: Did + commit: Cid + rev: string + since: string + blocks: bytes + ops: repoOp[] + blobs: Cid[] + time: Datetime }; /// Info message -alias info = { - name: string, - message?: string, +def type info = { + name: string + message?: string }; ``` @@ -249,7 +303,7 @@ alias info = { - Parameters: Like queries, subscriptions can have parameters - Return type: A union of message types that can be emitted -- Each message type must be defined as an alias or record +- Each message type must be defined as a def type or record - Message types can be local or imported from other lexicons - Subscriptions are long-lived WebSocket connections - No error block (errors are handled at the WebSocket protocol level) @@ -260,32 +314,32 @@ alias info = { /// Subscribe to chat messages for a stream subscription subscribeChat( /// The DID of the streamer - streamer: Did, + streamer: Did /// Optional cursor to resume from - cursor?: string, + cursor?: string ): message | delete | join | leave; /// Chat message payload -alias message = { - id: string, - text: string, - author: Did, - createdAt: Datetime, +def type message = { + id: string + text: string + author: Did + createdAt: Datetime }; /// Delete event payload -alias delete = { - id: string, +def type delete = { + id: string }; /// Join event payload -alias join = { - user: Did, +def type join = { + user: Did }; /// Leave event payload -alias leave = { - user: Did, +def type leave = { + user: Did }; ``` @@ -304,8 +358,8 @@ Queries and procedures can return: ```mlf record example { - required: string, - optional?: string, + required: string + optional?: string } ``` @@ -313,11 +367,11 @@ record example { ```mlf record example { - tags: string[], + tags: string[] items: string[] constrained { - minLength: 1, - maxLength: 10, - }, + minLength: 1 + maxLength: 10 + } } ``` @@ -328,10 +382,10 @@ Use the pipe operator `|`: ```mlf record example { // Closed union (only these types) - content: text | image | video, + content: text | image | video // Union of tokens - state: open | closed | pending, + state: open | closed | pending } ``` @@ -340,7 +394,7 @@ Open unions (allowing unknown types) use `_`: ```mlf record example { // Open union (can include unknown types) - content: text | image | _, + content: text | image | _ } ``` @@ -351,12 +405,12 @@ Reference local or external definitions: ```mlf // Local reference (same file) record post { - author: author, // References 'alias author' in same file + author: author // References 'def type author' in same file } // Cross-file reference record post { - profile: app.bsky.actor.profile, // References app/bsky/actor/profile.mlf + profile: app.bsky.actor.profile // References app/bsky/actor/profile.mlf } ``` @@ -370,23 +424,23 @@ When applying constraints, each constraint must be **at least as restrictive** a ```mlf // Valid: More restrictive constraints -alias shortString = string constrained { - maxLength: 100, +def type shortString = string constrained { + maxLength: 100 }; record post { // Can further constrain to 50 (more restrictive than 100) title: shortString constrained { - maxLength: 50, // ✓ Valid: 50 ≤ 100 - }, + maxLength: 50 // ✓ Valid: 50 ≤ 100 + } } // Invalid: Less restrictive constraints record invalid { // ERROR: Cannot expand to 200 (less restrictive than 100) content: shortString constrained { - maxLength: 200, // ✗ Invalid: 200 > 100 - }, + maxLength: 200 // ✗ Invalid: 200 > 100 + } } ``` @@ -403,28 +457,34 @@ record invalid { ```mlf field: string constrained { - minLength: 1, // Minimum byte length - maxLength: 1000, // Maximum byte length - minGraphemes: 1, // Minimum grapheme clusters - maxGraphemes: 100, // Maximum grapheme clusters - format: "uri", // Format validation - enum: ["a", "b", "c"], // Allowed values (closed set) - knownValues: [ // Known values (extensible set) - value1, - value2, - ], - default: "defaultValue", // Default value + minLength: 1 // Minimum byte length + maxLength: 1000 // Maximum byte length + minGraphemes: 1 // Minimum grapheme clusters + maxGraphemes: 100 // Maximum grapheme clusters + format: "uri" // Format validation + enum: ["a", "b", "c"] // Allowed values (closed set) - string literals + knownValues: [ // Known values (extensible set) - can be string literals OR token references + value1 // Token reference + "value2" // String literal + ] + default: "defaultValue" // Default value } ``` +**Note**: `enum`, `knownValues`, and `default` can accept either: +- **Literals**: `"open"`, `42`, `true` (string, integer, or boolean) +- **References**: `open`, `myType` (references to tokens, records, types, etc.) + +When using references, the identifier will be resolved to its string representation in the generated lexicon. + ### Integer Constraints ```mlf field: integer constrained { - minimum: 0, - maximum: 100, - enum: [1, 2, 3], - default: 1, + minimum: 0 + maximum: 100 + enum: [1, 2, 3] + default: 1 } ``` @@ -432,8 +492,8 @@ field: integer constrained { ```mlf field: string[] constrained { - minLength: 1, - maxLength: 10, + minLength: 1 + maxLength: 10 } ``` @@ -441,8 +501,8 @@ field: string[] constrained { ```mlf field: blob constrained { - accept: ["image/png", "image/jpeg"], // MIME types - maxSize: 1000000, // Bytes + accept: ["image/png", "image/jpeg"] // MIME types + maxSize: 1000000 // Bytes } ``` @@ -450,7 +510,7 @@ field: blob constrained { ```mlf field: boolean constrained { - default: false, + default: false } ``` @@ -464,7 +524,7 @@ Use `///` for documentation (appears in generated docs/code): /// A user profile record record profile { /// The user's display name - displayName?: string, + displayName?: string } ``` @@ -484,7 +544,7 @@ Three forms of annotations are supported: ```mlf @deprecated record oldRecord { - field: string, + field: string } ``` @@ -493,7 +553,7 @@ record oldRecord { @since(1, 2, 0) @doc("https://example.com/docs") record example { - field: string, + field: string } ``` @@ -507,7 +567,7 @@ Arguments can be: @validate(min: 0, max: 100, strict: true) @codegen(language: "rust", derive: "Debug, Clone") record example { - field: integer, + field: integer } ``` @@ -515,12 +575,13 @@ record example { Annotations can be placed on: - Records -- Aliases +- Inline Types +- Def Types - Tokens - Queries - Procedures - Subscriptions -- Fields within records/aliases +- Fields within records/types ```mlf /// A user profile @@ -528,11 +589,11 @@ Annotations can be placed on: record profile { /// User's DID @indexed - did: Did, + did: Did /// Display name @sensitive(pii: true) - displayName?: string, + displayName?: string } ``` @@ -567,28 +628,6 @@ record experimentalFeature { /* ... */ } **Note:** The interpretation of annotations is entirely up to the tooling consuming the MLF. Different tools may support different annotation sets. -## Namespaces - -Organize related definitions within namespaces: - -```mlf -namespace .actor { - record profile { - displayName?: string, - } - - query getProfile( - actor: AtIdentifier, - ): profile; -} - -namespace .feed { - record post { - text: string, - } -} -``` - ## Use Statements Import definitions from other lexicons: @@ -614,7 +653,7 @@ After importing, use the short name: use app.bsky.actor.profile; record myThing { - author: profile, // Instead of app.bsky.actor.profile + author: profile // Instead of app.bsky.actor.profile } ``` @@ -641,7 +680,7 @@ When resolving cross-file references: ### File Path Convention -Lexicons follow a directory structure matching their NSID: +The lexicon NSID is determined by the file path. Lexicons can follow a directory structure matching their NSID: ``` lexicons/ @@ -656,7 +695,7 @@ lexicons/ thing.mlf → com.example.thing ``` -Or flat with dots in filename: +Or use a flat structure with dots in the filename: ``` lexicons/ app.bsky.actor.profile.mlf @@ -664,6 +703,8 @@ lexicons/ com.example.thing.mlf ``` +In both cases, the NSID is derived from the file path, not from internal declarations. + ## CLI Commands ```bash @@ -701,65 +742,65 @@ token closed; /// An issue in a repository record issue { /// The repository this issue belongs to - repo: AtUri, + repo: AtUri /// Issue title title: string constrained { - minGraphemes: 1, - maxGraphemes: 200, - }, + minGraphemes: 1 + maxGraphemes: 200 + } /// Issue body (markdown) body?: string constrained { - maxGraphemes: 10000, - }, + maxGraphemes: 10000 + } /// Issue state state: string constrained { knownValues: [ - open, - closed, - ], - default: "open", - }, + open + closed + ] + default: "open" + } /// Creation timestamp - createdAt: Datetime, + createdAt: Datetime } /// A comment on an issue record comment { /// The issue this comment belongs to - issue: AtUri, + issue: AtUri /// Comment body (markdown) body: string constrained { - minGraphemes: 1, - maxGraphemes: 10000, - }, + minGraphemes: 1 + maxGraphemes: 10000 + } /// Creation timestamp - createdAt: Datetime, + createdAt: Datetime /// Optional reply target - replyTo?: AtUri, + replyTo?: AtUri } /// Get an issue by URI query getIssue( /// Issue AT-URI - uri: AtUri, + uri: AtUri ): issue | error { /// Issue not found - NotFound, + NotFound }; /// Create a new issue procedure createIssue( - repo: AtUri, - title: string, - body?: string, + repo: AtUri + title: string + body?: string ): { - uri: AtUri, - cid: Cid, + uri: AtUri + cid: Cid } | error { /// Repository not found - RepoNotFound, + RepoNotFound /// Title too long - TitleTooLong, + TitleTooLong }; ``` @@ -773,9 +814,9 @@ MLF compiles to standard ATProto JSON Lexicons: ```mlf record post { text: string constrained { - maxLength: 300, - }, - createdAt: Datetime, + maxLength: 300 + } + createdAt: Datetime } ``` @@ -812,7 +853,7 @@ record post { **MLF:** ```mlf subscription subscribeRepos( - cursor?: integer, + cursor?: integer ): commit | identity; ``` @@ -872,9 +913,17 @@ Lexicons are versioned at the NSID level. MLF files should include version metad ### Reserved Keywords ``` -alias, as, blob, boolean, bytes, constrained, error, integer, -namespace, null, number, procedure, query, record, string, -subscription, token, unknown, use +as, blob, boolean, bytes, constrained, def, error, inline, integer, +null, number, procedure, query, record, string, subscription, token, +type, unknown, use +``` + +### Reserved Names + +The following names cannot be used as item names: + +``` +main, defs ``` ### Raw Identifiers @@ -882,9 +931,9 @@ subscription, token, unknown, use To use a reserved keyword as an identifier, wrap it in backticks: ```mlf -alias `record` = { - `record`: com.atproto.repo.strongRef, - `error`: string, +def type `record` = { + `record`: com.atproto.repo.strongRef + `error`: string }; ``` diff --git a/mlf-cli/src/generate/lexicon.rs b/mlf-cli/src/generate/lexicon.rs index 04e93c8..6f9f010 100644 --- a/mlf-cli/src/generate/lexicon.rs +++ b/mlf-cli/src/generate/lexicon.rs @@ -86,9 +86,30 @@ pub fn run(input_patterns: Vec, output_dir: PathBuf, flat: bool) -> Resu } }; - let namespace = extract_namespace(&file_path, &lexicon); + let namespace = extract_namespace(&file_path); - let json_lexicon = mlf_codegen::generate_lexicon(&namespace, &lexicon); + // Create workspace with prelude for inline type resolution + let mut workspace = match mlf_lang::Workspace::with_prelude() { + Ok(ws) => ws, + Err(e) => { + errors.push((file_path.display().to_string(), format!("Failed to load prelude: {:?}", e))); + continue; + } + }; + + // Add the module to the workspace + if let Err(e) = workspace.add_module(namespace.clone(), lexicon.clone()) { + errors.push((file_path.display().to_string(), format!("Failed to add module: {:?}", e))); + continue; + } + + // Resolve types + if let Err(e) = workspace.resolve() { + errors.push((file_path.display().to_string(), format!("Type resolution error: {:?}", e))); + continue; + } + + let json_lexicon = mlf_codegen::generate_lexicon(&namespace, &lexicon, &workspace); let output_path = if flat { output_dir.join(format!("{}.json", namespace)) @@ -131,18 +152,8 @@ pub fn run(input_patterns: Vec, output_dir: PathBuf, flat: bool) -> Resu Ok(()) } -fn extract_namespace(file_path: &Path, lexicon: &mlf_lang::ast::Lexicon) -> String { - use mlf_lang::ast::Item; - - for item in &lexicon.items { - if let Item::Namespace(ns) = item { - if ns.name.name.starts_with('.') { - continue; - } - return ns.name.name.clone(); - } - } - +fn extract_namespace(file_path: &Path) -> String { + // Namespace is derived solely from the filename file_path .file_stem() .and_then(|s| s.to_str()) diff --git a/mlf-codegen/Cargo.toml b/mlf-codegen/Cargo.toml index 97c4311..969073e 100644 --- a/mlf-codegen/Cargo.toml +++ b/mlf-codegen/Cargo.toml @@ -5,4 +5,4 @@ edition = "2024" [dependencies] mlf-lang = { path = "../mlf-lang" } -serde_json = "1" +serde_json = { version = "1", features = ["preserve_order"] } diff --git a/mlf-codegen/src/lib.rs b/mlf-codegen/src/lib.rs index f0bb92f..ab2dd9d 100644 --- a/mlf-codegen/src/lib.rs +++ b/mlf-codegen/src/lib.rs @@ -1,37 +1,66 @@ use mlf_lang::ast::*; +use mlf_lang::Workspace; use serde_json::{json, Map, Value}; use std::collections::HashMap; -pub fn generate_lexicon(namespace: &str, lexicon: &Lexicon) -> Value { +pub fn generate_lexicon(namespace: &str, lexicon: &Lexicon, workspace: &Workspace) -> Value { let usage_counts = analyze_type_usage(lexicon); + // Extract the last segment of the namespace to determine main + let namespace_parts: Vec<&str> = namespace.split('.').collect(); + let expected_main_name = namespace_parts.last().copied().unwrap_or(""); + let is_defs_namespace = expected_main_name == "defs"; + + // Count main-eligible items (records, queries, procedures, subscriptions) + let main_eligible_count = lexicon.items.iter().filter(|item| { + matches!(item, Item::Record(_) | Item::Query(_) | Item::Procedure(_) | Item::Subscription(_)) + }).count(); + let mut defs = Map::new(); - let mut main_def: Option = None; for item in &lexicon.items { match item { Item::Record(record) => { - let record_json = generate_record_json(record, &usage_counts); - main_def = Some(record_json); + let record_json = generate_record_json(record, &usage_counts, workspace, namespace); + // If there's only one main-eligible item, it becomes "main" + if main_eligible_count == 1 || (!is_defs_namespace && record.name.name == expected_main_name) { + defs.insert("main".to_string(), record_json); + } else { + defs.insert(record.name.name.clone(), record_json); + } } Item::Query(query) => { - let query_json = generate_query_json(query, &usage_counts); - main_def = Some(query_json); + let query_json = generate_query_json(query, &usage_counts, workspace, namespace); + if main_eligible_count == 1 || (!is_defs_namespace && query.name.name == expected_main_name) { + defs.insert("main".to_string(), query_json); + } else { + defs.insert(query.name.name.clone(), query_json); + } } Item::Procedure(procedure) => { - let procedure_json = generate_procedure_json(procedure, &usage_counts); - main_def = Some(procedure_json); + let procedure_json = generate_procedure_json(procedure, &usage_counts, workspace, namespace); + if main_eligible_count == 1 || (!is_defs_namespace && procedure.name.name == expected_main_name) { + defs.insert("main".to_string(), procedure_json); + } else { + defs.insert(procedure.name.name.clone(), procedure_json); + } } Item::Subscription(subscription) => { - let subscription_json = generate_subscription_json(subscription, &usage_counts); - main_def = Some(subscription_json); - } - Item::Alias(alias) => { - if should_hoist_alias(&alias.name.name, &usage_counts) { - let alias_json = generate_alias_json(alias, &usage_counts); - defs.insert(alias.name.name.clone(), alias_json); + let subscription_json = generate_subscription_json(subscription, &usage_counts, workspace, namespace); + if main_eligible_count == 1 || (!is_defs_namespace && subscription.name.name == expected_main_name) { + defs.insert("main".to_string(), subscription_json); + } else { + defs.insert(subscription.name.name.clone(), subscription_json); } } + Item::DefType(def_type) => { + let def_type_json = generate_def_type_json(def_type, &usage_counts, workspace, namespace); + defs.insert(def_type.name.name.clone(), def_type_json); + } + Item::InlineType(_) => { + // Inline types are never added to defs - they expand at point of use + // TODO: inline expansion will be handled by workspace/cross-file resolution + } Item::Token(token) => { let token_json = json!({ "type": "token", @@ -43,15 +72,11 @@ pub fn generate_lexicon(namespace: &str, lexicon: &Lexicon) -> Value { } } - if let Some(main) = main_def { - defs.insert("main".to_string(), main); - } - - json!({ - "lexicon": 1, - "id": namespace, - "defs": defs - }) + let mut root = Map::new(); + root.insert("lexicon".to_string(), json!(1)); + root.insert("id".to_string(), json!(namespace)); + root.insert("defs".to_string(), json!(defs)); + Value::Object(root) } fn analyze_type_usage(lexicon: &Lexicon) -> HashMap { @@ -92,8 +117,11 @@ fn analyze_type_usage(lexicon: &Lexicon) -> HashMap { } count_type_references(&subscription.messages, &mut usage_counts); } - Item::Alias(alias) => { - count_type_references(&alias.ty, &mut usage_counts); + Item::InlineType(inline_type) => { + count_type_references(&inline_type.ty, &mut usage_counts); + } + Item::DefType(def_type) => { + count_type_references(&def_type.ty, &mut usage_counts); } _ => {} } @@ -121,15 +149,12 @@ fn count_type_references(ty: &Type, counts: &mut HashMap) { count_type_references(&field.ty, counts); } } + Type::Parenthesized { inner, .. } => count_type_references(inner, counts), Type::Constrained { base, .. } => count_type_references(base, counts), _ => {} } } -fn should_hoist_alias(name: &str, usage_counts: &HashMap) -> bool { - usage_counts.get(name).map_or(false, |&count| count > 1) -} - fn extract_docs(docs: &[DocComment]) -> String { docs.iter() .map(|d| d.text.trim()) @@ -137,7 +162,7 @@ fn extract_docs(docs: &[DocComment]) -> String { .join("\n") } -fn generate_record_json(record: &Record, usage_counts: &HashMap) -> Value { +fn generate_record_json(record: &Record, usage_counts: &HashMap, workspace: &Workspace, current_namespace: &str) -> Value { let mut required = Vec::new(); let mut properties = Map::new(); @@ -146,7 +171,7 @@ fn generate_record_json(record: &Record, usage_counts: &HashMap) required.push(field.name.name.clone()); } - let field_json = generate_type_json(&field.ty, usage_counts); + let field_json = generate_type_json(&field.ty, usage_counts, workspace, current_namespace); properties.insert(field.name.name.clone(), field_json); } @@ -164,7 +189,7 @@ fn generate_record_json(record: &Record, usage_counts: &HashMap) }) } -fn generate_query_json(query: &Query, usage_counts: &HashMap) -> Value { +fn generate_query_json(query: &Query, usage_counts: &HashMap, workspace: &Workspace, current_namespace: &str) -> Value { let mut params_properties = Map::new(); let mut params_required = Vec::new(); @@ -172,29 +197,35 @@ fn generate_query_json(query: &Query, usage_counts: &HashMap) -> if !param.optional { params_required.push(param.name.name.clone()); } - let param_json = generate_type_json(¶m.ty, usage_counts); + let mut param_json = generate_type_json(¶m.ty, usage_counts, workspace, current_namespace); + // Add description if the parameter has doc comments + if !param.docs.is_empty() { + if let Some(obj) = param_json.as_object_mut() { + obj.insert("description".to_string(), json!(extract_docs(¶m.docs))); + } + } params_properties.insert(param.name.name.clone(), param_json); } let params = if !params_properties.is_empty() { - json!({ - "type": "params", - "required": params_required, - "properties": params_properties - }) + let mut params_obj = Map::new(); + params_obj.insert("type".to_string(), json!("params")); + params_obj.insert("required".to_string(), json!(params_required)); + params_obj.insert("properties".to_string(), json!(params_properties)); + Value::Object(params_obj) } else { - json!({ - "type": "params", - "properties": {} - }) + let mut params_obj = Map::new(); + params_obj.insert("type".to_string(), json!("params")); + params_obj.insert("properties".to_string(), json!({})); + Value::Object(params_obj) }; let output = match &query.returns { ReturnType::Type(ty) => { - json!({ - "encoding": "application/json", - "schema": generate_type_json(ty, usage_counts) - }) + let mut output_obj = Map::new(); + output_obj.insert("encoding".to_string(), json!("application/json")); + output_obj.insert("schema".to_string(), generate_type_json(ty, usage_counts, workspace, current_namespace)); + Value::Object(output_obj) } ReturnType::TypeWithErrors { success, errors, .. } => { let mut error_defs = Map::new(); @@ -207,23 +238,23 @@ fn generate_query_json(query: &Query, usage_counts: &HashMap) -> ); } - json!({ - "encoding": "application/json", - "schema": generate_type_json(success, usage_counts), - "errors": error_defs - }) + let mut output_obj = Map::new(); + output_obj.insert("encoding".to_string(), json!("application/json")); + output_obj.insert("schema".to_string(), generate_type_json(success, usage_counts, workspace, current_namespace)); + output_obj.insert("errors".to_string(), json!(error_defs)); + Value::Object(output_obj) } }; - json!({ - "type": "query", - "description": extract_docs(&query.docs), - "parameters": params, - "output": output - }) + let mut query_obj = Map::new(); + query_obj.insert("type".to_string(), json!("query")); + query_obj.insert("description".to_string(), json!(extract_docs(&query.docs))); + query_obj.insert("parameters".to_string(), params); + query_obj.insert("output".to_string(), output); + Value::Object(query_obj) } -fn generate_procedure_json(procedure: &Procedure, usage_counts: &HashMap) -> Value { +fn generate_procedure_json(procedure: &Procedure, usage_counts: &HashMap, workspace: &Workspace, current_namespace: &str) -> Value { let mut params_properties = Map::new(); let mut params_required = Vec::new(); @@ -231,29 +262,30 @@ fn generate_procedure_json(procedure: &Procedure, usage_counts: &HashMap { - json!({ - "encoding": "application/json", - "schema": generate_type_json(ty, usage_counts) - }) + let mut output_obj = Map::new(); + output_obj.insert("encoding".to_string(), json!("application/json")); + output_obj.insert("schema".to_string(), generate_type_json(ty, usage_counts, workspace, current_namespace)); + Value::Object(output_obj) } ReturnType::TypeWithErrors { success, errors, .. } => { let mut error_defs = Map::new(); @@ -266,30 +298,29 @@ fn generate_procedure_json(procedure: &Procedure, usage_counts: &HashMap, + workspace: &Workspace, + current_namespace: &str, ) -> Value { let mut params_properties = Map::new(); let mut params_required = Vec::new(); @@ -298,7 +329,7 @@ fn generate_subscription_json( if !param.optional { params_required.push(param.name.name.clone()); } - let param_json = generate_type_json(¶m.ty, usage_counts); + let param_json = generate_type_json(¶m.ty, usage_counts, workspace, current_namespace); params_properties.insert(param.name.name.clone(), param_json); } @@ -313,7 +344,7 @@ fn generate_subscription_json( }; let message = json!({ - "schema": generate_type_json(&subscription.messages, usage_counts) + "schema": generate_type_json(&subscription.messages, usage_counts, workspace, current_namespace) }); let mut result = json!({ @@ -329,35 +360,55 @@ fn generate_subscription_json( result } -fn generate_alias_json(alias: &Alias, usage_counts: &HashMap) -> Value { - generate_type_json(&alias.ty, usage_counts) +fn generate_def_type_json(def_type: &DefType, usage_counts: &HashMap, workspace: &Workspace, current_namespace: &str) -> Value { + generate_type_json(&def_type.ty, usage_counts, workspace, current_namespace) } -fn generate_type_json(ty: &Type, usage_counts: &HashMap) -> Value { +fn generate_type_json(ty: &Type, usage_counts: &HashMap, workspace: &Workspace, current_namespace: &str) -> Value { match ty { Type::Primitive { kind, .. } => generate_primitive_json(*kind), Type::Reference { path, .. } => { + // Try to resolve this reference in the workspace + if let Some(resolved_ty) = workspace.resolve_type_reference(path) { + // Check if this is an inline type by looking in the workspace + if workspace.is_inline_type(path) { + // Inline type: expand it recursively + return generate_type_json(&resolved_ty, usage_counts, workspace, current_namespace); + } + } + + // Not an inline type (or couldn't resolve) - generate a ref if path.segments.len() == 1 { let name = &path.segments[0].name; - if should_hoist_alias(name, usage_counts) { - json!({ "ref": format!("#{}", name) }) - } else { - json!({ "ref": format!("#{}", name) }) - } + json!({ + "type": "ref", + "ref": format!("#{}", name) + }) } else { - json!({ "ref": path.to_string() }) + // Multi-segment path ref + let namespace = path.segments[..path.segments.len()-1] + .iter() + .map(|s| s.name.as_str()) + .collect::>() + .join("."); + let def_name = &path.segments.last().unwrap().name; + + json!({ + "type": "ref", + "ref": format!("{}#{}", namespace, def_name) + }) } } Type::Array { inner, .. } => { json!({ "type": "array", - "items": generate_type_json(inner, usage_counts) + "items": generate_type_json(inner, usage_counts, workspace, current_namespace) }) } Type::Union { types, .. } => { let refs: Vec = types .iter() - .map(|t| generate_type_json(t, usage_counts)) + .map(|t| generate_type_json(t, usage_counts, workspace, current_namespace)) .collect(); json!({ "type": "union", @@ -374,18 +425,22 @@ fn generate_type_json(ty: &Type, usage_counts: &HashMap) -> Value } properties.insert( field.name.name.clone(), - generate_type_json(&field.ty, usage_counts), + generate_type_json(&field.ty, usage_counts, workspace, current_namespace), ); } - json!({ - "type": "object", - "required": required, - "properties": properties - }) + let mut obj = Map::new(); + obj.insert("type".to_string(), json!("object")); + obj.insert("required".to_string(), json!(required)); + obj.insert("properties".to_string(), json!(properties)); + Value::Object(obj) + } + Type::Parenthesized { inner, .. } => { + // Parentheses are just for grouping - unwrap and process inner type + generate_type_json(inner, usage_counts, workspace, current_namespace) } Type::Constrained { base, constraints, .. } => { - let mut base_json = generate_type_json(base, usage_counts); + let mut base_json = generate_type_json(base, usage_counts, workspace, current_namespace); if let Some(obj) = base_json.as_object_mut() { for constraint in constraints { @@ -437,10 +492,23 @@ fn apply_constraint_to_json(obj: &mut Map, constraint: &Constrain obj.insert("format".to_string(), json!(value)); } Constraint::Enum { values, .. } => { - obj.insert("enum".to_string(), json!(values)); + let enum_vals: Vec = values + .iter() + .map(|v| match v { + mlf_lang::ast::ValueRef::Literal(s) => s.clone(), + mlf_lang::ast::ValueRef::Reference(path) => path.to_string(), + }) + .collect(); + obj.insert("enum".to_string(), json!(enum_vals)); } Constraint::KnownValues { values, .. } => { - let known_vals: Vec = values.iter().map(|path| path.to_string()).collect(); + let known_vals: Vec = values + .iter() + .map(|v| match v { + mlf_lang::ast::ValueRef::Literal(s) => s.clone(), + mlf_lang::ast::ValueRef::Reference(path) => path.to_string(), + }) + .collect(); obj.insert("knownValues".to_string(), json!(known_vals)); } Constraint::Accept { mimes, .. } => { @@ -454,8 +522,18 @@ fn apply_constraint_to_json(obj: &mut Map, constraint: &Constrain ConstraintValue::String(s) => json!(s), ConstraintValue::Integer(i) => json!(i), ConstraintValue::Boolean(b) => json!(b), + ConstraintValue::Reference(path) => json!(path.to_string()), }; obj.insert("default".to_string(), default_val); } + Constraint::Const { value, .. } => { + let const_val = match value { + ConstraintValue::String(s) => json!(s), + ConstraintValue::Integer(i) => json!(i), + ConstraintValue::Boolean(b) => json!(b), + ConstraintValue::Reference(path) => json!(path.to_string()), + }; + obj.insert("const".to_string(), const_val); + } } } diff --git a/mlf-diagnostics/src/lib.rs b/mlf-diagnostics/src/lib.rs index 3b74eb6..8a94645 100644 --- a/mlf-diagnostics/src/lib.rs +++ b/mlf-diagnostics/src/lib.rs @@ -150,6 +150,9 @@ fn format_validation_error(f: &mut fmt::Formatter<'_>, error: &ValidationError) ValidationError::ConstraintTooPermissive { message, .. } => { write!(f, "Constraint is too permissive: {}", message) } + ValidationError::ReservedName { name, .. } => { + write!(f, "Reserved name '{}' cannot be used as an item name", name) + } } } @@ -160,6 +163,7 @@ fn get_error_code(error: &ValidationError) -> &'static str { ValidationError::InvalidConstraint { .. } => "mlf::invalid_constraint", ValidationError::TypeMismatch { .. } => "mlf::type_mismatch", ValidationError::ConstraintTooPermissive { .. } => "mlf::constraint_too_permissive", + ValidationError::ReservedName { .. } => "mlf::reserved_name", } } @@ -192,6 +196,10 @@ fn get_error_labels(error: &ValidationError) -> Vec { ValidationError::ConstraintTooPermissive { span, message } => { vec![LabeledSpan::at(span.start..span.end, message.clone())] } + ValidationError::ReservedName { span, name } => vec![LabeledSpan::at( + span.start..span.end, + format!("'{}' is a reserved name and cannot be used", name), + )], } } diff --git a/mlf-lang/src/ast.rs b/mlf-lang/src/ast.rs index 8a5006e..34af890 100644 --- a/mlf-lang/src/ast.rs +++ b/mlf-lang/src/ast.rs @@ -31,12 +31,12 @@ pub struct Lexicon { #[derive(Debug, Clone, PartialEq)] pub enum Item { Record(Record), - Alias(Alias), + InlineType(InlineType), + DefType(DefType), Token(Token), Query(Query), Procedure(Procedure), Subscription(Subscription), - Namespace(Namespace), Use(Use), } @@ -44,12 +44,12 @@ impl Spanned for Item { fn span(&self) -> Span { match self { Item::Record(r) => r.span, - Item::Alias(a) => a.span, + Item::InlineType(i) => i.span, + Item::DefType(d) => d.span, Item::Token(t) => t.span, Item::Query(q) => q.span, Item::Procedure(p) => p.span, Item::Subscription(s) => s.span, - Item::Namespace(n) => n.span, Item::Use(u) => u.span, } } @@ -106,9 +106,19 @@ pub struct Field { pub span: Span, } -/// A type alias +/// An inline type definition (expands at point of use) #[derive(Debug, Clone, PartialEq)] -pub struct Alias { +pub struct InlineType { + pub docs: Vec, + pub annotations: Vec, + pub name: Ident, + pub ty: Type, + pub span: Span, +} + +/// A def type definition (becomes a named def in lexicon) +#[derive(Debug, Clone, PartialEq)] +pub struct DefType { pub docs: Vec, pub annotations: Vec, pub name: Ident, @@ -171,6 +181,15 @@ pub enum ReturnType { }, } +impl ReturnType { + pub fn span(&self) -> Span { + match self { + ReturnType::Type(ty) => ty.span(), + ReturnType::TypeWithErrors { span, .. } => *span, + } + } +} + /// An error definition in a query/procedure #[derive(Debug, Clone, PartialEq)] pub struct ErrorDef { @@ -179,14 +198,6 @@ pub struct ErrorDef { pub span: Span, } -/// A namespace block -#[derive(Debug, Clone, PartialEq)] -pub struct Namespace { - pub name: Ident, // e.g., ".actor" - pub items: Vec, - pub span: Span, -} - /// A use statement #[derive(Debug, Clone, PartialEq)] pub struct Use { @@ -241,6 +252,8 @@ pub enum Type { Union { types: Vec, span: Span }, /// Object type (inline) Object { fields: Vec, span: Span }, + /// Parenthesized type (for grouping, e.g., (A | B)[]) + Parenthesized { inner: Box, span: Span }, /// Constrained type Constrained { base: Box, @@ -259,6 +272,7 @@ impl Spanned for Type { Type::Array { span, .. } => *span, Type::Union { span, .. } => *span, Type::Object { span, .. } => *span, + Type::Parenthesized { span, .. } => *span, Type::Constrained { span, .. } => *span, Type::Unknown { span } => *span, } @@ -286,8 +300,8 @@ pub enum Constraint { MinGraphemes { value: usize, span: Span }, MaxGraphemes { value: usize, span: Span }, Format { value: String, span: Span }, - Enum { values: Vec, span: Span }, - KnownValues { values: Vec, span: Span }, + Enum { values: Vec, span: Span }, + KnownValues { values: Vec, span: Span }, // Numeric constraints Minimum { value: i64, span: Span }, @@ -297,8 +311,9 @@ pub enum Constraint { Accept { mimes: Vec, span: Span }, MaxSize { value: usize, span: Span }, - // Default value + // Value constraints Default { value: ConstraintValue, span: Span }, + Const { value: ConstraintValue, span: Span }, } impl Spanned for Constraint { @@ -316,14 +331,35 @@ impl Spanned for Constraint { Constraint::Accept { span, .. } => *span, Constraint::MaxSize { span, .. } => *span, Constraint::Default { span, .. } => *span, + Constraint::Const { span, .. } => *span, } } } -/// A value in a constraint default +/// A value in a constraint default - can be a literal or reference #[derive(Debug, Clone, PartialEq)] pub enum ConstraintValue { String(String), Integer(i64), Boolean(bool), + /// Reference to a named item (token, record, alias, etc.) + Reference(Path), +} + +/// A value reference in enum/knownValues constraints - can be a string literal or reference to any named item +#[derive(Debug, Clone, PartialEq)] +pub enum ValueRef { + /// String literal (e.g., "open") + Literal(String), + /// Reference to a named item - token, record, alias, etc. (e.g., open) + Reference(Path), +} + +impl core::fmt::Display for ValueRef { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + ValueRef::Literal(s) => write!(f, "\"{}\"", s), + ValueRef::Reference(path) => write!(f, "{}", path.to_string()), + } + } } diff --git a/mlf-lang/src/error.rs b/mlf-lang/src/error.rs index 6d11057..4c5389f 100644 --- a/mlf-lang/src/error.rs +++ b/mlf-lang/src/error.rs @@ -17,6 +17,7 @@ pub enum ValidationError { InvalidConstraint { message: String, span: Span }, TypeMismatch { expected: String, found: String, span: Span }, ConstraintTooPermissive { message: String, span: Span }, + ReservedName { name: String, span: Span }, } #[derive(Debug, Clone, Default)] diff --git a/mlf-lang/src/lexer.rs b/mlf-lang/src/lexer.rs index d3ac224..cbdff52 100644 --- a/mlf-lang/src/lexer.rs +++ b/mlf-lang/src/lexer.rs @@ -15,13 +15,14 @@ use crate::span::Span; #[derive(Debug, Clone, PartialEq)] pub enum Token { // Keywords - Alias, As, Blob, Boolean, Bytes, Constrained, + Def, Error, + Inline, Integer, Namespace, Null, @@ -32,6 +33,7 @@ pub enum Token { String, Subscription, Token, + Type, Unknown, Use, @@ -68,13 +70,14 @@ pub enum Token { impl core::fmt::Display for Token { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { match self { - Token::Alias => write!(f, "alias"), Token::As => write!(f, "as"), Token::Blob => write!(f, "blob"), Token::Boolean => write!(f, "boolean"), Token::Bytes => write!(f, "bytes"), Token::Constrained => write!(f, "constrained"), + Token::Def => write!(f, "def"), Token::Error => write!(f, "error"), + Token::Inline => write!(f, "inline"), Token::Integer => write!(f, "integer"), Token::Namespace => write!(f, "namespace"), Token::Null => write!(f, "null"), @@ -85,6 +88,7 @@ impl core::fmt::Display for Token { Token::String => write!(f, "string"), Token::Subscription => write!(f, "subscription"), Token::Token => write!(f, "token"), + Token::Type => write!(f, "type"), Token::Unknown => write!(f, "unknown"), Token::Use => write!(f, "use"), Token::Ident(s) => write!(f, "{}", s), @@ -139,14 +143,15 @@ fn identifier(input: &str) -> IResult<&str, Token> { )).parse(input)?; let token = match name { - "alias" => Token::Alias, "as" => Token::As, "blob" => Token::Blob, "boolean" => Token::Boolean, "bytes" => Token::Bytes, "constrained" => Token::Constrained, + "def" => Token::Def, "error" => Token::Error, "false" => Token::False, + "inline" => Token::Inline, "integer" => Token::Integer, "namespace" => Token::Namespace, "null" => Token::Null, @@ -158,6 +163,7 @@ fn identifier(input: &str) -> IResult<&str, Token> { "subscription" => Token::Subscription, "token" => Token::Token, "true" => Token::True, + "type" => Token::Type, "unknown" => Token::Unknown, "use" => Token::Use, _ => Token::Ident(name.into()), diff --git a/mlf-lang/src/parser.rs b/mlf-lang/src/parser.rs index 824f62a..41b6d2d 100644 --- a/mlf-lang/src/parser.rs +++ b/mlf-lang/src/parser.rs @@ -61,6 +61,46 @@ impl Parser { } } + fn parse_field_name(&mut self) -> Result { + let current = self.current(); + // Field names can be identifiers or keywords + let name = match ¤t.token { + LexToken::Ident(n) => n.clone(), + LexToken::Record => "record".into(), + LexToken::Token => "token".into(), + LexToken::Inline => "inline".into(), + LexToken::Def => "def".into(), + LexToken::Type => "type".into(), + LexToken::Query => "query".into(), + LexToken::Procedure => "procedure".into(), + LexToken::Subscription => "subscription".into(), + LexToken::Error => "error".into(), + LexToken::Use => "use".into(), + LexToken::As => "as".into(), + LexToken::String => "string".into(), + LexToken::Integer => "integer".into(), + LexToken::Number => "number".into(), + LexToken::Boolean => "boolean".into(), + LexToken::Null => "null".into(), + LexToken::Unknown => "unknown".into(), + LexToken::Constrained => "constrained".into(), + LexToken::True => "true".into(), + LexToken::False => "false".into(), + _ => { + return Err(ParseError::Syntax { + message: alloc::format!("Expected field name, found {}", current.token), + span: current.span, + }); + } + }; + let ident = Ident { + name, + span: current.span, + }; + self.advance(); + Ok(ident) + } + fn parse_path(&mut self) -> Result { let mut segments = Vec::new(); let start = self.current().span.start; @@ -105,20 +145,26 @@ pub fn parse_lexicon(input: &str) -> Result { impl Parser { fn parse_item(&mut self) -> Result { - while matches!(self.current().token, LexToken::DocComment(_)) { + let mut doc_comments = Vec::new(); + while let LexToken::DocComment(comment) = &self.current().token { + let span = self.current().span; + doc_comments.push(DocComment { + text: comment.clone(), + span, + }); self.advance(); } let annotations = self.parse_annotations()?; match &self.current().token { - LexToken::Record => self.parse_record(annotations), - LexToken::Alias => self.parse_alias(annotations), - LexToken::Token => self.parse_token(annotations), - LexToken::Query => self.parse_query(annotations), - LexToken::Procedure => self.parse_procedure(annotations), - LexToken::Subscription => self.parse_subscription(annotations), - LexToken::Namespace => self.parse_namespace(), + LexToken::Record => self.parse_record(doc_comments, annotations), + LexToken::Inline => self.parse_inline_type(doc_comments, annotations), + LexToken::Def => self.parse_def_type(doc_comments, annotations), + LexToken::Token => self.parse_token(doc_comments, annotations), + LexToken::Query => self.parse_query(doc_comments, annotations), + LexToken::Procedure => self.parse_procedure(doc_comments, annotations), + LexToken::Subscription => self.parse_subscription(doc_comments, annotations), LexToken::Use => self.parse_use(), _ => Err(ParseError::Syntax { message: alloc::format!("Expected item definition, found {}", self.current().token), @@ -219,33 +265,32 @@ impl Parser { } } - fn parse_record(&mut self, annotations: Vec) -> Result { + fn parse_record(&mut self, docs: Vec, annotations: Vec) -> Result { let start = self.expect(LexToken::Record)?; let name = self.parse_ident()?; self.expect(LexToken::LeftBrace)?; let mut fields = Vec::new(); - let mut doc_comments = Vec::new(); + let mut field_docs = Vec::new(); while !matches!(self.current().token, LexToken::RightBrace) { if let LexToken::DocComment(comment) = &self.current().token { let span = self.current().span; - doc_comments.push(DocComment { + field_docs.push(DocComment { text: comment.clone(), span, }); self.advance(); } else { - fields.push(self.parse_field(doc_comments.clone())?); - doc_comments.clear(); + fields.push(self.parse_field(field_docs.clone())?); + field_docs.clear(); } } let end = self.expect(LexToken::RightBrace)?; - self.expect(LexToken::Semicolon)?; Ok(Item::Record(Record { - docs: Vec::new(), + docs, annotations, name, fields, @@ -255,7 +300,7 @@ impl Parser { fn parse_field(&mut self, docs: Vec) -> Result { let annotations = self.parse_annotations()?; - let name = self.parse_ident()?; + let name = self.parse_field_name()?; let optional = if matches!(self.current().token, LexToken::Question) { self.advance(); @@ -266,7 +311,14 @@ impl Parser { self.expect(LexToken::Colon)?; let ty = self.parse_type()?; - self.expect(LexToken::Comma)?; + + // Comma is required (unless we're at the end) + if !matches!(self.current().token, LexToken::RightBrace) { + self.expect(LexToken::Comma)?; + } else if matches!(self.current().token, LexToken::Comma) { + // Allow trailing comma + self.advance(); + } let span = Span::new(name.span.start, ty.span().end); @@ -280,15 +332,33 @@ impl Parser { }) } - fn parse_alias(&mut self, annotations: Vec) -> Result { - let start = self.expect(LexToken::Alias)?; + fn parse_inline_type(&mut self, docs: Vec, annotations: Vec) -> Result { + let start = self.expect(LexToken::Inline)?; + self.expect(LexToken::Type)?; + let name = self.parse_ident()?; + self.expect(LexToken::Equals)?; + let ty = self.parse_type()?; + let end = self.expect(LexToken::Semicolon)?; + + Ok(Item::InlineType(InlineType { + docs, + annotations, + name, + ty, + span: Span::new(start.start, end.end), + })) + } + + fn parse_def_type(&mut self, docs: Vec, annotations: Vec) -> Result { + let start = self.expect(LexToken::Def)?; + self.expect(LexToken::Type)?; let name = self.parse_ident()?; self.expect(LexToken::Equals)?; let ty = self.parse_type()?; let end = self.expect(LexToken::Semicolon)?; - Ok(Item::Alias(Alias { - docs: Vec::new(), + Ok(Item::DefType(DefType { + docs, annotations, name, ty, @@ -296,20 +366,20 @@ impl Parser { })) } - fn parse_token(&mut self, annotations: Vec) -> Result { + fn parse_token(&mut self, docs: Vec, annotations: Vec) -> Result { let start = self.expect(LexToken::Token)?; let name = self.parse_ident()?; let end = self.expect(LexToken::Semicolon)?; Ok(Item::Token(Token { - docs: Vec::new(), + docs, annotations, name, span: Span::new(start.start, end.end), })) } - fn parse_query(&mut self, annotations: Vec) -> Result { + fn parse_query(&mut self, docs: Vec, annotations: Vec) -> Result { let start = self.expect(LexToken::Query)?; let name = self.parse_ident()?; self.expect(LexToken::LeftParen)?; @@ -351,7 +421,7 @@ impl Parser { let end = self.expect(LexToken::Semicolon)?; Ok(Item::Query(Query { - docs: Vec::new(), + docs, annotations, name, params, @@ -360,7 +430,7 @@ impl Parser { })) } - fn parse_procedure(&mut self, annotations: Vec) -> Result { + fn parse_procedure(&mut self, docs: Vec, annotations: Vec) -> Result { let start = self.expect(LexToken::Procedure)?; let name = self.parse_ident()?; self.expect(LexToken::LeftParen)?; @@ -402,7 +472,7 @@ impl Parser { let end = self.expect(LexToken::Semicolon)?; Ok(Item::Procedure(Procedure { - docs: Vec::new(), + docs, annotations, name, params, @@ -411,7 +481,7 @@ impl Parser { })) } - fn parse_subscription(&mut self, annotations: Vec) -> Result { + fn parse_subscription(&mut self, docs: Vec, annotations: Vec) -> Result { let start = self.expect(LexToken::Subscription)?; let name = self.parse_ident()?; self.expect(LexToken::LeftParen)?; @@ -426,7 +496,7 @@ impl Parser { let end = self.expect(LexToken::Semicolon)?; Ok(Item::Subscription(Subscription { - docs: Vec::new(), + docs, annotations, name, params, @@ -435,25 +505,6 @@ impl Parser { })) } - fn parse_namespace(&mut self) -> Result { - let start = self.expect(LexToken::Namespace)?; - let path = self.parse_path()?; - - // Convert path to a single identifier with dotted name - let name = Ident { - name: path.segments.iter().map(|s| s.name.as_str()).collect::>().join("."), - span: path.span, - }; - - let end = self.expect(LexToken::Semicolon)?; - - Ok(Item::Namespace(Namespace { - name, - items: Vec::new(), - span: Span::new(start.start, end.end), - })) - } - fn parse_use(&mut self) -> Result { let start = self.expect(LexToken::Use)?; let path = self.parse_path()?; @@ -480,36 +531,47 @@ impl Parser { fn parse_params(&mut self) -> Result, ParseError> { let mut params = Vec::new(); + let mut doc_comments = Vec::new(); while !matches!(self.current().token, LexToken::RightParen) { - let annotations = self.parse_annotations()?; - let name = self.parse_ident()?; - - let optional = if matches!(self.current().token, LexToken::Question) { + if let LexToken::DocComment(comment) = &self.current().token { + let span = self.current().span; + doc_comments.push(DocComment { + text: comment.clone(), + span, + }); self.advance(); - true } else { - false - }; + let annotations = self.parse_annotations()?; + let name = self.parse_ident()?; - self.expect(LexToken::Colon)?; - let ty = self.parse_type()?; + let optional = if matches!(self.current().token, LexToken::Question) { + self.advance(); + true + } else { + false + }; - let span = Span::new(name.span.start, ty.span().end); + self.expect(LexToken::Colon)?; + let ty = self.parse_type()?; - params.push(Field { - docs: Vec::new(), - annotations, - name, - ty, - optional, - span, - }); + let span = Span::new(name.span.start, ty.span().end); - if matches!(self.current().token, LexToken::Comma) { - self.advance(); - } else { - break; + params.push(Field { + docs: doc_comments.clone(), + annotations, + name, + ty, + optional, + span, + }); + doc_comments.clear(); + + if matches!(self.current().token, LexToken::Comma) { + self.advance(); + } else { + break; + } } } @@ -533,7 +595,15 @@ impl Parser { } else { let name = self.parse_ident()?; let span = name.span; - self.expect(LexToken::Comma)?; + + // Comma is required (unless we're at the end) + if !matches!(self.current().token, LexToken::RightBrace) { + self.expect(LexToken::Comma)?; + } else if matches!(self.current().token, LexToken::Comma) { + // Allow trailing comma + self.advance(); + } + errors.push(ErrorDef { docs: doc_comments.clone(), name, @@ -644,9 +714,20 @@ impl Parser { LexToken::LeftBrace => { self.advance(); let mut fields = Vec::new(); + let mut doc_comments = Vec::new(); while !matches!(self.current().token, LexToken::RightBrace) { - fields.push(self.parse_field(Vec::new())?); + if let LexToken::DocComment(comment) = &self.current().token { + let span = self.current().span; + doc_comments.push(DocComment { + text: comment.clone(), + span, + }); + self.advance(); + } else { + fields.push(self.parse_field(doc_comments.clone())?); + doc_comments.clear(); + } } let end = self.expect(LexToken::RightBrace)?; @@ -655,6 +736,15 @@ impl Parser { span: Span::new(start, end.end), } } + LexToken::LeftParen => { + self.advance(); + let inner = self.parse_type()?; + let end = self.expect(LexToken::RightParen)?; + Type::Parenthesized { + inner: alloc::boxed::Box::new(inner), + span: Span::new(start, end.end), + } + } _ => { return Err(ParseError::Syntax { message: alloc::format!("Expected type, found {}", current.token), @@ -682,10 +772,12 @@ impl Parser { while !matches!(self.current().token, LexToken::RightBrace) { constraints.push(self.parse_constraint()?); - if matches!(self.current().token, LexToken::Comma) { + // Comma is required (unless we're at the end) + if !matches!(self.current().token, LexToken::RightBrace) { + self.expect(LexToken::Comma)?; + } else if matches!(self.current().token, LexToken::Comma) { + // Allow trailing comma self.advance(); - } else { - break; } } @@ -777,18 +869,25 @@ impl Parser { while !matches!(self.current().token, LexToken::RightBracket) { let current = self.current(); - match ¤t.token { + // Accept either string literals or identifier paths + let value_ref = match ¤t.token { LexToken::StringLit(s) => { - values.push(s.clone()); + let v = ValueRef::Literal(s.clone()); self.advance(); + v + } + LexToken::Ident(_) => { + let path = self.parse_path()?; + ValueRef::Reference(path) } _ => { return Err(ParseError::Syntax { - message: alloc::format!("Expected string literal in enum"), + message: alloc::format!("Expected string literal or identifier in enum"), span: current.span, }); } - } + }; + values.push(value_ref); if matches!(self.current().token, LexToken::Comma) { self.advance(); @@ -914,7 +1013,26 @@ impl Parser { let mut values = Vec::new(); while !matches!(self.current().token, LexToken::RightBracket) { - values.push(self.parse_path()?); + let current = self.current(); + // Accept either string literals or identifier paths + let value_ref = match ¤t.token { + LexToken::StringLit(s) => { + let v = ValueRef::Literal(s.clone()); + self.advance(); + v + } + LexToken::Ident(_) => { + let path = self.parse_path()?; + ValueRef::Reference(path) + } + _ => { + return Err(ParseError::Syntax { + message: alloc::format!("Expected string literal or identifier in knownValues"), + span: current.span, + }); + } + }; + values.push(value_ref); if matches!(self.current().token, LexToken::Comma) { self.advance(); @@ -959,9 +1077,13 @@ impl Parser { self.advance(); v } + LexToken::Ident(_) => { + let path = self.parse_path()?; + ConstraintValue::Reference(path) + } _ => { return Err(ParseError::Syntax { - message: alloc::format!("Expected string, integer, or boolean for default"), + message: alloc::format!("Expected string, integer, boolean, or identifier for default"), span: current_span, }); } @@ -971,6 +1093,46 @@ impl Parser { span: Span::new(start, end_span), } } + "const" => { + use crate::ast::ConstraintValue; + let end_span = current_span.end; + let value = match ¤t.token { + LexToken::StringLit(s) => { + let v = ConstraintValue::String(s.clone()); + self.advance(); + v + } + LexToken::IntLit(i) => { + let v = ConstraintValue::Integer(*i); + self.advance(); + v + } + LexToken::True => { + let v = ConstraintValue::Boolean(true); + self.advance(); + v + } + LexToken::False => { + let v = ConstraintValue::Boolean(false); + self.advance(); + v + } + LexToken::Ident(_) => { + let path = self.parse_path()?; + ConstraintValue::Reference(path) + } + _ => { + return Err(ParseError::Syntax { + message: alloc::format!("Expected string, integer, boolean, or identifier for const"), + span: current_span, + }); + } + }; + Constraint::Const { + value, + span: Span::new(start, end_span), + } + } _ => { return Err(ParseError::Syntax { message: alloc::format!("Unknown constraint: {}", name.name), diff --git a/mlf-lang/src/workspace.rs b/mlf-lang/src/workspace.rs index b7f8aed..8052080 100644 --- a/mlf-lang/src/workspace.rs +++ b/mlf-lang/src/workspace.rs @@ -141,14 +141,19 @@ impl Workspace { fn typecheck_item(&self, namespace: &str, item: &Item) -> Result<(), ValidationErrors> { match item { - Item::Alias(a) => self.typecheck_alias(namespace, a), + Item::InlineType(i) => self.typecheck_inline_type(namespace, i), + Item::DefType(d) => self.typecheck_def_type(namespace, d), Item::Record(r) => self.typecheck_record(namespace, r), _ => Ok(()), } } - fn typecheck_alias(&self, namespace: &str, alias: &Alias) -> Result<(), ValidationErrors> { - self.typecheck_type(namespace, &alias.ty) + fn typecheck_inline_type(&self, namespace: &str, inline_type: &InlineType) -> Result<(), ValidationErrors> { + self.typecheck_type(namespace, &inline_type.ty) + } + + fn typecheck_def_type(&self, namespace: &str, def_type: &DefType) -> Result<(), ValidationErrors> { + self.typecheck_type(namespace, &def_type.ty) } fn typecheck_record(&self, namespace: &str, record: &Record) -> Result<(), ValidationErrors> { @@ -209,6 +214,7 @@ impl Workspace { Err(errors) } } + Type::Parenthesized { inner, .. } => self.typecheck_type(namespace, inner), Type::Constrained { base, constraints, span } => { let mut errors = ValidationErrors::new(); @@ -273,6 +279,7 @@ impl Workspace { } Constraint::KnownValues { .. } => {} Constraint::Default { .. } => {} + Constraint::Const { .. } => {} } } @@ -434,6 +441,7 @@ impl Workspace { all_constraints.extend(self.get_base_constraints(base)); all_constraints } + Type::Parenthesized { inner, .. } => self.get_base_constraints(inner), Type::Reference { path, .. } => { if let Some(resolved_ty) = self.resolve_type_reference(path) { self.get_base_constraints(&resolved_ty) @@ -449,6 +457,7 @@ impl Workspace { match ty { Type::Primitive { kind, .. } => Some(*kind), Type::Constrained { base, .. } => self.get_base_primitive(base), + Type::Parenthesized { inner, .. } => self.get_base_primitive(inner), Type::Reference { path, .. } => { if let Some(resolved_ty) = self.resolve_type_reference(path) { self.get_base_primitive(&resolved_ty) @@ -460,17 +469,21 @@ impl Workspace { } } - fn resolve_type_reference(&self, path: &Path) -> Option { + pub fn resolve_type_reference(&self, path: &Path) -> Option { if path.segments.len() == 1 { let name = &path.segments[0].name; for (_, module) in &self.modules { if let Some(Symbol::Alias { .. }) = module.symbols.types.get(name) { for item in &module.lexicon.items { - if let Item::Alias(a) = item { - if a.name.name == *name { - return Some(a.ty.clone()); + match item { + Item::InlineType(i) if i.name.name == *name => { + return Some(i.ty.clone()); + } + Item::DefType(d) if d.name.name == *name => { + return Some(d.ty.clone()); } + _ => {} } } } @@ -485,10 +498,14 @@ impl Workspace { if let Some(module) = self.modules.get(&target_namespace) { for item in &module.lexicon.items { - if let Item::Alias(a) = item { - if a.name.name == *type_name { - return Some(a.ty.clone()); + match item { + Item::InlineType(i) if i.name.name == *type_name => { + return Some(i.ty.clone()); } + Item::DefType(d) if d.name.name == *type_name => { + return Some(d.ty.clone()); + } + _ => {} } } } @@ -497,6 +514,43 @@ impl Workspace { None } + pub fn is_inline_type(&self, path: &Path) -> bool { + if path.segments.len() == 1 { + let name = &path.segments[0].name; + + for (_, module) in &self.modules { + if let Some(Symbol::Alias { .. }) = module.symbols.types.get(name) { + for item in &module.lexicon.items { + if let Item::InlineType(i) = item { + if i.name.name == *name { + return true; + } + } + } + } + } + } else { + let target_namespace = path.segments[..path.segments.len() - 1] + .iter() + .map(|s| s.name.as_str()) + .collect::>() + .join("."); + let type_name = &path.segments[path.segments.len() - 1].name; + + if let Some(module) = self.modules.get(&target_namespace) { + for item in &module.lexicon.items { + if let Item::InlineType(i) = item { + if i.name.name == *type_name { + return true; + } + } + } + } + } + + false + } + fn resolve_imports(&mut self) -> Result<(), ValidationErrors> { let mut errors = ValidationErrors::new(); @@ -624,6 +678,14 @@ impl Workspace { for item in &lexicon.items { match item { Item::Record(r) => { + // Check for reserved names + if r.name.name == "main" || r.name.name == "defs" { + errors.push(crate::error::ValidationError::ReservedName { + name: r.name.name.clone(), + span: r.name.span, + }); + } + if let Some(existing) = symbols.types.get(&r.name.name) { errors.push(crate::error::ValidationError::DuplicateDefinition { name: r.name.name.clone(), @@ -640,24 +702,65 @@ impl Workspace { ); } } - Item::Alias(a) => { - if let Some(existing) = symbols.types.get(&a.name.name) { + Item::InlineType(i) => { + // Check for reserved names + if i.name.name == "main" || i.name.name == "defs" { + errors.push(crate::error::ValidationError::ReservedName { + name: i.name.name.clone(), + span: i.name.span, + }); + } + + if let Some(existing) = symbols.types.get(&i.name.name) { + errors.push(crate::error::ValidationError::DuplicateDefinition { + name: i.name.name.clone(), + first_span: existing.span(), + second_span: i.name.span, + }); + } else { + symbols.types.insert( + i.name.name.clone(), + Symbol::Alias { + name: i.name.name.clone(), + span: i.name.span, + }, + ); + } + } + Item::DefType(d) => { + // Check for reserved names + if d.name.name == "main" || d.name.name == "defs" { + errors.push(crate::error::ValidationError::ReservedName { + name: d.name.name.clone(), + span: d.name.span, + }); + } + + if let Some(existing) = symbols.types.get(&d.name.name) { errors.push(crate::error::ValidationError::DuplicateDefinition { - name: a.name.name.clone(), + name: d.name.name.clone(), first_span: existing.span(), - second_span: a.name.span, + second_span: d.name.span, }); } else { symbols.types.insert( - a.name.name.clone(), + d.name.name.clone(), Symbol::Alias { - name: a.name.name.clone(), - span: a.name.span, + name: d.name.name.clone(), + span: d.name.span, }, ); } } Item::Token(t) => { + // Check for reserved names + if t.name.name == "main" || t.name.name == "defs" { + errors.push(crate::error::ValidationError::ReservedName { + name: t.name.name.clone(), + span: t.name.span, + }); + } + if let Some(existing) = symbols.types.get(&t.name.name) { errors.push(crate::error::ValidationError::DuplicateDefinition { name: t.name.name.clone(), @@ -674,10 +777,34 @@ impl Workspace { ); } } - Item::Query(_) | Item::Procedure(_) | Item::Subscription(_) => { - // These don't define types, so skip + Item::Query(q) => { + // Check for reserved names + if q.name.name == "main" || q.name.name == "defs" { + errors.push(crate::error::ValidationError::ReservedName { + name: q.name.name.clone(), + span: q.name.span, + }); + } } - Item::Namespace(_) | Item::Use(_) => { + Item::Procedure(p) => { + // Check for reserved names + if p.name.name == "main" || p.name.name == "defs" { + errors.push(crate::error::ValidationError::ReservedName { + name: p.name.name.clone(), + span: p.name.span, + }); + } + } + Item::Subscription(s) => { + // Check for reserved names + if s.name.name == "main" || s.name.name == "defs" { + errors.push(crate::error::ValidationError::ReservedName { + name: s.name.name.clone(), + span: s.name.span, + }); + } + } + Item::Use(_) => { // Handled separately } } @@ -709,11 +836,12 @@ impl Workspace { fn resolve_item(&self, namespace: &str, item: &Item) -> Result<(), ValidationErrors> { match item { Item::Record(r) => self.resolve_record(namespace, r), - Item::Alias(a) => self.resolve_alias(namespace, a), + Item::InlineType(i) => self.resolve_inline_type(namespace, i), + Item::DefType(d) => self.resolve_def_type(namespace, d), Item::Query(q) => self.resolve_query(namespace, q), Item::Procedure(p) => self.resolve_procedure(namespace, p), Item::Subscription(s) => self.resolve_subscription(namespace, s), - Item::Token(_) | Item::Namespace(_) | Item::Use(_) => Ok(()), + Item::Token(_) | Item::Use(_) => Ok(()), } } @@ -733,8 +861,12 @@ impl Workspace { } } - fn resolve_alias(&self, namespace: &str, alias: &Alias) -> Result<(), ValidationErrors> { - self.resolve_type(namespace, &alias.ty) + fn resolve_inline_type(&self, namespace: &str, inline_type: &InlineType) -> Result<(), ValidationErrors> { + self.resolve_type(namespace, &inline_type.ty) + } + + fn resolve_def_type(&self, namespace: &str, def_type: &DefType) -> Result<(), ValidationErrors> { + self.resolve_type(namespace, &def_type.ty) } fn resolve_query(&self, namespace: &str, query: &Query) -> Result<(), ValidationErrors> { @@ -850,6 +982,9 @@ impl Workspace { Err(errors) } } + Type::Parenthesized { inner, .. } => { + self.resolve_type(namespace, inner) + } Type::Constrained { base, .. } => { self.resolve_type(namespace, base) } diff --git a/mlf-validation/src/lib.rs b/mlf-validation/src/lib.rs index 1a1f51d..2f502e1 100644 --- a/mlf-validation/src/lib.rs +++ b/mlf-validation/src/lib.rs @@ -97,6 +97,9 @@ impl<'a> RecordValidator<'a> { Type::Union { types, .. } => { self.validate_union(value, types, path, errors); } + Type::Parenthesized { inner, .. } => { + self.validate_against_type(value, inner, path, errors); + } Type::Reference { path: ref_path, .. } => { // Try to resolve reference if let Some(resolved_type) = self.resolve_reference(ref_path) { @@ -112,14 +115,18 @@ impl<'a> RecordValidator<'a> { } fn resolve_reference(&self, path: &Path) -> Option { - // Simple resolution: look for aliases with matching name + // Simple resolution: look for inline/def types with matching name if path.segments.len() == 1 { let name = &path.segments[0].name; for item in &self.lexicon.items { - if let Item::Alias(alias) = item { - if alias.name.name == *name { - return Some(alias.ty.clone()); + match item { + Item::InlineType(i) if i.name.name == *name => { + return Some(i.ty.clone()); + } + Item::DefType(d) if d.name.name == *name => { + return Some(d.ty.clone()); } + _ => {} } } } @@ -300,10 +307,14 @@ impl<'a> RecordValidator<'a> { } Constraint::Enum { values, .. } => { if let Some(s) = value.as_str() { - if !values.contains(&s.to_string()) { + let enum_strings: Vec = values.iter().map(|v| match v { + mlf_lang::ast::ValueRef::Literal(lit) => lit.clone(), + mlf_lang::ast::ValueRef::Reference(path) => path.to_string(), + }).collect(); + if !enum_strings.contains(&s.to_string()) { errors.push(ValidationError { path: path.to_string(), - message: format!("Value '{}' not in enum: {:?}", s, values), + message: format!("Value '{}' not in enum: {:?}", s, enum_strings), }); } } @@ -345,6 +356,9 @@ impl<'a> RecordValidator<'a> { Constraint::Default { .. } => { // Default values are used when field is missing, not for validation } + Constraint::Const { .. } => { + // Const values are enforced at compile time, not runtime validation + } } } } diff --git a/mlf-wasm/src/lib.rs b/mlf-wasm/src/lib.rs index 1c9c2e5..35e42fb 100644 --- a/mlf-wasm/src/lib.rs +++ b/mlf-wasm/src/lib.rs @@ -106,8 +106,41 @@ pub fn generate_lexicon(source: &str, namespace: &str) -> JsValue { } }; + // Create workspace with prelude for type resolution + let mut workspace = match mlf_lang::Workspace::with_prelude() { + Ok(ws) => ws, + Err(e) => { + let result = GenerateResult { + success: false, + lexicon: None, + error: Some(format!("Failed to load prelude: {:?}", e)), + }; + return serde_wasm_bindgen::to_value(&result).unwrap(); + } + }; + + // Add the module to workspace for resolution + if let Err(e) = workspace.add_module(namespace.to_string(), lexicon.clone()) { + let result = GenerateResult { + success: false, + lexicon: None, + error: Some(format!("Failed to add module: {:?}", e)), + }; + return serde_wasm_bindgen::to_value(&result).unwrap(); + } + + // Resolve types (expands inline types from prelude) + if let Err(e) = workspace.resolve() { + let result = GenerateResult { + success: false, + lexicon: None, + error: Some(format!("Type resolution error: {:?}", e)), + }; + return serde_wasm_bindgen::to_value(&result).unwrap(); + } + // Generate JSON lexicon - let json_lexicon = mlf_codegen::generate_lexicon(namespace, &lexicon); + let json_lexicon = mlf_codegen::generate_lexicon(namespace, &lexicon, &workspace); match serde_json::to_string_pretty(&json_lexicon) { Ok(json_str) => { diff --git a/resources/prelude.mlf b/resources/prelude.mlf index 5fc1271..02394c0 100644 --- a/resources/prelude.mlf +++ b/resources/prelude.mlf @@ -1,43 +1,43 @@ -alias AtIdentifier = string constrained { +inline type AtIdentifier = string constrained { format: "at-identifier", }; -alias AtUri = string constrained { +inline type AtUri = string constrained { format: "at-uri", }; -alias Cid = string constrained { +inline type Cid = string constrained { format: "cid", }; -alias Datetime = string constrained { +inline type Datetime = string constrained { format: "datetime", }; -alias Did = string constrained { +inline type Did = string constrained { format: "did", }; -alias Handle = string constrained { +inline type Handle = string constrained { format: "handle", }; -alias Nsid = string constrained { +inline type Nsid = string constrained { format: "nsid", }; -alias Tid = string constrained { +inline type Tid = string constrained { format: "tid", }; -alias RecordKey = string constrained { +inline type RecordKey = string constrained { format: "record-key", }; -alias Uri = string constrained { +inline type Uri = string constrained { format: "uri", }; -alias Language = string constrained { +inline type Language = string constrained { format: "language", }; diff --git a/tree-sitter-mlf/grammar.js b/tree-sitter-mlf/grammar.js index c630482..faa1aad 100644 --- a/tree-sitter-mlf/grammar.js +++ b/tree-sitter-mlf/grammar.js @@ -20,10 +20,10 @@ module.exports = grammar({ source_file: $ => repeat($.item), item: $ => choice( - $.namespace_declaration, $.use_statement, $.record_definition, - $.alias_definition, + $.inline_type_definition, + $.def_type_definition, $.token_definition, $.query_definition, $.procedure_definition, @@ -34,15 +34,6 @@ module.exports = grammar({ doc_comment: $ => token(seq('///', /.*/)), comment: $ => token(seq('//', /.*/)), - // Namespace - namespace_declaration: $ => seq( - 'namespace', - field('name', $.namespace_identifier), - ';' - ), - - namespace_identifier: $ => /[a-z][a-z0-9]*(\.[a-z][a-z0-9]*)*/, - // Use statements use_statement: $ => seq( 'use', @@ -54,8 +45,7 @@ module.exports = grammar({ record_definition: $ => seq( 'record', field('name', $.identifier), - field('body', $.record_body), - ';' + field('body', $.record_body) ), record_body: $ => seq( @@ -73,9 +63,20 @@ module.exports = grammar({ ',' ), - // Alias definition - alias_definition: $ => seq( - 'alias', + // Inline type definition + inline_type_definition: $ => seq( + 'inline', + 'type', + field('name', $.identifier), + '=', + field('type', $.type), + ';' + ), + + // Def type definition + def_type_definition: $ => seq( + 'def', + 'type', field('name', $.identifier), '=', field('type', $.type), diff --git a/tree-sitter-mlf/src/grammar.json b/tree-sitter-mlf/src/grammar.json index bf18b48..1ea4a4f 100644 --- a/tree-sitter-mlf/src/grammar.json +++ b/tree-sitter-mlf/src/grammar.json @@ -25,7 +25,11 @@ }, { "type": "SYMBOL", - "name": "alias_definition" + "name": "inline_type_definition" + }, + { + "type": "SYMBOL", + "name": "def_type_definition" }, { "type": "SYMBOL", @@ -225,12 +229,53 @@ } ] }, - "alias_definition": { + "inline_type_definition": { + "type": "SEQ", + "members": [ + { + "type": "STRING", + "value": "inline" + }, + { + "type": "STRING", + "value": "type" + }, + { + "type": "FIELD", + "name": "name", + "content": { + "type": "SYMBOL", + "name": "identifier" + } + }, + { + "type": "STRING", + "value": "=" + }, + { + "type": "FIELD", + "name": "type", + "content": { + "type": "SYMBOL", + "name": "type" + } + }, + { + "type": "STRING", + "value": ";" + } + ] + }, + "def_type_definition": { "type": "SEQ", "members": [ { "type": "STRING", - "value": "alias" + "value": "def" + }, + { + "type": "STRING", + "value": "type" }, { "type": "FIELD", diff --git a/tree-sitter-mlf/src/node-types.json b/tree-sitter-mlf/src/node-types.json index 1a7cef9..d07acd2 100644 --- a/tree-sitter-mlf/src/node-types.json +++ b/tree-sitter-mlf/src/node-types.json @@ -1,30 +1,4 @@ [ - { - "type": "alias_definition", - "named": true, - "fields": { - "name": { - "multiple": false, - "required": true, - "types": [ - { - "type": "identifier", - "named": true - } - ] - }, - "type": { - "multiple": false, - "required": true, - "types": [ - { - "type": "type", - "named": true - } - ] - } - } - }, { "type": "array_literal", "named": true, @@ -155,6 +129,32 @@ ] } }, + { + "type": "def_type_definition", + "named": true, + "fields": { + "name": { + "multiple": false, + "required": true, + "types": [ + { + "type": "identifier", + "named": true + } + ] + }, + "type": { + "multiple": false, + "required": true, + "types": [ + { + "type": "type", + "named": true + } + ] + } + } + }, { "type": "error_definition", "named": true, @@ -222,6 +222,32 @@ "named": true, "fields": {} }, + { + "type": "inline_type_definition", + "named": true, + "fields": { + "name": { + "multiple": false, + "required": true, + "types": [ + { + "type": "identifier", + "named": true + } + ] + }, + "type": { + "multiple": false, + "required": true, + "types": [ + { + "type": "type", + "named": true + } + ] + } + } + }, { "type": "item", "named": true, @@ -231,7 +257,11 @@ "required": true, "types": [ { - "type": "alias_definition", + "type": "def_type_definition", + "named": true + }, + { + "type": "inline_type_definition", "named": true }, { @@ -700,10 +730,6 @@ "type": "`", "named": false }, - { - "type": "alias", - "named": false - }, { "type": "blob", "named": false @@ -724,6 +750,10 @@ "type": "constrained", "named": false }, + { + "type": "def", + "named": false + }, { "type": "doc_comment", "named": true @@ -732,6 +762,10 @@ "type": "false", "named": false }, + { + "type": "inline", + "named": false + }, { "type": "integer", "named": false @@ -770,11 +804,11 @@ }, { "type": "string", - "named": true + "named": false }, { "type": "string", - "named": false + "named": true }, { "type": "subscription", @@ -792,6 +826,10 @@ "type": "true", "named": false }, + { + "type": "type", + "named": false + }, { "type": "unknown", "named": false diff --git a/website/content/docs/syntax.md b/website/content/docs/syntax.md index 78b4cd6..89b1211 100644 --- a/website/content/docs/syntax.md +++ b/website/content/docs/syntax.md @@ -15,17 +15,60 @@ weight = 2 ``` ### File Naming Convention -Files should follow the lexicon NSID: +The file path determines the lexicon NSID. Files should follow the lexicon NSID structure: - `com.example.forum.thread.mlf` → Lexicon NSID: `com.example.forum.thread` - `com.example.user.profile.mlf` → Lexicon NSID: `com.example.user.profile` +The lexicon NSID is derived solely from the filename, not from any internal declarations. + ## Basic Structure Every MLF file can contain: -- Namespace declarations - Use statements (imports) -- Type definitions (record, alias, token, query, procedure, subscription) +- Type definitions (record, inline type, def type, token, query, procedure, subscription) + +## Syntax Rules + +### Semicolons + +- **Records** do NOT have semicolons after the closing brace `}` +- All other definitions require semicolons: + - `use` statements end with `;` + - `token` definitions end with `;` + - `inline type` definitions end with `;` + - `def type` definitions end with `;` + - `query` definitions end with `;` + - `procedure` definitions end with `;` + - `subscription` definitions end with `;` + +### Commas + +Commas are **required** between items, with **trailing commas allowed**: + +**Record fields:** +```mlf +record example { + field1: string, + field2: integer, // trailing comma allowed +} +``` + +**Constraints:** +```mlf +title: string constrained { + maxLength: 200, + minLength: 1, // trailing comma allowed +} +``` + +**Error definitions:** +```mlf +query getThread(): thread | error { + NotFound, + BadRequest, // trailing comma allowed +} +``` ## Primitive Types @@ -63,32 +106,46 @@ Records define structured data types stored in repositories: record thread { /// Thread title title: string constrained { - maxLength: 200, - minLength: 1, - }, + maxLength: 200 + minLength: 1 + } /// Thread body - body?: string, // Optional field + body?: string // Optional field /// Thread creation timestamp - createdAt: Datetime, + createdAt: Datetime +} +``` + +## Type Definitions + +MLF supports two kinds of type definitions: + +### Inline Types + +Expanded at the point of use, never appear in generated lexicon defs: + +```mlf +inline type AtIdentifier = string constrained { + format "at-identifier" }; ``` -## Aliases +### Def Types -Type aliases define reusable object shapes: +Become named definitions in the lexicon's defs block: ```mlf -alias replyRef = { - root: AtUri, - parent: AtUri, +def type replyRef = { + root: AtUri + parent: AtUri }; record thread { - reply?: replyRef, -}; + reply?: replyRef +} ``` -If used in multiple places, they will be hoisted to a def. If only used once, they will be inlined. +Use `inline type` for type aliases that should be expanded inline (like primitive type wrappers). Use `def type` for types that should be referenced by name in the generated lexicon. ## Tokens @@ -103,10 +160,10 @@ token closed; record issue { state: string constrained { - knownValues: [open, closed], - default: "open", - }, -}; + knownValues: [open, closed] + default: "open" + } +} ``` Tokens must have doc comments describing their purpose. @@ -117,18 +174,18 @@ Add validation constraints to types: ```mlf title: string constrained { - maxLength: 200, - minLength: 1, -}; + maxLength: 200 + minLength: 1 +} age: integer constrained { - minimum: 0, - maximum: 150, -}; + minimum: 0 + maximum: 150 +} status: string constrained { - enum: ["draft", "published", "archived"], -}; + enum: ["draft", "published", "archived"] +} ``` ### String Constraints @@ -136,10 +193,28 @@ status: string constrained { - `maxLength` / `minLength` - Length in bytes - `maxGraphemes` / `minGraphemes` - Length in grapheme clusters - `format` - Format validation (datetime, uri, did, handle, etc.) -- `enum` - Allowed values (closed set) -- `knownValues` - Known values (extensible set, can reference tokens) +- `enum` - Allowed values (closed set) - accepts string literals or token references +- `knownValues` - Known values (extensible set) - accepts string literals or token references - `default` - Default value +**enum, knownValues, and default** can use either literals or references: +```mlf +// String literals +status: string constrained { + knownValues: ["open", "closed", "pending"] + default: "open" +} + +// References to named items (tokens, aliases, records, etc.) +token open; +token closed; + +status: string constrained { + knownValues: [open, closed] // References tokens defined above + default: open // References the token +} +``` + ### Integer Constraints - `minimum` / `maximum` - Min/max values @@ -150,8 +225,8 @@ status: string constrained { ```mlf tags: string[] constrained { - minLength: 1, - maxLength: 10, + minLength: 1 + maxLength: 10 } ``` @@ -159,8 +234,8 @@ tags: string[] constrained { ```mlf avatar: blob constrained { - accept: ["image/png", "image/jpeg"], - maxSize: 1000000, // bytes + accept: ["image/png", "image/jpeg"] + maxSize: 1000000 // bytes } ``` @@ -168,7 +243,7 @@ avatar: blob constrained { ```mlf field: boolean constrained { - default: false, + default: false } ``` @@ -177,16 +252,16 @@ field: boolean constrained { Constraints can only make types **more restrictive**, never less restrictive: ```mlf -alias shortString = string constrained { - maxLength: 100, +def type shortString = string constrained { + maxLength: 100 }; record post { // Valid: 50 is more restrictive than 100 title: shortString constrained { - maxLength: 50, - }, -}; + maxLength: 50 + } +} ``` **Refinement rules:** @@ -231,8 +306,8 @@ Inline object types: ```mlf metadata: { - version: integer, - timestamp: Datetime, + version: integer + timestamp: Datetime } ``` @@ -244,7 +319,7 @@ Queries are read-only HTTP endpoints (GET): /// Get a user profile query getProfile( /// The actor's DID or handle - actor: AtIdentifier, + actor: AtIdentifier ): profile; ``` @@ -252,12 +327,12 @@ With errors: ```mlf query getThread( - uri: AtUri, + uri: AtUri ): thread | error { /// Thread not found - NotFound, + NotFound /// Invalid request - BadRequest, + BadRequest }; ``` @@ -268,14 +343,14 @@ Procedures are write operations (POST): ```mlf /// Create a new thread procedure createThread( - title: string, - body: string, + title: string + body: string ): { - uri: AtUri, - cid: Cid, + uri: AtUri + cid: Cid } | error { /// Title too long - TitleTooLong, + TitleTooLong }; ``` @@ -287,25 +362,25 @@ Subscriptions are WebSocket-based event streams: /// Subscribe to repository events subscription subscribeRepos( /// Optional cursor for resuming - cursor?: integer, + cursor?: integer ): commit | identity | handle; ``` -Message types must be defined as aliases or records: +Message types must be defined as def types or records: ```mlf /// Commit message -alias commit = { - seq: integer, - repo: Did, - commit: Cid, - time: Datetime, +def type commit = { + seq: integer + repo: Did + commit: Cid + time: Datetime }; /// Identity message -alias identity = { - did: Did, - handle: Handle, +def type identity = { + did: Did + handle: Handle }; ``` @@ -319,8 +394,8 @@ Use `///` for documentation (appears in generated docs/code): /// A forum thread record thread { /// Thread title - title: string, -}; + title: string +} ``` ### Regular Comments @@ -330,8 +405,8 @@ Use `//` for comments that won't appear in output: ```mlf // This is a regular comment record example { - field: string, // inline comment -}; + field: string // inline comment +} ``` ## Annotations @@ -342,7 +417,7 @@ Annotations use `@` and provide metadata for external tooling: ```mlf @deprecated record oldRecord { - field: string, + field: string } ``` @@ -351,7 +426,7 @@ record oldRecord { @since(1, 2, 0) @doc("https://example.com/docs") record example { - field: string, + field: string } ``` @@ -361,14 +436,14 @@ record example { @table(name: "threads", indexes: "did,createdAt") record thread { @indexed - did: Did, + did: Did @sensitive(pii: true) - title: string, + title: string } ``` -Annotations can be placed on records, aliases, tokens, queries, procedures, subscriptions, and fields. +Annotations can be placed on records, inline types, def types, tokens, queries, procedures, subscriptions, and fields. ## Imports @@ -397,39 +472,7 @@ After importing, use the short name: use com.example.user.profile; record thread { - author: profile, // Instead of com.example.user.profile -} -``` - -## Namespaces - -Organize related definitions: - -```mlf -namespace com.example.forum.thread; - -record thread { - title: string, -}; -``` - -Or use nested namespaces: - -```mlf -namespace .forum { - record thread { - title: string, - } - - query getThread( - uri: AtUri, - ): thread; -} - -namespace .user { - record profile { - displayName: string, - } + author: profile // Instead of com.example.user.profile } ``` @@ -440,16 +483,16 @@ Reference local or external definitions: ```mlf // Local reference (same file) record thread { - author: author, // References 'alias author' in same file + author: author // References 'def type author' in same file } // Cross-file reference record thread { - profile: com.example.user.profile, // References com/example/user/profile.mlf + profile: com.example.user.profile // References com/example/user/profile.mlf } ``` -**Note:** All references use dotted notation. The `#` character is NOT used for references. +All references use dotted notation. ## Optional Fields @@ -457,9 +500,9 @@ Use `?` to mark fields as optional: ```mlf record thread { - title: string, // Required - body?: string, // Optional - tags?: string[], // Optional array + title: string // Required + body?: string // Optional + tags?: string[] // Optional array } ``` @@ -468,9 +511,9 @@ record thread { Use backticks to escape reserved keywords when you need to use them as identifiers: ```mlf -alias `record` = { - `record`: com.atproto.repo.strongRef, - `error`: string, +def type `record` = { + `record`: com.atproto.repo.strongRef + `error`: string }; ``` @@ -509,42 +552,42 @@ token closed; record thread { /// Thread title title: string constrained { - minGraphemes: 1, - maxGraphemes: 200, - }, + minGraphemes: 1 + maxGraphemes: 200 + } /// Thread body (markdown) body?: string constrained { - maxGraphemes: 10000, - }, + maxGraphemes: 10000 + } /// Thread state state: string constrained { - knownValues: [open, closed], - default: "open", - }, + knownValues: [open, closed] + default: "open" + } /// Author profile - author: profile, + author: profile /// Creation timestamp - createdAt: Datetime, -}; + createdAt: Datetime +} /// Get a thread by URI query getThread( /// Thread AT-URI - uri: AtUri, + uri: AtUri ): thread | error { /// Thread not found - NotFound, + NotFound }; /// Create a new thread procedure createThread( - title: string, - body?: string, + title: string + body?: string ): { - uri: AtUri, - cid: Cid, + uri: AtUri + cid: Cid } | error { /// Title too long - TitleTooLong, + TitleTooLong }; ``` diff --git a/website/sass/style.scss b/website/sass/style.scss index 0c29007..afb8dff 100644 --- a/website/sass/style.scss +++ b/website/sass/style.scss @@ -381,7 +381,9 @@ body:has(.playground-page) footer { padding: 0.75rem 1rem; background: var(--bg-alt); border-bottom: 1px solid var(--border); - height: 3rem; + min-height: 3rem; + flex-wrap: wrap; + gap: 0.5rem; } .panel-header h3 { @@ -390,6 +392,29 @@ body:has(.playground-page) footer { margin: 0; } +.file-path-container { + display: flex; + align-items: center; + flex: 1; +} + +.file-path-container input { + flex: 1; + padding: 0.375rem 0.75rem; + background: var(--bg); + border: 1px solid var(--border); + border-radius: 0.25rem; + color: var(--text); + font-size: 0.875rem; + font-family: 'SF Mono', Monaco, 'Cascadia Code', 'Roboto Mono', Consolas, 'Courier New', monospace; + min-width: 0; +} + +.file-path-container input:focus { + outline: none; + border-color: var(--accent); +} + .tabs { display: flex; gap: 0.5rem; @@ -443,43 +468,87 @@ textarea::placeholder { color: #718096; } -.shiki-editor-container { +.editor-container-with-lines { width: 100%; flex: 1; min-height: 0; - overflow: auto; background: var(--code-bg); + display: flex; + flex-direction: row; } -.shiki-editor { - min-height: 100%; +.editor-wrapper { + position: relative; + flex: 1; + min-height: 0; + overflow: hidden; +} + +.highlight-backdrop { + position: absolute; + top: 0; + left: 0; + right: 0; + bottom: 0; + padding: 1rem; + font-family: 'SF Mono', 'Monaco', 'Inconsolata', 'Fira Code', 'Menlo', monospace; + font-size: 0.875rem; + line-height: 1.6; + white-space: pre; + overflow: hidden; + pointer-events: none; + z-index: 1; +} + +.highlight-backdrop span { + font-family: inherit; + font-size: inherit; + line-height: inherit; +} + +.mlf-textarea { + position: absolute; + top: 0; + left: 0; + right: 0; + bottom: 0; + width: 100%; + height: 100%; padding: 1rem; border: none; - font-family: 'Atkinson Hyperlegible Mono', 'SF Mono', 'Monaco', 'Inconsolata', 'Fira Code', 'Menlo', monospace; + font-family: 'SF Mono', 'Monaco', 'Inconsolata', 'Fira Code', 'Menlo', monospace; font-size: 0.875rem; line-height: 1.6; - color: var(--code-text); + resize: none; + background: transparent; + color: transparent; white-space: pre; tab-size: 4; - caret-color: var(--text); + overflow: auto; + z-index: 2; + caret-color: #fff; + -webkit-text-fill-color: transparent; } -.shiki-editor:focus { +.mlf-textarea:focus { outline: none; } -.shiki-editor span { - font-family: inherit; - font-size: inherit; - line-height: inherit; +.mlf-textarea::selection { + background: rgba(255, 255, 255, 0.2); } -.shiki-output-container { +.shiki-output-outer-container { width: 100%; flex: 1; min-height: 0; - overflow: auto; background: var(--code-bg); + display: flex; + flex-direction: row; +} + +.shiki-output-container { + min-height: 100%; padding: 1rem; font-family: 'Atkinson Hyperlegible Mono', 'SF Mono', 'Monaco', 'Inconsolata', 'Fira Code', 'Menlo', monospace; font-size: 0.875rem; @@ -836,3 +905,34 @@ footer { color: var(--accent); text-decoration: none; } + +/* Line numbers for code editor */ +.line-numbers { + flex-shrink: 0; + min-width: 3rem; + padding: 1rem 0.5rem 1rem 1rem; + text-align: right; + user-select: none; + color: var(--text-muted); + background: var(--code-bg); + border-right: 1px solid var(--border); + overflow: hidden; +} + +.line-number { + font-family: 'Atkinson Hyperlegible Mono', 'SF Mono', 'Monaco', 'Inconsolata', 'Fira Code', 'Menlo', monospace; + font-size: 0.875rem; + line-height: 1.6; +} + +.editor-wrapper { + flex: 1; + overflow: auto; + min-width: 0; +} + +.output-wrapper { + flex: 1; + overflow: auto; + min-width: 0; +} diff --git a/website/static/js/app.js b/website/static/js/app.js index 838471b..26ab42c 100644 --- a/website/static/js/app.js +++ b/website/static/js/app.js @@ -35,11 +35,11 @@ const mlfGrammar = { patterns: [ { name: 'keyword.control.mlf', - match: '\\b(namespace|use|record|alias|token|query|procedure|subscription|throws|constrained)\\b' + match: '\\b(use|record|inline|def|type|token|query|procedure|subscription|throws|constrained)\\b' }, { name: 'keyword.other.mlf', - match: '\\b(main)\\b' + match: '\\b(main|defs)\\b' } ] }, @@ -144,45 +144,74 @@ function initEditor() { const textarea = document.getElementById('mlf-editor'); const initialCode = textarea.value; - // Create editor container + // Create editor container with line numbers and highlighting editorContainer = document.createElement('div'); - editorContainer.className = 'shiki-editor-container'; + editorContainer.className = 'editor-container-with-lines'; - const editor = document.createElement('div'); - editor.className = 'shiki-editor'; - editor.contentEditable = 'true'; - editor.spellcheck = false; - editor.id = 'shiki-editor'; + // Create line numbers + const lineNumbers = document.createElement('div'); + lineNumbers.className = 'line-numbers'; + lineNumbers.id = 'line-numbers'; - editorContainer.appendChild(editor); + // Create wrapper for textarea and highlight layer + const editorWrapper = document.createElement('div'); + editorWrapper.className = 'editor-wrapper'; - // Replace textarea with editor - textarea.style.display = 'none'; - textarea.parentNode.insertBefore(editorContainer, textarea); + // Create highlight backdrop + const highlightBackdrop = document.createElement('div'); + highlightBackdrop.className = 'highlight-backdrop'; + highlightBackdrop.id = 'highlight-backdrop'; - // Set initial content + // Style the textarea + textarea.className = 'mlf-textarea'; + + // Insert container before textarea, then build the structure + const parent = textarea.parentNode; + parent.insertBefore(editorContainer, textarea); + + editorWrapper.appendChild(highlightBackdrop); + editorWrapper.appendChild(textarea); + + editorContainer.appendChild(lineNumbers); + editorContainer.appendChild(editorWrapper); + + // Set initial line numbers and highlighting + updateLineNumbers(initialCode); updateHighlighting(initialCode); - // Convert JSON output textarea to highlighted div + // Convert JSON output textarea to highlighted div with line numbers const jsonTextarea = document.getElementById('lexicon-result'); + + const jsonOuterContainer = document.createElement('div'); + jsonOuterContainer.className = 'shiki-output-outer-container'; + jsonOuterContainer.id = 'lexicon-result-outer-container'; + + const jsonLineNumbers = document.createElement('div'); + jsonLineNumbers.className = 'line-numbers'; + jsonLineNumbers.id = 'json-line-numbers'; + + const jsonWrapper = document.createElement('div'); + jsonWrapper.className = 'output-wrapper'; + const jsonContainer = document.createElement('div'); jsonContainer.className = 'shiki-output-container'; jsonContainer.id = 'lexicon-result-container'; + jsonWrapper.appendChild(jsonContainer); + jsonOuterContainer.appendChild(jsonLineNumbers); + jsonOuterContainer.appendChild(jsonWrapper); + jsonTextarea.style.display = 'none'; - jsonTextarea.parentNode.insertBefore(jsonContainer, jsonTextarea); + jsonTextarea.parentNode.insertBefore(jsonOuterContainer, jsonTextarea); } function updateHighlighting(code) { - if (!highlighter || !editorContainer) return; - - const editor = editorContainer.querySelector('.shiki-editor'); - if (!editor) return; + if (!highlighter) return; - // Store cursor position - const cursorOffset = getCaretPosition(editor); + const backdrop = document.getElementById('highlight-backdrop'); + if (!backdrop) return; - // Update highlighted content + // Update highlighted content in backdrop const html = highlighter.codeToHtml(code, { lang: 'mlf', theme: 'dracula' @@ -194,15 +223,22 @@ function updateHighlighting(code) { const codeElement = temp.querySelector('code'); if (codeElement) { - editor.innerHTML = codeElement.innerHTML; + backdrop.innerHTML = codeElement.innerHTML; } else { - editor.textContent = code; + backdrop.textContent = code; } +} - // Restore cursor position - if (document.activeElement === editor) { - setCaretPosition(editor, cursorOffset); - } +function updateLineNumbers(code) { + const lineNumbers = document.getElementById('line-numbers'); + if (!lineNumbers) return; + + const lines = code.split('\n').length; + const lineNumbersHtml = Array.from({ length: lines }, (_, i) => + `
${i + 1}
` + ).join(''); + + lineNumbers.innerHTML = lineNumbersHtml; } function getCaretPosition(element) { @@ -256,8 +292,8 @@ function setCaretPosition(element, offset) { } function getEditorContent() { - const editor = editorContainer?.querySelector('.shiki-editor'); - return editor ? editor.textContent : ''; + const textarea = document.getElementById('mlf-editor'); + return textarea ? textarea.value : ''; } function updateJsonOutput(jsonString) { @@ -284,11 +320,27 @@ function updateJsonOutput(jsonString) { } else { container.textContent = formatted; } + + // Update line numbers for JSON output + updateJsonLineNumbers(formatted); } catch (e) { container.textContent = jsonString; + updateJsonLineNumbers(jsonString); } } +function updateJsonLineNumbers(code) { + const lineNumbers = document.getElementById('json-line-numbers'); + if (!lineNumbers) return; + + const lines = code.split('\n').length; + const lineNumbersHtml = Array.from({ length: lines }, (_, i) => + `
${i + 1}
` + ).join(''); + + lineNumbers.innerHTML = lineNumbersHtml; +} + function hidePlayground() { const playground = document.querySelector('.playground-container'); if (playground) { @@ -307,17 +359,45 @@ function setupEventListeners() { }); // Editor input with debounce - const editor = editorContainer?.querySelector('.shiki-editor'); - if (editor) { - let timeout; - editor.addEventListener('input', () => { - clearTimeout(timeout); - timeout = setTimeout(() => { - const code = getEditorContent(); - updateHighlighting(code); + const textarea = document.getElementById('mlf-editor'); + if (textarea) { + let checkTimeout; + + textarea.addEventListener('input', () => { + const code = textarea.value; + + // Update line numbers and highlighting immediately + updateLineNumbers(code); + updateHighlighting(code); + + // Clear previous timeout + clearTimeout(checkTimeout); + + // Debounce validation/check only + checkTimeout = setTimeout(() => { handleCheck(); }, 500); }); + + // Synchronize scroll between textarea, line numbers, and backdrop + const lineNumbers = document.getElementById('line-numbers'); + const backdrop = document.getElementById('highlight-backdrop'); + if (lineNumbers && backdrop) { + textarea.addEventListener('scroll', () => { + lineNumbers.scrollTop = textarea.scrollTop; + // Use transform to scroll the backdrop in sync + backdrop.style.transform = `translate(-${textarea.scrollLeft}px, -${textarea.scrollTop}px)`; + }); + } + } + + // Synchronize scroll between JSON output and line numbers + const jsonWrapper = document.querySelector('.output-wrapper'); + const jsonLineNumbers = document.getElementById('json-line-numbers'); + if (jsonWrapper && jsonLineNumbers) { + jsonWrapper.addEventListener('scroll', () => { + jsonLineNumbers.scrollTop = jsonWrapper.scrollTop; + }); } // Auto-validate on record input change (debounced) @@ -329,6 +409,16 @@ function setupEventListeners() { timeout = setTimeout(handleValidate, 500); }); } + + // File path input change triggers re-generation + const filePathInput = document.getElementById('file-path'); + if (filePathInput) { + let timeout; + filePathInput.addEventListener('input', () => { + clearTimeout(timeout); + timeout = setTimeout(handleCheck, 500); + }); + } } function switchTab(tabName) { @@ -343,12 +433,44 @@ function switchTab(tabName) { }); } +function extractNamespaceFromPath(filePath) { + // Remove leading/trailing slashes + filePath = filePath.trim().replace(/^\/+|\/+$/g, ''); + + // Validate it ends with .mlf + if (!filePath.endsWith('.mlf')) { + return null; + } + + // Remove the .mlf extension + const withoutExt = filePath.slice(0, -4); + + // Replace slashes with dots to get the namespace + const namespace = withoutExt.replace(/\//g, '.'); + + // Validate namespace format (should be valid NSID segments) + // Basic validation: only alphanumeric, dots, and hyphens + if (!/^[a-z0-9][a-z0-9.-]*[a-z0-9]$/.test(namespace)) { + return null; + } + + return namespace; +} + function handleCheck() { if (!wasm) { return; } const source = getEditorContent(); + const filePath = document.getElementById('file-path').value; + + // Extract namespace from file path + const namespace = extractNamespaceFromPath(filePath); + if (!namespace) { + showError('Invalid file path. Must be a valid path ending in .mlf (e.g., com/example/app/thread.mlf)'); + return; + } try { // Check the MLF source @@ -357,10 +479,6 @@ function handleCheck() { if (checkResult.success) { hideError(); - // Generate lexicon - extract namespace from source or use default - const namespaceMatch = source.match(/namespace\s+([\w.]+)/); - const namespace = namespaceMatch ? namespaceMatch[1] : 'com.example.post'; - const generateResult = wasm.generate_lexicon(source, namespace); if (generateResult.success) { diff --git a/website/syntaxes/mlf.sublime-syntax b/website/syntaxes/mlf.sublime-syntax index 66be980..37c755b 100644 --- a/website/syntaxes/mlf.sublime-syntax +++ b/website/syntaxes/mlf.sublime-syntax @@ -29,9 +29,9 @@ contexts: pop: true keywords: - - match: '\b(namespace|use|record|alias|token|query|procedure|subscription|throws|constrained)\b' + - match: '\b(namespace|use|record|inline|def|type|token|query|procedure|subscription|throws|constrained)\b' scope: keyword.control.mlf - - match: '\b(main)\b' + - match: '\b(main|defs)\b' scope: keyword.other.mlf types: diff --git a/website/templates/playground.html b/website/templates/playground.html index 47fd272..4391625 100644 --- a/website/templates/playground.html +++ b/website/templates/playground.html @@ -5,7 +5,9 @@
-

MLF Source

+
+ +
+}
-

Output

-- 2.51.2