From 52d38883d78b177226c8e69cef8dd741b5fdd01b Mon Sep 17 00:00:00 2001 From: Matt Stavola Date: Fri, 10 Oct 2025 03:18:05 -0400 Subject: [PATCH] Update namespacing and imports --- .mlf/.gitignore | 2 - README.md | 78 +- SPEC.md | 975 ------------------ mlf-cli/src/check.rs | 258 ++++- mlf-cli/src/generate/code.rs | 203 +++- mlf-cli/src/generate/lexicon.rs | 182 +++- mlf-cli/src/generate/mlf.rs | 158 ++- mlf-cli/src/generate/mlf.rs.backup | 941 +++++++++++++++++ mlf-cli/src/generate/mod.rs | 10 +- mlf-cli/src/main.rs | 49 +- mlf-codegen/src/lib.rs | 107 +- mlf-diagnostics/src/lib.rs | 145 ++- mlf-lang/src/error.rs | 15 +- mlf-lang/src/lexer.rs | 26 + mlf-lang/src/parser.rs | 215 +++- mlf-lang/src/workspace.rs | 715 ++++++++++--- std/com/atproto/admin/defs.mlf | 4 +- tree-sitter-mlf/grammar.js | 23 +- tree-sitter-mlf/queries/highlights.scm | 15 +- tree-sitter-mlf/src/grammar.json | 168 +++ tree-sitter-mlf/src/node-types.json | 81 +- tree-sitter-mlf/test.mlf | 13 + website/content/docs/cli/02-configuration.md | 113 +- website/content/docs/cli/04-check.md | 14 - website/content/docs/cli/06-generate.md | 191 +++- .../content/docs/language-guide/08-imports.md | 234 ++++- .../content/docs/language-guide/09-prelude.md | 12 +- .../docs/language-guide/10-important-info.md | 124 ++- .../docs/language-guide/11-annotations.md | 164 +++ .../docs/language-guide/11-lexicon-mapping.md | 529 ++++++++++ website/syntaxes/mlf.sublime-syntax | 6 +- 31 files changed, 4260 insertions(+), 1510 deletions(-) delete mode 100644 .mlf/.gitignore delete mode 100644 SPEC.md create mode 100644 mlf-cli/src/generate/mlf.rs.backup create mode 100644 website/content/docs/language-guide/11-annotations.md create mode 100644 website/content/docs/language-guide/11-lexicon-mapping.md diff --git a/.mlf/.gitignore b/.mlf/.gitignore deleted file mode 100644 index d6b7ef3..0000000 --- a/.mlf/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore diff --git a/README.md b/README.md index 562d676..02d40de 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ A human-friendly DSL for ATProto Lexicons -**This is a work in progress, things are subject to break and change** +*This is a work in progress, things are subject to break and change* ## What it looks like @@ -22,9 +22,9 @@ def type replyRef = { }; ``` -## Getting started +## Installation -### Install +Right now you can only install mlf from source: ```bash # Install with all code generators (default: TypeScript, Go, Rust) @@ -37,74 +37,14 @@ cargo install --path mlf-cli --no-default-features --features typescript,go cargo install --path mlf-cli --no-default-features ``` -### Generate code from MLF - -```bash -# Generate TypeScript types -mlf generate code -g typescript -i examples/**/*.mlf -o output/ - -# Generate Go structs -mlf generate code -g go -i examples/**/*.mlf -o output/ - -# Generate Rust structs with serde -mlf generate code -g rust -i examples/**/*.mlf -o output/ - -# Generate JSON lexicons (always available) -mlf generate code -g json -i examples/**/*.mlf -o output/ -# Or use the legacy command: -mlf generate lexicon -i examples/**/*.mlf -o output/ -``` - -### Validate MLF files - -```bash -mlf check examples/app.bsky.feed.post.mlf -``` - -### Validate JSON records - -```bash -mlf validate examples/app.bsky.feed.post.mlf record.json -``` - -### Convert JSON lexicons to MLF - -Convert existing ATProto JSON lexicons to MLF format: - -```bash -# Convert a single lexicon -mlf generate mlf -i my-lexicon.json -o ./ - -# Convert multiple lexicons -mlf generate mlf -i "dist/lexicons/**/*.json" -o src/lexicons/ -``` - -This is useful for: -- Migrating existing JSON lexicons to the MLF format -- Learning MLF syntax by comparing JSON and MLF -- Working with lexicons from external sources +## Documentation -## Project layout +Visit the [MLF website](https://mlf.lol/docs) for comprehensive documentation, guides, and examples. -``` -mlf/ -├── mlf-cli/ # Command-line app -├── mlf-lang/ # Parser and lexer (no_std compatible) -├── mlf-codegen/ # Core code generation with plugin system -├── codegen-plugins/ # Language-specific code generators -│ ├── mlf-codegen-typescript/ # TypeScript generator -│ ├── mlf-codegen-go/ # Go generator -│ └── mlf-codegen-rust/ # Rust generator -├── mlf-validation/ # Lexicon validation -├── mlf-diagnostics/ # Fancy error reporting -├── mlf-wasm/ # WASM bindings for browser use -├── tree-sitter-mlf/ # Tree-sitter grammar for syntax highlighting -└── website/ # Docs and playground - └── mlf-playground-wasm/ # Playground WASM with all generators -``` +## Architecture -## Documentation +Please review [ARCHITECTURE.md](ARCHITECTURE.md) for an overview of how the project is structured. -Full documentation available at the [MLF website](https://mlf.lol) (or run `just serve` in `website/`). +## License -See [SPEC.md](SPEC.md) for the complete language specification. +MIT diff --git a/SPEC.md b/SPEC.md deleted file mode 100644 index ed9ee51..0000000 --- a/SPEC.md +++ /dev/null @@ -1,975 +0,0 @@ -# MLF (Matt's Lexicon Format) Specification - -## Overview - -MLF is a domain-specific language (DSL) for writing ATProto Lexicons with 100% fidelity to the [AT Protocol Lexicon specification](https://atproto.com/specs/lexicon). It provides a more ergonomic, type-safe syntax for defining records, queries, procedures, and types. - -## Design Goals - -1. **100% ATProto Fidelity**: Every valid ATProto Lexicon can be represented in MLF -2. **Human-Readable**: Clear, concise syntax that's easy to read and write -3. **no_std Compatible**: Core parser can run in constrained environments -4. **Tooling-Friendly**: Enable validation, code generation, and formatting - -## File Structure - -### File Extension -- `.mlf` - MLF source files - -### Shebang (Optional) -```mlf -#!/usr/bin/env mlf -``` - -The `#` character is reserved for shebangs only and is not used elsewhere in the syntax. - -### File Naming Convention -The file path determines the lexicon NSID. Files should follow the lexicon NSID structure: -- `app.bsky.feed.post.mlf` → Lexicon NSID: `app.bsky.feed.post` -- `sh.tangled.repo.issue.mlf` → Lexicon NSID: `sh.tangled.repo.issue` - -The lexicon NSID is derived solely from the filename, not from any internal namespace declarations. - -## Core Concepts - -### NSIDs (Namespaced Identifiers) - -NSIDs use dotted notation: -``` -app.bsky.feed.post -com.example.thing -sh.tangled.repo.issue -``` - -- Format: `authority.name(.name)*` -- Authority: Typically a reversed domain name -- Segments: Lowercase letters, numbers, hyphens (no underscores) - -### Lexicon Resolution - -References to definitions can be: - -1. **Local (same file)**: Just use the name - ```mlf - record myRecord { - field: myType // References type in same file - } - - def type myType = { /* ... */ } - ``` - -2. **Cross-file (different lexicon)**: Use full dotted path - ```mlf - record myRecord { - profile: app.bsky.actor.profile // References app/bsky/actor/profile.mlf - author: com.example.user.author // References com/example/user/author.mlf - } - ``` - -**Note**: The `#` character is NOT used for references. All references use dotted notation. - -### Syntax Rules - -#### Semicolons - -All definitions require semicolons: -- `record` definitions end with `};` -- `use` statements end with `;` -- `token` definitions end with `;` -- `inline type` definitions end with `;` -- `def type` definitions end with `;` -- `query` definitions end with `;` -- `procedure` definitions end with `;` -- `subscription` definitions end with `;` - -#### Commas - -Commas are **required** between items, with **trailing commas allowed**: - -- **Record fields**: Commas required between fields, trailing comma allowed - ```mlf - record example { - field1: string, - field2: integer, // trailing comma allowed - } - ``` - -- **Constraints**: Commas required between constraint properties, trailing comma allowed - ```mlf - title: string constrained { - maxLength: 200, - minLength: 1, // trailing comma allowed - } - ``` - -- **Error definitions**: Commas required between errors, trailing comma allowed - ```mlf - query getThread(): thread | error { - NotFound, - BadRequest, // trailing comma allowed - } - ``` - -## Type System - -### Primitive Types - -```mlf -null // Null value -boolean // True or false -integer // 64-bit integer -string // UTF-8 string -bytes // Byte array -``` - -**Note:** ATProto Lexicons do not support floating-point numbers. Only `integer` is available for numeric values. - -### Special String Formats - -Defined in `prelude.mlf` and available everywhere: - -```mlf -Did // Decentralized Identifier (did:*) -AtUri // AT-URI (at://...) -AtIdentifier // Either a DID or Handle -Handle // Handle identifier (domain name) -Datetime // ISO 8601 datetime -Uri // Generic URI -Cid // Content Identifier -Nsid // Namespaced Identifier -Tid // Timestamp Identifier -RecordKey // Record key -Language // BCP 47 language code -``` - -### Blob Types - -```mlf -blob // Generic blob -``` - -With constraints: -```mlf -avatar: blob constrained { - accept: ["image/png", "image/jpeg"] - maxSize: 1000000 // bytes -} -``` - -### Unknown Type - -```mlf -unknown // Represents any value, used for forward compatibility -``` - -## Definitions - -### Records - -Records are the primary data structure, stored in repositories: - -```mlf -record post { - text!: string constrained { - maxLength: 300 - maxGraphemes: 300 - } - createdAt!: Datetime - reply: replyRef // Optional field (default) -} -``` - -### Type Definitions - -MLF supports two kinds of type definitions: - -**Inline Types** - Expanded at the point of use, never appear in generated lexicon defs: - -```mlf -inline type AtIdentifier = string constrained { - format "at-identifier" -}; -``` - -**Def Types** - Become named definitions in the lexicon's defs block: - -```mlf -def type ReplyRef = { - root!: AtUri - parent!: AtUri -}; -``` - -Use `inline type` for type aliases that should be expanded inline (like primitive type wrappers). Use `def type` for types that should be referenced by name in the generated lexicon. - -### Tokens - -Tokens are named constants used in enums and unions: - -```mlf -/// Open state -token open; - -/// Closed state -token closed; - -record issue { - state!: string constrained { - knownValues: [ - open // References token defined above - closed - ] - default: "open" - } -} -``` - -Tokens must have doc comments describing their purpose. - -### Queries - -Queries are read-only HTTP endpoints (GET): - -```mlf -/// Get a user profile -query getProfile( - /// The actor's DID or handle - actor!: AtIdentifier - /// Optional viewer context (default) - viewer: Did -): profileView | error { - /// Profile not found - ProfileNotFound - /// Invalid request parameters - BadRequest -}; -``` - -### Procedures - -Procedures are write operations (POST): - -```mlf -/// Create a new post -procedure createPost( - text!: string - createdAt!: Datetime -): { - uri!: AtUri - cid!: Cid -} | error { - /// Text exceeds maximum length - TextTooLong -}; -``` - -### Subscriptions - -Subscriptions are WebSocket-based event streams that emit messages over time. They are used for real-time updates and event notifications. - -```mlf -/// Subscribe to repository events -subscription subscribeRepos( - /// Optional cursor for resuming from a specific point (default) - cursor: integer -): commit | identity | handle | migrate | tombstone | info; -``` - -**Message definitions** for subscriptions are defined as def types or records: - -```mlf -/// Commit message emitted by subscribeRepos -def type commit = { - seq!: integer - rebase!: boolean - tooBig!: boolean - repo!: Did - commit!: Cid - rev!: string - since!: string - blocks!: bytes - ops!: repoOp[] - blobs!: Cid[] - time!: Datetime -}; - -/// Info message -def type info = { - name!: string - message: string // Optional (default) -}; -``` - -**Subscription features:** - -- Parameters: Like queries, subscriptions can have parameters -- Return type: A union of message types that can be emitted -- Each message type must be defined as a def type or record -- Message types can be local or imported from other lexicons -- Subscriptions are long-lived WebSocket connections -- No error block (errors are handled at the WebSocket protocol level) - -**Example: Chat message subscription** - -```mlf -/// Subscribe to chat messages for a stream -subscription subscribeChat( - /// The DID of the streamer - streamer!: Did - /// Optional cursor to resume from (default) - cursor: string -): message | delete | join | leave; - -/// Chat message payload -def type message = { - id!: string - text!: string - author!: Did - createdAt!: Datetime -}; - -/// Delete event payload -def type delete = { - id!: string -}; - -/// Join event payload -def type join = { - user!: Did -}; - -/// Leave event payload -def type leave = { - user!: Did -}; -``` - -### Return Types - -Queries and procedures can return: - -1. **Simple success**: `(): returnType` -2. **Success with errors**: `(): successType | error { ErrorName, ... }` - - Each error must have a doc comment describing it -3. **Unknown/empty**: `(): unknown` - -## Type Modifiers - -### Optional and Required Fields - -Fields are **optional by default**. Use `!:` to mark a field as required: - -```mlf -record example { - optional: string // Optional (default) - required!: string // Required (marked with !) -} -``` - -### Arrays - -```mlf -record example { - tags: string[] - items: string[] constrained { - minLength: 1 - maxLength: 10 - } -} -``` - -### Unions - -Use the pipe operator `|`. Unions are **open by default** (allowing unknown types): - -```mlf -record example { - // Open union (default, can include unknown types) - content: text | image | video - - // Union of tokens (also open by default) - state: open | closed | pending -} -``` - -Closed unions (only allowing listed types) use `| !`: - -```mlf -record example { - // Closed union (marked with !, only these types allowed) - content: text | image | video | ! -} -``` - -### References - -Reference local or external definitions: - -```mlf -// Local reference (same file) -record post { - author: author // References 'def type author' in same file -} - -// Cross-file reference -record post { - profile: app.bsky.actor.profile // References app/bsky/actor/profile.mlf -} -``` - -## Constraints - -Constraints refine types by adding additional restrictions. A key principle is that constraints can only make types **more restrictive**, never less restrictive. This ensures type safety and proper substitutability. - -### Constraint Refinement Rules - -When applying constraints, each constraint must be **at least as restrictive** as any parent constraint: - -```mlf -// Valid: More restrictive constraints -def type shortString = string constrained { - maxLength: 100 -}; - -record post { - // Can further constrain to 50 (more restrictive than 100) - title: shortString constrained { - maxLength: 50 // ✓ Valid: 50 ≤ 100 - } -} - -// Invalid: Less restrictive constraints -record invalid { - // ERROR: Cannot expand to 200 (less restrictive than 100) - content: shortString constrained { - maxLength: 200 // ✗ Invalid: 200 > 100 - } -} -``` - -**Refinement rules by constraint type:** - -- **Numeric bounds**: `minimum` can only increase, `maximum` can only decrease -- **Length bounds**: `minLength`/`minGraphemes` can only increase, `maxLength`/`maxGraphemes` can only decrease -- **Enums**: Can only restrict to a subset of values -- **Known values**: Can add new values (extensible) but cannot remove specified ones -- **Format**: Cannot change once specified -- **Defaults**: Can be specified if not already set - -### String Constraints - -```mlf -field: string constrained { - minLength: 1 // Minimum byte length - maxLength: 1000 // Maximum byte length - minGraphemes: 1 // Minimum grapheme clusters - maxGraphemes: 100 // Maximum grapheme clusters - format: "uri" // Format validation - enum: ["a", "b", "c"] // Allowed values (closed set) - string literals - knownValues: [ // Known values (extensible set) - can be string literals OR token references - value1 // Token reference - "value2" // String literal - ] - default: "defaultValue" // Default value -} -``` - -**Note**: `enum`, `knownValues`, and `default` can accept either: -- **Literals**: `"open"`, `42`, `true` (string, integer, or boolean) -- **References**: `open`, `myType` (references to tokens, records, types, etc.) - -When using references, the identifier will be resolved to its string representation in the generated lexicon. - -### Integer Constraints - -```mlf -field: integer constrained { - minimum: 0 - maximum: 100 - enum: [1, 2, 3] - default: 1 -} -``` - -### Array Constraints - -```mlf -field: string[] constrained { - minLength: 1 - maxLength: 10 -} -``` - -### Blob Constraints - -```mlf -field: blob constrained { - accept: ["image/png", "image/jpeg"] // MIME types - maxSize: 1000000 // Bytes -} -``` - -### Boolean Constraints - -```mlf -field: boolean constrained { - default: false -} -``` - -## Comments - -### Documentation Comments - -Use `///` for documentation (appears in generated docs/code): - -```mlf -/// A user profile record -record profile { - /// The user's display name - displayName?: string -} -``` - -### Regular Comments - -Regular comments (`//`) are ignored when processing and will have no impact on any output. - -## Annotations - -Annotations use the `@` symbol and are metadata markers for external tooling. MLF itself assigns no semantic meaning to annotations - they are purely for tools, linters, code generators, and other processors to interpret. - -### Annotation Syntax - -Three forms of annotations are supported: - -**1. Simple annotation:** -```mlf -@deprecated -record oldRecord { - field: string -} -``` - -**2. Positional arguments:** -```mlf -@since(1, 2, 0) -@doc("https://example.com/docs") -record example { - field: string -} -``` - -Arguments can be: -- Strings: `"value"` -- Numbers: `42`, `3.14` -- Booleans: `true`, `false` - -**3. Named arguments:** -```mlf -@validate(min: 0, max: 100, strict: true) -@codegen(language: "rust", derive: "Debug, Clone") -record example { - field: integer -} -``` - -### Annotation Placement - -Annotations can be placed on: -- Records -- Inline Types -- Def Types -- Tokens -- Queries -- Procedures -- Subscriptions -- Fields within records/types - -```mlf -/// A user profile -@table(name: "profiles", indexes: "did,handle") -record profile { - /// User's DID - @indexed - did!: Did - - /// Display name (optional) - @sensitive(pii: true) - displayName: string -} -``` - -### Common Annotation Examples - -```mlf -// Deprecation -@deprecated -@deprecated(since: "2.0.0", replacement: "newRecord") -record oldRecord { /* ... */ } - -// Code generation hints -@derive("Debug, Clone, Serialize") -@table(name: "users") -record user { /* ... */ } - -// Validation -@validate(custom: "validateEmail") -@range(min: 0, max: 100) -field: integer - -// Documentation -@example("did:plc:abc123") -@see("https://atproto.com/specs/did") -field: Did - -// Versioning -@since(1, 0, 0) -@unstable -record experimentalFeature { /* ... */ } -``` - -**Note:** The interpretation of annotations is entirely up to the tooling consuming the MLF. Different tools may support different annotation sets. - -## Use Statements - -Import definitions from other lexicons: - -```mlf -// Named imports -use app.bsky.actor.{profile, profileView}; -use sh.tangled.repo.issue.{issue, open, closed}; - -// Alias entire namespace -use app.bsky.actor as Actor; - -// Wildcard import -use app.bsky.feed.*; - -// Mixed -use sh.tangled.repo.issue.{issue as IssueRecord, open, closed}; -``` - -After importing, use the short name: - -```mlf -use app.bsky.actor.profile; - -record myThing { - author: profile // Instead of app.bsky.actor.profile -} -``` - -## Lexicon Discovery & Resolution - -### File Discovery - -Tools discover lexicons explicit paths: Single file, list of files, or glob pattern - -```bash -mlf validate app.bsky.feed.post.mlf -mlf validate *.mlf -mlf validate "**/*.mlf" -``` - -### Resolution Order - -When resolving cross-file references: - -1. Current file (local definitions) -2. Explicitly imported lexicons (via `use`) -3. Configured lexicon paths -4. (Future) Remote fetch via ATProto - -### File Path Convention - -The lexicon NSID is determined by the file path. Lexicons can follow a directory structure matching their NSID: - -``` -lexicons/ - app/ - bsky/ - actor/ - profile.mlf → app.bsky.actor.profile - feed/ - post.mlf → app.bsky.feed.post - com/ - example/ - thing.mlf → com.example.thing -``` - -Or use a flat structure with dots in the filename: -``` -lexicons/ - app.bsky.actor.profile.mlf - app.bsky.feed.post.mlf - com.example.thing.mlf -``` - -In both cases, the NSID is derived from the file path, not from internal declarations. - -## CLI Commands - -```bash -# Generation -mlf generate code --input "**/*.mlf" --plugin rust src/* -mlf generate lexicon --input "**/*.mlf" lexicons/* -mlf generate example --input "**/*.mlf" --count 5 examples/* - -# Convert JSON lexicons to MLF -mlf generate mlf --input "lexicons/**/*.json" --output ./mlf/ - -# Validate lexicons -mlf validate -mlf validate "**/*.mlf" - -# Format lexicons -mlf fmt - -# Validate a record against a lexicon -mlf check --input app.bsky.feed.post.mlf ./record.json -``` - -### JSON to MLF Conversion - -The `mlf generate mlf` command converts ATProto JSON lexicons back to MLF format. This is useful for: - -- **Migration**: Converting existing JSON lexicons to MLF -- **Interoperability**: Working with lexicons from external sources -- **Learning**: Seeing how JSON lexicons map to MLF syntax -- **Comparison**: Generating MLF from JSON to compare with hand-written MLF - -The converter automatically: -- Converts format strings (did, datetime, handle) to prelude types (Did, Datetime, Handle) -- Properly formats required (`!`) and optional (default) fields -- Converts `namespace#name` references to `namespace.name` notation -- Generates clean, properly indented MLF with correct syntax - -## Examples - -### Complete Lexicon Example - -```mlf -#!/usr/bin/env mlf - -use app.bsky.actor.profile; - -/// Open issue state -token open; - -/// Closed issue state -token closed; - -/// An issue in a repository -record issue { - /// The repository this issue belongs to - repo!: AtUri - /// Issue title - title!: string constrained { - minGraphemes: 1 - maxGraphemes: 200 - } - /// Issue body (markdown) - body: string constrained { - maxGraphemes: 10000 - } - /// Issue state - state!: string constrained { - knownValues: [ - open - closed - ] - default: "open" - } - /// Creation timestamp - createdAt!: Datetime -} - -/// A comment on an issue -record comment { - /// The issue this comment belongs to - issue!: AtUri - /// Comment body (markdown) - body!: string constrained { - minGraphemes: 1 - maxGraphemes: 10000 - } - /// Creation timestamp - createdAt!: Datetime - /// Optional reply target - replyTo: AtUri -} - -/// Get an issue by URI -query getIssue( - /// Issue AT-URI - uri!: AtUri -): issue | error { - /// Issue not found - NotFound -}; - -/// Create a new issue -procedure createIssue( - repo!: AtUri - title!: string - body: string // Optional (default) -): { - uri!: AtUri - cid!: Cid -} | error { - /// Repository not found - RepoNotFound - /// Title too long - TitleTooLong -}; -``` - -## ATProto Mapping - -### MLF → JSON Lexicon - -MLF compiles to standard ATProto JSON Lexicons: - -**MLF:** -```mlf -record post { - text!: string constrained { - maxLength: 300 - } - createdAt!: Datetime -} -``` - -**JSON:** -```json -{ - "lexicon": 1, - "id": "app.bsky.feed.post", - "defs": { - "main": { - "type": "record", - "key": "tid", - "record": { - "type": "object", - "required": ["text", "createdAt"], - "properties": { - "text": { - "type": "string", - "maxLength": 300 - }, - "createdAt": { - "type": "string", - "format": "datetime" - } - } - } - } - } -} -``` - -### Subscription Mapping - -**MLF:** -```mlf -subscription subscribeRepos( - cursor: integer // Optional (default) -): commit | identity; -``` - -**JSON:** -```json -{ - "lexicon": 1, - "id": "com.atproto.sync.subscribeRepos", - "defs": { - "main": { - "type": "subscription", - "parameters": { - "type": "params", - "properties": { - "cursor": { - "type": "integer" - } - } - }, - "message": { - "schema": { - "type": "union", - "refs": ["#commit", "#identity"] - } - } - }, - "commit": { - "type": "object", - "required": ["seq", "repo", "commit"], - "properties": { - "seq": { "type": "integer" }, - "repo": { "type": "string", "format": "did" }, - "commit": { "type": "string", "format": "cid" } - } - } - } -} -``` - -## Future Considerations - -### Potential Extensions - -- **Version constraints**: Specify compatible lexicon versions in lexicon headers -- **Custom validation**: Pluggable validators beyond built-in constraints -- **Documentation generation**: Automatic API docs from MLF with annotation support -- **Standard annotation registry**: Common annotations like `@deprecated`, `@since`, `@internal` -- **Import resolution**: Remote lexicon fetching and caching -- **Type inference**: Automatic type inference for constrained types - -### Versioning - -Lexicons are versioned at the NSID level. MLF files should include version metadata in comments or future version declarations. - -## Appendix - -### Reserved Keywords - -``` -as, blob, boolean, bytes, constrained, def, error, inline, integer, -null, procedure, query, record, string, subscription, token, -type, unknown, use -``` - -### Reserved Names - -The following names cannot be used as item names: - -``` -main, defs -``` - -### Raw Identifiers - -To use a reserved keyword as an identifier, wrap it in backticks: - -```mlf -def type `record` = { - `record`: com.atproto.repo.strongRef - `error`: string -}; -``` - -This allows field names or type names to match reserved keywords when necessary for compatibility with existing schemas. - -### Constraint Keywords - -``` -accept, default, enum, format, knownValues, maxGraphemes, -maxLength, maxSize, maximum, minGraphemes, minLength, minimum -``` - -### Format Values - -``` -at-identifier, at-uri, cid, datetime, did, handle, language, -nsid, record-key, tid, uri -``` diff --git a/mlf-cli/src/check.rs b/mlf-cli/src/check.rs index 85f3a59..2ad7cce 100644 --- a/mlf-cli/src/check.rs +++ b/mlf-cli/src/check.rs @@ -37,20 +37,6 @@ pub enum CheckError { help: Option, }, - #[error("Failed to expand glob pattern")] - #[diagnostic(code(mlf::check::glob_error))] - GlobError { - #[source] - source: glob::GlobError, - }, - - #[error("Invalid glob pattern: {pattern}")] - #[diagnostic(code(mlf::check::invalid_glob))] - InvalidGlob { - pattern: String, - #[source] - source: glob::PatternError, - }, #[error("Record validation failed")] #[diagnostic(code(mlf::check::record_validation))] @@ -63,22 +49,27 @@ pub enum CheckError { ConfigError(#[from] ConfigError), } -pub fn run_check(input_patterns: Vec) -> Result<(), CheckError> { - // If no input patterns provided, use source directory from mlf.toml - let patterns = if input_patterns.is_empty() { - let current_dir = std::env::current_dir() - .map_err(|e| CheckError::ReadFile { - path: ".".to_string(), - source: e, - })?; +pub fn run_check(input_paths: Vec, explicit_root: Option) -> Result<(), CheckError> { + let current_dir = std::env::current_dir() + .map_err(|e| CheckError::ReadFile { + path: ".".to_string(), + source: e, + })?; + // Determine root directory and input paths + let (root_dir, file_paths) = if input_paths.is_empty() { + // No input provided: must use mlf.toml match find_project_root(¤t_dir) { Ok(project_root) => { let config_path = project_root.join("mlf.toml"); let config = MlfConfig::load(&config_path)?; - let source_pattern = format!("{}/**/*.mlf", config.source.directory); + let source_dir = project_root.join(&config.source.directory); + let root = explicit_root.unwrap_or_else(|| source_dir.clone()); println!("Using source directory from mlf.toml: {}", config.source.directory); - vec![source_pattern] + + // Collect all .mlf files from source directory + let files = collect_mlf_files(&source_dir)?; + (root, files) } Err(ConfigError::NotFound) => { return Err(CheckError::ValidationErrors { @@ -88,24 +79,45 @@ pub fn run_check(input_patterns: Vec) -> Result<(), CheckError> { Err(e) => return Err(CheckError::ConfigError(e)), } } else { - input_patterns - }; + // Input provided: determine root + let root = if let Some(explicit) = explicit_root { + // --root flag takes precedence + explicit + } else if let Ok(project_root) = find_project_root(¤t_dir) { + // Try to use mlf.toml source directory + let config_path = project_root.join("mlf.toml"); + if let Ok(config) = MlfConfig::load(&config_path) { + project_root.join(&config.source.directory) + } else { + current_dir.clone() + } + } else { + // Fall back to current directory + current_dir.clone() + }; - let mut file_paths = Vec::new(); + // Collect files from input paths + let mut files = Vec::new(); + for input_path in input_paths { + let path = if input_path.is_absolute() { + input_path + } else { + current_dir.join(input_path) + }; - for pattern in patterns { - if pattern.contains('*') || pattern.contains('?') { - for entry in glob::glob(&pattern).map_err(|source| CheckError::InvalidGlob { - pattern: pattern.clone(), - source, - })? { - let path = entry.map_err(|source| CheckError::GlobError { source })?; - file_paths.push(path); + if path.is_dir() { + files.extend(collect_mlf_files(&path)?); + } else if path.is_file() { + files.push(path); + } else { + return Err(CheckError::ReadFile { + path: path.display().to_string(), + source: std::io::Error::new(std::io::ErrorKind::NotFound, "Path not found"), + }); } - } else { - file_paths.push(PathBuf::from(pattern)); } - } + (root, files) + }; // Try to load cached lexicons from .mlf directory let current_dir = std::env::current_dir() @@ -148,20 +160,16 @@ pub fn run_check(input_patterns: Vec) -> Result<(), CheckError> { } }; - let namespace = file_path - .file_stem() - .and_then(|s| s.to_str()) - .unwrap_or("unknown") - .to_string(); + let namespace = extract_namespace(&file_path, &root_dir)?; if let Err(e) = workspace.add_module(namespace.clone(), lexicon.clone()) { - let diagnostic = ValidationDiagnostic::new(filename.clone(), source.clone(), e); + let diagnostic = ValidationDiagnostic::new(filename.clone(), source.clone(), namespace.clone(), e); eprintln!("{:?}", miette::Report::new(diagnostic)); had_parse_errors = true; continue; } - source_files.push((filename.clone(), source)); + source_files.push((filename.clone(), namespace.clone(), source)); println!("✓ {}: Parsed successfully", file_path.display()); } @@ -172,11 +180,79 @@ pub fn run_check(input_patterns: Vec) -> Result<(), CheckError> { } if let Err(e) = workspace.resolve() { - // Show all errors from the first source file - if let Some((filename, source)) = source_files.first() { - let diagnostic = ValidationDiagnostic::new(filename.clone(), source.clone(), e); - eprintln!("{:?}", miette::Report::new(diagnostic)); + // Collect all modules that have errors + let mut modules_with_errors: std::collections::BTreeMap, String)> = std::collections::BTreeMap::new(); + + // First, add all explicitly checked files + for (filename, namespace, source) in &source_files { + modules_with_errors.insert(namespace.clone(), (Some(filename.clone()), source.clone())); + } + + // Then, find any cached modules with errors and try to load their source + for error in &e.errors { + let error_namespace = mlf_diagnostics::get_error_module_namespace_str(error); + if !modules_with_errors.contains_key(error_namespace) { + let namespace_path = error_namespace.replace('.', "/"); + let mut source_loaded = false; + + // Try multiple locations for the source file + let mut possible_paths = vec![ + // Check in lexicons/ directory (common structure) + current_dir.join("lexicons").join(format!("{}.mlf", namespace_path)), + // Check in source directory from config + current_dir.join("src").join(format!("{}.mlf", namespace_path)), + // Check relative to current directory + current_dir.join(format!("{}.mlf", namespace_path)), + ]; + + // Add cache directory if available (lexicons are in lexicons/mlf/ subdirectory) + if let Some(cache_dir) = &mlf_cache_dir { + possible_paths.push(cache_dir.join("lexicons").join("mlf").join(format!("{}.mlf", namespace_path))); + } + + for path in possible_paths { + if let Ok(source) = std::fs::read_to_string(&path) { + modules_with_errors.insert( + error_namespace.to_string(), + (Some(path.display().to_string()), source) + ); + source_loaded = true; + break; + } + } + + if !source_loaded { + // Couldn't load source, add placeholder + modules_with_errors.insert( + error_namespace.to_string(), + (None, String::new()) + ); + } + } + } + + // Show diagnostics for all modules with errors + for (namespace, (filename_opt, source)) in &modules_with_errors { + // Only show diagnostic if this module has errors + let has_errors = e.errors.iter().any(|error| { + mlf_diagnostics::get_error_module_namespace_str(error) == namespace + }); + + if has_errors { + if let Some(filename) = filename_opt { + // Have source file, show full diagnostic + let diagnostic = ValidationDiagnostic::new(filename.clone(), source.clone(), namespace.clone(), e.clone()); + eprintln!("{:?}", miette::Report::new(diagnostic)); + } else { + // No source available, just list the errors + let error_count = e.errors.iter() + .filter(|err| mlf_diagnostics::get_error_module_namespace_str(err) == namespace) + .count(); + eprintln!("\n{}: {} error(s) (source not available)", namespace, error_count); + } + } } + return Err(CheckError::ValidationErrors { help: Some("Workspace validation failed".to_string()), }); @@ -235,3 +311,87 @@ pub fn validate(lexicon_path: PathBuf, record_path: PathBuf) -> Result<(), Check } } } + +/// Recursively collect all .mlf files from a directory +fn collect_mlf_files(dir: &std::path::Path) -> Result, CheckError> { + let mut files = Vec::new(); + + if !dir.exists() { + return Err(CheckError::ReadFile { + path: dir.display().to_string(), + source: std::io::Error::new(std::io::ErrorKind::NotFound, "Directory not found"), + }); + } + + fn visit_dirs(dir: &std::path::Path, files: &mut Vec) -> std::io::Result<()> { + if dir.is_dir() { + for entry in std::fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + if path.is_dir() { + visit_dirs(&path, files)?; + } else if path.extension().and_then(|s| s.to_str()) == Some("mlf") { + files.push(path); + } + } + } + Ok(()) + } + + visit_dirs(dir, &mut files).map_err(|source| CheckError::ReadFile { + path: dir.display().to_string(), + source, + })?; + + Ok(files) +} + +/// Extract namespace from file path relative to root directory +/// e.g., root=/project/lexicons, file=/project/lexicons/com/example/foo.mlf -> com.example.foo +fn extract_namespace(file_path: &std::path::Path, root_dir: &std::path::Path) -> Result { + // Get the canonical paths to handle . and .. correctly + let file_canonical = file_path.canonicalize().map_err(|source| CheckError::ReadFile { + path: file_path.display().to_string(), + source, + })?; + + let root_canonical = root_dir.canonicalize().map_err(|source| CheckError::ReadFile { + path: root_dir.display().to_string(), + source, + })?; + + // Get relative path from root to file + let relative_path = file_canonical.strip_prefix(&root_canonical) + .map_err(|_| CheckError::ValidationErrors { + help: Some(format!( + "File {} is not within root directory {}", + file_path.display(), + root_dir.display() + )), + })?; + + // Convert path to namespace + let mut components = Vec::new(); + for component in relative_path.components() { + if let std::path::Component::Normal(os_str) = component { + if let Some(s) = os_str.to_str() { + components.push(s); + } + } + } + + // Remove .mlf extension from last component + if let Some(last) = components.last_mut() { + if let Some(stem) = last.strip_suffix(".mlf") { + *last = stem; + } + } + + if components.is_empty() { + return Err(CheckError::ValidationErrors { + help: Some(format!("Could not extract namespace from path: {}", file_path.display())), + }); + } + + Ok(components.join(".")) +} diff --git a/mlf-cli/src/generate/code.rs b/mlf-cli/src/generate/code.rs index 90dc514..32cffbb 100644 --- a/mlf-cli/src/generate/code.rs +++ b/mlf-cli/src/generate/code.rs @@ -21,21 +21,6 @@ pub enum GenerateError { source: std::io::Error, }, - #[error("Failed to expand glob pattern")] - #[diagnostic(code(mlf::generate::glob_error))] - GlobError { - #[source] - source: glob::GlobError, - }, - - #[error("Invalid glob pattern: {pattern}")] - #[diagnostic(code(mlf::generate::invalid_glob))] - InvalidGlob { - pattern: String, - #[source] - source: glob::PatternError, - }, - #[error("Generator '{name}' not found")] #[diagnostic(code(mlf::generate::generator_not_found))] #[help("Available generators: {}", available.join(", "))] @@ -51,11 +36,46 @@ pub enum GenerateError { } pub fn run( - generator_name: String, - input_patterns: Vec, - output_dir: PathBuf, + generator_name: Option, + input_paths: Vec, + output_dir: Option, + root: Option, flat: bool, ) -> Result<(), GenerateError> { + let current_dir = std::env::current_dir().map_err(|source| GenerateError::WriteOutput { + path: "current directory".to_string(), + source, + })?; + + // Load mlf.toml if available + let project_root = crate::config::find_project_root(¤t_dir).ok(); + let config = project_root + .as_ref() + .and_then(|root| { + let config_path = root.join("mlf.toml"); + crate::config::MlfConfig::load(&config_path).ok() + }); + + // Determine generator name + let generator_name = if let Some(explicit) = generator_name { + explicit + } else if let Some(cfg) = &config { + // Find first non-lexicon, non-mlf output in mlf.toml + cfg.output + .iter() + .find(|o| o.r#type != "lexicon" && o.r#type != "mlf") + .map(|o| o.r#type.clone()) + .ok_or_else(|| GenerateError::GeneratorNotFound { + name: "any".to_string(), + available: vec!["No code generator outputs configured in mlf.toml. Either add an output configuration or provide --generator flag.".to_string()], + })? + } else { + return Err(GenerateError::GeneratorNotFound { + name: "any".to_string(), + available: vec!["No mlf.toml found and no --generator flag provided. Either create a mlf.toml or provide --generator flag.".to_string()], + }); + }; + // Find the generator let generators = mlf_codegen::plugin::generators(); let generator = generators @@ -72,19 +92,65 @@ pub fn run( println!("Using generator: {} ({})", generator.name(), generator.description()); println!("Output extension: {}\n", generator.file_extension()); + // Determine output directory + let output_dir = if let Some(explicit) = output_dir { + explicit + } else if let Some(cfg) = &config { + // Find output matching the generator type + cfg.output + .iter() + .find(|o| o.r#type == generator_name) + .map(|o| PathBuf::from(&o.directory)) + .ok_or_else(|| GenerateError::WriteOutput { + path: "mlf.toml".to_string(), + source: std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("No output configured for generator '{}' in mlf.toml", generator_name) + ), + })? + } else { + return Err(GenerateError::WriteOutput { + path: "mlf.toml".to_string(), + source: std::io::Error::new( + std::io::ErrorKind::NotFound, + "No mlf.toml found and no --output flag provided" + ), + }); + }; + + // Determine root directory + let root_dir = if let Some(explicit) = root { + explicit + } else if let Some(cfg) = &config { + project_root.as_ref().unwrap().join(&cfg.source.directory) + } else { + current_dir.clone() + }; + + // Determine input paths + let input_paths = if input_paths.is_empty() { + if let Some(cfg) = &config { + vec![project_root.as_ref().unwrap().join(&cfg.source.directory)] + } else { + return Err(GenerateError::WriteOutput { + path: "input".to_string(), + source: std::io::Error::new( + std::io::ErrorKind::NotFound, + "No input files specified and no mlf.toml found" + ), + }); + } + } else { + input_paths + }; + // Collect input files let mut file_paths = Vec::new(); - for pattern in input_patterns { - if pattern.contains('*') || pattern.contains('?') { - for entry in glob::glob(&pattern).map_err(|source| GenerateError::InvalidGlob { - pattern: pattern.clone(), - source, - })? { - let path = entry.map_err(|source| GenerateError::GlobError { source })?; - file_paths.push(path); - } - } else { - file_paths.push(PathBuf::from(pattern)); + for path in input_paths { + if path.is_dir() { + file_paths.extend(collect_mlf_files(&path)?); + } else if path.is_file() && path.extension().and_then(|s| s.to_str()) == Some("mlf") { + file_paths.push(path); } } @@ -116,7 +182,16 @@ pub fn run( } }; - let namespace = extract_namespace(&file_path); + let namespace = match extract_namespace(&file_path, &root_dir) { + Ok(ns) => ns, + Err(e) => { + errors.push(( + file_path.display().to_string(), + format!("Failed to extract namespace: {}", e), + )); + continue; + } + }; // Create workspace with standard library and .mlf cache let mlf_cache_dir = crate::config::find_project_root(&std::env::current_dir().unwrap()) @@ -220,33 +295,77 @@ pub fn run( Ok(()) } -fn extract_namespace(file_path: &Path) -> String { - // Extract namespace from path components - // e.g., com/atproto/admin/defs.mlf -> com.atproto.admin.defs +/// Collect all .mlf files recursively from a directory +fn collect_mlf_files(dir: &Path) -> Result, GenerateError> { + let mut files = Vec::new(); + + for entry in std::fs::read_dir(dir).map_err(|source| GenerateError::WriteOutput { + path: dir.display().to_string(), + source, + })? { + let entry = entry.map_err(|source| GenerateError::WriteOutput { + path: dir.display().to_string(), + source, + })?; + + let path = entry.path(); + + if path.is_dir() { + files.extend(collect_mlf_files(&path)?); + } else if path.extension().and_then(|s| s.to_str()) == Some("mlf") { + files.push(path); + } + } + + Ok(files) +} + +fn extract_namespace(file_path: &Path, root_dir: &Path) -> Result { + // Canonicalize both paths for comparison + let file_canonical = file_path.canonicalize()?; + let root_canonical = root_dir.canonicalize()?; + + // Get the relative path from root to file + let relative_path = file_canonical + .strip_prefix(&root_canonical) + .map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::Other, + format!( + "File path {} is not under root directory {}", + file_path.display(), + root_dir.display() + ), + ) + })?; - let mut components = Vec::new(); + // Convert path components to namespace parts + let mut namespace_parts = Vec::new(); - for component in file_path.components() { + for component in relative_path.components() { match component { std::path::Component::Normal(os_str) => { if let Some(s) = os_str.to_str() { - components.push(s); + namespace_parts.push(s); } } - _ => continue, // Skip ., .., /, etc. + _ => continue, } } - // Remove the .mlf extension from the last component if present - if let Some(last) = components.last_mut() { + // Remove .mlf extension from the last component if present + if let Some(last) = namespace_parts.last_mut() { if let Some(stem) = last.strip_suffix(".mlf") { *last = stem; } } - if components.is_empty() { - return "unknown".to_string(); + if namespace_parts.is_empty() { + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + format!("Could not extract namespace from path: {}", file_path.display()), + )); } - components.join(".") + Ok(namespace_parts.join(".")) } diff --git a/mlf-cli/src/generate/lexicon.rs b/mlf-cli/src/generate/lexicon.rs index 4ce503d..90d21f7 100644 --- a/mlf-cli/src/generate/lexicon.rs +++ b/mlf-cli/src/generate/lexicon.rs @@ -29,36 +29,85 @@ pub enum GenerateError { source: std::io::Error, }, - #[error("Failed to expand glob pattern")] - #[diagnostic(code(mlf::generate::glob_error))] - GlobError { - #[source] - source: glob::GlobError, - }, - - #[error("Invalid glob pattern: {pattern}")] - #[diagnostic(code(mlf::generate::invalid_glob))] - InvalidGlob { - pattern: String, - #[source] - source: glob::PatternError, - }, } -pub fn run(input_patterns: Vec, output_dir: PathBuf, flat: bool) -> Result<(), GenerateError> { +pub fn run(input_paths: Vec, output_dir: Option, explicit_root: Option, flat: bool) -> Result<(), GenerateError> { + let current_dir = std::env::current_dir() + .map_err(|e| GenerateError::WriteOutput { + path: ".".to_string(), + source: e, + })?; + + // Load mlf.toml if available + let project_root = crate::config::find_project_root(¤t_dir).ok(); + let config = project_root + .as_ref() + .and_then(|root| { + let config_path = root.join("mlf.toml"); + crate::config::MlfConfig::load(&config_path).ok() + }); + + // Determine output directory + let output_dir = if let Some(explicit) = output_dir { + explicit + } else if let Some(cfg) = &config { + // Find first lexicon output in mlf.toml + cfg.output + .iter() + .find(|o| o.r#type == "lexicon") + .map(|o| PathBuf::from(&o.directory)) + .ok_or_else(|| GenerateError::ParseLexicon { + path: "mlf.toml".to_string(), + help: Some("No lexicon output configured in mlf.toml. Either add an output configuration or provide --output flag.".to_string()), + })? + } else { + return Err(GenerateError::ParseLexicon { + path: "mlf.toml".to_string(), + help: Some("No mlf.toml found and no --output flag provided. Either create a mlf.toml or provide --output flag.".to_string()), + }); + }; + + // Determine root directory + let root_dir = if let Some(explicit) = explicit_root { + explicit + } else if let Some(cfg) = &config { + project_root.as_ref().unwrap().join(&cfg.source.directory) + } else { + current_dir.clone() + }; + + // Determine input paths + let input_paths = if input_paths.is_empty() { + if let Some(cfg) = &config { + vec![project_root.as_ref().unwrap().join(&cfg.source.directory)] + } else { + return Err(GenerateError::ParseLexicon { + path: "input".to_string(), + help: Some("No input files specified and no mlf.toml found. Either provide input files or create a mlf.toml.".to_string()), + }); + } + } else { + input_paths + }; + + // Collect files from input paths let mut file_paths = Vec::new(); + for input_path in input_paths { + let path = if input_path.is_absolute() { + input_path + } else { + current_dir.join(input_path) + }; - for pattern in input_patterns { - if pattern.contains('*') || pattern.contains('?') { - for entry in glob::glob(&pattern).map_err(|source| GenerateError::InvalidGlob { - pattern: pattern.clone(), - source, - })? { - let path = entry.map_err(|source| GenerateError::GlobError { source })?; - file_paths.push(path); - } + if path.is_dir() { + file_paths.extend(collect_mlf_files(&path)?); + } else if path.is_file() { + file_paths.push(path); } else { - file_paths.push(PathBuf::from(pattern)); + return Err(GenerateError::ReadFile { + path: path.display().to_string(), + source: std::io::Error::new(std::io::ErrorKind::NotFound, "Path not found"), + }); } } @@ -87,7 +136,7 @@ pub fn run(input_patterns: Vec, output_dir: PathBuf, flat: bool) -> Resu } }; - let namespace = extract_namespace(&file_path); + let namespace = extract_namespace(&file_path, &root_dir)?; // Create workspace with standard library and .mlf cache for inline type resolution let mlf_cache_dir = crate::config::find_project_root(&std::env::current_dir().unwrap()) @@ -157,24 +206,76 @@ pub fn run(input_patterns: Vec, output_dir: PathBuf, flat: bool) -> Resu Ok(()) } -fn extract_namespace(file_path: &Path) -> String { - // Extract namespace from path components - // e.g., com/atproto/admin/defs.mlf -> com.atproto.admin.defs +/// Recursively collect all .mlf files from a directory +fn collect_mlf_files(dir: &Path) -> Result, GenerateError> { + let mut files = Vec::new(); - let mut components = Vec::new(); + if !dir.exists() { + return Err(GenerateError::ReadFile { + path: dir.display().to_string(), + source: std::io::Error::new(std::io::ErrorKind::NotFound, "Directory not found"), + }); + } - for component in file_path.components() { - match component { - std::path::Component::Normal(os_str) => { - if let Some(s) = os_str.to_str() { - components.push(s); + fn visit_dirs(dir: &Path, files: &mut Vec) -> std::io::Result<()> { + if dir.is_dir() { + for entry in std::fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + if path.is_dir() { + visit_dirs(&path, files)?; + } else if path.extension().and_then(|s| s.to_str()) == Some("mlf") { + files.push(path); } } - _ => continue, // Skip ., .., /, etc. + } + Ok(()) + } + + visit_dirs(dir, &mut files).map_err(|source| GenerateError::ReadFile { + path: dir.display().to_string(), + source, + })?; + + Ok(files) +} + +/// Extract namespace from file path relative to root directory +/// e.g., root=/project/lexicons, file=/project/lexicons/com/example/foo.mlf -> com.example.foo +fn extract_namespace(file_path: &Path, root_dir: &Path) -> Result { + // Get the canonical paths to handle . and .. correctly + let file_canonical = file_path.canonicalize().map_err(|source| GenerateError::ReadFile { + path: file_path.display().to_string(), + source, + })?; + + let root_canonical = root_dir.canonicalize().map_err(|source| GenerateError::ReadFile { + path: root_dir.display().to_string(), + source, + })?; + + // Get relative path from root to file + let relative_path = file_canonical.strip_prefix(&root_canonical) + .map_err(|_| GenerateError::ParseLexicon { + path: file_path.display().to_string(), + help: Some(format!( + "File {} is not within root directory {}", + file_path.display(), + root_dir.display() + )), + })?; + + // Convert path to namespace + let mut components = Vec::new(); + for component in relative_path.components() { + if let std::path::Component::Normal(os_str) = component { + if let Some(s) = os_str.to_str() { + components.push(s); + } } } - // Remove the .mlf extension from the last component if present + // Remove .mlf extension from last component if let Some(last) = components.last_mut() { if let Some(stem) = last.strip_suffix(".mlf") { *last = stem; @@ -182,8 +283,11 @@ fn extract_namespace(file_path: &Path) -> String { } if components.is_empty() { - return "unknown".to_string(); + return Err(GenerateError::ParseLexicon { + path: file_path.display().to_string(), + help: Some("Could not extract namespace from path".to_string()), + }); } - components.join(".") + Ok(components.join(".")) } diff --git a/mlf-cli/src/generate/mlf.rs b/mlf-cli/src/generate/mlf.rs index 774b050..17f4715 100644 --- a/mlf-cli/src/generate/mlf.rs +++ b/mlf-cli/src/generate/mlf.rs @@ -51,7 +51,39 @@ pub enum MlfGenerateError { }, } -pub fn run(input_patterns: Vec, output_dir: PathBuf) -> Result<(), MlfGenerateError> { +pub fn run(input_patterns: Vec, output_dir: Option) -> Result<(), MlfGenerateError> { + let current_dir = std::env::current_dir().map_err(|source| MlfGenerateError::WriteOutput { + path: "current directory".to_string(), + source, + })?; + + // Load mlf.toml if available + let project_root = crate::config::find_project_root(¤t_dir).ok(); + let config = project_root + .as_ref() + .and_then(|root| { + let config_path = root.join("mlf.toml"); + crate::config::MlfConfig::load(&config_path).ok() + }); + + // Determine output directory + let output_dir = if let Some(explicit) = output_dir { + explicit + } else if let Some(cfg) = &config { + // Find first mlf output in mlf.toml + cfg.output + .iter() + .find(|o| o.r#type == "mlf") + .map(|o| PathBuf::from(&o.directory)) + .ok_or_else(|| MlfGenerateError::InvalidLexicon { + message: "No mlf output configured in mlf.toml. Either add an output configuration or provide --output flag.".to_string(), + })? + } else { + return Err(MlfGenerateError::InvalidLexicon { + message: "No mlf.toml found and no --output flag provided. Either create a mlf.toml or provide --output flag.".to_string(), + }); + }; + let mut file_paths = Vec::new(); for pattern in input_patterns { @@ -179,6 +211,11 @@ pub fn generate_mlf_from_json(json: &Value) -> Result } })?; + // Create a context to pass the current namespace to type generation + let ctx = ConversionContext { + current_namespace: nsid.to_string(), + }; + // Process all definitions for (name, def) in defs { let def_type = def.get("type").and_then(|v| v.as_str()).ok_or_else(|| { @@ -189,22 +226,22 @@ pub fn generate_mlf_from_json(json: &Value) -> Result match def_type { "record" => { - let mlf = generate_record(name, def, last_segment)?; + let mlf = generate_record(name, def, last_segment, &ctx)?; output.push_str(&mlf); output.push('\n'); } "query" => { - let mlf = generate_query(name, def, last_segment)?; + let mlf = generate_query(name, def, last_segment, &ctx)?; output.push_str(&mlf); output.push('\n'); } "procedure" => { - let mlf = generate_procedure(name, def, last_segment)?; + let mlf = generate_procedure(name, def, last_segment, &ctx)?; output.push_str(&mlf); output.push('\n'); } "subscription" => { - let mlf = generate_subscription(name, def, last_segment)?; + let mlf = generate_subscription(name, def, last_segment, &ctx)?; output.push_str(&mlf); output.push('\n'); } @@ -213,20 +250,22 @@ pub fn generate_mlf_from_json(json: &Value) -> Result output.push_str(&mlf); output.push('\n'); } - "object" => { - let mlf = generate_def_type(name, def, last_segment)?; + _ => { + // All other types (object, string, array, union, etc.) are treated as def type + let mlf = generate_def_type(name, def, last_segment, &ctx)?; output.push_str(&mlf); output.push('\n'); } - _ => { - // Unknown type, skip - } } } Ok(output) } +struct ConversionContext { + current_namespace: String, +} + /// Reserved words in MLF that need to be escaped const RESERVED_WORDS: &[&str] = &[ "main", "record", "query", "procedure", "subscription", "token", "def", "type", "use", @@ -243,7 +282,7 @@ fn escape_name(name: &str) -> String { } } -fn generate_record(name: &str, def: &Value, last_segment: &str) -> Result { +fn generate_record(name: &str, def: &Value, last_segment: &str, ctx: &ConversionContext) -> Result { let mut output = String::new(); // Add doc comment if present @@ -255,6 +294,11 @@ fn generate_record(name: &str, def: &Value, last_segment: &str) -> Result Result Result Result { +fn generate_query(name: &str, def: &Value, last_segment: &str, ctx: &ConversionContext) -> Result { let mut output = String::new(); // Add doc comment @@ -325,6 +369,11 @@ fn generate_query(name: &str, def: &Value, last_segment: &str) -> Result Result Result Result Result { +fn generate_procedure(name: &str, def: &Value, last_segment: &str, ctx: &ConversionContext) -> Result { let mut output = String::new(); // Add doc comment @@ -412,6 +461,11 @@ fn generate_procedure(name: &str, def: &Value, last_segment: &str) -> Result Result Result Result Result { +fn generate_subscription(name: &str, def: &Value, last_segment: &str, ctx: &ConversionContext) -> Result { let mut output = String::new(); // Add doc comment @@ -505,6 +559,11 @@ fn generate_subscription(name: &str, def: &Value, last_segment: &str) -> Result< } } + // Add @main annotation for "main" definitions + if name == "main" { + output.push_str("@main\n"); + } + let subscription_name = if name == "main" { escape_name(last_segment) } else { @@ -532,7 +591,7 @@ fn generate_subscription(name: &str, def: &Value, last_segment: &str) -> Result< .map(|(param_name, param_def)| { let is_required = required.contains(¶m_name.as_str()); let required_marker = if is_required { "!" } else { "" }; - let param_type = generate_type(param_def).unwrap_or_else(|_| "unknown".to_string()); + let param_type = generate_type(param_def, ctx).unwrap_or_else(|_| "unknown".to_string()); let escaped_param_name = escape_name(param_name); format!("{}{}: {}", escaped_param_name, required_marker, param_type) @@ -549,7 +608,7 @@ fn generate_subscription(name: &str, def: &Value, last_segment: &str) -> Result< // Message types if let Some(message) = def.get("message").and_then(|v| v.as_object()) { if let Some(schema) = message.get("schema") { - let message_type = generate_type(schema)?; + let message_type = generate_type(schema, ctx)?; output.push_str(&format!(": {}", message_type)); } } @@ -575,9 +634,23 @@ fn generate_token(name: &str, def: &Value) -> Result { Ok(output) } -fn generate_def_type(name: &str, def: &Value, last_segment: &str) -> Result { +fn generate_def_type(name: &str, def: &Value, last_segment: &str, ctx: &ConversionContext) -> Result { let mut output = String::new(); + // Add doc comment if present + if let Some(desc) = def.get("description").and_then(|v| v.as_str()) { + if !desc.is_empty() { + for line in desc.lines() { + output.push_str(&format!("/// {}\n", line)); + } + } + } + + // Add @main annotation for "main" definitions + if name == "main" { + output.push_str("@main\n"); + } + // Use last segment of NSID for "main" definitions let def_name = if name == "main" { escape_name(last_segment) @@ -586,14 +659,14 @@ fn generate_def_type(name: &str, def: &Value, last_segment: &str) -> Result Result { +fn generate_type_with_indent(type_def: &Value, indent_level: usize, ctx: &ConversionContext) -> Result { let type_name = type_def.get("type").and_then(|v| v.as_str()); match type_name { @@ -631,7 +704,7 @@ fn generate_type_with_indent(type_def: &Value, indent_level: usize) -> Result Result generate_type(type_def), + _ => generate_type(type_def, ctx), } } -fn generate_type(type_def: &Value) -> Result { +fn generate_type(type_def: &Value, ctx: &ConversionContext) -> Result { let type_name = type_def.get("type").and_then(|v| v.as_str()); match type_name { @@ -735,7 +808,7 @@ fn generate_type(type_def: &Value) -> Result { .unwrap_or("unknown") .to_string() } else { - generate_type(items)? + generate_type(items, ctx)? }; let mut result = format!("{}[]", item_type); @@ -773,7 +846,7 @@ fn generate_type(type_def: &Value) -> Result { let is_required = required.contains(&field_name.as_str()); let required_marker = if is_required { "!" } else { "" }; - let field_type = generate_type(field_def)?; + let field_type = generate_type(field_def, ctx)?; let escaped_field_name = escape_name(field_name); output.push_str(&format!( " {}{}: {},\n", @@ -793,7 +866,7 @@ fn generate_type(type_def: &Value) -> Result { let type_strs: Vec = refs .iter() - .map(|r| generate_type(r).unwrap_or_else(|_| "unknown".to_string())) + .map(|r| generate_type(r, ctx).unwrap_or_else(|_| "unknown".to_string())) .collect(); let mut result = type_strs.join(" | "); @@ -807,11 +880,26 @@ fn generate_type(type_def: &Value) -> Result { } Some("ref") => { if let Some(ref_str) = type_def.get("ref").and_then(|v| v.as_str()) { - // Convert refs: strip leading # and convert remaining # to . - // "#audio" -> "audio" (local ref, just the name) - // "com.example#foo" -> "com.example.foo" (external ref) - let clean_ref = ref_str.trim_start_matches('#').replace('#', "."); - Ok(clean_ref) + // Handle references: + // "#defName" -> "defName" (local reference, same file) + // "namespace.id#defName" -> Check if same namespace, if so use "defName", else use full path + + if let Some(stripped) = ref_str.strip_prefix('#') { + // Local reference: #defName -> defName + Ok(stripped.to_string()) + } else if let Some((namespace, def_name)) = ref_str.split_once('#') { + // Check if this references the current namespace + if namespace == ctx.current_namespace { + // Same namespace - use just the def name + Ok(def_name.to_string()) + } else { + // Different namespace - use full NSID format + Ok(format!("{}.{}", namespace, def_name)) + } + } else { + // No # at all - shouldn't happen in valid lexicons, but handle gracefully + Ok(ref_str.to_string()) + } } else { Err(MlfGenerateError::InvalidLexicon { message: "Missing 'ref' in ref type".to_string(), diff --git a/mlf-cli/src/generate/mlf.rs.backup b/mlf-cli/src/generate/mlf.rs.backup new file mode 100644 index 0000000..a58612c --- /dev/null +++ b/mlf-cli/src/generate/mlf.rs.backup @@ -0,0 +1,941 @@ +use miette::Diagnostic; +use serde_json::Value; +use std::path::PathBuf; +use thiserror::Error; + +#[derive(Error, Debug, Diagnostic)] +pub enum MlfGenerateError { + #[error("Failed to read file: {path}")] + #[diagnostic(code(mlf::generate::read_file))] + #[allow(dead_code)] + ReadFile { + path: String, + #[source] + source: std::io::Error, + }, + + #[error("Failed to parse JSON: {path}")] + #[diagnostic(code(mlf::generate::parse_json))] + #[allow(dead_code)] + ParseJson { + path: String, + #[source] + source: serde_json::Error, + }, + + #[error("Failed to write output: {path}")] + #[diagnostic(code(mlf::generate::write_output))] + WriteOutput { + path: String, + #[source] + source: std::io::Error, + }, + + #[error("Invalid lexicon format: {message}")] + #[diagnostic(code(mlf::generate::invalid_lexicon))] + InvalidLexicon { message: String }, + + #[error("Failed to expand glob pattern")] + #[diagnostic(code(mlf::generate::glob_error))] + GlobError { + #[source] + source: glob::GlobError, + }, + + #[error("Invalid glob pattern: {pattern}")] + #[diagnostic(code(mlf::generate::invalid_glob))] + InvalidGlob { + pattern: String, + #[source] + source: glob::PatternError, + }, +} + +pub fn run(input_patterns: Vec, output_dir: PathBuf) -> Result<(), MlfGenerateError> { + let mut file_paths = Vec::new(); + + for pattern in input_patterns { + if pattern.contains('*') || pattern.contains('?') { + for entry in glob::glob(&pattern).map_err(|source| MlfGenerateError::InvalidGlob { + pattern: pattern.clone(), + source, + })? { + let path = entry.map_err(|source| MlfGenerateError::GlobError { source })?; + file_paths.push(path); + } + } else { + file_paths.push(PathBuf::from(pattern)); + } + } + + std::fs::create_dir_all(&output_dir).map_err(|source| MlfGenerateError::WriteOutput { + path: output_dir.display().to_string(), + source, + })?; + + let mut errors = Vec::new(); + let mut success_count = 0; + + for file_path in file_paths { + let source = match std::fs::read_to_string(&file_path) { + Ok(s) => s, + Err(source) => { + errors.push(( + file_path.display().to_string(), + format!("Failed to read file: {}", source), + )); + continue; + } + }; + + let json: Value = match serde_json::from_str(&source) { + Ok(j) => j, + Err(source) => { + errors.push(( + file_path.display().to_string(), + format!("Failed to parse JSON: {}", source), + )); + continue; + } + }; + + let mlf_content = match generate_mlf_from_json(&json) { + Ok(content) => content, + Err(e) => { + errors.push((file_path.display().to_string(), format!("{:?}", e))); + continue; + } + }; + + // Extract namespace from JSON "id" field + let namespace = json + .get("id") + .and_then(|v| v.as_str()) + .ok_or_else(|| MlfGenerateError::InvalidLexicon { + message: "Missing 'id' field in lexicon".to_string(), + })?; + + // Create output path from namespace + let mut output_path = output_dir.clone(); + for segment in namespace.split('.') { + output_path.push(segment); + } + if let Err(source) = std::fs::create_dir_all(&output_path.parent().unwrap()) { + errors.push(( + file_path.display().to_string(), + format!("Failed to create directory: {}", source), + )); + continue; + } + output_path.set_extension("mlf"); + + if let Err(source) = std::fs::write(&output_path, mlf_content) { + errors.push(( + output_path.display().to_string(), + format!("Failed to write file: {}", source), + )); + continue; + } + + println!("Generated: {}", output_path.display()); + success_count += 1; + } + + if !errors.is_empty() { + eprintln!( + "\n{} file(s) generated successfully, {} error(s) encountered:\n", + success_count, + errors.len() + ); + for (path, error) in &errors { + eprintln!(" {} - {}", path, error); + } + eprintln!(); + return Err(MlfGenerateError::InvalidLexicon { + message: format!("{} errors total", errors.len()), + }); + } + + println!("\nSuccessfully generated {} file(s)", success_count); + Ok(()) +} + +pub fn generate_mlf_from_json(json: &Value) -> Result { + let mut output = String::new(); + + // Extract NSID to get the last segment for "main" definitions + let nsid = json + .get("id") + .and_then(|v| v.as_str()) + .ok_or_else(|| MlfGenerateError::InvalidLexicon { + message: "Missing 'id' field in lexicon".to_string(), + })?; + + let last_segment = nsid.split('.').last().unwrap_or("main"); + + let defs = json.get("defs").and_then(|v| v.as_object()).ok_or_else(|| { + MlfGenerateError::InvalidLexicon { + message: "Missing or invalid 'defs' field".to_string(), + } + })?; + + // Create a context to pass the current namespace to type generation + let ctx = ConversionContext { + current_namespace: nsid.to_string(), + }; + + // Process all definitions + for (name, def) in defs { + let def_type = def.get("type").and_then(|v| v.as_str()).ok_or_else(|| { + MlfGenerateError::InvalidLexicon { + message: format!("Missing 'type' field for definition '{}'", name), + } + })?; + + match def_type { + "record" => { + let mlf = generate_record(name, def, last_segment, &ctx)?; + output.push_str(&mlf); + output.push('\n'); + } + "query" => { + let mlf = generate_query(name, def, last_segment, &ctx)?; + output.push_str(&mlf); + output.push('\n'); + } + "procedure" => { + let mlf = generate_procedure(name, def, last_segment, &ctx)?; + output.push_str(&mlf); + output.push('\n'); + } + "subscription" => { + let mlf = generate_subscription(name, def, last_segment, &ctx)?; + output.push_str(&mlf); + output.push('\n'); + } + "token" => { + let mlf = generate_token(name, def)?; + output.push_str(&mlf); + output.push('\n'); + } + "object" => { + let mlf = generate_def_type(name, def, last_segment, &ctx)?; + output.push_str(&mlf); + output.push('\n'); + } + _ => { + // Unknown type, skip + } + } + } + + Ok(output) +} + +struct ConversionContext { + current_namespace: String, +} + +/// Reserved words in MLF that need to be escaped +const RESERVED_WORDS: &[&str] = &[ + "main", "record", "query", "procedure", "subscription", "token", "def", "type", "use", + "pub", "alias", "namespace", "constrained", "error", "unit", "null", "boolean", + "integer", "string", "bytes", "blob", "unknown", "array", "object", "union", "ref", +]; + +/// Escape a name if it's a reserved word +fn escape_name(name: &str) -> String { + if RESERVED_WORDS.contains(&name) { + format!("`{}`", name) + } else { + name.to_string() + } +} + +fn generate_record(name: &str, def: &Value, last_segment: &str, ctx: &ConversionContext) -> Result { + let mut output = String::new(); + + // Add doc comment if present + if let Some(desc) = def.get("description").and_then(|v| v.as_str()) { + if !desc.is_empty() { + for line in desc.lines() { + output.push_str(&format!("/// {}\n", line)); + } + } + } + + // Add @main annotation for "main" definitions + if name == "main" { + output.push_str("@main\n"); + } + + // Use last segment of NSID for "main" definitions + let record_name = if name == "main" { + escape_name(last_segment) + } else { + escape_name(name) + }; + + output.push_str(&format!("record {} {{\n", record_name)); + + // Get the record object + let record_obj = def.get("record").and_then(|v| v.as_object()).ok_or_else(|| { + MlfGenerateError::InvalidLexicon { + message: format!("Missing 'record' field in record definition '{}'", name), + } + })?; + + let properties = record_obj + .get("properties") + .and_then(|v| v.as_object()) + .ok_or_else(|| MlfGenerateError::InvalidLexicon { + message: format!("Missing 'properties' in record '{}'", name), + })?; + + let required = record_obj + .get("required") + .and_then(|v| v.as_array()) + .map(|arr| { + arr.iter() + .filter_map(|v| v.as_str()) + .collect::>() + }) + .unwrap_or_default(); + + for (field_name, field_def) in properties { + // Add field doc comment + if let Some(desc) = field_def.get("description").and_then(|v| v.as_str()) { + if !desc.is_empty() { + for line in desc.lines() { + output.push_str(&format!(" /// {}\n", line)); + } + } + } + + let is_required = required.contains(&field_name.as_str()); + let required_marker = if is_required { "!" } else { "" }; + + let field_type = generate_type(field_def)?; + let escaped_field_name = escape_name(field_name); + output.push_str(&format!( + " {}{}: {},\n", + escaped_field_name, required_marker, field_type + )); + } + + output.push_str("}\n"); + Ok(output) +} + +fn generate_query(name: &str, def: &Value, last_segment: &str, ctx: &ConversionContext) -> Result { + let mut output = String::new(); + + // Add doc comment + if let Some(desc) = def.get("description").and_then(|v| v.as_str()) { + if !desc.is_empty() { + for line in desc.lines() { + output.push_str(&format!("/// {}\n", line)); + } + } + } + + // Add @main annotation for "main" definitions + if name == "main" { + output.push_str("@main\n"); + } + + let query_name = if name == "main" { + escape_name(last_segment) + } else { + escape_name(name) + }; + output.push_str(&format!("query {}", query_name)); + + // Parameters + output.push('('); + if let Some(params) = def.get("parameters").and_then(|v| v.as_object()) { + let properties = params.get("properties").and_then(|v| v.as_object()); + let required = params + .get("required") + .and_then(|v| v.as_array()) + .map(|arr| { + arr.iter() + .filter_map(|v| v.as_str()) + .collect::>() + }) + .unwrap_or_default(); + + if let Some(props) = properties { + let param_strs: Vec = props + .iter() + .map(|(param_name, param_def)| { + let is_required = required.contains(¶m_name.as_str()); + let required_marker = if is_required { "!" } else { "" }; + let param_type = generate_type(param_def).unwrap_or_else(|_| "unknown".to_string()); + let escaped_param_name = escape_name(param_name); + + // Add doc comment inline if present + let mut result = String::new(); + if let Some(desc) = param_def.get("description").and_then(|v| v.as_str()) { + if !desc.is_empty() { + result.push_str(&format!("\n /// {}\n ", desc)); + } + } + result.push_str(&format!("{}{}: {}", escaped_param_name, required_marker, param_type)); + result + }) + .collect(); + + if !param_strs.is_empty() { + output.push_str(¶m_strs.join(",")); + } + } + } + output.push(')'); + + // Output type + if let Some(output_obj) = def.get("output").and_then(|v| v.as_object()) { + if let Some(schema) = output_obj.get("schema") { + let return_type = generate_type(schema)?; + output.push_str(&format!(": {}", return_type)); + + // Check for errors + if let Some(errors) = output_obj.get("errors").and_then(|v| v.as_object()) { + output.push_str(" | error {\n"); + for (error_name, error_def) in errors { + if let Some(desc) = error_def.get("description").and_then(|v| v.as_str()) { + if !desc.is_empty() { + output.push_str(&format!(" /// {}\n", desc)); + } + } + output.push_str(&format!(" {},\n", error_name)); + } + output.push('}'); + } + } + } + + output.push_str(";\n"); + Ok(output) +} + +fn generate_procedure(name: &str, def: &Value, last_segment: &str, ctx: &ConversionContext) -> Result { + let mut output = String::new(); + + // Add doc comment + if let Some(desc) = def.get("description").and_then(|v| v.as_str()) { + if !desc.is_empty() { + for line in desc.lines() { + output.push_str(&format!("/// {}\n", line)); + } + } + } + + // Add @main annotation for "main" definitions + if name == "main" { + output.push_str("@main\n"); + } + + let procedure_name = if name == "main" { + escape_name(last_segment) + } else { + escape_name(name) + }; + output.push_str(&format!("procedure {}", procedure_name)); + + // Input parameters + output.push('('); + if let Some(input) = def.get("input").and_then(|v| v.as_object()) { + if let Some(schema) = input.get("schema").and_then(|v| v.as_object()) { + let properties = schema.get("properties").and_then(|v| v.as_object()); + let required = schema + .get("required") + .and_then(|v| v.as_array()) + .map(|arr| { + arr.iter() + .filter_map(|v| v.as_str()) + .collect::>() + }) + .unwrap_or_default(); + + if let Some(props) = properties { + let param_strs: Vec = props + .iter() + .map(|(param_name, param_def)| { + let is_required = required.contains(¶m_name.as_str()); + let required_marker = if is_required { "!" } else { "" }; + let param_type = + generate_type(param_def).unwrap_or_else(|_| "unknown".to_string()); + let escaped_param_name = escape_name(param_name); + + // Add doc comment inline if present + let mut result = String::new(); + if let Some(desc) = param_def.get("description").and_then(|v| v.as_str()) { + if !desc.is_empty() { + result.push_str(&format!("\n /// {}\n ", desc)); + } + } + result.push_str(&format!( + "{}{}: {}", + escaped_param_name, required_marker, param_type + )); + result + }) + .collect(); + + if !param_strs.is_empty() { + output.push_str(¶m_strs.join(",")); + } + } + } + } + output.push(')'); + + // Output type + if let Some(output_obj) = def.get("output").and_then(|v| v.as_object()) { + if let Some(schema) = output_obj.get("schema") { + let return_type = generate_type(schema)?; + output.push_str(&format!(": {}", return_type)); + + // Check for errors + if let Some(errors) = output_obj.get("errors").and_then(|v| v.as_object()) { + output.push_str(" | error {\n"); + for (error_name, error_def) in errors { + if let Some(desc) = error_def.get("description").and_then(|v| v.as_str()) { + if !desc.is_empty() { + output.push_str(&format!(" /// {}\n", desc)); + } + } + output.push_str(&format!(" {},\n", error_name)); + } + output.push('}'); + } + } + } + + output.push_str(";\n"); + Ok(output) +} + +fn generate_subscription(name: &str, def: &Value, last_segment: &str, ctx: &ConversionContext) -> Result { + let mut output = String::new(); + + // Add doc comment + if let Some(desc) = def.get("description").and_then(|v| v.as_str()) { + if !desc.is_empty() { + for line in desc.lines() { + output.push_str(&format!("/// {}\n", line)); + } + } + } + + // Add @main annotation for "main" definitions + if name == "main" { + output.push_str("@main\n"); + } + + let subscription_name = if name == "main" { + escape_name(last_segment) + } else { + escape_name(name) + }; + output.push_str(&format!("subscription {}", subscription_name)); + + // Parameters + output.push('('); + if let Some(params) = def.get("parameters").and_then(|v| v.as_object()) { + let properties = params.get("properties").and_then(|v| v.as_object()); + let required = params + .get("required") + .and_then(|v| v.as_array()) + .map(|arr| { + arr.iter() + .filter_map(|v| v.as_str()) + .collect::>() + }) + .unwrap_or_default(); + + if let Some(props) = properties { + let param_strs: Vec = props + .iter() + .map(|(param_name, param_def)| { + let is_required = required.contains(¶m_name.as_str()); + let required_marker = if is_required { "!" } else { "" }; + let param_type = generate_type(param_def).unwrap_or_else(|_| "unknown".to_string()); + let escaped_param_name = escape_name(param_name); + + format!("{}{}: {}", escaped_param_name, required_marker, param_type) + }) + .collect(); + + if !param_strs.is_empty() { + output.push_str(¶m_strs.join(", ")); + } + } + } + output.push(')'); + + // Message types + if let Some(message) = def.get("message").and_then(|v| v.as_object()) { + if let Some(schema) = message.get("schema") { + let message_type = generate_type(schema)?; + output.push_str(&format!(": {}", message_type)); + } + } + + output.push_str(";\n"); + Ok(output) +} + +fn generate_token(name: &str, def: &Value) -> Result { + let mut output = String::new(); + + // Add doc comment + if let Some(desc) = def.get("description").and_then(|v| v.as_str()) { + if !desc.is_empty() { + for line in desc.lines() { + output.push_str(&format!("/// {}\n", line)); + } + } + } + + let escaped_name = escape_name(name); + output.push_str(&format!("token {};\n", escaped_name)); + Ok(output) +} + +fn generate_def_type(name: &str, def: &Value, last_segment: &str, ctx: &ConversionContext) -> Result { + let mut output = String::new(); + + // Add @main annotation for "main" definitions + if name == "main" { + output.push_str("@main\n"); + } + + // Use last segment of NSID for "main" definitions + let def_name = if name == "main" { + escape_name(last_segment) + } else { + escape_name(name) + }; + + output.push_str(&format!("def type {} = ", def_name)); + let type_str = generate_type_with_indent(def, 0)?; + output.push_str(&type_str); + output.push_str(";\n"); + + Ok(output) +} + +fn generate_type_with_indent(type_def: &Value, indent_level: usize, ctx: &ConversionContext) -> Result { + let type_name = type_def.get("type").and_then(|v| v.as_str()); + + match type_name { + Some("object") => { + let indent = " ".repeat(indent_level); + let field_indent = " ".repeat(indent_level + 1); + + let mut output = String::from("{\n"); + let properties = type_def + .get("properties") + .and_then(|v| v.as_object()) + .ok_or_else(|| MlfGenerateError::InvalidLexicon { + message: "Missing 'properties' in object type".to_string(), + })?; + + let required = type_def + .get("required") + .and_then(|v| v.as_array()) + .map(|arr| { + arr.iter() + .filter_map(|v| v.as_str()) + .collect::>() + }) + .unwrap_or_default(); + + for (field_name, field_def) in properties { + // Add field doc comment + if let Some(desc) = field_def.get("description").and_then(|v| v.as_str()) { + if !desc.is_empty() { + for line in desc.lines() { + output.push_str(&format!("{}/// {}\n", field_indent, line)); + } + } + } + + let is_required = required.contains(&field_name.as_str()); + let required_marker = if is_required { "!" } else { "" }; + let field_type = generate_type_with_indent(field_def, indent_level + 1)?; + let escaped_field_name = escape_name(field_name); + output.push_str(&format!( + "{}{}{}: {},\n", + field_indent, escaped_field_name, required_marker, field_type + )); + } + + output.push_str(&format!("{}}}", indent)); + Ok(output) + } + _ => generate_type(type_def), + } +} + +fn generate_type(type_def: &Value, ctx: &ConversionContext) -> Result { + let type_name = type_def.get("type").and_then(|v| v.as_str()); + + match type_name { + Some("null") => Ok("null".to_string()), + Some("boolean") => Ok("boolean".to_string()), + Some("integer") => { + let mut result = "integer".to_string(); + result = apply_constraints(result, type_def); + Ok(result) + } + Some("string") => { + // Check if this is a format string that maps to a prelude type + if let Some(format) = type_def.get("format").and_then(|v| v.as_str()) { + let prelude_type = match format { + "did" => "Did", + "at-uri" => "AtUri", + "at-identifier" => "AtIdentifier", + "handle" => "Handle", + "datetime" => "Datetime", + "uri" => "Uri", + "cid" => "Cid", + "nsid" => "Nsid", + "tid" => "Tid", + "record-key" => "RecordKey", + "language" => "Language", + _ => { + // Unknown format, fall through to normal string with constraints + let mut result = "string".to_string(); + result = apply_constraints(result, type_def); + return Ok(result); + } + }; + // If it's a known prelude type with only the format constraint, use the prelude type directly + // Check if there are other constraints besides format + let has_other_constraints = type_def.get("minLength").is_some() + || type_def.get("maxLength").is_some() + || type_def.get("minGraphemes").is_some() + || type_def.get("maxGraphemes").is_some() + || type_def.get("enum").is_some() + || type_def.get("knownValues").is_some() + || type_def.get("default").is_some(); + + if !has_other_constraints { + return Ok(prelude_type.to_string()); + } + } + + let mut result = "string".to_string(); + result = apply_constraints(result, type_def); + Ok(result) + } + Some("bytes") => Ok("bytes".to_string()), + Some("blob") => { + let mut result = "blob".to_string(); + result = apply_constraints(result, type_def); + Ok(result) + } + Some("unknown") => Ok("unknown".to_string()), + Some("array") => { + let items = type_def.get("items").ok_or_else(|| { + MlfGenerateError::InvalidLexicon { + message: "Missing 'items' in array type".to_string(), + } + })?; + + // Check if items have constraints + let items_obj = items.as_object(); + let has_item_constraints = items_obj.map_or(false, |obj| { + obj.contains_key("minLength") || + obj.contains_key("maxLength") || + obj.contains_key("minGraphemes") || + obj.contains_key("maxGraphemes") || + obj.contains_key("minimum") || + obj.contains_key("maximum") || + obj.contains_key("enum") || + obj.contains_key("knownValues") || + obj.contains_key("default") + }); + + let item_type = if has_item_constraints { + // If item has constraints, we need to wrap in parentheses to apply constraints before [] + // For now, just generate the base type without item constraints + // TODO: Consider generating a type alias for complex constrained items + items.get("type") + .and_then(|t| t.as_str()) + .unwrap_or("unknown") + .to_string() + } else { + generate_type(items)? + }; + + let mut result = format!("{}[]", item_type); + result = apply_constraints(result, type_def); + Ok(result) + } + Some("object") => { + let mut output = String::from("{\n"); + let properties = type_def + .get("properties") + .and_then(|v| v.as_object()) + .ok_or_else(|| MlfGenerateError::InvalidLexicon { + message: "Missing 'properties' in object type".to_string(), + })?; + + let required = type_def + .get("required") + .and_then(|v| v.as_array()) + .map(|arr| { + arr.iter() + .filter_map(|v| v.as_str()) + .collect::>() + }) + .unwrap_or_default(); + + for (field_name, field_def) in properties { + // Add field doc comment + if let Some(desc) = field_def.get("description").and_then(|v| v.as_str()) { + if !desc.is_empty() { + for line in desc.lines() { + output.push_str(&format!(" /// {}\n", line)); + } + } + } + + let is_required = required.contains(&field_name.as_str()); + let required_marker = if is_required { "!" } else { "" }; + let field_type = generate_type(field_def)?; + let escaped_field_name = escape_name(field_name); + output.push_str(&format!( + " {}{}: {},\n", + escaped_field_name, required_marker, field_type + )); + } + + output.push_str(" }"); + Ok(output) + } + Some("union") => { + let refs = type_def.get("refs").and_then(|v| v.as_array()).ok_or_else(|| { + MlfGenerateError::InvalidLexicon { + message: "Missing 'refs' in union type".to_string(), + } + })?; + + let type_strs: Vec = refs + .iter() + .map(|r| generate_type(r).unwrap_or_else(|_| "unknown".to_string())) + .collect(); + + let mut result = type_strs.join(" | "); + + // Check if closed + if type_def.get("closed").and_then(|v| v.as_bool()).unwrap_or(false) { + result.push_str(" | !"); + } + + Ok(result) + } + Some("ref") => { + if let Some(ref_str) = type_def.get("ref").and_then(|v| v.as_str()) { + // Handle references: + // "#defName" -> "defName" (local reference, same file) + // "namespace.id#defName" -> Check if same namespace, if so use "defName", else use full path + + if let Some(stripped) = ref_str.strip_prefix('#') { + // Local reference: #defName -> defName + Ok(stripped.to_string()) + } else if let Some((namespace, def_name)) = ref_str.split_once('#') { + // Check if this is the current namespace + // For now, we'll just use the def name if it's the same namespace + // Note: This requires passing context through, which we'll add + // For external refs, we keep the full NSID format + Ok(format!("{}.{}", namespace, def_name)) + } else { + // No # at all - shouldn't happen in valid lexicons, but handle gracefully + Ok(ref_str.to_string()) + } + } else { + Err(MlfGenerateError::InvalidLexicon { + message: "Missing 'ref' in ref type".to_string(), + }) + } + } + _ => Ok("unknown".to_string()), + } +} + +fn apply_constraints(mut type_str: String, type_def: &Value) -> String { + let mut constraints = Vec::new(); + + if let Some(min_length) = type_def.get("minLength").and_then(|v| v.as_i64()) { + constraints.push(format!("minLength: {}", min_length)); + } + if let Some(max_length) = type_def.get("maxLength").and_then(|v| v.as_i64()) { + constraints.push(format!("maxLength: {}", max_length)); + } + if let Some(min_graphemes) = type_def.get("minGraphemes").and_then(|v| v.as_i64()) { + constraints.push(format!("minGraphemes: {}", min_graphemes)); + } + if let Some(max_graphemes) = type_def.get("maxGraphemes").and_then(|v| v.as_i64()) { + constraints.push(format!("maxGraphemes: {}", max_graphemes)); + } + if let Some(minimum) = type_def.get("minimum").and_then(|v| v.as_i64()) { + constraints.push(format!("minimum: {}", minimum)); + } + if let Some(maximum) = type_def.get("maximum").and_then(|v| v.as_i64()) { + constraints.push(format!("maximum: {}", maximum)); + } + if let Some(format) = type_def.get("format").and_then(|v| v.as_str()) { + constraints.push(format!("format: \"{}\"", format)); + } + if let Some(enum_vals) = type_def.get("enum").and_then(|v| v.as_array()) { + let vals: Vec = enum_vals + .iter() + .filter_map(|v| v.as_str()) + .map(|s| format!("\"{}\"", s)) + .collect(); + constraints.push(format!("enum: [{}]", vals.join(", "))); + } + if let Some(known_vals) = type_def.get("knownValues").and_then(|v| v.as_array()) { + let vals: Vec = known_vals + .iter() + .filter_map(|v| v.as_str()) + .map(|s| format!("\"{}\"", s)) + .collect(); + constraints.push(format!("knownValues: [{}]", vals.join(", "))); + } + if let Some(accept) = type_def.get("accept").and_then(|v| v.as_array()) { + let mimes: Vec = accept + .iter() + .filter_map(|v| v.as_str()) + .map(|s| format!("\"{}\"", s)) + .collect(); + constraints.push(format!("accept: [{}]", mimes.join(", "))); + } + if let Some(max_size) = type_def.get("maxSize").and_then(|v| v.as_i64()) { + constraints.push(format!("maxSize: {}", max_size)); + } + if let Some(default) = type_def.get("default") { + let default_str = match default { + Value::String(s) => format!("\"{}\"", s), + Value::Number(n) => n.to_string(), + Value::Bool(b) => b.to_string(), + _ => "null".to_string(), + }; + constraints.push(format!("default: {}", default_str)); + } + + if !constraints.is_empty() { + type_str.push_str(" constrained {\n"); + for constraint in &constraints { + type_str.push_str(&format!(" {},\n", constraint)); + } + type_str.push_str(" }"); + } + + type_str +} diff --git a/mlf-cli/src/generate/mod.rs b/mlf-cli/src/generate/mod.rs index 02f53cd..5238e54 100644 --- a/mlf-cli/src/generate/mod.rs +++ b/mlf-cli/src/generate/mod.rs @@ -31,9 +31,9 @@ pub fn run_all() -> Result<(), std::io::Error> { println!("Running {} output configuration(s)...", config.output.len()); - // Build input pattern from source directory - let source_pattern = format!("{}/**/*.mlf", config.source.directory); - let input_patterns = vec![source_pattern]; + // Build input path from source directory + let source_dir = project_root.join(&config.source.directory); + let input_paths = vec![source_dir.clone()]; let mut errors = Vec::new(); let mut success_count = 0; @@ -46,7 +46,7 @@ pub fn run_all() -> Result<(), std::io::Error> { let result = match output_type.as_str() { "lexicon" => { - lexicon::run(input_patterns.clone(), output_dir, false) + lexicon::run(input_paths.clone(), Some(output_dir), Some(source_dir.clone()), false) .map_err(|e| format!("{}", e)) } "mlf" => { @@ -57,7 +57,7 @@ pub fn run_all() -> Result<(), std::io::Error> { } generator_type => { // Assume it's a code generator (typescript, go, rust, etc.) - code::run(generator_type.to_string(), input_patterns.clone(), output_dir, false) + code::run(Some(generator_type.to_string()), input_paths.clone(), Some(output_dir), Some(source_dir.clone()), false) .map_err(|e| format!("{}", e)) } }; diff --git a/mlf-cli/src/main.rs b/mlf-cli/src/main.rs index 0429568..20af32c 100644 --- a/mlf-cli/src/main.rs +++ b/mlf-cli/src/main.rs @@ -37,8 +37,11 @@ enum Commands { }, Check { - #[arg(help = "MLF lexicon file(s) to validate (glob patterns supported). If omitted, checks source directory from mlf.toml")] - input: Vec, + #[arg(help = "MLF lexicon file(s) or directory to validate. If omitted, checks source directory from mlf.toml")] + input: Vec, + + #[arg(long, help = "Root directory for namespace calculation (defaults to mlf.toml source directory or current directory)")] + root: Option, }, Validate { @@ -66,24 +69,30 @@ enum Commands { #[derive(Subcommand)] enum GenerateCommands { Lexicon { - #[arg(short, long, help = "Input MLF files (glob patterns supported)")] - input: Vec, + #[arg(short, long, help = "Input MLF file(s) or directory. If omitted, uses source directory from mlf.toml")] + input: Vec, - #[arg(short, long, help = "Output directory")] - output: PathBuf, + #[arg(short, long, help = "Output directory. If omitted, uses first lexicon output from mlf.toml")] + output: Option, + + #[arg(long, help = "Root directory for namespace calculation (defaults to mlf.toml source directory or current directory)")] + root: Option, #[arg(long, help = "Use flat file structure (e.g., app.bsky.post.json)")] flat: bool, }, Code { - #[arg(short, long, help = "Generator to use (json, typescript, go, rust)")] - generator: String, + #[arg(short, long, help = "Generator to use (typescript, go, rust, etc.). If omitted, uses first code output from mlf.toml")] + generator: Option, - #[arg(short, long, help = "Input MLF files (glob patterns supported)")] - input: Vec, + #[arg(short, long, help = "Input MLF file(s) or directory. If omitted, uses source directory from mlf.toml")] + input: Vec, + + #[arg(short, long, help = "Output directory. If omitted, uses matching output from mlf.toml")] + output: Option, - #[arg(short, long, help = "Output directory")] - output: PathBuf, + #[arg(long, help = "Root directory for namespace calculation (defaults to mlf.toml source directory or current directory)")] + root: Option, #[arg(long, help = "Use flat file structure (e.g., app.bsky.post.ts)")] flat: bool, @@ -92,8 +101,8 @@ enum GenerateCommands { #[arg(short, long, help = "Input JSON lexicon files (glob patterns supported)")] input: Vec, - #[arg(short, long, help = "Output directory")] - output: PathBuf, + #[arg(short, long, help = "Output directory. If omitted, uses first mlf output from mlf.toml")] + output: Option, }, } @@ -104,18 +113,18 @@ fn main() { Commands::Init { yes } => { init::run_init(yes).into_diagnostic() } - Commands::Check { input } => { - check::run_check(input).into_diagnostic() + Commands::Check { input, root } => { + check::run_check(input, root).into_diagnostic() } Commands::Validate { lexicon, record } => { check::validate(lexicon, record).into_diagnostic() } Commands::Generate { command } => match command { - Some(GenerateCommands::Lexicon { input, output, flat }) => { - generate::lexicon::run(input, output, flat).into_diagnostic() + Some(GenerateCommands::Lexicon { input, output, root, flat }) => { + generate::lexicon::run(input, output, root, flat).into_diagnostic() } - Some(GenerateCommands::Code { generator, input, output, flat }) => { - generate::code::run(generator, input, output, flat).into_diagnostic() + Some(GenerateCommands::Code { generator, input, output, root, flat }) => { + generate::code::run(generator, input, output, root, flat).into_diagnostic() } Some(GenerateCommands::Mlf { input, output }) => { generate::mlf::run(input, output).into_diagnostic() diff --git a/mlf-codegen/src/lib.rs b/mlf-codegen/src/lib.rs index 5aaea78..c0a0f24 100644 --- a/mlf-codegen/src/lib.rs +++ b/mlf-codegen/src/lib.rs @@ -68,6 +68,10 @@ pub mod plugin { pub use plugin::{CodeGenerator, GeneratorContext}; +fn has_main_annotation(annotations: &[Annotation]) -> bool { + annotations.iter().any(|ann| ann.name.name == "main") +} + pub fn generate_lexicon(namespace: &str, lexicon: &Lexicon, workspace: &Workspace) -> Value { let usage_counts = analyze_type_usage(lexicon); @@ -76,10 +80,26 @@ pub fn generate_lexicon(namespace: &str, lexicon: &Lexicon, workspace: &Workspac let expected_main_name = namespace_parts.last().copied().unwrap_or(""); let is_defs_namespace = expected_main_name == "defs"; - // Count main-eligible items (records, queries, procedures, subscriptions) - let main_eligible_count = lexicon.items.iter().filter(|item| { - matches!(item, Item::Record(_) | Item::Query(_) | Item::Procedure(_) | Item::Subscription(_)) - }).count(); + // Count main-eligible items (records, queries, procedures, subscriptions, def types) without @main + let main_eligible_items: Vec<&Item> = lexicon.items.iter() + .filter(|item| { + matches!(item, Item::Record(_) | Item::Query(_) | Item::Procedure(_) | Item::Subscription(_) | Item::DefType(_)) + }) + .collect(); + + let main_eligible_count = main_eligible_items.len(); + + // Check if any item has @main annotation + let has_explicit_main = main_eligible_items.iter().any(|item| { + match item { + Item::Record(r) => has_main_annotation(&r.annotations), + Item::Query(q) => has_main_annotation(&q.annotations), + Item::Procedure(p) => has_main_annotation(&p.annotations), + Item::Subscription(s) => has_main_annotation(&s.annotations), + Item::DefType(d) => has_main_annotation(&d.annotations), + _ => false, + } + }); let mut defs = Map::new(); @@ -87,8 +107,17 @@ pub fn generate_lexicon(namespace: &str, lexicon: &Lexicon, workspace: &Workspac match item { Item::Record(record) => { let record_json = generate_record_json(record, &usage_counts, workspace, namespace); - // If there's only one main-eligible item, it becomes "main" - if main_eligible_count == 1 || (!is_defs_namespace && record.name.name == expected_main_name) { + + // Check if this should be main + let is_main = if has_explicit_main { + // If @main is used explicitly, only that item is main + has_main_annotation(&record.annotations) + } else { + // Otherwise use heuristics: single item or name matches namespace + main_eligible_count == 1 || (!is_defs_namespace && record.name.name == expected_main_name) + }; + + if is_main { defs.insert("main".to_string(), record_json); } else { defs.insert(record.name.name.clone(), record_json); @@ -96,7 +125,14 @@ pub fn generate_lexicon(namespace: &str, lexicon: &Lexicon, workspace: &Workspac } Item::Query(query) => { let query_json = generate_query_json(query, &usage_counts, workspace, namespace); - if main_eligible_count == 1 || (!is_defs_namespace && query.name.name == expected_main_name) { + + let is_main = if has_explicit_main { + has_main_annotation(&query.annotations) + } else { + main_eligible_count == 1 || (!is_defs_namespace && query.name.name == expected_main_name) + }; + + if is_main { defs.insert("main".to_string(), query_json); } else { defs.insert(query.name.name.clone(), query_json); @@ -104,7 +140,14 @@ pub fn generate_lexicon(namespace: &str, lexicon: &Lexicon, workspace: &Workspac } Item::Procedure(procedure) => { let procedure_json = generate_procedure_json(procedure, &usage_counts, workspace, namespace); - if main_eligible_count == 1 || (!is_defs_namespace && procedure.name.name == expected_main_name) { + + let is_main = if has_explicit_main { + has_main_annotation(&procedure.annotations) + } else { + main_eligible_count == 1 || (!is_defs_namespace && procedure.name.name == expected_main_name) + }; + + if is_main { defs.insert("main".to_string(), procedure_json); } else { defs.insert(procedure.name.name.clone(), procedure_json); @@ -112,7 +155,14 @@ pub fn generate_lexicon(namespace: &str, lexicon: &Lexicon, workspace: &Workspac } Item::Subscription(subscription) => { let subscription_json = generate_subscription_json(subscription, &usage_counts, workspace, namespace); - if main_eligible_count == 1 || (!is_defs_namespace && subscription.name.name == expected_main_name) { + + let is_main = if has_explicit_main { + has_main_annotation(&subscription.annotations) + } else { + main_eligible_count == 1 || (!is_defs_namespace && subscription.name.name == expected_main_name) + }; + + if is_main { defs.insert("main".to_string(), subscription_json); } else { defs.insert(subscription.name.name.clone(), subscription_json); @@ -120,7 +170,19 @@ pub fn generate_lexicon(namespace: &str, lexicon: &Lexicon, workspace: &Workspac } Item::DefType(def_type) => { let def_type_json = generate_def_type_json(def_type, &usage_counts, workspace, namespace); - defs.insert(def_type.name.name.clone(), def_type_json); + + // Check if this should be main + let is_main = if has_explicit_main { + has_main_annotation(&def_type.annotations) + } else { + main_eligible_count == 1 || (!is_defs_namespace && def_type.name.name == expected_main_name) + }; + + if is_main { + defs.insert("main".to_string(), def_type_json); + } else { + defs.insert(def_type.name.name.clone(), def_type_json); + } } Item::InlineType(_) => { // Inline types are never added to defs - they expand at point of use @@ -138,6 +200,7 @@ pub fn generate_lexicon(namespace: &str, lexicon: &Lexicon, workspace: &Workspac } let mut root = Map::new(); + root.insert("$type".to_string(), json!("com.atproto.lexicon.schema")); root.insert("lexicon".to_string(), json!(1)); root.insert("id".to_string(), json!(namespace)); root.insert("defs".to_string(), json!(defs)); @@ -473,14 +536,34 @@ fn generate_type_json(ty: &Type, usage_counts: &HashMap, workspac } // Not an inline type (or couldn't resolve) - generate a ref - if path.segments.len() == 1 { + // First, try to get the fully resolved namespace for this type + if let Some(full_namespace) = workspace.resolve_reference_namespace(path, current_namespace) { + // We have the full namespace where this type is defined + if full_namespace == current_namespace { + // It's in the current namespace - use local reference + let type_name = path.segments.last().unwrap().name.as_str(); + json!({ + "type": "ref", + "ref": format!("#{}", type_name) + }) + } else { + // It's in a different namespace - use full reference + let type_name = path.segments.last().unwrap().name.as_str(); + json!({ + "type": "ref", + "ref": format!("{}#{}", full_namespace, type_name) + }) + } + } else if path.segments.len() == 1 { + // Couldn't resolve namespace - fall back to heuristic + // Single segment likely means local reference let name = &path.segments[0].name; json!({ "type": "ref", "ref": format!("#{}", name) }) } else { - // Multi-segment path ref + // Multi-segment path ref - use as-is let namespace = path.segments[..path.segments.len()-1] .iter() .map(|s| s.name.as_str()) diff --git a/mlf-diagnostics/src/lib.rs b/mlf-diagnostics/src/lib.rs index 8a94645..0d03267 100644 --- a/mlf-diagnostics/src/lib.rs +++ b/mlf-diagnostics/src/lib.rs @@ -70,13 +70,15 @@ impl Diagnostic for ParseDiagnostic { #[derive(Debug)] pub struct ValidationDiagnostic { source_code: NamedSource, + module_namespace: String, errors: ValidationErrors, } impl ValidationDiagnostic { - pub fn new(filename: String, source: String, errors: ValidationErrors) -> Self { + pub fn new(filename: String, source: String, module_namespace: String, errors: ValidationErrors) -> Self { Self { source_code: NamedSource::new(filename, source), + module_namespace, errors, } } @@ -84,10 +86,31 @@ impl ValidationDiagnostic { impl fmt::Display for ValidationDiagnostic { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if self.errors.len() == 1 { - format_validation_error(f, &self.errors.errors[0]) + // Filter to only errors that belong to this module + let errors_in_this_module: Vec<&ValidationError> = self + .errors + .errors + .iter() + .filter(|error| get_error_module_namespace(error) == self.module_namespace) + .collect(); + + if errors_in_this_module.len() == 1 { + format_validation_error(f, errors_in_this_module[0]) + } else if errors_in_this_module.is_empty() { + write!( + f, + "Found {} validation error(s) in other modules in the workspace", + self.errors.len() + ) + } else if errors_in_this_module.len() == self.errors.len() { + write!(f, "Found {} validation error(s)", self.errors.len()) } else { - write!(f, "Found {} validation errors", self.errors.len()) + write!( + f, + "Found {} validation error(s) in this module ({} total in workspace)", + errors_in_this_module.len(), + self.errors.len() + ) } } } @@ -96,8 +119,16 @@ impl std::error::Error for ValidationDiagnostic {} impl Diagnostic for ValidationDiagnostic { fn code<'a>(&'a self) -> Option> { - if self.errors.len() == 1 { - Some(Box::new(get_error_code(&self.errors.errors[0]))) + // Filter to only errors in this module + let errors_in_this_module: Vec<&ValidationError> = self + .errors + .errors + .iter() + .filter(|error| get_error_module_namespace(error) == self.module_namespace) + .collect(); + + if errors_in_this_module.len() == 1 { + Some(Box::new(get_error_code(errors_in_this_module[0]))) } else { Some(Box::new("mlf::validation")) } @@ -108,9 +139,16 @@ impl Diagnostic for ValidationDiagnostic { } fn labels(&self) -> Option + '_>> { - let labels: Vec = self + // Filter to only errors that belong to this module + let errors_in_this_module: Vec<&ValidationError> = self .errors .errors + .iter() + .filter(|error| get_error_module_namespace(error) == self.module_namespace) + .collect(); + + // Get all labels from those errors + let labels: Vec = errors_in_this_module .iter() .flat_map(|error| get_error_labels(error)) .collect(); @@ -123,8 +161,16 @@ impl Diagnostic for ValidationDiagnostic { } fn help<'a>(&'a self) -> Option> { - if self.errors.len() == 1 { - get_error_help(&self.errors.errors[0]) + // Filter to only errors in this module + let errors_in_this_module: Vec<&ValidationError> = self + .errors + .errors + .iter() + .filter(|error| get_error_module_namespace(error) == self.module_namespace) + .collect(); + + if errors_in_this_module.len() == 1 { + get_error_help(errors_in_this_module[0]) } else { None } @@ -153,6 +199,23 @@ fn format_validation_error(f: &mut fmt::Formatter<'_>, error: &ValidationError) ValidationError::ReservedName { name, .. } => { write!(f, "Reserved name '{}' cannot be used as an item name", name) } + ValidationError::AmbiguousMain { name, namespace_suffix, .. } => { + write!( + f, + "Ambiguous main definition for '{}' in namespace ending with '{}'. Use @main to disambiguate", + name, namespace_suffix + ) + } + ValidationError::MultipleMain { name, .. } => { + write!(f, "Multiple items named '{}' marked with @main. Only one can be @main", name) + } + ValidationError::ConflictNotAllowed { name, namespace_suffix, .. } => { + write!( + f, + "Name conflict for '{}' is not allowed. Conflicts are only allowed when the name matches the namespace suffix ('{}')", + name, namespace_suffix + ) + } } } @@ -164,6 +227,9 @@ fn get_error_code(error: &ValidationError) -> &'static str { ValidationError::TypeMismatch { .. } => "mlf::type_mismatch", ValidationError::ConstraintTooPermissive { .. } => "mlf::constraint_too_permissive", ValidationError::ReservedName { .. } => "mlf::reserved_name", + ValidationError::AmbiguousMain { .. } => "mlf::ambiguous_main", + ValidationError::MultipleMain { .. } => "mlf::multiple_main", + ValidationError::ConflictNotAllowed { .. } => "mlf::conflict_not_allowed", } } @@ -173,6 +239,7 @@ fn get_error_labels(error: &ValidationError) -> Vec { first_span, second_span, name, + .. } => vec![ LabeledSpan::at( first_span.start..first_span.end, @@ -183,23 +250,57 @@ fn get_error_labels(error: &ValidationError) -> Vec { format!("'{}' redefined here", name), ), ], - ValidationError::UndefinedReference { span, name } => vec![LabeledSpan::at( + ValidationError::UndefinedReference { span, name, .. } => vec![LabeledSpan::at( span.start..span.end, format!("'{}' is not defined", name), )], - ValidationError::InvalidConstraint { span, message } => { + ValidationError::InvalidConstraint { span, message, .. } => { vec![LabeledSpan::at(span.start..span.end, message.clone())] } ValidationError::TypeMismatch { span, .. } => { vec![LabeledSpan::at(span.start..span.end, "type mismatch here")] } - ValidationError::ConstraintTooPermissive { span, message } => { + ValidationError::ConstraintTooPermissive { span, message, .. } => { vec![LabeledSpan::at(span.start..span.end, message.clone())] } - ValidationError::ReservedName { span, name } => vec![LabeledSpan::at( + ValidationError::ReservedName { span, name, .. } => vec![LabeledSpan::at( span.start..span.end, format!("'{}' is a reserved name and cannot be used", name), )], + ValidationError::AmbiguousMain { + first_span, + second_span, + name, + .. + } => vec![ + LabeledSpan::at( + first_span.start..first_span.end, + format!("'{}' defined here", name), + ), + LabeledSpan::at( + second_span.start..second_span.end, + format!("'{}' also defined here", name), + ), + ], + ValidationError::MultipleMain { + first_span, + second_span, + name, + .. + } => vec![ + LabeledSpan::at( + first_span.start..first_span.end, + format!("'{}' marked as @main here", name), + ), + LabeledSpan::at( + second_span.start..second_span.end, + format!("'{}' also marked as @main here", name), + ), + ], + ValidationError::ConflictNotAllowed { span, name, .. } => vec![LabeledSpan::at( + span.start..span.end, + format!("'{}' conflicts with another definition", name), + )], } } @@ -243,3 +344,21 @@ fn get_error_help(error: &ValidationError) -> Option> { _ => None, } } + +pub fn get_error_module_namespace_str(error: &ValidationError) -> &str { + match error { + ValidationError::DuplicateDefinition { module_namespace, .. } => module_namespace, + ValidationError::UndefinedReference { module_namespace, .. } => module_namespace, + ValidationError::InvalidConstraint { module_namespace, .. } => module_namespace, + ValidationError::TypeMismatch { module_namespace, .. } => module_namespace, + ValidationError::ConstraintTooPermissive { module_namespace, .. } => module_namespace, + ValidationError::ReservedName { module_namespace, .. } => module_namespace, + ValidationError::AmbiguousMain { module_namespace, .. } => module_namespace, + ValidationError::MultipleMain { module_namespace, .. } => module_namespace, + ValidationError::ConflictNotAllowed { module_namespace, .. } => module_namespace, + } +} + +fn get_error_module_namespace(error: &ValidationError) -> &str { + get_error_module_namespace_str(error) +} diff --git a/mlf-lang/src/error.rs b/mlf-lang/src/error.rs index 4c5389f..21ce115 100644 --- a/mlf-lang/src/error.rs +++ b/mlf-lang/src/error.rs @@ -12,12 +12,15 @@ pub enum ParseError { #[derive(Debug, Clone, PartialEq)] pub enum ValidationError { - DuplicateDefinition { name: String, first_span: Span, second_span: Span }, - UndefinedReference { name: String, span: Span }, - InvalidConstraint { message: String, span: Span }, - TypeMismatch { expected: String, found: String, span: Span }, - ConstraintTooPermissive { message: String, span: Span }, - ReservedName { name: String, span: Span }, + DuplicateDefinition { name: String, first_span: Span, second_span: Span, module_namespace: String }, + UndefinedReference { name: String, span: Span, module_namespace: String }, + InvalidConstraint { message: String, span: Span, module_namespace: String }, + TypeMismatch { expected: String, found: String, span: Span, module_namespace: String }, + ConstraintTooPermissive { message: String, span: Span, module_namespace: String }, + ReservedName { name: String, span: Span, module_namespace: String }, + AmbiguousMain { name: String, namespace_suffix: String, first_span: Span, second_span: Span, module_namespace: String }, + MultipleMain { name: String, first_span: Span, second_span: Span, module_namespace: String }, + ConflictNotAllowed { name: String, namespace_suffix: String, span: Span, module_namespace: String }, } #[derive(Debug, Clone, Default)] diff --git a/mlf-lang/src/lexer.rs b/mlf-lang/src/lexer.rs index a6cfb80..c2dbdc0 100644 --- a/mlf-lang/src/lexer.rs +++ b/mlf-lang/src/lexer.rs @@ -421,4 +421,30 @@ record foo"#; assert_eq!(tokens[1].token, Token::Record); assert_eq!(tokens[2].token, Token::Ident("foo".into())); } + + #[test] + fn test_raw_identifiers() { + let input = "`record` `type` `string`"; + let tokens = tokenize(input).unwrap(); + assert_eq!(tokens[0].token, Token::Ident("record".into())); + assert_eq!(tokens[1].token, Token::Ident("type".into())); + assert_eq!(tokens[2].token, Token::Ident("string".into())); + } + + #[test] + fn test_raw_identifier_in_field() { + let input = "def type foo = { `record`: string, };"; + let tokens = tokenize(input).unwrap(); + assert_eq!(tokens[0].token, Token::Def); + assert_eq!(tokens[1].token, Token::Type); + assert_eq!(tokens[2].token, Token::Ident("foo".into())); + assert_eq!(tokens[3].token, Token::Equals); + assert_eq!(tokens[4].token, Token::LeftBrace); + assert_eq!(tokens[5].token, Token::Ident("record".into())); // Should be Ident, not Record keyword + assert_eq!(tokens[6].token, Token::Colon); + assert_eq!(tokens[7].token, Token::String); + assert_eq!(tokens[8].token, Token::Comma); + assert_eq!(tokens[9].token, Token::RightBrace); + assert_eq!(tokens[10].token, Token::Semicolon); + } } diff --git a/mlf-lang/src/parser.rs b/mlf-lang/src/parser.rs index 599b7a6..bc91a34 100644 --- a/mlf-lang/src/parser.rs +++ b/mlf-lang/src/parser.rs @@ -100,15 +100,55 @@ impl Parser { Ok(ident) } + fn parse_ident_or_keyword(&mut self) -> Result { + let current = self.current(); + // Path segments can be identifiers or keywords + let name = match ¤t.token { + LexToken::Ident(n) => n.clone(), + LexToken::Record => "record".into(), + LexToken::Token => "token".into(), + LexToken::Inline => "inline".into(), + LexToken::Def => "def".into(), + LexToken::Type => "type".into(), + LexToken::Query => "query".into(), + LexToken::Procedure => "procedure".into(), + LexToken::Subscription => "subscription".into(), + LexToken::Error => "error".into(), + LexToken::Use => "use".into(), + LexToken::As => "as".into(), + LexToken::String => "string".into(), + LexToken::Integer => "integer".into(), + LexToken::Boolean => "boolean".into(), + LexToken::Null => "null".into(), + LexToken::Unknown => "unknown".into(), + LexToken::Constrained => "constrained".into(), + LexToken::Blob => "blob".into(), + LexToken::Bytes => "bytes".into(), + LexToken::Namespace => "namespace".into(), + _ => { + return Err(ParseError::Syntax { + message: alloc::format!("Expected identifier, found {}", current.token), + span: current.span, + }); + } + }; + let ident = Ident { + name, + span: current.span, + }; + self.advance(); + Ok(ident) + } + fn parse_path(&mut self) -> Result { let mut segments = Vec::new(); let start = self.current().span.start; - segments.push(self.parse_ident()?); + segments.push(self.parse_ident_or_keyword()?); while matches!(self.current().token, LexToken::Dot) { self.advance(); - segments.push(self.parse_ident()?); + segments.push(self.parse_ident_or_keyword()?); } let end = segments.last().unwrap().span.end; @@ -530,7 +570,58 @@ impl Parser { let start = self.expect(LexToken::Use)?; let path = self.parse_path()?; - let imports = if matches!(self.current().token, LexToken::As) { + let imports = if matches!(self.current().token, LexToken::LeftBrace) { + // use namespace { items } + self.advance(); // consume { + let mut items = Vec::new(); + + while !matches!(self.current().token, LexToken::RightBrace) { + // Check for 'main' keyword + let current = self.current(); + let name_ident = if let LexToken::Ident(name) = ¤t.token { + if name == "main" { + // Create a special identifier for main + let ident = Ident { + name: "main".into(), + span: current.span, + }; + self.advance(); + ident + } else { + self.parse_ident()? + } + } else { + self.parse_ident()? + }; + + // Check for alias (as keyword) + let alias = if matches!(self.current().token, LexToken::As) { + self.advance(); + Some(self.parse_ident()?) + } else { + None + }; + + items.push(UseItem { + name: name_ident, + alias, + }); + + // Comma is optional before closing brace + if matches!(self.current().token, LexToken::Comma) { + self.advance(); + } else if !matches!(self.current().token, LexToken::RightBrace) { + return Err(ParseError::Syntax { + message: alloc::format!("Expected comma or closing brace, found {}", self.current().token), + span: self.current().span, + }); + } + } + + self.expect(LexToken::RightBrace)?; + UseImports::Items(items) + } else if matches!(self.current().token, LexToken::As) { + // use namespace as alias self.advance(); let alias = self.parse_ident()?; UseImports::Items(alloc::vec![UseItem { @@ -538,6 +629,7 @@ impl Parser { alias: Some(alias), }]) } else { + // use namespace (imports all) UseImports::All }; @@ -1502,4 +1594,121 @@ mod tests { _ => panic!("Expected subscription"), } } + + #[test] + fn test_parse_use_with_main() { + let input = "use com.example.thread { main };"; + let result = parse_lexicon(input); + assert!(result.is_ok()); + let lexicon = result.unwrap(); + assert_eq!(lexicon.items.len(), 1); + match &lexicon.items[0] { + Item::Use(u) => { + assert_eq!(u.path.to_string(), "com.example.thread"); + match &u.imports { + UseImports::Items(items) => { + assert_eq!(items.len(), 1); + assert_eq!(items[0].name.name, "main"); + assert!(items[0].alias.is_none()); + } + _ => panic!("Expected UseImports::Items"), + } + } + _ => panic!("Expected use statement"), + } + } + + #[test] + fn test_parse_use_with_main_and_alias() { + let input = "use com.example.thread { main as ThreadRecord };"; + let result = parse_lexicon(input); + assert!(result.is_ok()); + let lexicon = result.unwrap(); + assert_eq!(lexicon.items.len(), 1); + match &lexicon.items[0] { + Item::Use(u) => { + assert_eq!(u.path.to_string(), "com.example.thread"); + match &u.imports { + UseImports::Items(items) => { + assert_eq!(items.len(), 1); + assert_eq!(items[0].name.name, "main"); + assert_eq!(items[0].alias.as_ref().unwrap().name, "ThreadRecord"); + } + _ => panic!("Expected UseImports::Items"), + } + } + _ => panic!("Expected use statement"), + } + } + + #[test] + fn test_parse_use_with_multiple_items() { + let input = "use com.example { main, foo, bar as Baz };"; + let result = parse_lexicon(input); + assert!(result.is_ok()); + let lexicon = result.unwrap(); + assert_eq!(lexicon.items.len(), 1); + match &lexicon.items[0] { + Item::Use(u) => { + assert_eq!(u.path.to_string(), "com.example"); + match &u.imports { + UseImports::Items(items) => { + assert_eq!(items.len(), 3); + assert_eq!(items[0].name.name, "main"); + assert!(items[0].alias.is_none()); + assert_eq!(items[1].name.name, "foo"); + assert!(items[1].alias.is_none()); + assert_eq!(items[2].name.name, "bar"); + assert_eq!(items[2].alias.as_ref().unwrap().name, "Baz"); + } + _ => panic!("Expected UseImports::Items"), + } + } + _ => panic!("Expected use statement"), + } + } + + #[test] + fn test_parse_keywords_in_paths() { + let input = r#"def type test = { + field1: app.bsky.embed.record, + field2: com.example.query, + field3: com.example.string.type, + };"#; + let result = parse_lexicon(input); + assert!(result.is_ok()); + let lexicon = result.unwrap(); + assert_eq!(lexicon.items.len(), 1); + match &lexicon.items[0] { + Item::DefType(d) => { + assert_eq!(d.name.name, "test"); + match &d.ty { + Type::Object { fields, .. } => { + assert_eq!(fields.len(), 3); + // Verify the paths contain keywords + match &fields[0].ty { + Type::Reference { path, .. } => { + assert_eq!(path.to_string(), "app.bsky.embed.record"); + } + _ => panic!("Expected reference type"), + } + match &fields[1].ty { + Type::Reference { path, .. } => { + assert_eq!(path.to_string(), "com.example.query"); + } + _ => panic!("Expected reference type"), + } + match &fields[2].ty { + Type::Reference { path, .. } => { + assert_eq!(path.to_string(), "com.example.string.type"); + } + _ => panic!("Expected reference type"), + } + } + _ => panic!("Expected object type"), + } + } + _ => panic!("Expected def type"), + } + } } diff --git a/mlf-lang/src/workspace.rs b/mlf-lang/src/workspace.rs index 4b25724..08d1b2a 100644 --- a/mlf-lang/src/workspace.rs +++ b/mlf-lang/src/workspace.rs @@ -54,6 +54,7 @@ impl Workspace { errors.push(ValidationError::InvalidConstraint { message: alloc::format!("Failed to parse prelude: {:?}", e), span: crate::span::Span::new(0, 0), + module_namespace: "prelude".to_string(), }); errors })?; @@ -71,6 +72,7 @@ impl Workspace { errors.push(ValidationError::InvalidConstraint { message: alloc::format!("Failed to parse prelude: {:?}", e), span: crate::span::Span::new(0, 0), + module_namespace: "prelude".to_string(), }); errors })?; @@ -84,20 +86,14 @@ impl Workspace { for file in dir.files() { if let Some(path_str) = file.path().to_str() { if path_str.ends_with(".mlf") && !path_str.ends_with("prelude.mlf") { - // Convert file path to namespace - // e.g., "com/atproto/repo/defs.mlf" -> "com.atproto.repo" - // e.g., "com/atproto/lexicon/schema.mlf" -> "com.atproto.lexicon" - let mut namespace = path_str + // Convert file path to namespace (including filename) + // e.g., "com/atproto/repo/defs.mlf" -> "com.atproto.repo.defs" + // e.g., "com/atproto/lexicon/schema.mlf" -> "com.atproto.lexicon.schema" + let namespace = path_str .strip_suffix(".mlf") .unwrap_or(path_str) .replace('/', "."); - // Strip the filename part - files are just containers, not part of the NSID - // Get the directory path as the namespace - if let Some(last_dot) = namespace.rfind('.') { - namespace = namespace[..last_dot].to_string(); - } - if let Some(contents) = file.contents_utf8() { let lexicon = crate::parser::parse_lexicon(contents) .map_err(|e| { @@ -105,6 +101,7 @@ impl Workspace { errors.push(ValidationError::InvalidConstraint { message: alloc::format!("Failed to parse {} (file: {}): {:?}", namespace, path_str, e), span: crate::span::Span::new(0, 0), + module_namespace: namespace.clone(), }); errors })?; @@ -264,6 +261,7 @@ impl Workspace { errors.push(ValidationError::InvalidConstraint { message: "Union type must have at least one member".into(), span: *span, + module_namespace: namespace.to_string(), }); } @@ -296,11 +294,11 @@ impl Workspace { errors.append(&mut base_errors); } - if let Err(mut constraint_errors) = self.typecheck_constraints(base, constraints, *span) { + if let Err(mut constraint_errors) = self.typecheck_constraints(namespace, base, constraints, *span) { errors.append(&mut constraint_errors); } - if let Err(mut refinement_errors) = self.check_constraint_refinement(base, constraints) { + if let Err(mut refinement_errors) = self.check_constraint_refinement(namespace, base, constraints) { errors.append(&mut refinement_errors); } @@ -313,7 +311,7 @@ impl Workspace { } } - fn typecheck_constraints(&self, base: &Type, constraints: &[Constraint], _span: Span) -> Result<(), ValidationErrors> { + fn typecheck_constraints(&self, namespace: &str, base: &Type, constraints: &[Constraint], _span: Span) -> Result<(), ValidationErrors> { let mut errors = ValidationErrors::new(); let base_kind = self.get_base_primitive(base); @@ -329,6 +327,7 @@ impl Workspace { errors.push(ValidationError::InvalidConstraint { message: alloc::format!("Length constraint can only be applied to string or array types"), span: *span, + module_namespace: namespace.to_string(), }); } } @@ -340,6 +339,7 @@ impl Workspace { errors.push(ValidationError::InvalidConstraint { message: alloc::format!("String constraint on non-string type"), span: *span, + module_namespace: namespace.to_string(), }); } } @@ -349,6 +349,7 @@ impl Workspace { errors.push(ValidationError::InvalidConstraint { message: alloc::format!("Numeric constraint on non-numeric type"), span: *span, + module_namespace: namespace.to_string(), }); } } @@ -358,6 +359,7 @@ impl Workspace { errors.push(ValidationError::InvalidConstraint { message: alloc::format!("Blob constraint on non-blob type"), span: *span, + module_namespace: namespace.to_string(), }); } } @@ -374,7 +376,7 @@ impl Workspace { } } - fn check_constraint_refinement(&self, base: &Type, new_constraints: &[Constraint]) -> Result<(), ValidationErrors> { + fn check_constraint_refinement(&self, namespace: &str, base: &Type, new_constraints: &[Constraint]) -> Result<(), ValidationErrors> { let base_constraints = self.get_base_constraints(base); if base_constraints.is_empty() { @@ -395,6 +397,7 @@ impl Workspace { new_max, base_max ), span: *span, + module_namespace: namespace.to_string(), }); } } @@ -410,6 +413,7 @@ impl Workspace { new_min, base_min ), span: *span, + module_namespace: namespace.to_string(), }); } } @@ -425,6 +429,7 @@ impl Workspace { new_max, base_max ), span: *span, + module_namespace: namespace.to_string(), }); } } @@ -440,6 +445,7 @@ impl Workspace { new_min, base_min ), span: *span, + module_namespace: namespace.to_string(), }); } } @@ -455,6 +461,7 @@ impl Workspace { new_max, base_max ), span: *span, + module_namespace: namespace.to_string(), }); } } @@ -470,6 +477,7 @@ impl Workspace { new_min, base_min ), span: *span, + module_namespace: namespace.to_string(), }); } } @@ -485,6 +493,7 @@ impl Workspace { new_max, base_max ), span: *span, + module_namespace: namespace.to_string(), }); } } @@ -501,6 +510,7 @@ impl Workspace { new_val ), span: *span, + module_namespace: namespace.to_string(), }); } } @@ -580,6 +590,7 @@ impl Workspace { .join("."); let type_name = &path.segments[path.segments.len() - 1].name; + // First try: normal resolution (namespace + type) if let Some(module) = self.modules.get(&target_namespace) { for item in &module.lexicon.items { match item { @@ -593,6 +604,25 @@ impl Workspace { } } } + + // Second try: implicit main resolution + let full_namespace = path.to_string(); + let namespace_suffix = full_namespace.split('.').last().unwrap_or(&full_namespace); + if namespace_suffix == type_name { + if let Some(module) = self.modules.get(&full_namespace) { + for item in &module.lexicon.items { + match item { + Item::InlineType(i) if i.name.name == *type_name => { + return Some(i.ty.clone()); + } + Item::DefType(d) if d.name.name == *type_name => { + return Some(d.ty.clone()); + } + _ => {} + } + } + } + } } None @@ -635,6 +665,71 @@ impl Workspace { false } + /// Resolve a type reference to its actual namespace + /// Returns the namespace where the type is defined, or None if not found + pub fn resolve_reference_namespace(&self, path: &Path, current_namespace: &str) -> Option { + if path.segments.len() == 1 { + let name = &path.segments[0].name; + + // Check current module first + if let Some(module) = self.modules.get(current_namespace) { + // Check if it's a local type + if module.symbols.types.contains_key(name) { + return Some(current_namespace.to_string()); + } + + // Check if it's imported + if let Some(imported) = module.imports.mappings.get(name) { + // Build the full namespace from the original path + // original_path is Vec like ["place", "stream", "key", "key"] + // We want "place.stream.key" (drop the last segment which is the type name) + if imported.original_path.len() > 1 { + let namespace = imported.original_path[..imported.original_path.len() - 1].join("."); + return Some(namespace); + } else { + // Edge case: just a single segment, assume it's in the current namespace + return Some(current_namespace.to_string()); + } + } + } + + // Check prelude + if let Some(module) = self.modules.get("prelude") { + if module.symbols.types.contains_key(name) { + return Some("prelude".to_string()); + } + } + + return None; + } else { + // Multi-segment path: resolve normally + let target_namespace = path.segments[..path.segments.len() - 1] + .iter() + .map(|s| s.name.as_str()) + .collect::>() + .join("."); + let type_name = &path.segments[path.segments.len() - 1].name; + + // First try: normal resolution + if let Some(module) = self.modules.get(&target_namespace) { + if module.symbols.types.contains_key(type_name) { + return Some(target_namespace); + } + } + + // Second try: implicit main resolution + let full_namespace = path.to_string(); + if let Some(module) = self.modules.get(&full_namespace) { + let namespace_suffix = full_namespace.split('.').last().unwrap_or(&full_namespace); + if namespace_suffix == type_name && module.symbols.types.contains_key(type_name) { + return Some(full_namespace); + } + } + + None + } + } + fn resolve_imports(&mut self) -> Result<(), ValidationErrors> { let mut errors = ValidationErrors::new(); @@ -662,25 +757,33 @@ impl Workspace { fn resolve_use_statement(&mut self, current_namespace: &str, use_stmt: &Use) -> Result<(), ValidationErrors> { let mut errors = ValidationErrors::new(); - let (target_namespace, type_name_opt) = match &use_stmt.imports { + // Determine if this is: + // 1. use namespace; (UseImports::All) + // 2. use namespace { items }; (UseImports::Items with path=namespace) + // 3. use namespace.typename as alias; (UseImports::Items with path=namespace.typename, single item) + + let (target_namespace, is_single_type_import) = match &use_stmt.imports { UseImports::All => { - (use_stmt.path.to_string(), None) + // use namespace; + (use_stmt.path.to_string(), false) } - UseImports::Items(_) => { - if use_stmt.path.segments.len() < 2 { - errors.push(ValidationError::UndefinedReference { - name: use_stmt.path.to_string(), - span: use_stmt.path.span, - }); - return Err(errors); + UseImports::Items(items) => { + // Check if this is the old syntax: use namespace.typename as alias + // In this case, path has >=2 segments and items has 1 item whose name matches the last segment + if items.len() == 1 + && use_stmt.path.segments.len() >= 2 + && items[0].name.name == use_stmt.path.segments.last().unwrap().name { + // Old syntax: use namespace.typename as alias + let namespace = use_stmt.path.segments[..use_stmt.path.segments.len() - 1] + .iter() + .map(|s| s.name.as_str()) + .collect::>() + .join("."); + (namespace, true) + } else { + // New syntax: use namespace { items }; + (use_stmt.path.to_string(), false) } - let namespace = use_stmt.path.segments[..use_stmt.path.segments.len() - 1] - .iter() - .map(|s| s.name.as_str()) - .collect::>() - .join("."); - let type_name = use_stmt.path.segments.last().unwrap().name.clone(); - (namespace, Some(type_name)) } }; @@ -688,12 +791,14 @@ impl Workspace { errors.push(ValidationError::UndefinedReference { name: target_namespace.clone(), span: use_stmt.path.span, + module_namespace: current_namespace.to_string(), }); return Err(errors); } let imports_to_add: Vec<(String, ImportedSymbol)> = match &use_stmt.imports { UseImports::All => { + // Import all types from the namespace let target_module = self.modules.get(&target_namespace).unwrap(); target_module.symbols.types.keys() .map(|type_name| { @@ -709,34 +814,67 @@ impl Workspace { .collect() } UseImports::Items(items) => { + // Import specific items from the namespace let target_module = self.modules.get(&target_namespace).unwrap(); let mut imports = Vec::new(); - let type_name = type_name_opt.as_ref().unwrap(); - if !target_module.symbols.types.contains_key(type_name) { - errors.push(ValidationError::UndefinedReference { - name: alloc::format!("{}.{}", target_namespace, type_name), - span: use_stmt.path.span, - }); - } else { - for item in items { - let local_name = if let Some(alias) = &item.alias { - alias.name.clone() - } else { - type_name.clone() - }; + // Get the namespace suffix for main resolution + let namespace_suffix = target_namespace.split('.').last().unwrap_or(&target_namespace); - imports.push(( - local_name.clone(), - ImportedSymbol { - original_path: use_stmt.path.segments.iter() - .map(|s| s.name.clone()) - .collect(), - local_name, - }, - )); + for item in items { + // Determine the actual type name to look up + let type_name = if item.name.name == "main" { + // Special case: "main" keyword resolves to the namespace suffix + namespace_suffix.to_string() + } else { + item.name.name.clone() + }; + + // Check if the type exists in the target module + if !target_module.symbols.types.contains_key(&type_name) { + errors.push(ValidationError::UndefinedReference { + name: alloc::format!("{}.{}", target_namespace, item.name.name), + span: item.name.span, + module_namespace: current_namespace.to_string(), + }); + continue; } + + // Determine the local name (alias or original name) + let local_name = if let Some(alias) = &item.alias { + alias.name.clone() + } else if item.name.name == "main" { + // If importing main without alias, bind to namespace suffix + namespace_suffix.to_string() + } else { + type_name.clone() + }; + + // Build the original_path correctly based on import type + let original_path = if is_single_type_import { + // Old syntax: use namespace.typename as alias + // Path segments already include the type name + use_stmt.path.segments.iter() + .map(|s| s.name.clone()) + .collect() + } else { + // New syntax: use namespace { items }; + // Need to append the type name to the namespace + use_stmt.path.segments.iter() + .map(|s| s.name.clone()) + .chain(core::iter::once(type_name.clone())) + .collect() + }; + + imports.push(( + local_name.clone(), + ImportedSymbol { + original_path, + local_name, + }, + )); } + imports } }; @@ -753,144 +891,281 @@ impl Workspace { } } - fn build_symbol_table(_namespace: &str, lexicon: &Lexicon) -> Result { + fn has_main_annotation(annotations: &[Annotation]) -> bool { + annotations.iter().any(|ann| ann.name.name == "main") + } + + fn is_main_eligible_item(item: &Item) -> bool { + matches!(item, Item::Record(_) | Item::Query(_) | Item::Procedure(_) | Item::Subscription(_) | Item::DefType(_)) + } + + fn build_symbol_table(namespace: &str, lexicon: &Lexicon) -> Result { let mut symbols = SymbolTable { types: BTreeMap::new(), }; let mut errors = ValidationErrors::new(); + // Extract namespace suffix for conflict checking + let namespace_suffix = namespace.split('.').last().unwrap_or(namespace); + + // First pass: group items by name to detect duplicates + let mut items_by_name: BTreeMap> = BTreeMap::new(); + for item in &lexicon.items { - match item { - Item::Record(r) => { - // Check for reserved names - if r.name.name == "main" || r.name.name == "defs" { - errors.push(crate::error::ValidationError::ReservedName { - name: r.name.name.clone(), - span: r.name.span, - }); - } + let name = match item { + Item::Record(r) => Some(r.name.name.as_str()), + Item::InlineType(i) => Some(i.name.name.as_str()), + Item::DefType(d) => Some(d.name.name.as_str()), + Item::Token(t) => Some(t.name.name.as_str()), + Item::Query(q) => Some(q.name.name.as_str()), + Item::Procedure(p) => Some(p.name.name.as_str()), + Item::Subscription(s) => Some(s.name.name.as_str()), + Item::Use(_) => None, + }; - if let Some(existing) = symbols.types.get(&r.name.name) { - errors.push(crate::error::ValidationError::DuplicateDefinition { - name: r.name.name.clone(), - first_span: existing.span(), - second_span: r.name.span, - }); - } else { + if let Some(name) = name { + items_by_name.entry(name.to_string()).or_insert_with(Vec::new).push(item); + } + } + + // Second pass: validate each name group + for (name, items) in items_by_name { + // Check for reserved names + if name == "main" || name == "defs" { + for item in &items { + let span = match item { + Item::Record(r) => r.name.span, + Item::InlineType(i) => i.name.span, + Item::DefType(d) => d.name.span, + Item::Token(t) => t.name.span, + Item::Query(q) => q.name.span, + Item::Procedure(p) => p.name.span, + Item::Subscription(s) => s.name.span, + Item::Use(_) => continue, + }; + errors.push(crate::error::ValidationError::ReservedName { + name: name.clone(), + span, + module_namespace: namespace.to_string(), + }); + } + continue; + } + + if items.len() == 1 { + // No duplicates, just add to symbol table + let item = items[0]; + match item { + Item::Record(r) => { symbols.types.insert( - r.name.name.clone(), + name.clone(), Symbol::Record { - name: r.name.name.clone(), + name: name.clone(), span: r.name.span, }, ); } - } - Item::InlineType(i) => { - // Check for reserved names - if i.name.name == "main" || i.name.name == "defs" { - errors.push(crate::error::ValidationError::ReservedName { - name: i.name.name.clone(), - span: i.name.span, - }); - } - - if let Some(existing) = symbols.types.get(&i.name.name) { - errors.push(crate::error::ValidationError::DuplicateDefinition { - name: i.name.name.clone(), - first_span: existing.span(), - second_span: i.name.span, - }); - } else { + Item::InlineType(i) => { symbols.types.insert( - i.name.name.clone(), + name.clone(), Symbol::Alias { - name: i.name.name.clone(), + name: name.clone(), span: i.name.span, }, ); } - } - Item::DefType(d) => { - // Check for reserved names - if d.name.name == "main" || d.name.name == "defs" { - errors.push(crate::error::ValidationError::ReservedName { - name: d.name.name.clone(), - span: d.name.span, - }); - } - - if let Some(existing) = symbols.types.get(&d.name.name) { - errors.push(crate::error::ValidationError::DuplicateDefinition { - name: d.name.name.clone(), - first_span: existing.span(), - second_span: d.name.span, - }); - } else { + Item::DefType(d) => { symbols.types.insert( - d.name.name.clone(), + name.clone(), Symbol::Alias { - name: d.name.name.clone(), + name: name.clone(), span: d.name.span, }, ); } - } - Item::Token(t) => { - // Check for reserved names - if t.name.name == "main" || t.name.name == "defs" { - errors.push(crate::error::ValidationError::ReservedName { - name: t.name.name.clone(), - span: t.name.span, - }); - } - - if let Some(existing) = symbols.types.get(&t.name.name) { - errors.push(crate::error::ValidationError::DuplicateDefinition { - name: t.name.name.clone(), - first_span: existing.span(), - second_span: t.name.span, - }); - } else { + Item::Token(t) => { symbols.types.insert( - t.name.name.clone(), + name.clone(), Symbol::Token { - name: t.name.name.clone(), + name: name.clone(), span: t.name.span, }, ); } + _ => {} // Queries, procedures, subscriptions not added to symbol table } - Item::Query(q) => { - // Check for reserved names - if q.name.name == "main" || q.name.name == "defs" { - errors.push(crate::error::ValidationError::ReservedName { - name: q.name.name.clone(), - span: q.name.span, - }); + } else { + // Duplicates found - apply @main rules + // Check if name matches namespace suffix + if name != namespace_suffix { + // Conflict not allowed - name doesn't match suffix + for (idx, item) in items.iter().enumerate() { + if idx > 0 { + let span = match item { + Item::Record(r) => r.name.span, + Item::InlineType(i) => i.name.span, + Item::DefType(d) => d.name.span, + Item::Token(t) => t.name.span, + Item::Query(q) => q.name.span, + Item::Procedure(p) => p.name.span, + Item::Subscription(s) => s.name.span, + Item::Use(_) => continue, + }; + errors.push(crate::error::ValidationError::ConflictNotAllowed { + name: name.clone(), + namespace_suffix: namespace_suffix.to_string(), + span, + module_namespace: namespace.to_string(), + }); + } } + continue; } - Item::Procedure(p) => { - // Check for reserved names - if p.name.name == "main" || p.name.name == "defs" { - errors.push(crate::error::ValidationError::ReservedName { - name: p.name.name.clone(), - span: p.name.span, - }); + + // Check if any inline types are involved (not allowed to conflict) + let has_inline = items.iter().any(|item| matches!(item, Item::InlineType(_))); + if has_inline { + for (idx, item) in items.iter().enumerate() { + if idx > 0 { + let span = match item { + Item::Record(r) => r.name.span, + Item::InlineType(i) => i.name.span, + Item::DefType(d) => d.name.span, + Item::Token(t) => t.name.span, + Item::Query(q) => q.name.span, + Item::Procedure(p) => p.name.span, + Item::Subscription(s) => s.name.span, + Item::Use(_) => continue, + }; + errors.push(crate::error::ValidationError::DuplicateDefinition { + name: name.clone(), + first_span: match items[0] { + Item::Record(r) => r.name.span, + Item::InlineType(i) => i.name.span, + Item::DefType(d) => d.name.span, + Item::Token(t) => t.name.span, + Item::Query(q) => q.name.span, + Item::Procedure(p) => p.name.span, + Item::Subscription(s) => s.name.span, + Item::Use(_) => continue, + }, + second_span: span, + module_namespace: namespace.to_string(), + }); + } } + continue; } - Item::Subscription(s) => { - // Check for reserved names - if s.name.name == "main" || s.name.name == "defs" { - errors.push(crate::error::ValidationError::ReservedName { - name: s.name.name.clone(), - span: s.name.span, - }); + + // Check @main annotations + let items_with_main: Vec<&Item> = items.iter() + .filter(|item| { + let annotations = match item { + Item::Record(r) => &r.annotations, + Item::DefType(d) => &d.annotations, + Item::Query(q) => &q.annotations, + Item::Procedure(p) => &p.annotations, + Item::Subscription(s) => &s.annotations, + _ => return false, + }; + Self::has_main_annotation(annotations) + }) + .copied() + .collect(); + + if items_with_main.is_empty() { + // No @main annotation - ambiguous + errors.push(crate::error::ValidationError::AmbiguousMain { + name: name.clone(), + namespace_suffix: namespace_suffix.to_string(), + first_span: match items[0] { + Item::Record(r) => r.name.span, + Item::DefType(d) => d.name.span, + Item::Query(q) => q.name.span, + Item::Procedure(p) => p.name.span, + Item::Subscription(s) => s.name.span, + _ => continue, + }, + second_span: match items[1] { + Item::Record(r) => r.name.span, + Item::DefType(d) => d.name.span, + Item::Query(q) => q.name.span, + Item::Procedure(p) => p.name.span, + Item::Subscription(s) => s.name.span, + _ => continue, + }, + module_namespace: namespace.to_string(), + }); + } else if items_with_main.len() > 1 { + // Multiple @main annotations + errors.push(crate::error::ValidationError::MultipleMain { + name: name.clone(), + first_span: match items_with_main[0] { + Item::Record(r) => r.name.span, + Item::DefType(d) => d.name.span, + Item::Query(q) => q.name.span, + Item::Procedure(p) => p.name.span, + Item::Subscription(s) => s.name.span, + _ => continue, + }, + second_span: match items_with_main[1] { + Item::Record(r) => r.name.span, + Item::DefType(d) => d.name.span, + Item::Query(q) => q.name.span, + Item::Procedure(p) => p.name.span, + Item::Subscription(s) => s.name.span, + _ => continue, + }, + module_namespace: namespace.to_string(), + }); + } else { + // Exactly one @main - valid conflict + // Add the non-main item to symbol table (only defs/aliases) + for item in &items { + let has_main = match item { + Item::Record(r) => Self::has_main_annotation(&r.annotations), + Item::DefType(d) => Self::has_main_annotation(&d.annotations), + Item::Query(q) => Self::has_main_annotation(&q.annotations), + Item::Procedure(p) => Self::has_main_annotation(&p.annotations), + Item::Subscription(s) => Self::has_main_annotation(&s.annotations), + _ => false, + }; + + if !has_main { + // Add non-main item to symbol table + match item { + Item::DefType(d) => { + symbols.types.insert( + name.clone(), + Symbol::Alias { + name: name.clone(), + span: d.name.span, + }, + ); + } + Item::Record(r) => { + symbols.types.insert( + name.clone(), + Symbol::Record { + name: name.clone(), + span: r.name.span, + }, + ); + } + Item::Token(t) => { + symbols.types.insert( + name.clone(), + Symbol::Token { + name: name.clone(), + span: t.name.span, + }, + ); + } + _ => {} + } + } } } - Item::Use(_) => { - // Handled separately - } } } @@ -1109,6 +1384,7 @@ impl Workspace { errors.push(crate::error::ValidationError::UndefinedReference { name: name.clone(), span, + module_namespace: current_namespace.to_string(), }); return Err(errors); } @@ -1120,16 +1396,29 @@ impl Workspace { .join("."); let type_name = &path.segments[path.segments.len() - 1].name; + // First try: normal resolution (namespace + type) if let Some(module) = self.modules.get(&target_namespace) { if module.symbols.types.contains_key(type_name) { return Ok(()); } } + // Second try: implicit main resolution + // If com.atproto.repo.strongRef fails, try treating the full path as a namespace + // and look for a type named "strongRef" (matching the namespace suffix) + let full_namespace = &full_path; + if let Some(module) = self.modules.get(full_namespace) { + let namespace_suffix = full_namespace.split('.').last().unwrap_or(full_namespace); + if namespace_suffix == type_name && module.symbols.types.contains_key(type_name) { + return Ok(()); + } + } + let mut errors = ValidationErrors::new(); errors.push(crate::error::ValidationError::UndefinedReference { name: full_path, span, + module_namespace: current_namespace.to_string(), }); Err(errors) } @@ -1604,4 +1893,126 @@ mod tests { // Check that std modules are loaded assert!(ws.modules.len() > 1, "Should have more than just prelude"); } + + #[test] + fn test_main_annotation_valid_conflict() { + let mut ws = Workspace::new(); + + let input = r#" + @main + record thread { + title!: string, + } + + def type thread = { + id!: string, + }; + "#; + let lexicon = parse_lexicon(input).unwrap(); + let result = ws.add_module("com.example.thread".into(), lexicon); + assert!(result.is_ok()); + } + + #[test] + fn test_main_annotation_ambiguous() { + let mut ws = Workspace::new(); + + let input = r#" + record thread { + title!: string, + } + + def type thread = { + id!: string, + }; + "#; + let lexicon = parse_lexicon(input).unwrap(); + let result = ws.add_module("com.example.thread".into(), lexicon); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors.errors.iter().any(|e| matches!(e, ValidationError::AmbiguousMain { .. }))); + } + + #[test] + fn test_main_annotation_multiple() { + let mut ws = Workspace::new(); + + let input = r#" + @main + record thread { + title!: string, + } + + @main + def type thread = { + id!: string, + }; + "#; + let lexicon = parse_lexicon(input).unwrap(); + let result = ws.add_module("com.example.thread".into(), lexicon); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors.errors.iter().any(|e| matches!(e, ValidationError::MultipleMain { .. }))); + } + + #[test] + fn test_main_annotation_conflict_not_allowed() { + let mut ws = Workspace::new(); + + let input = r#" + @main + record post { + text!: string, + } + + def type post = { + id!: string, + }; + "#; + let lexicon = parse_lexicon(input).unwrap(); + let result = ws.add_module("com.example.thread".into(), lexicon); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors.errors.iter().any(|e| matches!(e, ValidationError::ConflictNotAllowed { .. }))); + } + + #[test] + fn test_main_annotation_inline_type_conflict() { + let mut ws = Workspace::new(); + + let input = r#" + inline type thread = string; + + def type thread = { + id!: string, + }; + "#; + let lexicon = parse_lexicon(input).unwrap(); + let result = ws.add_module("com.example.thread".into(), lexicon); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors.errors.iter().any(|e| matches!(e, ValidationError::DuplicateDefinition { .. }))); + } + + #[test] + fn test_main_annotation_query_valid() { + let mut ws = Workspace::new(); + + let input = r#" + @main + query getThread(id!: string): threadData; + + def type thread = { + id!: string, + title!: string, + }; + + def type threadData = { + thread!: thread, + }; + "#; + let lexicon = parse_lexicon(input).unwrap(); + let result = ws.add_module("com.example.thread".into(), lexicon); + assert!(result.is_ok()); + } } diff --git a/std/com/atproto/admin/defs.mlf b/std/com/atproto/admin/defs.mlf index 6a948b7..7ccc336 100644 --- a/std/com/atproto/admin/defs.mlf +++ b/std/com/atproto/admin/defs.mlf @@ -11,8 +11,8 @@ def type accountView = { email: string, relatedRecords: [unknown], indexedAt!: Datetime, - invitedBy: com.atproto.server.inviteCode, - invites: [com.atproto.server.inviteCode], + invitedBy: com.atproto.server.defs.inviteCode, + invites: [com.atproto.server.defs.inviteCode], invitesDisabled: boolean, emailConfirmedAt: Datetime, inviteNote: string, diff --git a/tree-sitter-mlf/grammar.js b/tree-sitter-mlf/grammar.js index 0edcca1..e9d16d0 100644 --- a/tree-sitter-mlf/grammar.js +++ b/tree-sitter-mlf/grammar.js @@ -17,7 +17,8 @@ module.exports = grammar({ ], conflicts: $ => [ - [$.type, $.union_type] + [$.type, $.union_type], + [$.type_path] ], rules: { @@ -42,9 +43,29 @@ module.exports = grammar({ use_statement: $ => seq( 'use', field('path', $.type_path), + optional(choice( + seq('as', field('alias', $.identifier)), + seq('.', '*'), + seq('.', field('imports', $.import_block)) + )), ';' ), + import_block: $ => seq( + '{', + optional(seq( + $.import_item, + repeat(seq(',', $.import_item)), + optional(',') + )), + '}' + ), + + import_item: $ => seq( + field('name', choice('main', $.identifier)), + optional(seq('as', field('alias', $.identifier))) + ), + // Record definition record_definition: $ => seq( 'record', diff --git a/tree-sitter-mlf/queries/highlights.scm b/tree-sitter-mlf/queries/highlights.scm index fe55fa0..ead1ec2 100644 --- a/tree-sitter-mlf/queries/highlights.scm +++ b/tree-sitter-mlf/queries/highlights.scm @@ -1,6 +1,8 @@ ; Keywords [ "use" + "as" + "main" "record" "def" "inline" @@ -28,6 +30,17 @@ (reference_type (type_path) @type) +; Import paths +(use_statement + path: (type_path) @namespace) + +; Import items +(import_item + name: (identifier) @type) + +(import_item + alias: (identifier) @type) + ; Function/method names (query_definition name: (identifier) @function) @@ -79,7 +92,7 @@ ":" "=" "|" - "?" + "*" ] @operator ; Delimiters diff --git a/tree-sitter-mlf/src/grammar.json b/tree-sitter-mlf/src/grammar.json index 5b62441..3189030 100644 --- a/tree-sitter-mlf/src/grammar.json +++ b/tree-sitter-mlf/src/grammar.json @@ -92,12 +92,177 @@ "name": "type_path" } }, + { + "type": "CHOICE", + "members": [ + { + "type": "CHOICE", + "members": [ + { + "type": "SEQ", + "members": [ + { + "type": "STRING", + "value": "as" + }, + { + "type": "FIELD", + "name": "alias", + "content": { + "type": "SYMBOL", + "name": "identifier" + } + } + ] + }, + { + "type": "SEQ", + "members": [ + { + "type": "STRING", + "value": "." + }, + { + "type": "STRING", + "value": "*" + } + ] + }, + { + "type": "SEQ", + "members": [ + { + "type": "STRING", + "value": "." + }, + { + "type": "FIELD", + "name": "imports", + "content": { + "type": "SYMBOL", + "name": "import_block" + } + } + ] + } + ] + }, + { + "type": "BLANK" + } + ] + }, { "type": "STRING", "value": ";" } ] }, + "import_block": { + "type": "SEQ", + "members": [ + { + "type": "STRING", + "value": "{" + }, + { + "type": "CHOICE", + "members": [ + { + "type": "SEQ", + "members": [ + { + "type": "SYMBOL", + "name": "import_item" + }, + { + "type": "REPEAT", + "content": { + "type": "SEQ", + "members": [ + { + "type": "STRING", + "value": "," + }, + { + "type": "SYMBOL", + "name": "import_item" + } + ] + } + }, + { + "type": "CHOICE", + "members": [ + { + "type": "STRING", + "value": "," + }, + { + "type": "BLANK" + } + ] + } + ] + }, + { + "type": "BLANK" + } + ] + }, + { + "type": "STRING", + "value": "}" + } + ] + }, + "import_item": { + "type": "SEQ", + "members": [ + { + "type": "FIELD", + "name": "name", + "content": { + "type": "CHOICE", + "members": [ + { + "type": "STRING", + "value": "main" + }, + { + "type": "SYMBOL", + "name": "identifier" + } + ] + } + }, + { + "type": "CHOICE", + "members": [ + { + "type": "SEQ", + "members": [ + { + "type": "STRING", + "value": "as" + }, + { + "type": "FIELD", + "name": "alias", + "content": { + "type": "SYMBOL", + "name": "identifier" + } + } + ] + }, + { + "type": "BLANK" + } + ] + } + ] + }, "record_definition": { "type": "SEQ", "members": [ @@ -1009,6 +1174,9 @@ [ "type", "union_type" + ], + [ + "type_path" ] ], "precedences": [], diff --git a/tree-sitter-mlf/src/node-types.json b/tree-sitter-mlf/src/node-types.json index 1ec5ce2..57b0fa3 100644 --- a/tree-sitter-mlf/src/node-types.json +++ b/tree-sitter-mlf/src/node-types.json @@ -222,6 +222,51 @@ "named": true, "fields": {} }, + { + "type": "import_block", + "named": true, + "fields": {}, + "children": { + "multiple": true, + "required": false, + "types": [ + { + "type": "import_item", + "named": true + } + ] + } + }, + { + "type": "import_item", + "named": true, + "fields": { + "alias": { + "multiple": false, + "required": false, + "types": [ + { + "type": "identifier", + "named": true + } + ] + }, + "name": { + "multiple": false, + "required": true, + "types": [ + { + "type": "identifier", + "named": true + }, + { + "type": "main", + "named": false + } + ] + } + } + }, { "type": "inline_type_definition", "named": true, @@ -650,6 +695,26 @@ "type": "use_statement", "named": true, "fields": { + "alias": { + "multiple": false, + "required": false, + "types": [ + { + "type": "identifier", + "named": true + } + ] + }, + "imports": { + "multiple": false, + "required": false, + "types": [ + { + "type": "import_block", + "named": true + } + ] + }, "path": { "multiple": false, "required": true, @@ -674,6 +739,10 @@ "type": ")", "named": false }, + { + "type": "*", + "named": false + }, { "type": ",", "named": false @@ -706,6 +775,10 @@ "type": "`", "named": false }, + { + "type": "as", + "named": false + }, { "type": "blob", "named": false @@ -750,6 +823,10 @@ "type": "integer", "named": false }, + { + "type": "main", + "named": false + }, { "type": "null", "named": false @@ -772,11 +849,11 @@ }, { "type": "string", - "named": false + "named": true }, { "type": "string", - "named": true + "named": false }, { "type": "subscription", diff --git a/tree-sitter-mlf/test.mlf b/tree-sitter-mlf/test.mlf index 7f31959..10fd6c7 100644 --- a/tree-sitter-mlf/test.mlf +++ b/tree-sitter-mlf/test.mlf @@ -1,3 +1,14 @@ +// Test imports - new syntax with dot +use com.example.forum.profile; +use com.example.thread.{ main }; +use com.example.types.{ author, postRef }; +use com.example.post.{ main as Post }; +use com.example.user.{ main as User, userMeta }; + +// Test wildcard and alias +use com.example.forum.*; +use com.example.types as Types; + /// A simple post record record post { /// Post text @@ -7,4 +18,6 @@ record post { }, /// Creation timestamp createdAt!: Datetime, + author: profile, + thread: Post, } diff --git a/website/content/docs/cli/02-configuration.md b/website/content/docs/cli/02-configuration.md index 26cf8c8..d359f57 100644 --- a/website/content/docs/cli/02-configuration.md +++ b/website/content/docs/cli/02-configuration.md @@ -41,7 +41,9 @@ directory = "./lexicons" This directory is used by: - `mlf check` (when run without arguments) -- `mlf generate` (when run without arguments) +- `mlf generate` commands (when run without `--input` or `--root`) + +The source directory also serves as the default **root** for namespace calculation. For example, a file at `./lexicons/com/example/thread.mlf` will have the namespace `com.example.thread`. ### Output Configurations @@ -63,6 +65,10 @@ directory = "./pkg/lexicons" [[output]] type = "rust" directory = "./src/lexicons" + +[[output]] +type = "mlf" +directory = "./converted" ``` **Supported types:** @@ -70,6 +76,7 @@ directory = "./src/lexicons" - `"typescript"` - Generate TypeScript types - `"go"` - Generate Go structs - `"rust"` - Generate Rust structs +- `"mlf"` - Convert JSON lexicons to MLF When you run `mlf generate` without arguments, it will generate all configured outputs. @@ -96,7 +103,13 @@ When run without arguments, checks all files in the source directory: ```bash mlf check -# Equivalent to: mlf check "./lexicons/**/*.mlf" +# Uses: input=./lexicons, root=./lexicons +``` + +You can override with explicit arguments: + +```bash +mlf check ./custom-lexicons --root ./custom-lexicons ``` ### `mlf generate` @@ -108,6 +121,45 @@ mlf generate # Runs all [[output]] configurations ``` +### `mlf generate lexicon` + +When run without arguments, uses configuration defaults: + +```bash +mlf generate lexicon +# Uses: input=./lexicons, output=first lexicon output, root=./lexicons +``` + +Override with explicit arguments: + +```bash +mlf generate lexicon -i ./src -o ./dist --root ./src +``` + +### `mlf generate code` + +When run without arguments, uses configuration defaults: + +```bash +mlf generate code +# Uses: generator=first non-lexicon output, input=./lexicons, output=matching directory +``` + +Override with explicit arguments: + +```bash +mlf generate code -g typescript -i ./src -o ./types --root ./src +``` + +### `mlf generate mlf` + +When run without `--output`, uses configuration defaults: + +```bash +mlf generate mlf -i external.json +# Uses: output=first mlf output +``` + ### `mlf fetch` When run without arguments, fetches all dependencies: @@ -124,6 +176,31 @@ mlf fetch stream.place --save # Downloads lexicons AND adds to dependencies array ``` +## Understanding Namespace Calculation + +The `source.directory` in `mlf.toml` acts as the **root** for namespace calculation. File paths relative to this root become the namespace. + +**Example:** + +```toml +[source] +directory = "./lexicons" +``` + +| File Path | Namespace | +|-----------|-----------| +| `./lexicons/com/example/thread.mlf` | `com.example.thread` | +| `./lexicons/com/example/types/post.mlf` | `com.example.types.post` | +| `./lexicons/app/bsky/feed/post.mlf` | `app.bsky.feed.post` | + +If your files are in a different location, use the `--root` flag: + +```bash +mlf generate lexicon -i ./src/schemas -o ./dist --root ./src/schemas +``` + +Now `./src/schemas/com/example/thread.mlf` → namespace `com.example.thread` + ## Complete Example Here's a complete `mlf.toml` for a TypeScript project using ATProto lexicons: @@ -197,8 +274,9 @@ The `.mlf` directory is automatically added to `.gitignore`, so fetched lexicons 1. **Commit `mlf.toml`** - Version control your configuration 2. **Don't commit `.mlf/`** - Let each developer fetch dependencies 3. **Use semantic namespaces** - Organize lexicons by domain -4. **Multiple outputs** - Generate both lexicons and code simultaneously -5. **CI/CD integration** - Run `mlf check` in your CI pipeline +4. **Set consistent root** - Keep your source directory as the root for namespace calculation +5. **Multiple outputs** - Generate both lexicons and code simultaneously +6. **CI/CD integration** - Run `mlf check` in your CI pipeline ## Override Configuration @@ -206,11 +284,34 @@ You can always override configuration with explicit arguments: ```bash # Override source directory -mlf check "./other-lexicons/**/*.mlf" +mlf check ./other-lexicons --root ./other-lexicons # Override output -mlf generate lexicon -i custom.mlf -o ./custom-output/ +mlf generate lexicon -i custom.mlf -o ./custom-output/ --root ./ + +# Override generator +mlf generate code -g go -i ./lexicons -o ./go-types --root ./lexicons # Fetch specific namespace (ignoring dependencies list) mlf fetch stream.place ``` + +## Multiple Projects + +If you have multiple MLF projects, each can have its own `mlf.toml`: + +``` +my-app/ +├── mlf.toml +├── lexicons/ +│ └── com/example/app/... +└── dist/ + +shared-lexicons/ +├── mlf.toml +├── lexicons/ +│ └── com/example/shared/... +└── dist/ +``` + +Commands always use the `mlf.toml` in the current directory or nearest parent directory. diff --git a/website/content/docs/cli/04-check.md b/website/content/docs/cli/04-check.md index 2930004..9086e40 100644 --- a/website/content/docs/cli/04-check.md +++ b/website/content/docs/cli/04-check.md @@ -168,20 +168,6 @@ The check command automatically loads lexicons from `.mlf/lexicons/mlf/` if they **Solution:** Correct the field type to match the schema -## Integration with CI/CD - -Use `mlf check` in your continuous integration pipeline: - -```yaml -# GitHub Actions example -- name: Validate MLF Lexicons - run: | - mlf fetch - mlf check -``` - -This ensures all lexicons remain valid as your project evolves. - ## Tips 1. **Use configuration** - Set up `mlf.toml` to avoid typing paths repeatedly diff --git a/website/content/docs/cli/06-generate.md b/website/content/docs/cli/06-generate.md index f78418e..37598c8 100644 --- a/website/content/docs/cli/06-generate.md +++ b/website/content/docs/cli/06-generate.md @@ -13,20 +13,25 @@ The `mlf generate` command converts MLF files to various output formats includin mlf generate # Generate JSON lexicons -mlf generate lexicon -i -o +mlf generate lexicon [OPTIONS] # Generate code in a specific language -mlf generate code -g -i -o +mlf generate code [OPTIONS] # Convert JSON lexicons to MLF -mlf generate mlf -i -o +mlf generate mlf -i [OPTIONS] ``` +All generate commands can use defaults from `mlf.toml`, making them easier to run without arguments. + ## Generate All Outputs When run without a subcommand, `mlf generate` uses your `mlf.toml` configuration to generate all specified outputs: ```toml +[source] +directory = "./lexicons" + [[output]] type = "lexicon" directory = "./dist/lexicons" @@ -67,32 +72,68 @@ Generated: ./src/types/com/example/thread.ts Generate ATProto JSON lexicons from MLF files. ```bash -mlf generate lexicon -i -o [OPTIONS] +mlf generate lexicon [OPTIONS] ``` **Options:** -- `-i, --input ` - Input MLF files (glob patterns supported, can be specified multiple times) -- `-o, --output ` - Output directory (required) +- `-i, --input ` - Input MLF file(s) or directory (defaults to `source.directory` from mlf.toml) +- `-o, --output ` - Output directory (defaults to first `type = "lexicon"` output from mlf.toml) +- `--root ` - Root directory for namespace calculation (defaults to `source.directory` from mlf.toml) - `--flat` - Use flat file structure (e.g., `com.example.thread.json`) -**Examples:** +### Using mlf.toml Defaults + +With this configuration: + +```toml +[source] +directory = "./lexicons" + +[[output]] +type = "lexicon" +directory = "./dist/lexicons" +``` + +You can run: + +```bash +mlf generate lexicon +# Uses: input=./lexicons, output=./dist/lexicons, root=./lexicons +``` + +### Explicit Arguments + +You can override defaults with explicit arguments: ```bash # Generate with folder structure -mlf generate lexicon -i thread.mlf -o lexicons/ +mlf generate lexicon -i thread.mlf -o lexicons/ --root ./ # Creates: lexicons/com/example/thread.json # Generate with flat structure -mlf generate lexicon -i thread.mlf -o lexicons/ --flat +mlf generate lexicon -i thread.mlf -o lexicons/ --root ./ --flat # Creates: lexicons/com.example.thread.json -# Generate from multiple files -mlf generate lexicon -i thread.mlf -i reply.mlf -o lexicons/ +# Generate from directory +mlf generate lexicon -i ./src/lexicons -o dist/lexicons/ --root ./src/lexicons +``` + +### Understanding --root + +The `--root` flag tells MLF where to calculate namespaces from. For example: + +```bash +# File: ./lexicons/com/example/thread.mlf +mlf generate lexicon -i ./lexicons -o ./dist --root ./lexicons +# Namespace: com.example.thread (relative to ./lexicons) -# Generate from glob pattern -mlf generate lexicon -i "src/**/*.mlf" -o dist/lexicons/ +# File: ./src/lexicons/com/example/thread.mlf +mlf generate lexicon -i ./src/lexicons -o ./dist --root ./src/lexicons +# Namespace: com.example.thread (relative to ./src/lexicons) ``` +Without `--root`, it defaults to the `source.directory` from mlf.toml or the current directory. + --- ## Generate Code @@ -100,28 +141,61 @@ mlf generate lexicon -i "src/**/*.mlf" -o dist/lexicons/ Generate code in various programming languages from MLF files. ```bash -mlf generate code -g -i -o [OPTIONS] +mlf generate code [OPTIONS] ``` **Options:** -- `-g, --generator ` - Generator to use (required): `json`, `typescript`, `go`, or `rust` -- `-i, --input ` - Input MLF files (glob patterns supported, can be specified multiple times) -- `-o, --output ` - Output directory (required) +- `-g, --generator ` - Generator to use (defaults to first non-lexicon output from mlf.toml) +- `-i, --input ` - Input MLF file(s) or directory (defaults to `source.directory` from mlf.toml) +- `-o, --output ` - Output directory (defaults to matching output from mlf.toml) +- `--root ` - Root directory for namespace calculation (defaults to `source.directory` from mlf.toml) - `--flat` - Use flat file structure **Available Generators:** | Generator | Output | Features | |-----------|--------|----------| -| `json` | `.json` | AT Protocol JSON lexicons (always available) | | `typescript` | `.ts` | TypeScript interfaces with JSDoc, optional fields with `?` | | `go` | `.go` | Go structs with JSON tags, proper capitalization | | `rust` | `.rs` | Rust structs with serde, `Option` for optional fields | +### Using mlf.toml Defaults + +With this configuration: + +```toml +[source] +directory = "./lexicons" + +[[output]] +type = "typescript" +directory = "./src/types" +``` + +You can run: + +```bash +mlf generate code +# Uses: generator=typescript, input=./lexicons, output=./src/types +``` + +### Explicit Arguments + +```bash +# Generate TypeScript +mlf generate code -g typescript -i thread.mlf -o src/types/ --root ./ + +# Generate Go +mlf generate code -g go -i ./lexicons -o pkg/models/ --root ./lexicons + +# Generate Rust with flat structure +mlf generate code -g rust -i ./src -o ./generated --root ./src --flat +``` + ### TypeScript Example ```bash -mlf generate code -g typescript -i thread.mlf -o src/types/ +mlf generate code -g typescript -i thread.mlf -o src/types/ --root ./ ``` **Input MLF:** @@ -160,7 +234,7 @@ export interface Main { ### Go Example ```bash -mlf generate code -g go -i thread.mlf -o pkg/models/ +mlf generate code -g go -i thread.mlf -o pkg/models/ --root ./ ``` **Generated Go:** @@ -184,7 +258,7 @@ type Main struct { ### Rust Example ```bash -mlf generate code -g rust -i thread.mlf -o src/models/ +mlf generate code -g rust -i thread.mlf -o src/models/ --root ./ ``` **Generated Rust:** @@ -215,14 +289,31 @@ pub struct Main { Convert ATProto JSON lexicons back to MLF format. ```bash -mlf generate mlf -i -o +mlf generate mlf -i [OPTIONS] ``` **Options:** -- `-i, --input ` - Input JSON lexicon files (glob patterns supported, can be specified multiple times) -- `-o, --output ` - Output directory (required) +- `-i, --input ` - Input JSON lexicon files (required, can be specified multiple times) +- `-o, --output ` - Output directory (defaults to first `type = "mlf"` output from mlf.toml) + +### Using mlf.toml Defaults + +With this configuration: + +```toml +[[output]] +type = "mlf" +directory = "./converted" +``` + +You can run: + +```bash +mlf generate mlf -i external-lexicon.json +# Uses: output=./converted +``` -**Examples:** +### Examples ```bash # Convert a single JSON lexicon to MLF @@ -232,8 +323,8 @@ mlf generate mlf -i com.example.thread.json -o ./lexicons/ # Convert multiple JSON lexicons mlf generate mlf -i lexicon1.json -i lexicon2.json -o ./mlf/ -# Convert using glob pattern -mlf generate mlf -i "dist/lexicons/**/*.json" -o ./src/ +# Convert from directory +mlf generate mlf -i dist/lexicons/com/example/thread.json -o ./src/ ``` **Features:** @@ -275,12 +366,13 @@ mlf generate mlf -i "dist/lexicons/**/*.json" -o ./src/ **Generated MLF:** ```mlf -record main { +@main +record thread { title!: string constrained { maxLength: 200, }, createdAt!: Datetime, -}; +} ``` --- @@ -293,7 +385,7 @@ All generators preserve documentation comments from MLF: ```mlf /// This is a user profile -def Profile = { +def type Profile = { /// The user's display name displayName: string, }; @@ -333,10 +425,45 @@ MLF types are mapped to appropriate language types: --- +## Multiple Output Targets + +You can configure multiple generators in `mlf.toml`: + +```toml +[source] +directory = "./lexicons" + +[[output]] +type = "lexicon" +directory = "./dist/lexicons" + +[[output]] +type = "typescript" +directory = "./src/types" + +[[output]] +type = "go" +directory = "./pkg/lexicons" + +[[output]] +type = "rust" +directory = "./rust-client/src/lexicons" +``` + +Then run: + +```bash +mlf generate +``` + +This generates all four output types from the same MLF source files. + +--- + ## Tips -1. **Use configuration** - Set up `mlf.toml` for multi-output generation -2. **Commit generated code** - If it's part of your build artifacts +1. **Use configuration** - Set up `mlf.toml` to avoid repetitive arguments +2. **Set explicit root** - Use `--root` when your file structure doesn't match your namespaces 3. **Regenerate often** - Run `mlf generate` after any lexicon changes 4. **Use flat mode** - For simpler directory structures 5. **Multiple generators** - Generate multiple languages from the same MLF files diff --git a/website/content/docs/language-guide/08-imports.md b/website/content/docs/language-guide/08-imports.md index 92c56e5..a5599eb 100644 --- a/website/content/docs/language-guide/08-imports.md +++ b/website/content/docs/language-guide/08-imports.md @@ -7,7 +7,7 @@ As your schemas grow, you'll want to split them across multiple files and reuse ## Basic Import -Import a definition from another file: +Import a specific definition from another file: ```mlf use com.example.forum.profile; @@ -17,16 +17,23 @@ record post { } ``` -This imports the `profile` record from `com/example/forum/profile.mlf`. +This imports the `profile` definition from `com/example/forum/profile.mlf`. ## How Imports Work -The namespace matches the file path: +The import path consists of the file's namespace plus the definition name: -| File Path | Namespace | Import Statement | +| File Path | Definition | Import Statement | |-----------|-----------|------------------| -| `com/example/forum/user.mlf` | `com.example.forum.user` | `use com.example.forum.user;` | -| `com/example/forum/post.mlf` | `com.example.forum.post` | `use com.example.forum.post;` | +| `com/example/forum/user.mlf` | `record user { ... }` | `use com.example.forum.user;` | +| `com/example/forum/post.mlf` | `def type postMeta = { ... }` | `use com.example.forum.post.postMeta;` | + +**Key point:** You import specific definitions by their full path: `namespace.definitionName` + +For example: +- File `com/example/forum/post.mlf` has namespace `com.example.forum.post` +- To import `postMeta` from that file: `use com.example.forum.post.postMeta;` +- Or using the main definition: `use com.example.forum.post;` (imports the record named `post`) ## What Can Be Imported @@ -61,12 +68,10 @@ record comment { ## Multiple Imports -Import multiple definitions with separate `use` statements: +Import multiple definitions from the same namespace with a single statement using `.{ }`: ```mlf -use com.example.forum.author; -use com.example.forum.timestamp; -use com.example.forum.location; +use com.example.forum.{ author, timestamp, location }; record post { author: author, @@ -75,6 +80,198 @@ record post { } ``` +This works for any namespace, regardless of how many levels deep: + +```mlf +use com.example.forum.types.{ author, postRef }; + +record comment { + author: author, + replyTo: postRef, +} +``` + +Or import them separately: + +```mlf +use com.example.forum.author; +use com.example.forum.timestamp; +use com.example.forum.location; +``` + +## Renaming Imports + +Sometimes you need to rename an imported definition to avoid conflicts or improve clarity. Use the `as` keyword: + +```mlf +use com.example.types.author as ForumAuthor; +use com.social.types.author as SocialAuthor; + +record crossPost { + forumAuthor: ForumAuthor, + socialAuthor: SocialAuthor, +} +``` + +This is useful when: +- Two imports have the same name +- You want a shorter or clearer name +- You're dealing with naming conflicts + +You can rename multiple imports at once: + +```mlf +use com.example.types.{ author as ForumAuthor, postRef as PostReference }; + +record crossPost { + author: ForumAuthor, + ref: PostReference, +} +``` + +You can also rename when there's a local definition with the same name: + +```mlf +// Local definition +def type thread = { + localId!: string, +} + +// Import with rename to avoid conflict +use com.example.types.thread as ExternalThread; + +record post { + localThread: thread, // Local def + externalThread: ExternalThread, // Imported type +} +``` + +## Importing Main Definitions + +Every MLF file has a "main" definition - the primary export. You can import it using just the file's namespace: + +```mlf +use com.example.thread; // Imports the main definition, bound as "thread" +``` + +This is shorthand for `use com.example.thread.{ main }`. + +**Note:** The `@main` annotation is only needed when there's a naming conflict (see [Important Info](/docs/language-guide/important-info/#the-main-annotation)). Otherwise, the main definition is determined automatically. + +### Example + +**File: `com/example/thread.mlf`** +```mlf +/// The main thread record +record thread { + title!: string, + body!: string, +} + +/// Thread metadata +def type threadMeta = { + id!: string, + viewCount!: integer, +} +``` + +**File: `com/example/post.mlf`** +```mlf +// Import the main thread record +use com.example.thread; + +// Import the threadMeta definition +use com.example.thread.threadMeta; + +record post { + thread: thread, // The main record + meta: threadMeta, // The def type + text!: string, +} +``` + +### Explicit Main Import + +You can also explicitly import main definitions using `.{ }`: + +```mlf +use com.example.thread.{ main }; + +record post { + thread: thread, // Bound as "thread" (the last segment of the namespace) +} +``` + +### Importing and Renaming Main + +You can rename the main definition when importing: + +```mlf +use com.example.thread.{ main as Thread }; + +record post { + thread: Thread, +} +``` + +Or import both main and other definitions together: + +```mlf +use com.example.thread.{ main as Thread, threadMeta as Meta }; + +record post { + thread: Thread, + meta: Meta, +} +``` + +## Wildcard Imports + +Import all definitions from a namespace with `.*`: + +```mlf +use com.example.forum.*; + +record post { + author: author, // All definitions from com.example.forum + postRef: postRef, // are now available +} +``` + +**Note:** Wildcard imports bring all public definitions into scope, which can lead to naming conflicts. Use with caution. + +## Namespace Aliasing + +Alias an entire namespace for shorter references: + +```mlf +use com.example.forum as Forum; + +record post { + author: Forum.author, + ref: Forum.postRef, +} +``` + +This is useful for: +- Avoiding naming conflicts +- Shortening long namespace paths +- Making code more readable + +## Import Syntax Summary + +Here's what MLF supports for imports: + +| Import Type | Syntax Example | +|-------------|----------------| +| **Single import** | `use com.example.forum.profile;` | +| **Multiple imports** | `use com.example.forum.{ author, postRef };` | +| **Main import** | `use com.example.thread;` or `use com.example.thread.{ main };` | +| **With renaming** | `use com.example.post.{ main as Post };` | +| **Mixed imports** | `use com.example.thread.{ main as Thread, threadMeta };` | +| **Wildcard imports** | `use com.example.forum.*;` | +| **Namespace aliasing** | `use com.example.forum as Forum;` | + ## Organizing Files Common organization patterns: @@ -94,8 +291,9 @@ com/ com/ example/ forum/ - author.mlf - postRef.mlf + types/ + author.mlf + postRef.mlf post.mlf comment.mlf ``` @@ -145,7 +343,7 @@ record post { Here's a well-organized multi-file lexicon: -**File: `com/example/forum/author.mlf`** +**File: `com/example/forum/types/author.mlf`** ```mlf /// Basic author information def type author = { @@ -155,7 +353,7 @@ def type author = { }; ``` -**File: `com/example/forum/postRef.mlf`** +**File: `com/example/forum/types/postRef.mlf`** ```mlf /// Reference to a post def type postRef = { @@ -166,8 +364,7 @@ def type postRef = { **File: `com/example/forum/post.mlf`** ```mlf -use com.example.forum.author; -use com.example.forum.postRef; +use com.example.forum.types.{ author, postRef }; /// A forum post record post { @@ -190,15 +387,14 @@ record post { /// Get a post by URI query getPost( uri: AtUri -):post | error { +): post | error { NotFound, }; ``` **File: `com/example/forum/comment.mlf`** ```mlf -use com.example.forum.author; -use com.example.forum.postRef; +use com.example.forum.types.{ author, postRef }; /// A comment on a post record comment { diff --git a/website/content/docs/language-guide/09-prelude.md b/website/content/docs/language-guide/09-prelude.md index 877587e..f1c1588 100644 --- a/website/content/docs/language-guide/09-prelude.md +++ b/website/content/docs/language-guide/09-prelude.md @@ -76,12 +76,12 @@ The standard library includes all `com.atproto.*` types, which you can reference // Use fully qualified names record myPost { reference: com.atproto.repo.strongRef, - labels: [com.atproto.label.label], + labels: [com.atproto.label.defs.label], } // Or import them use com.atproto.repo.strongRef; -use com.atproto.label.label; +use com.atproto.label.defs.label; record myPost { reference: strongRef, @@ -112,10 +112,10 @@ record post { } ``` -**`com.atproto.label.label`** - Content labels for moderation: +**`com.atproto.label.defs.label`** - Content labels for moderation: ```mlf record post { - labels: [com.atproto.label.label], + labels: [com.atproto.label.defs.label], } ``` @@ -245,6 +245,4 @@ You've now learned all the core features of MLF! You can define records, add con ## What's Next? -Read the [Important Info](/docs/language-guide/10-important-info/) section to understand how MLF maps to ATProto Lexicons, especially the rules for the `"main"` definition. - -Then check out the [Playground](/playground/) to experiment with MLF, or read the [CLI documentation](/docs/cli/) to learn how to compile your lexicons. +Next, learn about [Annotations](/docs/language-guide/annotations/) to add metadata for code generators and tooling. diff --git a/website/content/docs/language-guide/10-important-info.md b/website/content/docs/language-guide/10-important-info.md index 9c38d72..1f7af32 100644 --- a/website/content/docs/language-guide/10-important-info.md +++ b/website/content/docs/language-guide/10-important-info.md @@ -5,6 +5,20 @@ weight = 10 This section covers important details about how MLF maps to ATProto Lexicons. +## Shebang Support + +MLF files can optionally include a shebang for direct execution: + +```mlf +#!/usr/bin/env mlf + +record post { + text: string, +} +``` + +The `#` character is **only** used for shebangs at the start of files. It has no other meaning in MLF syntax. + ## The "main" Definition In ATProto Lexicons, each lexicon has a `defs` object where definitions are stored. One special definition is called `"main"` - it's the primary definition for that lexicon. @@ -105,7 +119,6 @@ Generates: ### Supporting Definitions These are **never** `"main"` - they're always named defs: -- `def type` definitions - `token` definitions - `inline type` definitions (don't appear in output at all) @@ -151,6 +164,108 @@ def type postRef = { When the NSID ends with `defs`, all items become named defs (no `"main"`). +## The @main Annotation + +Sometimes you need both a main definition **and** a def with the same name. This happens when the name matches your namespace suffix. + +### Why Would You Need This? + +Consider `app.bsky.embed.external`. You might want: +1. A **main record** called `external` (the primary export) +2. A **def type** called `external` (metadata about externals) + +Normally, duplicate names aren't allowed. But when the name matches the namespace suffix (the last part), you can use `@main`: + +```mlf +// File: app.bsky.embed.external.mlf + +/// The main external embed record +@main +record external { + external!: externalDetail, +} + +/// External link details +def type externalDetail = { + uri!: Uri, + title!: string, + description!: string, +} +``` + +### Rules + +1. **Duplicates only allowed when name matches namespace suffix** + - ✅ `com.example.thread` can have two items named "thread" + - ❌ `com.example.post` cannot have two items named "thread" + +2. **Must use @main to disambiguate** + - One item must have `@main` + - Only one item can have `@main` + +3. **Works with records, queries, procedures, subscriptions + defs** + - ✅ `@main record thread` + `def type thread` + - ✅ `@main query getThread` + `def type thread` + - ❌ `inline type thread` + anything (inline types can't be main) + +### Example: Thread Types + +```mlf +// File: com.example.thread.mlf + +/// The main thread record +@main +record thread { + title!: string, + body!: string, + author!: Did, + createdAt!: Datetime, +} + +/// Thread metadata +def type thread = { + id!: string, + viewCount!: integer, + replyCount!: integer, +} + +record reply { + threadMeta: thread, // References the def type, not the record + text!: string, +} +``` + +### When @main Isn't Needed + +If you only have one record/query/procedure/subscription in a file, it automatically becomes main: + +```mlf +// No @main needed - this automatically becomes the main definition +record post { + text!: string, + createdAt!: Datetime, +} + +def type author = { + did!: Did, + handle!: Handle, +} +``` + +### Error: Missing @main + +```mlf +// ERROR: Which one is main? +record thread { + title!: string, +} + +def type thread = { + id!: string, +} +// This will fail - you must add @main to one of them +``` + ## NSID and File Path Mapping The file path **is** the NSID. MLF derives the lexicon NSID from the file path: @@ -216,5 +331,10 @@ def type `record` = { // Type name "record" escaped - **Single main-eligible item** → automatically becomes `"main"` - **Name matches last NSID segment** → becomes `"main"` - **Neither condition met** → all items become named defs -- **Supporting definitions** (def type, token) → always named defs +- **Supporting definitions** (token, inline type) → always named defs - **File path** → determines the NSID +- **Shebang support** → optional `#!/usr/bin/env mlf` at file start + +## What's Next? + +Finally, explore [Lexicon Mapping](/docs/language-guide/lexicon-mapping/) to see how MLF constructs map to ATProto Lexicon JSON format. diff --git a/website/content/docs/language-guide/11-annotations.md b/website/content/docs/language-guide/11-annotations.md new file mode 100644 index 0000000..c7a1d15 --- /dev/null +++ b/website/content/docs/language-guide/11-annotations.md @@ -0,0 +1,164 @@ ++++ +title = "Annotations" +weight = 9 ++++ + +Annotations use the `@` symbol and provide metadata for external tooling. MLF itself assigns no semantic meaning to most annotations - they're purely for tools, linters, code generators, and other processors. + +## Annotation Syntax + +Three forms of annotations are supported: + +### Simple Annotation + +```mlf +@deprecated +record oldRecord { + field: string, +} +``` + +### Positional Arguments + +```mlf +@since(1, 2, 0) +@doc("https://example.com/docs") +record example { + field: string, +} +``` + +Arguments can be: +- **Strings**: `"value"` +- **Numbers**: `42`, `3.14` +- **Booleans**: `true`, `false` + +### Named Arguments + +```mlf +@validate(min: 0, max: 100, strict: true) +@codegen(language: "rust", derive: "Debug, Clone") +record example { + field: integer, +} +``` + +## Annotation Placement + +Annotations can be placed on: + +- Records +- Def Types +- Inline Types +- Tokens +- Queries +- Procedures +- Subscriptions +- Fields within records/types + +**Example:** + +```mlf +/// A user profile +@table(name: "profiles", indexes: "did,handle") +record profile { + /// User's DID + @indexed + did!: Did, + + /// Display name (optional) + @sensitive(pii: true) + displayName: string, +} +``` + +## MLF Annotations vs Generator Annotations + +MLF distinguishes between two categories: + +### 1. MLF Annotations + +Built into the MLF language and affect compilation/validation. These are **bare annotations** without any namespace prefix: + +**`@main`** - Marks an item as the main definition when there's ambiguity: + +```mlf +// File: com/example/thread.mlf +@main +record thread { + title!: string, +} + +// This def shares the same name but is not main +def type thread = { + id!: string, + viewCount!: integer, +} +``` + +See [Important Info](/docs/language-guide/important-info/#the-main-definition) for more details on the `@main` annotation. + +### 2. Generator Annotations + +Used by code generators and external tools. These have no effect on MLF compilation and **must** be namespaced with the generator name: + +```mlf +@rust:derive("Debug, Clone, Serialize") +@typescript:export +@go:tag(json: "custom_name") +record example { + field: string, +} +``` + +**Generator namespacing rules:** +- All generator annotations must have a namespace prefix (e.g., `@rust:foo`) +- Use `@all:annotation` to apply an annotation to all generators +- Bare annotations (without `:`) are reserved for MLF itself + +**Common generator namespaces:** +- `@rust:*` - Rust code generator annotations +- `@typescript:*` - TypeScript code generator annotations +- `@go:*` - Go code generator annotations +- `@python:*` - Python code generator annotations +- `@all:*` - Applies to all generators + +## Custom Annotations + +You can define your own annotations for custom tooling: + +```mlf +@myapp:cache(ttl: 3600) +@myapp:permission("read:public") +query getProfile(actor!: Did): profile; + +@myapp:audit_log +@myapp:rate_limit(requests: 100, window: 60) +procedure updateProfile(data!: profile): unit; +``` + +The interpretation is entirely up to your tooling. + +## Annotation Processing + +Annotations are preserved in the MLF AST and can be accessed by: + +- Code generators +- Linters +- Documentation generators +- Build tools +- Custom processors + +Each tool decides which annotations to support and how to interpret them. + +## Best Practices + +1. **Always namespace generator annotations** - Use `@generator:name` for all generator-specific annotations +2. **Use `@all:` for cross-generator annotations** - When an annotation should apply to all generators +3. **Document custom annotations** - Keep a registry of annotations your project uses +4. **Be consistent** - Use the same annotation patterns across your codebase +5. **Don't overuse** - Annotations should augment, not replace, good design + +## What's Next? + +Next, read the [Important Info](/docs/language-guide/important-info/) section to understand critical details about how MLF maps to ATProto Lexicons. diff --git a/website/content/docs/language-guide/11-lexicon-mapping.md b/website/content/docs/language-guide/11-lexicon-mapping.md new file mode 100644 index 0000000..1c5a658 --- /dev/null +++ b/website/content/docs/language-guide/11-lexicon-mapping.md @@ -0,0 +1,529 @@ ++++ +title = "Lexicon Mapping" +weight = 11 ++++ + +This page explains how MLF constructs map to ATProto Lexicon JSON format. Understanding this mapping helps you work with existing lexicons and understand what MLF generates. + +## Basic Record + +MLF provides a cleaner syntax for ATProto records: + +**MLF:** +```mlf +// File: com/example/forum/post.mlf +record post { + text!: string constrained { + maxLength: 300, + }, + createdAt!: Datetime, +} +``` + +**Generated JSON:** +```json +{ + "lexicon": 1, + "id": "com.example.forum.post", + "defs": { + "main": { + "type": "record", + "key": "tid", + "record": { + "type": "object", + "required": ["text", "createdAt"], + "properties": { + "text": { + "type": "string", + "maxLength": 300 + }, + "createdAt": { + "type": "string", + "format": "datetime" + } + } + } + } + } +} +``` + +**Key mappings:** +- MLF file path → JSON `id` field +- `record post` → `"main": { "type": "record" }` +- `field!:` → included in `required` array +- `field:` (no `!`) → optional field (not in `required`) +- `Datetime` → `{ "type": "string", "format": "datetime" }` + +## Query Definition + +**MLF:** +```mlf +// File: com/example/forum/getPost.mlf +query getPost( + uri!: AtUri, +): post | error { + NotFound, + BadRequest, +}; +``` + +**Generated JSON:** +```json +{ + "lexicon": 1, + "id": "com.example.forum.getPost", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["uri"], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "ref", + "ref": "#post" + } + }, + "errors": [ + { "name": "NotFound" }, + { "name": "BadRequest" } + ] + } + } +} +``` + +**Key mappings:** +- `query` → `"type": "query"` +- Parameters → `"parameters"` object with `"type": "params"` +- Return type → `"output"` with `"schema"` +- Error block → `"errors"` array + +## Procedure Definition + +**MLF:** +```mlf +// File: com/example/forum/createPost.mlf +procedure createPost( + text!: string, +): { + uri!: AtUri, + cid!: Cid, +} | error { + TextTooLong, +}; +``` + +**Generated JSON:** +```json +{ + "lexicon": 1, + "id": "com.example.forum.createPost", + "defs": { + "main": { + "type": "procedure", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["text"], + "properties": { + "text": { + "type": "string" + } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["uri", "cid"], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + } + } + } + }, + "errors": [ + { "name": "TextTooLong" } + ] + } + } +} +``` + +**Key mappings:** +- `procedure` → `"type": "procedure"` +- Parameters → `"input"` with inline schema +- Return object → `"output"` with inline schema + +## Subscription Definition + +**MLF:** +```mlf +// File: com/example/subscribeEvents.mlf +subscription subscribeEvents( + cursor: integer, +): commit | identity; + +def type commit = { + seq!: integer, + repo!: Did, + commit!: Cid, +}; + +def type identity = { + did!: Did, + handle!: Handle, +}; +``` + +**Generated JSON:** +```json +{ + "lexicon": 1, + "id": "com.example.subscribeEvents", + "defs": { + "main": { + "type": "subscription", + "parameters": { + "type": "params", + "properties": { + "cursor": { + "type": "integer" + } + } + }, + "message": { + "schema": { + "type": "union", + "refs": ["#commit", "#identity"] + } + } + }, + "commit": { + "type": "object", + "required": ["seq", "repo", "commit"], + "properties": { + "seq": { "type": "integer" }, + "repo": { "type": "string", "format": "did" }, + "commit": { "type": "string", "format": "cid" } + } + }, + "identity": { + "type": "object", + "required": ["did", "handle"], + "properties": { + "did": { "type": "string", "format": "did" }, + "handle": { "type": "string", "format": "handle" } + } + } + } +} +``` + +**Key mappings:** +- `subscription` → `"type": "subscription"` +- Parameters → `"parameters"` +- Message union → `"message": { "schema": { "type": "union" } }` +- `def type` → Named definition in `"defs"` + +## Type References + +MLF uses simplified reference syntax that maps to ATProto's `ref` format: + +**MLF:** +```mlf +// File: com/example/post.mlf +use com.example.types.author; + +record post { + author: author, // Imported type + metadata: postMetadata, // Local type +} + +def type postMetadata = { + views!: integer, +}; +``` + +**Generated JSON:** +```json +{ + "lexicon": 1, + "id": "com.example.post", + "defs": { + "main": { + "type": "record", + "record": { + "type": "object", + "properties": { + "author": { + "type": "ref", + "ref": "com.example.types#author" + }, + "metadata": { + "type": "ref", + "ref": "#postMetadata" + } + } + } + }, + "postMetadata": { + "type": "object", + "required": ["views"], + "properties": { + "views": { "type": "integer" } + } + } + } +} +``` + +**Reference rules:** +- Local references → `"#defName"` +- External references → `"namespace#defName"` +- Imported types → Resolved to full namespace + +## Unions + +**MLF:** +```mlf +// Open union (default - allows unknown types) +content: text | image | video + +// Closed union (only listed types) +content: text | image | video | ! +``` + +**Generated JSON:** +```json +{ + "openUnion": { + "type": "union", + "refs": ["#text", "#image", "#video"] + }, + "closedUnion": { + "type": "union", + "refs": ["#text", "#image", "#video"], + "closed": true + } +} +``` + +## Tokens + +**MLF:** +```mlf +token open; +token closed; + +record issue { + state!: string constrained { + knownValues: [open, closed], + }, +} +``` + +**Generated JSON:** +```json +{ + "defs": { + "main": { + "type": "record", + "record": { + "type": "object", + "required": ["state"], + "properties": { + "state": { + "type": "string", + "knownValues": ["open", "closed"] + } + } + } + }, + "open": { + "type": "token", + "description": "..." + }, + "closed": { + "type": "token", + "description": "..." + } + } +} +``` + +**Note:** Tokens are expanded to their string values in `knownValues` arrays. + +## Constraints + +MLF constraints map directly to ATProto validation rules: + +| MLF Constraint | JSON Field | +|----------------|------------| +| `maxLength: 100` | `"maxLength": 100` | +| `minLength: 1` | `"minLength": 1` | +| `maxGraphemes: 100` | `"maxGraphemes": 100` | +| `minGraphemes: 1` | `"minGraphemes": 1` | +| `minimum: 0` | `"minimum": 0` | +| `maximum: 100` | `"maximum": 100` | +| `enum: ["a", "b"]` | `"enum": ["a", "b"]` | +| `knownValues: [a, b]` | `"knownValues": ["a", "b"]` | +| `format: "uri"` | `"format": "uri"` | +| `default: "value"` | `"default": "value"` | +| `accept: ["image/png"]` | `"accept": ["image/png"]` | +| `maxSize: 1000000` | `"maxSize": 1000000` | + +## Prelude Types + +MLF prelude types are convenience wrappers around formatted strings: + +| MLF Type | JSON Representation | +|----------|---------------------| +| `Did` | `{ "type": "string", "format": "did" }` | +| `AtUri` | `{ "type": "string", "format": "at-uri" }` | +| `AtIdentifier` | `{ "type": "string", "format": "at-identifier" }` | +| `Handle` | `{ "type": "string", "format": "handle" }` | +| `Datetime` | `{ "type": "string", "format": "datetime" }` | +| `Uri` | `{ "type": "string", "format": "uri" }` | +| `Cid` | `{ "type": "string", "format": "cid" }` | +| `Nsid` | `{ "type": "string", "format": "nsid" }` | +| `Tid` | `{ "type": "string", "format": "tid" }` | +| `RecordKey` | `{ "type": "string", "format": "record-key" }` | +| `Language` | `{ "type": "string", "format": "language" }` | + +## Inline Types + +Inline types are expanded at the point of use and don't appear in the `defs` block: + +**MLF:** +```mlf +inline type ShortString = string constrained { + maxLength: 100, +}; + +record example { + title!: ShortString, +} +``` + +**Generated JSON:** +```json +{ + "defs": { + "main": { + "type": "record", + "record": { + "type": "object", + "required": ["title"], + "properties": { + "title": { + "type": "string", + "maxLength": 100 + } + } + } + } + } +} +``` + +**Note:** `ShortString` is expanded inline - it doesn't appear in `"defs"`. + +## Arrays + +**MLF:** +```mlf +tags: string[] +images: Uri[] constrained { + maxLength: 10, +} +``` + +**Generated JSON:** +```json +{ + "tags": { + "type": "array", + "items": { + "type": "string" + } + }, + "images": { + "type": "array", + "items": { + "type": "string", + "format": "uri" + }, + "maxLength": 10 + } +} +``` + +## Complete Example Comparison + +Here's a full lexicon showing MLF and its JSON output: + +**MLF (`com/example/forum/thread.mlf`):** +```mlf +use com.example.forum.types.author; + +token open; +token closed; + +record thread { + title!: string constrained { + maxGraphemes: 200, + }, + body!: string, + author!: author, + state!: string constrained { + knownValues: [open, closed], + default: "open", + }, + createdAt!: Datetime, + replies: integer constrained { + minimum: 0, + default: 0, + }, +} + +query getThread( + uri!: AtUri, +): thread | error { + NotFound, +}; +``` + +This generates a complete ATProto JSON lexicon with: +- Namespace derived from file path +- Main definition for the record +- Token definitions +- Query definition with parameters and errors +- All type references properly resolved +- Constraints mapped to validation rules + +## What's Next? + +You now understand how MLF maps to ATProto Lexicons! This knowledge helps when: +- Converting existing JSON lexicons to MLF +- Understanding generated lexicon output +- Debugging lexicon validation issues +- Working with the ATProto ecosystem diff --git a/website/syntaxes/mlf.sublime-syntax b/website/syntaxes/mlf.sublime-syntax index cc07180..b41c2ec 100644 --- a/website/syntaxes/mlf.sublime-syntax +++ b/website/syntaxes/mlf.sublime-syntax @@ -29,7 +29,7 @@ contexts: pop: true keywords: - - match: '\b(namespace|use|record|inline|def|type|token|query|procedure|subscription|throws|constrained)\b' + - match: '\b(namespace|use|as|record|inline|def|type|token|query|procedure|subscription|throws|constrained|error)\b' scope: keyword.control.mlf - match: '\b(main|defs)\b' scope: keyword.other.mlf @@ -65,11 +65,13 @@ contexts: operators: - match: '[{}()\[\]]' scope: punctuation.section.mlf - - match: '[,;:]' + - match: '[,;:.]' scope: punctuation.separator.mlf - match: '!' scope: keyword.operator.required.mlf - match: '\|' scope: keyword.operator.union.mlf + - match: '\*' + scope: keyword.operator.wildcard.mlf - match: '=' scope: keyword.operator.assignment.mlf -- 2.51.2