From e0604941c0ccc40eef0f3853629ab4821e8f3c0b Mon Sep 17 00:00:00 2001 From: Matt Stavola Date: Mon, 6 Oct 2025 22:23:08 -0400 Subject: [PATCH] Standard library --- Cargo.lock | 20 ++++ mlf-cli/src/check.rs | 4 +- mlf-lang/Cargo.toml | 1 + mlf-lang/src/lib.rs | 7 +- mlf-lang/src/workspace.rs | 105 +++++++++++++++++- std/com/atproto/admin/defs.mlf | 36 ++++++ std/com/atproto/identity/defs.mlf | 7 ++ std/com/atproto/label/defs.mlf | 52 +++++++++ std/com/atproto/lexicon/schema.mlf | 5 + std/com/atproto/moderation/defs.mlf | 68 ++++++++++++ std/com/atproto/repo/defs.mlf | 6 + std/com/atproto/repo/strongRef.mlf | 7 ++ std/com/atproto/server/defs.mlf | 16 +++ std/com/atproto/sync/defs.mlf | 3 + {resources => std}/prelude.mlf | 0 .../content/docs/language-guide/09-prelude.md | 76 ++++++++++--- 16 files changed, 395 insertions(+), 18 deletions(-) create mode 100644 std/com/atproto/admin/defs.mlf create mode 100644 std/com/atproto/identity/defs.mlf create mode 100644 std/com/atproto/label/defs.mlf create mode 100644 std/com/atproto/lexicon/schema.mlf create mode 100644 std/com/atproto/moderation/defs.mlf create mode 100644 std/com/atproto/repo/defs.mlf create mode 100644 std/com/atproto/repo/strongRef.mlf create mode 100644 std/com/atproto/server/defs.mlf create mode 100644 std/com/atproto/sync/defs.mlf rename {resources => std}/prelude.mlf (100%) diff --git a/Cargo.lock b/Cargo.lock index 9755166..c8adbe7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -220,6 +220,25 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "include_dir" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd" +dependencies = [ + "include_dir_macros", +] + +[[package]] +name = "include_dir_macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "indexmap" version = "2.11.4" @@ -367,6 +386,7 @@ dependencies = [ name = "mlf-lang" version = "0.1.0" dependencies = [ + "include_dir", "nom", ] diff --git a/mlf-cli/src/check.rs b/mlf-cli/src/check.rs index 2e790b3..e9f5653 100644 --- a/mlf-cli/src/check.rs +++ b/mlf-cli/src/check.rs @@ -74,9 +74,9 @@ pub fn run(input_patterns: Vec) -> Result<(), CheckError> { } } - let mut workspace = mlf_lang::Workspace::with_prelude().map_err(|e| { + let mut workspace = mlf_lang::Workspace::with_std().map_err(|e| { CheckError::ValidationErrors { - help: Some(format!("Failed to load prelude: {:?}", e)), + help: Some(format!("Failed to load standard library: {:?}", e)), } })?; diff --git a/mlf-lang/Cargo.toml b/mlf-lang/Cargo.toml index 94b9978..fd11355 100644 --- a/mlf-lang/Cargo.toml +++ b/mlf-lang/Cargo.toml @@ -6,6 +6,7 @@ license = "MIT" [dependencies] nom = { version = "8", default-features = false, features = ["alloc"] } +include_dir = "0.7" [features] default = ["std"] diff --git a/mlf-lang/src/lib.rs b/mlf-lang/src/lib.rs index 595e784..a21f3d1 100644 --- a/mlf-lang/src/lib.rs +++ b/mlf-lang/src/lib.rs @@ -16,4 +16,9 @@ pub use parser::parse_lexicon; pub use validate::validate_lexicon; pub use workspace::Workspace; -pub const PRELUDE: &str = include_str!("../../resources/prelude.mlf"); +// Standard library directory +use include_dir::{include_dir, Dir}; + +pub static STD_DIR: Dir<'static> = include_dir!("$CARGO_MANIFEST_DIR/../std"); + +pub const PRELUDE: &str = include_str!("../../std/prelude.mlf"); diff --git a/mlf-lang/src/workspace.rs b/mlf-lang/src/workspace.rs index d235fae..9d67016 100644 --- a/mlf-lang/src/workspace.rs +++ b/mlf-lang/src/workspace.rs @@ -61,6 +61,80 @@ impl Workspace { Ok(ws) } + pub fn with_std() -> Result { + let mut ws = Self::new(); + + // Add prelude + let prelude_lexicon = crate::parser::parse_lexicon(crate::PRELUDE) + .map_err(|e| { + let mut errors = ValidationErrors::new(); + errors.push(ValidationError::InvalidConstraint { + message: alloc::format!("Failed to parse prelude: {:?}", e), + span: crate::span::Span::new(0, 0), + }); + errors + })?; + ws.add_module("prelude".into(), prelude_lexicon)?; + + // Recursively load all .mlf files from the std directory + fn load_std_modules( + ws: &mut Workspace, + dir: &include_dir::Dir, + ) -> Result<(), ValidationErrors> { + for file in dir.files() { + if let Some(path_str) = file.path().to_str() { + if path_str.ends_with(".mlf") && !path_str.ends_with("prelude.mlf") { + // Convert file path to namespace + // e.g., "com/atproto/repo/defs.mlf" -> "com.atproto.repo" + // e.g., "com/atproto/lexicon/schema.mlf" -> "com.atproto.lexicon" + let mut namespace = path_str + .strip_suffix(".mlf") + .unwrap_or(path_str) + .replace('/', "."); + + // Strip the filename part - files are just containers, not part of the NSID + // Get the directory path as the namespace + if let Some(last_dot) = namespace.rfind('.') { + namespace = namespace[..last_dot].to_string(); + } + + if let Some(contents) = file.contents_utf8() { + let lexicon = crate::parser::parse_lexicon(contents) + .map_err(|e| { + let mut errors = ValidationErrors::new(); + errors.push(ValidationError::InvalidConstraint { + message: alloc::format!("Failed to parse {} (file: {}): {:?}", namespace, path_str, e), + span: crate::span::Span::new(0, 0), + }); + errors + })?; + + // If the module already exists, merge the items + if ws.modules.contains_key(&namespace) { + let module = ws.modules.get_mut(&namespace).unwrap(); + module.lexicon.items.extend(lexicon.items); + // Rebuild symbol table + module.symbols = Workspace::build_symbol_table(&namespace, &module.lexicon)?; + } else { + ws.add_module(namespace, lexicon)?; + } + } + } + } + } + + for subdir in dir.dirs() { + load_std_modules(ws, subdir)?; + } + + Ok(()) + } + + load_std_modules(&mut ws, &crate::STD_DIR)?; + + Ok(ws) + } + pub fn add_module(&mut self, namespace: String, lexicon: Lexicon) -> Result<(), ValidationErrors> { let symbols = Self::build_symbol_table(&namespace, &lexicon)?; @@ -247,8 +321,18 @@ impl Workspace { for constraint in constraints { match constraint { Constraint::MinLength { span, .. } - | Constraint::MaxLength { span, .. } - | Constraint::MinGraphemes { span, .. } + | Constraint::MaxLength { span, .. } => { + // MinLength/MaxLength can be applied to strings or arrays + let is_string = matches!(base_kind, Some(PrimitiveType::String)); + let is_array = matches!(base, Type::Array { .. }); + if !is_string && !is_array { + errors.push(ValidationError::InvalidConstraint { + message: alloc::format!("Length constraint can only be applied to string or array types"), + span: *span, + }); + } + } + Constraint::MinGraphemes { span, .. } | Constraint::MaxGraphemes { span, .. } | Constraint::Format { span, .. } | Constraint::Enum { span, .. } => { @@ -1495,4 +1579,21 @@ mod tests { let result = ws.resolve(); assert!(result.is_err()); } + + #[test] + fn test_with_std() { + let ws = Workspace::with_std().unwrap(); + + // Check that prelude is loaded + assert!(ws.modules.contains_key("prelude")); + + // Print all loaded modules for debugging + eprintln!("Loaded modules:"); + for (name, _) in &ws.modules { + eprintln!(" - {}", name); + } + + // Check that std modules are loaded + assert!(ws.modules.len() > 1, "Should have more than just prelude"); + } } diff --git a/std/com/atproto/admin/defs.mlf b/std/com/atproto/admin/defs.mlf new file mode 100644 index 0000000..0b925af --- /dev/null +++ b/std/com/atproto/admin/defs.mlf @@ -0,0 +1,36 @@ +// com.atproto.admin.defs + +def type statusAttr = { + applied: boolean, + ref?: string, +}; + +def type accountView = { + did: Did, + handle: Handle, + email?: string, + relatedRecords?: [unknown], + indexedAt: Datetime, + invitedBy?: com.atproto.server.inviteCode, + invites?: [com.atproto.server.inviteCode], + invitesDisabled?: boolean, + emailConfirmedAt?: Datetime, + inviteNote?: string, + deactivatedAt?: Datetime, + threatSignatures?: [threatSignature], +}; + +def type repoRef = { + did: Did, +}; + +def type repoBlobRef = { + did: Did, + cid: Cid, + recordUri?: AtUri, +}; + +def type threatSignature = { + property: string, + value: string, +}; diff --git a/std/com/atproto/identity/defs.mlf b/std/com/atproto/identity/defs.mlf new file mode 100644 index 0000000..92ea703 --- /dev/null +++ b/std/com/atproto/identity/defs.mlf @@ -0,0 +1,7 @@ +// com.atproto.identity.defs + +def type identityInfo = { + did: Did, + handle: Handle, + didDoc: unknown, +}; diff --git a/std/com/atproto/label/defs.mlf b/std/com/atproto/label/defs.mlf new file mode 100644 index 0000000..79570a9 --- /dev/null +++ b/std/com/atproto/label/defs.mlf @@ -0,0 +1,52 @@ +// com.atproto.label.defs + +def type label = { + ver?: integer, + src: Did, + uri: AtUri, + cid?: Cid, + val: string constrained { maxLength: 128 }, + neg?: boolean, + cts: Datetime, + exp?: Datetime, + sig?: bytes, +}; + +def type selfLabels = { + values: [selfLabel] constrained { maxLength: 10 }, +}; + +def type selfLabel = { + val: string constrained { maxLength: 128 }, +}; + +def type labelValueDefinition = { + identifier: string constrained { maxLength: 100, maxGraphemes: 100 }, + severity: string constrained { knownValues: ["inform", "alert", "none"] }, + blurs: string constrained { knownValues: ["content", "media", "none"] }, + defaultSetting?: string constrained { knownValues: ["ignore", "warn", "hide"], default: "warn" }, + adultOnly?: boolean, + locales: [labelValueDefinitionStrings], +}; + +def type labelValueDefinitionStrings = { + lang: Language, + name: string constrained { maxGraphemes: 64, maxLength: 640 }, + description: string constrained { maxGraphemes: 10000, maxLength: 100000 }, +}; + +def type labelValue = string constrained { + knownValues: [ + "!hide", + "!no-promote", + "!warn", + "!no-unauthenticated", + "dmca-violation", + "doxxing", + "porn", + "sexual", + "nudity", + "nsfl", + "gore", + ], +}; diff --git a/std/com/atproto/lexicon/schema.mlf b/std/com/atproto/lexicon/schema.mlf new file mode 100644 index 0000000..4af0d65 --- /dev/null +++ b/std/com/atproto/lexicon/schema.mlf @@ -0,0 +1,5 @@ +// com.atproto.lexicon.schema + +record schema { + lexicon: integer, +} diff --git a/std/com/atproto/moderation/defs.mlf b/std/com/atproto/moderation/defs.mlf new file mode 100644 index 0000000..339ab08 --- /dev/null +++ b/std/com/atproto/moderation/defs.mlf @@ -0,0 +1,68 @@ +// com.atproto.moderation.defs + +def type reasonType = string constrained { + knownValues: [ + "com.atproto.moderation.defs#reasonSpam", + "com.atproto.moderation.defs#reasonViolation", + "com.atproto.moderation.defs#reasonMisleading", + "com.atproto.moderation.defs#reasonSexual", + "com.atproto.moderation.defs#reasonRude", + "com.atproto.moderation.defs#reasonOther", + "com.atproto.moderation.defs#reasonAppeal", + "tools.ozone.report.defs#reasonAppeal", + "tools.ozone.report.defs#reasonViolenceAnimalWelfare", + "tools.ozone.report.defs#reasonViolenceThreats", + "tools.ozone.report.defs#reasonViolenceGraphicContent", + "tools.ozone.report.defs#reasonViolenceSelfHarm", + "tools.ozone.report.defs#reasonViolenceGlorification", + "tools.ozone.report.defs#reasonViolenceExtremistContent", + "tools.ozone.report.defs#reasonViolenceTrafficking", + "tools.ozone.report.defs#reasonViolenceOther", + "tools.ozone.report.defs#reasonSexualAbuseContent", + "tools.ozone.report.defs#reasonSexualNCII", + "tools.ozone.report.defs#reasonSexualSextortion", + "tools.ozone.report.defs#reasonSexualDeepfake", + "tools.ozone.report.defs#reasonSexualAnimal", + "tools.ozone.report.defs#reasonSexualUnlabeled", + "tools.ozone.report.defs#reasonSexualOther", + "tools.ozone.report.defs#reasonChildSafetyCSAM", + "tools.ozone.report.defs#reasonChildSafetyGroom", + "tools.ozone.report.defs#reasonChildSafetyMinorPrivacy", + "tools.ozone.report.defs#reasonChildSafetyEndangerment", + "tools.ozone.report.defs#reasonChildSafetyHarassment", + "tools.ozone.report.defs#reasonChildSafetyPromotion", + "tools.ozone.report.defs#reasonChildSafetyOther", + "tools.ozone.report.defs#reasonHarassmentTroll", + "tools.ozone.report.defs#reasonHarassmentTargeted", + "tools.ozone.report.defs#reasonHarassmentHateSpeech", + "tools.ozone.report.defs#reasonHarassmentDoxxing", + "tools.ozone.report.defs#reasonHarassmentOther", + "tools.ozone.report.defs#reasonMisleadingBot", + "tools.ozone.report.defs#reasonMisleadingImpersonation", + "tools.ozone.report.defs#reasonMisleadingSpam", + "tools.ozone.report.defs#reasonMisleadingScam", + "tools.ozone.report.defs#reasonMisleadingSyntheticContent", + "tools.ozone.report.defs#reasonMisleadingMisinformation", + "tools.ozone.report.defs#reasonMisleadingOther", + "tools.ozone.report.defs#reasonRuleSiteSecurity", + "tools.ozone.report.defs#reasonRuleStolenContent", + "tools.ozone.report.defs#reasonRuleProhibitedSales", + "tools.ozone.report.defs#reasonRuleBanEvasion", + "tools.ozone.report.defs#reasonRuleOther", + "tools.ozone.report.defs#reasonCivicElectoralProcess", + "tools.ozone.report.defs#reasonCivicDisclosure", + "tools.ozone.report.defs#reasonCivicInterference", + "tools.ozone.report.defs#reasonCivicMisinformation", + "tools.ozone.report.defs#reasonCivicImpersonation" + ] +}; + +token reasonSpam; +token reasonViolation; +token reasonMisleading; +token reasonSexual; +token reasonRude; +token reasonOther; +token reasonAppeal; + +def type subjectType = string constrained { knownValues: ["account", "record", "chat"] }; diff --git a/std/com/atproto/repo/defs.mlf b/std/com/atproto/repo/defs.mlf new file mode 100644 index 0000000..4e99585 --- /dev/null +++ b/std/com/atproto/repo/defs.mlf @@ -0,0 +1,6 @@ +// com.atproto.repo.defs + +def type commitMeta = { + cid: Cid, + rev: Tid, +}; diff --git a/std/com/atproto/repo/strongRef.mlf b/std/com/atproto/repo/strongRef.mlf new file mode 100644 index 0000000..d09ad65 --- /dev/null +++ b/std/com/atproto/repo/strongRef.mlf @@ -0,0 +1,7 @@ +// com.atproto.repo.strongRef +// A URI with a content-hash fingerprint. + +inline type strongRef = { + uri: AtUri, + cid: Cid, +}; diff --git a/std/com/atproto/server/defs.mlf b/std/com/atproto/server/defs.mlf new file mode 100644 index 0000000..15ea856 --- /dev/null +++ b/std/com/atproto/server/defs.mlf @@ -0,0 +1,16 @@ +// com.atproto.server.defs + +def type inviteCode = { + code: string, + available: integer, + disabled: boolean, + forAccount: string, + createdBy: string, + createdAt: Datetime, + uses: [inviteCodeUse], +}; + +def type inviteCodeUse = { + usedBy: Did, + usedAt: Datetime, +}; diff --git a/std/com/atproto/sync/defs.mlf b/std/com/atproto/sync/defs.mlf new file mode 100644 index 0000000..98b2663 --- /dev/null +++ b/std/com/atproto/sync/defs.mlf @@ -0,0 +1,3 @@ +// com.atproto.sync.defs + +def type hostStatus = string constrained { knownValues: ["active", "idle", "offline", "throttled", "banned"] }; diff --git a/resources/prelude.mlf b/std/prelude.mlf similarity index 100% rename from resources/prelude.mlf rename to std/prelude.mlf diff --git a/website/content/docs/language-guide/09-prelude.md b/website/content/docs/language-guide/09-prelude.md index 9aef57a..877587e 100644 --- a/website/content/docs/language-guide/09-prelude.md +++ b/website/content/docs/language-guide/09-prelude.md @@ -1,8 +1,15 @@ +++ -title = "Prelude" +title = "Standard Library" weight = 9 +++ +MLF ships with a standard library that includes commonly used types for working with the AT Protocol. The standard library consists of: + +1. **The Prelude** - Format types auto-imported into every file +2. **ATProto Types** - The full `com.atproto.*` namespace available for import + +## The Prelude + The prelude is a set of definitions automatically available in every MLF file. You don't need to import them - they're always there. ## String Format Types @@ -61,20 +68,63 @@ inline type Uri = string constrained { When you use `Datetime` in your record, it expands to `string` with `format: "datetime"`. -## Future: ATProto Types +## ATProto Standard Types -In the future, the prelude will also include all `com.atproto.*` definitions: +The standard library includes all `com.atproto.*` types, which you can reference using fully qualified names or imports: ```mlf -// Eventually, these will be in the prelude +// Use fully qualified names +record myPost { + reference: com.atproto.repo.strongRef, + labels: [com.atproto.label.label], +} + +// Or import them +use com.atproto.repo.strongRef; +use com.atproto.label.label; + record myPost { - // Reference standard ATProto types without importing - repo: com.atproto.sync.repo, - commit: com.atproto.sync.commit, + reference: strongRef, + labels: [label], +} +``` + +### Available ATProto Namespaces + +The standard library includes: +- `com.atproto.admin.*` - Administration and moderation types +- `com.atproto.identity.*` - Identity resolution types +- `com.atproto.label.*` - Content labeling types +- `com.atproto.lexicon.*` - Lexicon schema types +- `com.atproto.moderation.*` - Moderation action types +- `com.atproto.repo.*` - Repository and record types +- `com.atproto.server.*` - Server configuration types +- `com.atproto.sync.*` - Synchronization types + +### Common ATProto Types + +Here are some frequently used ATProto types: + +**`com.atproto.repo.strongRef`** - A URI with a content-hash fingerprint: +```mlf +record post { + replyTo?: com.atproto.repo.strongRef, +} +``` + +**`com.atproto.label.label`** - Content labels for moderation: +```mlf +record post { + labels: [com.atproto.label.label], } ``` -This will make it easier to reference standard ATProto types without manual imports. +**`com.atproto.server.inviteCode`** - Server invite codes: +```mlf +record account { + invitedBy?: com.atproto.server.inviteCode, +} +``` ## Using Prelude Types @@ -186,12 +236,12 @@ All the format types (`Did`, `Handle`, `Uri`, `Datetime`, `Language`, `AtIdentif ## Summary -The prelude provides: -- ✅ String format types for common patterns -- ✅ No imports needed - always available -- ✅ Eventually will include all `com.atproto.*` types +The standard library provides: +- ✅ **Prelude types** - String format types auto-imported into every file +- ✅ **ATProto types** - Full `com.atproto.*` namespace available via imports or fully qualified names +- ✅ No manual downloads or setup needed - ships with MLF -You've now learned all the core features of MLF! You can define records, add constraints, create custom types, use unions and tokens, define XRPC operations, import from other files, and use prelude types. +You've now learned all the core features of MLF! You can define records, add constraints, create custom types, use unions and tokens, define XRPC operations, import from other files, and use standard library types. ## What's Next? -- 2.51.2