From de4c5373e53fa202483df0954e476a5716d1f8fb Mon Sep 17 00:00:00 2001 From: Matt Stavola Date: Sat, 18 Apr 2026 01:35:23 -0400 Subject: [PATCH] Split manifest into its own collection with TID rkeys Each publish event now writes a fresh `lol.mlf.package/` record instead of clobbering a single `com.atproto.lexicon.schema/lol.mlf.package`. The schema definition and the manifest instance are different things in different collections, so they stop colliding when an mlf project dogfoods `lol.mlf.package` as a user lexicon. - mlf-atproto: new tid module (sortable base32 TID generator, CAS loop for monotonicity) - mlf-publish/manifest: drop the schema envelope; instance carries `$type = "lol.mlf.package"` and only instance fields - mlf-cli/publish: write manifest to `lol.mlf.package/` - mlf-cli/unpublish: list the manifest collection, pick the newest by TID, delete everything it lists + the full publish log --- mlf-atproto/src/lib.rs | 2 + mlf-atproto/src/tid.rs | 99 ++++++++++++++++++++++++ mlf-cli/src/publish.rs | 17 +++-- mlf-cli/src/unpublish.rs | 145 +++++++++++++++++++++--------------- mlf-publish/src/manifest.rs | 115 +++++++--------------------- mlf.toml | 4 + 6 files changed, 230 insertions(+), 152 deletions(-) create mode 100644 mlf-atproto/src/tid.rs diff --git a/mlf-atproto/src/lib.rs b/mlf-atproto/src/lib.rs index 41614b4..8478910 100644 --- a/mlf-atproto/src/lib.rs +++ b/mlf-atproto/src/lib.rs @@ -9,6 +9,7 @@ //! - [`records`] — typed wrappers for `getRecord` / `putRecord` / //! `listRecords` / `deleteRecord` //! - [`cid`] — client-side CID computation (DAG-CBOR + SHA-256 multihash) +//! - [`tid`] — sortable timestamp identifier generation (rkey) //! //! This crate is plumbing only — no MLF-specific domain logic lives here. //! Consumers (`mlf-lexicon-fetcher`, `mlf-publish`) build domain operations @@ -18,6 +19,7 @@ pub mod cid; pub mod identity; pub mod records; pub mod session; +pub mod tid; pub mod xrpc; pub use identity::{DnsResolver, MockDnsResolver, RealDnsResolver}; diff --git a/mlf-atproto/src/tid.rs b/mlf-atproto/src/tid.rs new file mode 100644 index 0000000..af1e821 --- /dev/null +++ b/mlf-atproto/src/tid.rs @@ -0,0 +1,99 @@ +//! ATProto TID (Timestamp Identifier) generation. +//! +//! A TID is a 64-bit integer encoded as 13 characters of a sortable +//! base32 alphabet. The layout (per https://atproto.com/specs/tid): +//! +//! - bit 63: always 0 (reserved) +//! - bits 62..10: 53-bit microseconds since the Unix epoch +//! - bits 9..0: 10-bit clock identifier (random per-process) +//! +//! Sortable base32 alphabet: `234567abcdefghijklmnopqrstuvwxyz`. +//! TIDs sort lexicographically in the same order as chronologically. + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{SystemTime, UNIX_EPOCH}; + +const ALPHABET: &[u8] = b"234567abcdefghijklmnopqrstuvwxyz"; + +/// Track the last microsecond we emitted so that two TIDs generated +/// in the same microsecond still sort in emission order. +static LAST_US: AtomicU64 = AtomicU64::new(0); + +/// Generate a fresh TID. Monotonically increasing within a process. +pub fn generate() -> String { + let now_us = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_micros() as u64) + .unwrap_or(0); + // CAS loop to install `max(now_us, prev + 1)` and read back the + // value we actually wrote. `fetch_update` returns the *previous* + // value, which is the wrong half of the trade for us. + let micros = loop { + let prev = LAST_US.load(Ordering::Relaxed); + let next = now_us.max(prev.saturating_add(1)); + if LAST_US + .compare_exchange_weak(prev, next, Ordering::SeqCst, Ordering::Relaxed) + .is_ok() + { + break next; + } + }; + let clock_id: u64 = rand_10_bits(); + let raw = ((micros & ((1 << 53) - 1)) << 10) | (clock_id & 0x3FF); + encode_base32_sortable(raw) +} + +fn rand_10_bits() -> u64 { + // A PID+nanos mix is good enough — the clock ID just needs to + // differ across concurrent processes, not be cryptographic. + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.subsec_nanos() as u64) + .unwrap_or(0); + let pid = std::process::id() as u64; + (nanos ^ pid.rotate_left(7)) & 0x3FF +} + +fn encode_base32_sortable(mut n: u64) -> String { + let mut buf = [0u8; 13]; + for slot in buf.iter_mut().rev() { + *slot = ALPHABET[(n & 0x1F) as usize]; + n >>= 5; + } + String::from_utf8(buf.to_vec()).expect("ASCII alphabet") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tid_is_13_chars_sortable_alphabet() { + let t = generate(); + assert_eq!(t.len(), 13); + assert!(t.bytes().all(|b| ALPHABET.contains(&b))); + } + + #[test] + fn tids_are_monotonic() { + let a = generate(); + let b = generate(); + let c = generate(); + assert!(a < b, "{a} < {b}"); + assert!(b < c, "{b} < {c}"); + } + + #[test] + fn tid_encodes_current_time_not_zero() { + // Regression: fetch_update returned the *previous* value, so + // the first call encoded zero micros (= "22222222222xx"). A + // correctly-populated TID has non-zero bits well above the + // bottom 10 (which hold the random clock id). + let tid = generate(); + let leading_zero_chars = tid.bytes().take_while(|&b| b == b'2').count(); + assert!( + leading_zero_chars < 5, + "{tid} has {leading_zero_chars} leading zero chars — time component wasn't populated", + ); + } +} diff --git a/mlf-cli/src/publish.rs b/mlf-cli/src/publish.rs index 0f064d7..3d2428b 100644 --- a/mlf-cli/src/publish.rs +++ b/mlf-cli/src/publish.rs @@ -20,7 +20,7 @@ use crate::config::{ use crate::credentials::{CredentialsFile, Scope}; use crate::remote_state::{RemoteState, RemoteStateError}; use miette::Diagnostic; -use mlf_atproto::{records, session}; +use mlf_atproto::{records, session, tid}; use mlf_plugin_host::discovery; use mlf_plugin_host::host::{HostError, PluginHandle}; use mlf_plugin_host::ui::{DenyInteractiveUi, TerminalUi, UiHandler}; @@ -250,7 +250,9 @@ async fn publish_package(pkg: &ResolvedPackage, opts: &PublishOpts) -> Result<() println!("\nPlan:"); print!("{}", plan.format_summary()); - if plan.is_empty() && !publish_cfg.manifest { + if plan.is_empty() { + // Nothing to publish → nothing worth recording in the publish log. + // The manifest represents a *publish event*; a no-op isn't one. println!("\n(no changes; nothing to publish)"); return Ok(()); } @@ -329,17 +331,22 @@ async fn publish_package(pkg: &ResolvedPackage, opts: &PublishOpts) -> Result<() println!(" ✓ {} {}", action.verb(), action.nsid()); } - // 7. Manifest. + // 7. Manifest. Each publish event writes a new instance at + // `lol.mlf.package/` so the PDS retains the full publish + // history. The schema definition for `lol.mlf.package` itself + // lives separately under `com.atproto.lexicon.schema` (published + // by the mlf project's own repo, not every caller's). if publish_cfg.manifest { - println!("Writing manifest (lol.mlf.package)..."); + let tid = tid::generate(); + println!("Writing manifest ({}/{tid})...", manifest::NSID); let manifest_record = build_manifest_record(&state, &plan); let _ = records::put_record( &http, &pds_url, &sess.access_jwt, &sess.did, - "com.atproto.lexicon.schema", manifest::NSID, + &tid, &manifest_record, ) .await diff --git a/mlf-cli/src/unpublish.rs b/mlf-cli/src/unpublish.rs index 5f7b052..7d0dc08 100644 --- a/mlf-cli/src/unpublish.rs +++ b/mlf-cli/src/unpublish.rs @@ -1,15 +1,16 @@ -//! `mlf unpublish` — delete every lexicon in the package's manifest, -//! plus the manifest record itself. Read the published manifest to -//! figure out what was published; if it's missing, refuse (we'd be -//! guessing which records belong to this workspace otherwise). +//! `mlf unpublish` — delete every lexicon in the package's latest +//! manifest, plus every manifest record in the publish log. Manifests +//! live in collection `lol.mlf.package` with TID rkeys; we read the +//! most-recent one to learn what NSIDs belong to us. If no manifest +//! exists we refuse — we'd be guessing which records to delete. use crate::config::{ConfigError, MlfConfig, find_project_root}; use crate::credentials::{CredentialsFile, Scope}; -use crate::remote_state::{RemoteState, RemoteStateError}; +use crate::remote_state::{RemoteStateError}; use dialoguer::{Confirm, theme::ColorfulTheme}; use miette::Diagnostic; use mlf_atproto::records::{self, RecordError}; -use mlf_atproto::session; +use mlf_atproto::{identity, session}; use mlf_publish::manifest; use std::collections::BTreeSet; use thiserror::Error; @@ -29,13 +30,11 @@ pub enum UnpublishError { NotPublishable, #[error( - "No manifest (`lol.mlf.package`) found on the PDS — refusing to guess which records belong to this workspace" + "No manifest records found in `lol.mlf.package` collection — refusing to guess which records belong to this workspace" )] #[diagnostic( code(mlf::unpublish::no_manifest), - help( - "Delete records manually via `goat lex unpublish`, or publish once first to create a manifest we can read back." - ) + help("Publish once first to create a manifest we can read back.") )] NoManifest, @@ -79,38 +78,10 @@ pub async fn run_unpublish(opts: UnpublishOpts) -> Result<(), UnpublishError> { if config.publish.is_none() { return Err(UnpublishError::NotPublishable); } + let package = config.package.clone(); - println!("Loading remote state..."); - let state = RemoteState::load().await?; - - // The manifest is what tells us "here's what this workspace published." - // If it's missing we don't know which records belong to us, so refuse. - let manifest_record = state - .remote - .get(manifest::NSID) - .ok_or(UnpublishError::NoManifest)? - .record_json - .clone(); - let to_delete = manifest_items(&manifest_record); - - if to_delete.is_empty() { - println!("Manifest lists zero records. Nothing to delete except the manifest itself."); - } else { - println!( - "Manifest lists {} record(s) published under `{}`:", - to_delete.len(), - state.package.name - ); - for nsid in &to_delete { - println!(" - {nsid}"); - } - } - - if !opts.yes && !confirm()? { - return Err(UnpublishError::Cancelled); - } - - // Credentials + session. + // Credentials + session. We need the session before we can read + // the publish log, which lives in the authed repo. let creds = load_credentials(&project_root)?; let pds_creds = creds.pds.ok_or(UnpublishError::NoPdsCreds)?; let handle = pds_creds.handle.clone().ok_or(UnpublishError::NoPdsCreds)?; @@ -122,10 +93,10 @@ pub async fn run_unpublish(opts: UnpublishOpts) -> Result<(), UnpublishError> { let pds_url = match pds_creds.extra.get("pds").and_then(|v| v.as_str()) { Some(url) => url.to_string(), None => { - let did = mlf_atproto::identity::resolve_handle_to_did(&http, &handle) + let did = identity::resolve_handle_to_did(&http, &handle) .await .map_err(|e| UnpublishError::Session(e.to_string()))?; - mlf_atproto::identity::resolve_did_to_pds(&http, &did) + identity::resolve_did_to_pds(&http, &did) .await .map_err(|e| UnpublishError::Session(e.to_string()))? } @@ -134,12 +105,46 @@ pub async fn run_unpublish(opts: UnpublishOpts) -> Result<(), UnpublishError> { .await .map_err(|e| UnpublishError::Session(e.to_string()))?; - let collection = "com.atproto.lexicon.schema"; + // List every manifest record (sorted newest-first by TID rkey). + println!("Reading publish log..."); + let manifest_records = records::list_all_records(&http, &pds_url, &sess.did, manifest::NSID) + .await + .map_err(|e| UnpublishError::Session(e.to_string()))?; + if manifest_records.is_empty() { + return Err(UnpublishError::NoManifest); + } + let (latest_rkey, latest) = latest_manifest(&manifest_records); + let to_delete = manifest_items(&latest.value); + if to_delete.is_empty() { + println!("Latest manifest lists zero records."); + } else { + println!( + "Latest manifest ({latest_rkey}) lists {} record(s) published under `{}`:", + to_delete.len(), + package.name + ); + for nsid in &to_delete { + println!(" - {nsid}"); + } + } + if manifest_records.len() > 1 { + println!( + "Publish log has {} manifest record(s) total — all will be removed.", + manifest_records.len() + ); + } + + if !opts.yes && !confirm()? { + return Err(UnpublishError::Cancelled); + } + + // Delete the schema records named in the latest manifest. for nsid in &to_delete { - // Only ever delete records in scope of the package — defence in depth - // against a corrupted or hand-edited manifest naming foreign NSIDs. - if !state.package.namespace_is_in_scope(nsid) && nsid != manifest::NSID { + // Defence in depth: only ever delete records inside this + // package's scope. A corrupted or hand-edited manifest pointing + // at foreign NSIDs gets ignored. + if !package.namespace_is_in_scope(nsid) { eprintln!("Skipping out-of-scope record `{nsid}`"); continue; } @@ -148,29 +153,51 @@ pub async fn run_unpublish(opts: UnpublishOpts) -> Result<(), UnpublishError> { &pds_url, &sess.access_jwt, &sess.did, - collection, + "com.atproto.lexicon.schema", nsid, ) .await?; println!(" ✓ deleted {nsid}"); } - // Finally, the manifest itself. - delete_one( - &http, - &pds_url, - &sess.access_jwt, - &sess.did, - collection, - manifest::NSID, - ) - .await?; - println!(" ✓ deleted {} (manifest)", manifest::NSID); + // Delete every manifest record — the entire publish log for this repo. + for r in &manifest_records { + let rkey = rkey_from_uri(&r.uri); + delete_one( + &http, + &pds_url, + &sess.access_jwt, + &sess.did, + manifest::NSID, + &rkey, + ) + .await?; + println!(" ✓ deleted {}/{rkey} (manifest)", manifest::NSID); + } println!("\n✓ Unpublish complete"); Ok(()) } +/// Pick the newest manifest. Manifest rkeys are TIDs, which sort +/// lexicographically in chronological order — so the highest rkey wins. +fn latest_manifest(records: &[records::Record]) -> (String, &records::Record) { + let mut best_idx = 0usize; + let mut best_rkey = rkey_from_uri(&records[0].uri); + for (i, r) in records.iter().enumerate().skip(1) { + let rkey = rkey_from_uri(&r.uri); + if rkey > best_rkey { + best_rkey = rkey; + best_idx = i; + } + } + (best_rkey, &records[best_idx]) +} + +fn rkey_from_uri(uri: &str) -> String { + uri.rsplit('/').next().unwrap_or(uri).to_string() +} + fn manifest_items(record: &serde_json::Value) -> BTreeSet { let Some(items) = record.get("published").and_then(|v| v.as_array()) else { return BTreeSet::new(); diff --git a/mlf-publish/src/manifest.rs b/mlf-publish/src/manifest.rs index 29b8f04..5e53509 100644 --- a/mlf-publish/src/manifest.rs +++ b/mlf-publish/src/manifest.rs @@ -1,30 +1,31 @@ //! `lol.mlf.package` manifest record construction. //! -//! The manifest is a single `com.atproto.lexicon.schema` record (rkey -//! `lol.mlf.package`) that acts as the durable pointer for a publish -//! event: a sorted list of `(nsid, cid)` pairs covering every lexicon -//! published in the release, plus a resolved-dependencies list. +//! The manifest is an **instance** of the `lol.mlf.package` record type, +//! written to collection `lol.mlf.package` with a TID rkey. Each publish +//! event gets its own immutable manifest record; the PDS retains the +//! full publish history. //! //! The record's own CID (computed by the PDS on `putRecord`) is the -//! deterministic identifier for the publish — "this version of this -//! package." There's deliberately no semver; mutation isn't supported. +//! deterministic identifier for this publish event — "this version of +//! this package." There's deliberately no semver; mutation isn't +//! supported. //! -//! Implementation note: we assemble the instance data (publishedAt, -//! tool, published[], resolvedDependencies[]) using the typed -//! [`mlf_generated_lexicon::lol::mlf::package::Package`] struct emitted by -//! `mlf generate` from `lexicons/lol/mlf/package.mlf` and owned by the -//! `mlf-generated-lexicon` crate. The `com.atproto.lexicon.schema` -//! envelope fields (`$type`, `lexicon`, `id`, `description`, `defs`) -//! are still hand-assembled: the atproto meta-schema only requires -//! `lexicon: integer`, so it's not worth generating a type for — the -//! envelope is stable and trivially verified by the -//! `check_meta_schema` validator. +//! The schema *definition* for `lol.mlf.package` lives separately at +//! `com.atproto.lexicon.schema/lol.mlf.package` — generated from the +//! `.mlf` source like any other lexicon, published by the mlf project +//! itself to its PDS. Third-party mlf users never write that record; +//! they only write manifest instances. +//! +//! Implementation note: the instance shape is driven by the typed +//! [`mlf_generated_lexicon::lol::mlf::package::Package`] struct emitted +//! from `lexicons/lol/mlf/package.mlf` — any schema change surfaces +//! here as a compile error. use mlf_generated_lexicon::lol::mlf::package::{Package, PublishedItem, ResolvedDependency}; -use serde_json::{Map, Value, json, to_value}; +use serde_json::{Map, Value, to_value}; -/// The NSID the manifest is published under (rkey == NSID, per the -/// ATProto lexicon spec). +/// The NSID for the manifest record type — also the collection name +/// where instances are written. pub const NSID: &str = "lol.mlf.package"; pub struct ManifestInputs<'a> { @@ -43,17 +44,15 @@ pub struct ManifestInputs<'a> { pub resolved_deps: &'a [(String, String)], } -/// Build the manifest record ready to be pushed via `putRecord`. The -/// return value includes the `$type` so the CID compute matches what the -/// PDS will compute on write. +/// Build the manifest instance record ready to be pushed via +/// `putRecord` to collection `lol.mlf.package`. The `$type` matches the +/// collection so the PDS can validate the instance against the schema. pub fn build(inputs: &ManifestInputs<'_>) -> Value { let mut published: Vec<(String, String)> = inputs.published.to_vec(); published.sort(); let mut deps: Vec<(String, String)> = inputs.resolved_deps.to_vec(); deps.sort(); - // Build the instance payload using the typed struct emitted by - // `mlf generate` — any schema change surfaces here as a compile error. let instance = Package { published_at: inputs.published_at.to_string(), tool: inputs.tool.to_string(), @@ -72,10 +71,6 @@ pub fn build(inputs: &ManifestInputs<'_>) -> Value { }, }; - // Merge the instance fields into a `com.atproto.lexicon.schema` - // record envelope. The envelope is hand-assembled because the - // atproto meta-schema is too permissive to generate useful types - // from (it only requires `lexicon: integer`). let instance_value = to_value(&instance).expect("Package serialises"); let instance_obj = instance_value .as_object() @@ -83,69 +78,13 @@ pub fn build(inputs: &ManifestInputs<'_>) -> Value { .clone(); let mut obj = Map::new(); - obj.insert( - "$type".into(), - Value::String("com.atproto.lexicon.schema".into()), - ); - obj.insert("lexicon".into(), Value::Number(1.into())); - obj.insert("id".into(), Value::String(NSID.into())); - obj.insert( - "description".into(), - Value::String("MLF publish manifest".into()), - ); - obj.insert("defs".into(), manifest_defs()); + obj.insert("$type".into(), Value::String(NSID.into())); for (k, v) in instance_obj { obj.insert(k, v); } Value::Object(obj) } -/// The `defs` block describing the instance shape. Kept in one place -/// so it's easy to bump if the on-PDS shape ever needs tightening. -/// The shape here matches what `mlf generate lexicon` would emit from -/// our `.mlf` source; keeping it in Rust avoids needing a second -/// generated artifact in the repo. -fn manifest_defs() -> Value { - json!({ - "main": { - "type": "record", - "key": "nsid", - "record": { - "type": "object", - "required": ["publishedAt", "tool", "published"], - "properties": { - "publishedAt": {"type": "string", "format": "datetime"}, - "tool": {"type": "string"}, - "published": { - "type": "array", - "items": {"type": "ref", "ref": "#PublishedItem"} - }, - "resolvedDependencies": { - "type": "array", - "items": {"type": "ref", "ref": "#ResolvedDependency"} - } - } - } - }, - "PublishedItem": { - "type": "object", - "required": ["nsid", "cid"], - "properties": { - "nsid": {"type": "string", "format": "nsid"}, - "cid": {"type": "string", "format": "cid"} - } - }, - "ResolvedDependency": { - "type": "object", - "required": ["nsid", "cid"], - "properties": { - "nsid": {"type": "string", "format": "nsid"}, - "cid": {"type": "string", "format": "cid"} - } - } - }) -} - #[cfg(test)] mod tests { use super::*; @@ -161,9 +100,9 @@ mod tests { ], resolved_deps: &[("com.atproto.repo.strongRef".into(), "bafy3".into())], }); - assert_eq!(v["$type"], "com.atproto.lexicon.schema"); - assert_eq!(v["id"], NSID); - assert_eq!(v["lexicon"], 1); + assert_eq!(v["$type"], NSID); + assert!(v.get("lexicon").is_none()); + assert!(v.get("defs").is_none()); // published list is sorted assert_eq!(v["published"][0]["nsid"], "com.example.other"); assert_eq!(v["published"][1]["nsid"], "com.example.thing"); diff --git a/mlf.toml b/mlf.toml index f6db9bf..b473304 100644 --- a/mlf.toml +++ b/mlf.toml @@ -1,6 +1,10 @@ [package] name = "lol.mlf" +[publish] +pds = "default" +dns = "cloudflare" + [source] directory = "./lexicons" -- 2.51.2