diff --git a/ufos/src/index_html.rs b/ufos/src/index_html.rs index 62ccb37..c70360b 100644 --- a/ufos/src/index_html.rs +++ b/ufos/src/index_html.rs @@ -8,7 +8,7 @@ pub const INDEX_HTML: &str = r#" + + +
+

+ TODO: pdsls jetstream link + Launch 🛸 UFOs app: Explore lexicons +

+ +
+ + + + + + + + diff --git a/ufos/src/index_html.rs b/ufos/src/index_html.rs index c70360b..051b0fd 100644 --- a/ufos/src/index_html.rs +++ b/ufos/src/index_html.rs @@ -2,7 +2,7 @@ pub const INDEX_HTML: &str = r#" - UFOs API Documentation + UFOs API documentation + + +
+

+ Launch who-am-i [todo] +

+ +
+ + + + + + + + -- 2.51.2 From 57c0d71d2ed8e74e2354df08acb99bb1fa66707c Mon Sep 17 00:00:00 2001 From: phil Date: Wed, 25 Jun 2025 17:05:31 -0400 Subject: [PATCH 028/134] wow fmt --- Makefile | 2 +- spacedust/src/consumer.rs | 18 ++++++---- spacedust/src/delay.rs | 4 +-- spacedust/src/lib.rs | 27 +++++++++----- spacedust/src/main.rs | 17 +++++---- spacedust/src/removable_delay_queue.rs | 26 ++++++++------ spacedust/src/server.rs | 36 ++++++++++--------- spacedust/src/subscriber.rs | 49 ++++++++++++-------------- who-am-i/Cargo.toml | 2 +- who-am-i/src/lib.rs | 2 +- who-am-i/src/main.rs | 2 +- who-am-i/src/oauth.rs | 31 ++++++++-------- who-am-i/src/server.rs | 34 +++++++++--------- 13 files changed, 135 insertions(+), 115 deletions(-) diff --git a/Makefile b/Makefile index 0fd5e48..4803c3c 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ test: cargo test --all-features fmt: - cargo fmt --package links --package constellation --package ufos + cargo fmt --package links --package constellation --package ufos --package spacedust --package who-am-i cargo +nightly fmt --package jetstream clippy: diff --git a/spacedust/src/consumer.rs b/spacedust/src/consumer.rs index bf78686..fdad8ac 100644 --- a/spacedust/src/consumer.rs +++ b/spacedust/src/consumer.rs @@ -1,5 +1,3 @@ -use std::sync::Arc; -use tokio_util::sync::CancellationToken; use crate::ClientMessage; use crate::error::ConsumerError; use crate::removable_delay_queue; @@ -8,7 +6,9 @@ use jetstream::{ events::{CommitOp, Cursor, EventKind}, }; use links::collect_links; +use std::sync::Arc; use tokio::sync::broadcast; +use tokio_util::sync::CancellationToken; const MAX_LINKS_PER_EVENT: usize = 100; @@ -61,12 +61,16 @@ pub async fn consume( }; // TODO: something a bit more robust - let at_uri = format!("at://{}/{}/{}", &*event.did, &*commit.collection, &*commit.rkey); + let at_uri = format!( + "at://{}/{}/{}", + &*event.did, &*commit.collection, &*commit.rkey + ); // TODO: keep a buffer and remove quick deletes to debounce notifs // for now we just drop all deletes eek if commit.operation == CommitOp::Delete { - d.remove_range((at_uri.clone(), 0)..=(at_uri.clone(), MAX_LINKS_PER_EVENT)).await; + d.remove_range((at_uri.clone(), 0)..=(at_uri.clone(), MAX_LINKS_PER_EVENT)) + .await; continue; } let Some(ref record) = commit.record else { @@ -86,7 +90,8 @@ pub async fn consume( if i >= MAX_LINKS_PER_EVENT { // todo: indicate if the link limit was reached (-> links omitted) log::warn!("consumer: event has too many links, ignoring the rest"); - metrics::counter!("consumer_dropped_links", "reason" => "too_many_links").increment(1); + metrics::counter!("consumer_dropped_links", "reason" => "too_many_links") + .increment(1); break; } let client_message = match ClientMessage::new_link(link, &at_uri, commit) { @@ -94,7 +99,8 @@ pub async fn consume( Err(e) => { // TODO indicate to clients that a link has been dropped log::warn!("consumer: failed to serialize link to json: {e:?}"); - metrics::counter!("consumer_dropped_links", "reason" => "failed_to_serialize").increment(1); + metrics::counter!("consumer_dropped_links", "reason" => "failed_to_serialize") + .increment(1); continue; } }; diff --git a/spacedust/src/delay.rs b/spacedust/src/delay.rs index 54eb97a..94044df 100644 --- a/spacedust/src/delay.rs +++ b/spacedust/src/delay.rs @@ -1,7 +1,7 @@ +use crate::error::DelayError; use crate::removable_delay_queue; -use tokio_util::sync::CancellationToken; use tokio::sync::broadcast; -use crate::error::DelayError; +use tokio_util::sync::CancellationToken; pub async fn to_broadcast( source: removable_delay_queue::Output<(String, usize), T>, diff --git a/spacedust/src/lib.rs b/spacedust/src/lib.rs index 049aa86..915f061 100644 --- a/spacedust/src/lib.rs +++ b/spacedust/src/lib.rs @@ -1,15 +1,15 @@ pub mod consumer; pub mod delay; pub mod error; +pub mod removable_delay_queue; pub mod server; pub mod subscriber; -pub mod removable_delay_queue; -use links::CollectedLink; use jetstream::events::CommitEvent; -use tokio_tungstenite::tungstenite::Message; +use links::CollectedLink; use serde::{Deserialize, Serialize}; use server::MultiSubscribeQuery; +use tokio_tungstenite::tungstenite::Message; #[derive(Debug)] pub struct FilterableProperties { @@ -32,7 +32,11 @@ pub struct ClientMessage { } impl ClientMessage { - pub fn new_link(link: CollectedLink, at_uri: &str, commit: &CommitEvent) -> Result { + pub fn new_link( + link: CollectedLink, + at_uri: &str, + commit: &CommitEvent, + ) -> Result { let subject_did = link.target.did(); let subject = link.target.into_string(); @@ -61,16 +65,23 @@ impl ClientMessage { let message = Message::Text(client_event_json.into()); - let properties = FilterableProperties { subject, subject_did, source }; + let properties = FilterableProperties { + subject, + subject_did, + source, + }; - Ok(ClientMessage { message, properties }) + Ok(ClientMessage { + message, + properties, + }) } } #[derive(Debug, Serialize)] -#[serde(rename_all="snake_case")] +#[serde(rename_all = "snake_case")] pub struct ClientEvent { - kind: &'static str, // "link" + kind: &'static str, // "link" origin: &'static str, // "live", "replay", "backfill" link: ClientLinkEvent, } diff --git a/spacedust/src/main.rs b/spacedust/src/main.rs index 5e138e1..54db30e 100644 --- a/spacedust/src/main.rs +++ b/spacedust/src/main.rs @@ -1,14 +1,14 @@ -use spacedust::error::MainTaskError; use spacedust::consumer; -use spacedust::server; use spacedust::delay; +use spacedust::error::MainTaskError; use spacedust::removable_delay_queue::removable_delay_queue; +use spacedust::server; use clap::Parser; use metrics_exporter_prometheus::PrometheusBuilder; +use std::time::Duration; use tokio::sync::broadcast; use tokio_util::sync::CancellationToken; -use std::time::Duration; /// Aggregate links in the at-mosphere #[derive(Parser, Debug, Clone)] @@ -80,15 +80,20 @@ async fn main() -> Result<(), String> { args.jetstream, None, args.jetstream_no_zstd, - consumer_shutdown + consumer_shutdown, ) - .await?; + .await?; Ok(()) }); let delay_shutdown = shutdown.clone(); tasks.spawn(async move { - delay::to_broadcast(delay_queue_receiver, consumer_delayed_sender, delay_shutdown).await?; + delay::to_broadcast( + delay_queue_receiver, + consumer_delayed_sender, + delay_shutdown, + ) + .await?; Ok(()) }); diff --git a/spacedust/src/removable_delay_queue.rs b/spacedust/src/removable_delay_queue.rs index cb2b09a..f528677 100644 --- a/spacedust/src/removable_delay_queue.rs +++ b/spacedust/src/removable_delay_queue.rs @@ -1,9 +1,9 @@ -use std::ops::RangeBounds; use std::collections::{BTreeMap, VecDeque}; -use std::time::{Duration, Instant}; -use tokio::sync::Mutex; +use std::ops::RangeBounds; use std::sync::Arc; +use std::time::{Duration, Instant}; use thiserror::Error; +use tokio::sync::Mutex; #[derive(Debug, Error)] pub enum EnqueueError { @@ -17,7 +17,7 @@ impl Key for T {} #[derive(Debug)] struct Queue { queue: VecDeque<(Instant, K)>, - items: BTreeMap + items: BTreeMap, } pub struct Input { @@ -49,7 +49,12 @@ impl Input { pub async fn remove_range(&self, range: impl RangeBounds) { let n = { let mut q = self.q.lock().await; - let keys = q.items.range(range).map(|(k, _)| k).cloned().collect::>(); + let keys = q + .items + .range(range) + .map(|(k, _)| k) + .cloned() + .collect::>(); for k in &keys { q.items.remove(k); } @@ -94,22 +99,21 @@ impl Output { } else { let overshoot = now.saturating_duration_since(expected_release); metrics::counter!("delay_queue_emit_total", "early" => "no").increment(1); - metrics::histogram!("delay_queue_emit_overshoot").record(overshoot.as_secs_f64()); + metrics::histogram!("delay_queue_emit_overshoot") + .record(overshoot.as_secs_f64()); } - return Some(item) + return Some(item); } else if Arc::strong_count(&self.q) == 1 { return None; } // the queue is *empty*, so we need to wait at least as long as the current delay tokio::time::sleep(self.delay).await; metrics::counter!("delay_queue_entirely_empty_total").increment(1); - }; + } } } -pub fn removable_delay_queue( - delay: Duration, -) -> (Input, Output) { +pub fn removable_delay_queue(delay: Duration) -> (Input, Output) { let q: Arc>> = Arc::new(Mutex::new(Queue { queue: VecDeque::new(), items: BTreeMap::new(), diff --git a/spacedust/src/server.rs b/spacedust/src/server.rs index 37bee28..a74b207 100644 --- a/spacedust/src/server.rs +++ b/spacedust/src/server.rs @@ -1,28 +1,26 @@ +use crate::ClientMessage; use crate::error::ServerError; use crate::subscriber::Subscriber; -use metrics::{histogram, counter}; -use std::sync::Arc; -use crate::ClientMessage; +use dropshot::{ + ApiDescription, ApiEndpointBodyContentType, Body, ConfigDropshot, ConfigLogging, + ConfigLoggingLevel, ExtractorMetadata, HttpError, HttpResponse, Query, RequestContext, + ServerBuilder, ServerContext, SharedExtractor, WebsocketConnection, channel, endpoint, +}; use http::{ - header::{ORIGIN, USER_AGENT}, Response, StatusCode, + header::{ORIGIN, USER_AGENT}, }; -use dropshot::{ - Body, - ApiDescription, ConfigDropshot, ConfigLogging, ConfigLoggingLevel, Query, RequestContext, - ServerBuilder, WebsocketConnection, channel, endpoint, HttpResponse, - ApiEndpointBodyContentType, ExtractorMetadata, HttpError, ServerContext, - SharedExtractor, -}; +use metrics::{counter, histogram}; +use std::sync::Arc; +use async_trait::async_trait; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use std::collections::HashSet; use tokio::sync::broadcast; use tokio::time::Instant; use tokio_tungstenite::tungstenite::protocol::{Role, WebSocketConfig}; use tokio_util::sync::CancellationToken; -use async_trait::async_trait; -use std::collections::HashSet; const INDEX_HTML: &str = include_str!("../static/index.html"); const FAVICON: &[u8] = include_bytes!("../static/favicon.ico"); @@ -30,7 +28,7 @@ const FAVICON: &[u8] = include_bytes!("../static/favicon.ico"); pub async fn serve( b: broadcast::Sender>, d: broadcast::Sender>, - shutdown: CancellationToken + shutdown: CancellationToken, ) -> Result<(), ServerError> { let config_logging = ConfigLogging::StderrTerminal { level: ConfigLoggingLevel::Info, @@ -65,7 +63,12 @@ pub async fn serve( ); let sub_shutdown = shutdown.clone(); - let ctx = Context { spec, b, d, shutdown: sub_shutdown }; + let ctx = Context { + spec, + b, + d, + shutdown: sub_shutdown, + }; let server = ServerBuilder::new(api, ctx, log) .config(ConfigDropshot { @@ -162,7 +165,6 @@ where // TODO: cors for HttpError - /// Serve index page as html #[endpoint { method = GET, @@ -316,7 +318,7 @@ async fn subscribe( upgraded.into_inner(), Role::Server, Some(WebSocketConfig::default().max_message_size( - Some(10 * 2_usize.pow(20)) // 10MiB, matching jetstream + Some(10 * 2_usize.pow(20)), // 10MiB, matching jetstream )), ) .await; diff --git a/spacedust/src/subscriber.rs b/spacedust/src/subscriber.rs index afa4717..2e6a963 100644 --- a/spacedust/src/subscriber.rs +++ b/spacedust/src/subscriber.rs @@ -1,16 +1,16 @@ use crate::error::SubscriberUpdateError; -use std::sync::Arc; -use tokio::time::interval; -use std::time::Duration; -use futures::StreamExt; -use crate::{ClientMessage, FilterableProperties, SubscriberSourcedMessage}; use crate::server::MultiSubscribeQuery; +use crate::{ClientMessage, FilterableProperties, SubscriberSourcedMessage}; +use dropshot::WebsocketConnectionRaw; use futures::SinkExt; +use futures::StreamExt; use std::error::Error; +use std::sync::Arc; +use std::time::Duration; use tokio::sync::broadcast::{self, error::RecvError}; +use tokio::time::interval; use tokio_tungstenite::{WebSocketStream, tungstenite::Message}; use tokio_util::sync::CancellationToken; -use dropshot::WebsocketConnectionRaw; const PING_PERIOD: Duration = Duration::from_secs(30); @@ -20,17 +20,14 @@ pub struct Subscriber { } impl Subscriber { - pub fn new( - query: MultiSubscribeQuery, - shutdown: CancellationToken, - ) -> Self { + pub fn new(query: MultiSubscribeQuery, shutdown: CancellationToken) -> Self { Self { query, shutdown } } pub async fn start( mut self, ws: WebSocketStream, - mut receiver: broadcast::Receiver> + mut receiver: broadcast::Receiver>, ) -> Result<(), Box> { let mut ping_state = None; let (mut ws_sender, mut ws_receiver) = ws.split(); @@ -83,6 +80,7 @@ impl Subscriber { // TODO: send client an explanation self.shutdown.cancel(); } + log::trace!("subscriber updated with opts: {:?}", self.query); }, Some(Ok(m)) => log::trace!("subscriber sent an unexpected message: {m:?}"), Some(Err(e)) => { @@ -122,36 +120,35 @@ impl Subscriber { Ok(()) } - fn filter( - &self, - properties: &FilterableProperties, - ) -> bool { + fn filter(&self, properties: &FilterableProperties) -> bool { let query = &self.query; // subject + subject DIDs are logical OR - if !( - query.wanted_subjects.is_empty() && query.wanted_subject_dids.is_empty() || - query.wanted_subjects.contains(&properties.subject) || - properties.subject_did.as_ref().map(|did| query.wanted_subject_dids.contains(did)).unwrap_or(false) - ) { // wowwww ^^ fix that - return false + if !(query.wanted_subjects.is_empty() && query.wanted_subject_dids.is_empty() + || query.wanted_subjects.contains(&properties.subject) + || properties + .subject_did + .as_ref() + .map(|did| query.wanted_subject_dids.contains(did)) + .unwrap_or(false)) + { + // wowwww ^^ fix that + return false; } // subjects together with sources are logical AND if !(query.wanted_sources.is_empty() || query.wanted_sources.contains(&properties.source)) { - return false + return false; } true } } - - impl MultiSubscribeQuery { pub fn update_from_raw(&mut self, s: &str) -> Result<(), SubscriberUpdateError> { - let SubscriberSourcedMessage::OptionsUpdate(opts) = serde_json::from_str(s) - .map_err(SubscriberUpdateError::FailedToParseMessage)?; + let SubscriberSourcedMessage::OptionsUpdate(opts) = + serde_json::from_str(s).map_err(SubscriberUpdateError::FailedToParseMessage)?; if opts.wanted_sources.len() > 1_000 { return Err(SubscriberUpdateError::TooManySourcesWanted); } diff --git a/who-am-i/Cargo.toml b/who-am-i/Cargo.toml index b15b2d3..47e6742 100644 --- a/who-am-i/Cargo.toml +++ b/who-am-i/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2024" [dependencies] -atrium-api = { version = "0.25.4", default-features = false, features = ["tokio", "agent"] } +atrium-api = { version = "0.25.4", default-features = false } atrium-identity = "0.1.5" atrium-oauth = "0.1.3" clap = { version = "4.5.40", features = ["derive"] } diff --git a/who-am-i/src/lib.rs b/who-am-i/src/lib.rs index 597444c..b112d21 100644 --- a/who-am-i/src/lib.rs +++ b/who-am-i/src/lib.rs @@ -3,5 +3,5 @@ mod oauth; mod server; pub use dns_resolver::HickoryDnsTxtResolver; +pub use oauth::{Client, authorize, client}; pub use server::serve; -pub use oauth::{Client, client, authorize}; diff --git a/who-am-i/src/main.rs b/who-am-i/src/main.rs index d4ebeee..6258eea 100644 --- a/who-am-i/src/main.rs +++ b/who-am-i/src/main.rs @@ -1,5 +1,5 @@ -use who_am_i::serve; use tokio_util::sync::CancellationToken; +use who_am_i::serve; #[tokio::main] async fn main() { diff --git a/who-am-i/src/oauth.rs b/who-am-i/src/oauth.rs index 37e797c..5bcd821 100644 --- a/who-am-i/src/oauth.rs +++ b/who-am-i/src/oauth.rs @@ -1,15 +1,14 @@ +use crate::HickoryDnsTxtResolver; use atrium_identity::{ did::{CommonDidResolver, CommonDidResolverConfig, DEFAULT_PLC_DIRECTORY_URL}, handle::{AtprotoHandleResolver, AtprotoHandleResolverConfig}, }; use atrium_oauth::{ - AuthorizeOptions, + AtprotoLocalhostClientMetadata, AuthorizeOptions, DefaultHttpClient, KnownScope, OAuthClient, + OAuthClientConfig, OAuthResolverConfig, Scope, store::{session::MemorySessionStore, state::MemoryStateStore}, - AtprotoLocalhostClientMetadata, DefaultHttpClient, KnownScope, OAuthClient, OAuthClientConfig, - OAuthResolverConfig, Scope, }; use std::sync::Arc; -use crate::HickoryDnsTxtResolver; pub type Client = OAuthClient< MemoryStateStore, @@ -23,9 +22,7 @@ pub fn client() -> Client { let config = OAuthClientConfig { client_metadata: AtprotoLocalhostClientMetadata { redirect_uris: Some(vec![String::from("http://127.0.0.1:9997/authorized")]), - scopes: Some(vec![ - Scope::Known(KnownScope::Atproto), - ]), + scopes: Some(vec![Scope::Known(KnownScope::Atproto)]), }, keys: None, resolver: OAuthResolverConfig { @@ -52,16 +49,16 @@ pub fn client() -> Client { } pub async fn authorize(client: &Client, handle: &str) -> String { - let Ok(url) = client.authorize( - handle, - AuthorizeOptions { - scopes: vec![ - Scope::Known(KnownScope::Atproto), - ], - ..Default::default() - }, - ) - .await else { + let Ok(url) = client + .authorize( + handle, + AuthorizeOptions { + scopes: vec![Scope::Known(KnownScope::Atproto)], + ..Default::default() + }, + ) + .await + else { panic!("failed to authorize"); }; url diff --git a/who-am-i/src/server.rs b/who-am-i/src/server.rs index 5a0fc1f..bb08d3c 100644 --- a/who-am-i/src/server.rs +++ b/who-am-i/src/server.rs @@ -1,17 +1,16 @@ - use atrium_api::agent::SessionManager; -use std::error::Error; -use metrics::{histogram, counter}; -use std::sync::Arc; +use dropshot::{ + ApiDescription, Body, ConfigDropshot, ConfigLogging, ConfigLoggingLevel, HttpError, + HttpResponse, HttpResponseSeeOther, Query, RequestContext, ServerBuilder, ServerContext, + endpoint, http_response_see_other, +}; use http::{ - header::{ORIGIN, USER_AGENT}, Response, StatusCode, + header::{ORIGIN, USER_AGENT}, }; -use dropshot::{ - Body, HttpResponseSeeOther, http_response_see_other, - ApiDescription, ConfigDropshot, ConfigLogging, ConfigLoggingLevel, RequestContext, - ServerBuilder, endpoint, HttpResponse, HttpError, ServerContext, Query, -}; +use metrics::{counter, histogram}; +use std::error::Error; +use std::sync::Arc; use atrium_oauth::CallbackParams; use schemars::JsonSchema; @@ -19,20 +18,17 @@ use serde::{Deserialize, Serialize}; use tokio::time::Instant; use tokio_util::sync::CancellationToken; -use crate::{Client, client, authorize}; +use crate::{Client, authorize, client}; const INDEX_HTML: &str = include_str!("../static/index.html"); const FAVICON: &[u8] = include_bytes!("../static/favicon.ico"); -pub async fn serve( - shutdown: CancellationToken -) -> Result<(), Box> { +pub async fn serve(shutdown: CancellationToken) -> Result<(), Box> { let config_logging = ConfigLogging::StderrTerminal { level: ConfigLoggingLevel::Info, }; - let log = config_logging - .to_logger("example-basic")?; + let log = config_logging.to_logger("example-basic")?; let mut api = ApiDescription::new(); api.register(index).unwrap(); @@ -58,7 +54,10 @@ pub async fn serve( .json()?, ); - let ctx = Context { spec, client: client().into() }; + let ctx = Context { + spec, + client: client().into(), + }; let server = ServerBuilder::new(api, ctx, log) .config(ConfigDropshot { @@ -153,7 +152,6 @@ where // TODO: cors for HttpError - /// Serve index page as html #[endpoint { method = GET, -- 2.51.2 From 56812cb05590435148bbf12a10cbb3ba622829f9 Mon Sep 17 00:00:00 2001 From: phil Date: Wed, 25 Jun 2025 18:21:49 -0400 Subject: [PATCH 029/134] dropshot -> axum whew ok --- Cargo.lock | 13 +- who-am-i/Cargo.toml | 9 +- who-am-i/src/main.rs | 4 +- who-am-i/src/server.rs | 293 +++++++---------------------------------- 4 files changed, 52 insertions(+), 267 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7beece2..e128f32 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -408,9 +408,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de45108900e1f9b9242f7f2e254aa3e2c029c921c258fe9e6b4217eeebd54288" +checksum = "021e862c184ae977658b36c4500f7feac3221ca5da43e3f25bd04ab6c79a29b5" dependencies = [ "axum-core", "bytes", @@ -4850,18 +4850,11 @@ dependencies = [ "atrium-api 0.25.4", "atrium-identity", "atrium-oauth", + "axum", "clap", - "dropshot", - "env_logger", "hickory-resolver", - "http", - "log", "metrics", - "schemars", - "semver", "serde", - "serde_json", - "serde_qs", "tokio", "tokio-util", ] diff --git a/who-am-i/Cargo.toml b/who-am-i/Cargo.toml index 47e6742..916c3ff 100644 --- a/who-am-i/Cargo.toml +++ b/who-am-i/Cargo.toml @@ -7,17 +7,10 @@ edition = "2024" atrium-api = { version = "0.25.4", default-features = false } atrium-identity = "0.1.5" atrium-oauth = "0.1.3" +axum = "0.8.4" clap = { version = "4.5.40", features = ["derive"] } -dropshot = "0.16.2" -env_logger = "0.11.8" hickory-resolver = "0.25.2" -http = "1.3.1" -log = "0.4.27" metrics = "0.24.2" -schemars = "0.8.22" -semver = "1.0.26" serde = { version = "1.0.219", features = ["derive"] } -serde_json = "1.0.140" -serde_qs = "1.0.0-rc.3" tokio = { version = "1.45.1", features = ["full", "macros"] } tokio-util = "0.7.15" diff --git a/who-am-i/src/main.rs b/who-am-i/src/main.rs index 6258eea..d340d12 100644 --- a/who-am-i/src/main.rs +++ b/who-am-i/src/main.rs @@ -3,8 +3,6 @@ use who_am_i::serve; #[tokio::main] async fn main() { - env_logger::init(); - let server_shutdown = CancellationToken::new(); - serve(server_shutdown).await.unwrap(); + serve(server_shutdown).await; } diff --git a/who-am-i/src/server.rs b/who-am-i/src/server.rs index bb08d3c..c2a44e1 100644 --- a/who-am-i/src/server.rs +++ b/who-am-i/src/server.rs @@ -1,21 +1,15 @@ use atrium_api::agent::SessionManager; -use dropshot::{ - ApiDescription, Body, ConfigDropshot, ConfigLogging, ConfigLoggingLevel, HttpError, - HttpResponse, HttpResponseSeeOther, Query, RequestContext, ServerBuilder, ServerContext, - endpoint, http_response_see_other, -}; -use http::{ - Response, StatusCode, - header::{ORIGIN, USER_AGENT}, +use atrium_oauth::CallbackParams; +use axum::{ + Router, + extract::{Query, State}, + response::{Html, Redirect}, + routing::get, }; -use metrics::{counter, histogram}; -use std::error::Error; -use std::sync::Arc; -use atrium_oauth::CallbackParams; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; -use tokio::time::Instant; +use serde::Deserialize; +use std::sync::Arc; +use tokio::net::TcpListener; use tokio_util::sync::CancellationToken; use crate::{Client, authorize, client}; @@ -23,248 +17,55 @@ use crate::{Client, authorize, client}; const INDEX_HTML: &str = include_str!("../static/index.html"); const FAVICON: &[u8] = include_bytes!("../static/favicon.ico"); -pub async fn serve(shutdown: CancellationToken) -> Result<(), Box> { - let config_logging = ConfigLogging::StderrTerminal { - level: ConfigLoggingLevel::Info, +pub async fn serve(shutdown: CancellationToken) { + let state = AppState { + client: Arc::new(client()), }; - let log = config_logging.to_logger("example-basic")?; - - let mut api = ApiDescription::new(); - api.register(index).unwrap(); - api.register(favicon).unwrap(); - api.register(openapi).unwrap(); - api.register(start_oauth).unwrap(); - api.register(finish_oauth).unwrap(); - - // TODO: put spec in a once cell / lazy lock thing? - let spec = Arc::new( - api.openapi( - "Who-am-i", - env!("CARGO_PKG_VERSION") - .parse() - .inspect_err(|e| { - eprintln!("failed to parse cargo package version for openapi: {e:?}") - }) - .unwrap_or(semver::Version::new(0, 0, 1)), - ) - .description("An atproto identity verifier that is very much not ready for real use") - .contact_name("part of @microcosm.blue") - .contact_url("https://microcosm.blue") - .json()?, - ); - - let ctx = Context { - spec, - client: client().into(), - }; - - let server = ServerBuilder::new(api, ctx, log) - .config(ConfigDropshot { - bind_address: "0.0.0.0:9997".parse().unwrap(), - ..Default::default() - }) - .start()?; - - tokio::select! { - s = server.wait_for_shutdown() => { - s?; - log::info!("server shut down normally."); - }, - _ = shutdown.cancelled() => { - log::info!("shutting down: closing server"); - server.close().await?; - }, - } - Ok(()) + let app = Router::new() + .route("/", get(|| async { Html(INDEX_HTML) })) + .route("/favicon.ico", get(|| async { FAVICON })) // todo MIME + .route("/auth", get(start_oauth)) + .route("/authorized", get(complete_oauth)) + .with_state(state); + + let listener = TcpListener::bind("0.0.0.0:9997") + .await + .expect("listener binding to work"); + + axum::serve(listener, app) + .with_graceful_shutdown(async move { shutdown.cancelled().await }) + .await + .unwrap(); } #[derive(Clone)] -struct Context { - pub spec: Arc, +struct AppState { pub client: Arc, } -async fn instrument_handler(ctx: &RequestContext, handler: H) -> Result -where - R: HttpResponse, - H: Future>, - T: ServerContext, -{ - let start = Instant::now(); - let result = handler.await; - let latency = start.elapsed(); - let status_code = match &result { - Ok(response) => response.status_code(), - Err(e) => e.status_code.as_status(), - } - .as_str() // just the number (.to_string()'s Display does eg `200 OK`) - .to_string(); - let endpoint = ctx.endpoint.operation_id.clone(); - let headers = ctx.request.headers(); - let origin = headers - .get(ORIGIN) - .and_then(|v| v.to_str().ok()) - .unwrap_or("") - .to_string(); - let ua = headers - .get(USER_AGENT) - .and_then(|v| v.to_str().ok()) - .map(|ua| { - if ua.starts_with("Mozilla/5.0 ") { - "browser" - } else { - ua - } - }) - .unwrap_or("") - .to_string(); - counter!("server_requests_total", - "endpoint" => endpoint.clone(), - "origin" => origin, - "ua" => ua, - "status_code" => status_code, - ) - .increment(1); - histogram!("server_handler_latency", "endpoint" => endpoint).record(latency.as_micros() as f64); - result -} - -use dropshot::{HttpResponseHeaders, HttpResponseOk}; - -pub type OkCorsResponse = Result>, HttpError>; - -/// Helper for constructing Ok responses: return OkCors(T).into() -/// (not happy with this yet) -pub struct OkCors(pub T); - -impl From> for OkCorsResponse -where - T: Serialize + JsonSchema + Send + Sync, -{ - fn from(ok: OkCors) -> OkCorsResponse { - let mut res = HttpResponseHeaders::new_unnamed(HttpResponseOk(ok.0)); - res.headers_mut() - .insert("access-control-allow-origin", "*".parse().unwrap()); - Ok(res) - } -} - -// TODO: cors for HttpError - -/// Serve index page as html -#[endpoint { - method = GET, - path = "/", - /* - * not useful to have this in openapi - */ - unpublished = true, -}] -async fn index(ctx: RequestContext) -> Result, HttpError> { - instrument_handler(&ctx, async { - Ok(Response::builder() - .status(StatusCode::OK) - .header(http::header::CONTENT_TYPE, "text/html") - .body(INDEX_HTML.into())?) - }) - .await -} - -/// Serve index page as html -#[endpoint { - method = GET, - path = "/favicon.ico", - /* - * not useful to have this in openapi - */ - unpublished = true, -}] -async fn favicon(ctx: RequestContext) -> Result, HttpError> { - instrument_handler(&ctx, async { - Ok(Response::builder() - .status(StatusCode::OK) - .header(http::header::CONTENT_TYPE, "image/x-icon") - .body(FAVICON.to_vec().into())?) - }) - .await -} - -/// Meta: get the openapi spec for this api -#[endpoint { - method = GET, - path = "/openapi", - /* - * not useful to have this in openapi - */ - unpublished = true, -}] -async fn openapi(ctx: RequestContext) -> OkCorsResponse { - instrument_handler(&ctx, async { - let spec = (*ctx.context().spec).clone(); - OkCors(spec).into() - }) - .await -} - -#[derive(Debug, Deserialize, JsonSchema)] -struct BeginOauthQuery { +#[derive(Debug, Deserialize)] +struct BeginOauthParams { handle: String, } -#[endpoint { - method = GET, - path = "/auth", -}] async fn start_oauth( - ctx: RequestContext, - query: Query, -) -> Result { - let BeginOauthQuery { handle } = query.into_inner(); - - instrument_handler(&ctx, async { - let Context { client, .. } = ctx.context(); - - let auth_url = authorize(client, &handle).await; - - http_response_see_other(auth_url) - }) - .await -} - -#[derive(Debug, Deserialize, JsonSchema)] -struct AuthorizedCallbackQuery { - code: String, - state: Option, - iss: Option, + State(state): State, + Query(params): Query, +) -> Redirect { + let AppState { client } = state; + let BeginOauthParams { handle } = params; + let auth_url = authorize(&client, &handle).await; + Redirect::to(&auth_url) } -impl From for CallbackParams { - fn from(q: AuthorizedCallbackQuery) -> Self { - let AuthorizedCallbackQuery { code, state, iss } = q; - Self { code, state, iss } - } -} -#[endpoint { - method = GET, - path = "/authorized", -}] -async fn finish_oauth( - ctx: RequestContext, - query: Query, -) -> Result, HttpError> { - instrument_handler(&ctx, async { - let Context { client, .. } = ctx.context(); - let params = query.into_inner(); - - let Ok((oauth_session, _)) = client.callback(params.into()).await else { - panic!("failed to do client callback"); - }; - let did = oauth_session.did().await.expect("a did to be present"); - - Ok(Response::builder() - .status(StatusCode::OK) - .header(http::header::CONTENT_TYPE, "text/html") - .body(format!("sup: {did:?}").into())?) - }) - .await +async fn complete_oauth( + State(state): State, + Query(params): Query, +) -> Html { + let AppState { client } = state; + let Ok((oauth_session, _)) = client.callback(params).await else { + panic!("failed to do client callback"); + }; + let did = oauth_session.did().await.expect("a did to be present"); + Html(format!("sup: {did:?}")) } -- 2.51.2 From b9be4800d85e9a6f1468ca0dba2ad77ecf73094c Mon Sep 17 00:00:00 2001 From: phil Date: Wed, 25 Jun 2025 20:56:15 -0400 Subject: [PATCH 030/134] wip, prompting flow with cookie --- Cargo.lock | 18 +++++++++++++++ who-am-i/Cargo.toml | 1 + who-am-i/src/server.rs | 45 ++++++++++++++++++++++++++++---------- who-am-i/static/login.html | 17 ++++++++++++++ 4 files changed, 70 insertions(+), 11 deletions(-) create mode 100644 who-am-i/static/login.html diff --git a/Cargo.lock b/Cargo.lock index e128f32..64f4c27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -469,6 +469,7 @@ dependencies = [ "axum", "axum-core", "bytes", + "cookie", "futures-util", "headers", "http", @@ -920,6 +921,22 @@ dependencies = [ "zstd", ] +[[package]] +name = "cookie" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" +dependencies = [ + "base64 0.22.1", + "hmac", + "percent-encoding", + "rand 0.8.5", + "sha2", + "subtle", + "time", + "version_check", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -4851,6 +4868,7 @@ dependencies = [ "atrium-identity", "atrium-oauth", "axum", + "axum-extra", "clap", "hickory-resolver", "metrics", diff --git a/who-am-i/Cargo.toml b/who-am-i/Cargo.toml index 916c3ff..a61d78f 100644 --- a/who-am-i/Cargo.toml +++ b/who-am-i/Cargo.toml @@ -8,6 +8,7 @@ atrium-api = { version = "0.25.4", default-features = false } atrium-identity = "0.1.5" atrium-oauth = "0.1.3" axum = "0.8.4" +axum-extra = { version = "0.10.1", features = ["cookie-signed", "typed-header"] } clap = { version = "4.5.40", features = ["derive"] } hickory-resolver = "0.25.2" metrics = "0.24.2" diff --git a/who-am-i/src/server.rs b/who-am-i/src/server.rs index c2a44e1..527c4f4 100644 --- a/who-am-i/src/server.rs +++ b/who-am-i/src/server.rs @@ -2,10 +2,11 @@ use atrium_api::agent::SessionManager; use atrium_oauth::CallbackParams; use axum::{ Router, - extract::{Query, State}, + extract::{FromRef, Query, State}, response::{Html, Redirect}, routing::get, }; +use axum_extra::extract::cookie::{Cookie, Key, SignedCookieJar}; use serde::Deserialize; use std::sync::Arc; @@ -14,17 +15,20 @@ use tokio_util::sync::CancellationToken; use crate::{Client, authorize, client}; -const INDEX_HTML: &str = include_str!("../static/index.html"); const FAVICON: &[u8] = include_bytes!("../static/favicon.ico"); +const INDEX_HTML: &str = include_str!("../static/index.html"); +const LOGIN_HTML: &str = include_str!("../static/login.html"); pub async fn serve(shutdown: CancellationToken) { let state = AppState { + key: Key::generate(), // TODO: via config client: Arc::new(client()), }; let app = Router::new() .route("/", get(|| async { Html(INDEX_HTML) })) .route("/favicon.ico", get(|| async { FAVICON })) // todo MIME + .route("/prompt", get(prompt)) .route("/auth", get(start_oauth)) .route("/authorized", get(complete_oauth)) .with_state(state); @@ -41,9 +45,25 @@ pub async fn serve(shutdown: CancellationToken) { #[derive(Clone)] struct AppState { + pub key: Key, pub client: Arc, } +impl FromRef for Key { + fn from_ref(state: &AppState) -> Self { + state.key.clone() + } +} + +async fn prompt(jar: SignedCookieJar) -> (SignedCookieJar, Html) { + let m = if let Some(did) = jar.get("did") { + format!("oh i know you: {did}") + } else { + LOGIN_HTML.into() + }; + (jar, Html(m)) +} + #[derive(Debug, Deserialize)] struct BeginOauthParams { handle: String, @@ -51,21 +71,24 @@ struct BeginOauthParams { async fn start_oauth( State(state): State, Query(params): Query, -) -> Redirect { - let AppState { client } = state; - let BeginOauthParams { handle } = params; - let auth_url = authorize(&client, &handle).await; - Redirect::to(&auth_url) + jar: SignedCookieJar, +) -> (SignedCookieJar, Redirect) { + // if any existing session was active, clear it first + let jar = jar.remove("did"); + + let auth_url = authorize(&state.client, ¶ms.handle).await; + (jar, Redirect::to(&auth_url)) } async fn complete_oauth( State(state): State, Query(params): Query, -) -> Html { - let AppState { client } = state; - let Ok((oauth_session, _)) = client.callback(params).await else { + jar: SignedCookieJar, +) -> (SignedCookieJar, Html) { + let Ok((oauth_session, _)) = state.client.callback(params).await else { panic!("failed to do client callback"); }; let did = oauth_session.did().await.expect("a did to be present"); - Html(format!("sup: {did:?}")) + let jar = jar.add(Cookie::new("did", did.to_string())); + (jar, Html(format!("sup: {did:?}"))) } diff --git a/who-am-i/static/login.html b/who-am-i/static/login.html new file mode 100644 index 0000000..c53f701 --- /dev/null +++ b/who-am-i/static/login.html @@ -0,0 +1,17 @@ + + + + + Who-am-i + + + + +
+ + +
+ + -- 2.51.2 From 08b772bf9f70797faafd225bccac484018df1c0f Mon Sep 17 00:00:00 2001 From: phil Date: Wed, 25 Jun 2025 22:44:40 -0400 Subject: [PATCH 031/134] host page blah blah --- who-am-i/demo/index.html | 14 ++++++++++++++ who-am-i/demo/serve | 4 ++++ who-am-i/src/server.rs | 13 +++++++++---- who-am-i/static/prompt-anon.html | 0 4 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 who-am-i/demo/index.html create mode 100755 who-am-i/demo/serve create mode 100644 who-am-i/static/prompt-anon.html diff --git a/who-am-i/demo/index.html b/who-am-i/demo/index.html new file mode 100644 index 0000000..0a11f1b --- /dev/null +++ b/who-am-i/demo/index.html @@ -0,0 +1,14 @@ + + + +

hey

+ + + + + + diff --git a/who-am-i/src/expiring_task_map.rs b/who-am-i/src/expiring_task_map.rs index d6717b6..050421c 100644 --- a/who-am-i/src/expiring_task_map.rs +++ b/who-am-i/src/expiring_task_map.rs @@ -3,21 +3,29 @@ use rand::{Rng, distr::Alphanumeric}; use std::sync::Arc; use std::time::Duration; use tokio::task::{JoinHandle, spawn}; -use tokio::time::sleep; // 0.8 +use tokio::time::sleep; +use tokio_util::sync::{CancellationToken, DropGuard}; #[derive(Clone)] -pub struct ExpiringTaskMap(Arc>); +pub struct ExpiringTaskMap(TaskMap); impl ExpiringTaskMap { pub fn new(expiration: Duration) -> Self { let map = TaskMap { - map: DashMap::new(), + map: Arc::new(DashMap::new()), expiration, }; - Self(Arc::new(map)) + Self(map) } - pub fn dispatch(&self, task: impl Future + Send + 'static) -> String { + pub fn dispatch(&self, task: F, cancel: CancellationToken) -> String + where + F: Future + Send + 'static, + { + let TaskMap { + ref map, + expiration, + } = self.0; let task_key: String = rand::rng() .sample_iter(&Alphanumeric) .take(24) @@ -25,16 +33,19 @@ impl ExpiringTaskMap { .collect(); // spawn a tokio task and put the join handle in the map for later retrieval - self.0.map.insert(task_key.clone(), spawn(task)); + map.insert(task_key.clone(), (cancel.clone().drop_guard(), spawn(task))); // spawn a second task to clean up the map in case it doesn't get claimed - spawn({ - let me = self.0.clone(); - let key = task_key.clone(); - async move { - sleep(me.expiration).await; - let _ = me.map.remove(&key); - // TODO: also use a cancellation token so taking and expiring can mutually cancel + let k = task_key.clone(); + let map = map.clone(); + spawn(async move { + if cancel + .run_until_cancelled(sleep(expiration)) + .await + .is_some() + { + map.remove(&k); + cancel.cancel(); } }); @@ -42,12 +53,13 @@ impl ExpiringTaskMap { } pub fn take(&self, key: &str) -> Option> { - eprintln!("trying to take..."); - self.0.map.remove(key).map(|(_, handle)| handle) + // when the _guard drops, the token gets cancelled for us + self.0.map.remove(key).map(|(_, (_guard, handle))| handle) } } +#[derive(Clone)] struct TaskMap { - map: DashMap>, + map: Arc)>>, expiration: Duration, } diff --git a/who-am-i/src/identity_resolver.rs b/who-am-i/src/identity_resolver.rs index dd78581..761c9f0 100644 --- a/who-am-i/src/identity_resolver.rs +++ b/who-am-i/src/identity_resolver.rs @@ -4,17 +4,19 @@ use atrium_identity::did::{CommonDidResolver, CommonDidResolverConfig, DEFAULT_P use atrium_oauth::DefaultHttpClient; use std::sync::Arc; -pub async fn resolve_identity(did: String) -> String { +pub async fn resolve_identity(did: String) -> Option { let http_client = Arc::new(DefaultHttpClient::default()); let resolver = CommonDidResolver::new(CommonDidResolverConfig { plc_directory_url: DEFAULT_PLC_DIRECTORY_URL.to_string(), http_client: Arc::clone(&http_client), }); let doc = resolver.resolve(&Did::new(did).unwrap()).await.unwrap(); // TODO: this is only half the resolution? or is atrium checking dns? - if let Some(aka) = doc.also_known_as { - if let Some(f) = aka.first() { - return f.to_string(); + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + doc.also_known_as.and_then(|mut aka| { + if aka.is_empty() { + None + } else { + Some(aka.remove(0)) } - } - "who knows".to_string() + }) } diff --git a/who-am-i/src/server.rs b/who-am-i/src/server.rs index 840d7c1..56a9d44 100644 --- a/who-am-i/src/server.rs +++ b/who-am-i/src/server.rs @@ -4,7 +4,7 @@ use axum::{ Router, extract::{FromRef, Query, State}, http::header::{HeaderMap, REFERER}, - response::{Html, IntoResponse, Redirect}, + response::{Html, IntoResponse, Json, Redirect}, routing::get, }; use axum_extra::extract::cookie::{Cookie, Key, SameSite, SignedCookieJar}; @@ -12,7 +12,7 @@ use axum_template::{RenderHtml, engine::Engine}; use handlebars::{Handlebars, handlebars_helper}; use serde::{Deserialize, Serialize}; -use serde_json::Value; +use serde_json::{Value, json}; use std::sync::Arc; use std::time::Duration; use tokio::net::TcpListener; @@ -34,7 +34,8 @@ struct AppState { pub key: Key, pub engine: AppEngine, pub client: Arc, - pub resolving: ExpiringTaskMap, + pub resolving: ExpiringTaskMap>, + pub shutdown: CancellationToken, } impl FromRef for Key { @@ -60,6 +61,7 @@ pub async fn serve(shutdown: CancellationToken, app_secret: String, dev: bool) { key: Key::from(app_secret.as_bytes()), // TODO: via config client: Arc::new(client()), resolving: ExpiringTaskMap::new(task_pickup_expiration), + shutdown: shutdown.clone(), }; let app = Router::new() @@ -89,7 +91,10 @@ struct Known { } async fn prompt( State(AppState { - engine, resolving, .. + engine, + resolving, + shutdown, + .. }): State, jar: SignedCookieJar, headers: HeaderMap, @@ -109,7 +114,8 @@ async fn prompt( let m = if let Some(did) = jar.get(DID_COOKIE_KEY) { let did = did.value_trimmed().to_string(); - let fetch_key = resolving.dispatch(resolve_identity(did.clone())); + let task_shutdown = shutdown.child_token(); + let fetch_key = resolving.dispatch(resolve_identity(did.clone()), task_shutdown); let json_did = Value::String(did); let json_fetch_key = Value::String(fetch_key); @@ -134,12 +140,19 @@ async fn user_info( State(AppState { resolving, .. }): State, Query(params): Query, ) -> impl IntoResponse { - // let fetch_key: [char; 16] = params.fetch_key.chars().collect::>().try_into().unwrap(); - let Some(handle) = resolving.take(¶ms.fetch_key) else { + let Some(task_handle) = resolving.take(¶ms.fetch_key) else { return "oops, task does not exist or is gone".into_response(); }; - let s = handle.await.unwrap(); - format!("sup: {s}").into_response() + if let Some(handle) = task_handle.await.unwrap() { + // TODO: get active state etc. + // ...but also, that's a bsky thing? + let Some(handle) = handle.strip_prefix("at://") else { + return "hmm, handle did not start with at://".into_response(); + }; + Json(json!({ "handle": handle })).into_response() + } else { + "no handle?".into_response() + } } #[derive(Debug, Deserialize)] diff --git a/who-am-i/templates/prompt-known.hbs b/who-am-i/templates/prompt-known.hbs index bdccddd..7edebbb 100644 --- a/who-am-i/templates/prompt-known.hbs +++ b/who-am-i/templates/prompt-known.hbs @@ -30,6 +30,10 @@ header { header > * { flex-basis: 33%; } +header > .empty { + font-size: 0.8rem; + opacity: 0.5; +} header > .title { text-align: center; } @@ -43,18 +47,81 @@ header > a.micro:hover { opacity: 1; } main { - padding: 0.25rem 0.5rem; background: #ccc; + display: flex; + flex-direction: column; flex-grow: 1; + padding: 0.25rem 0.5rem; } p { margin: 0.5rem 0; } + +#loader { + display: flex; + flex-grow: 1; + justify-content: center; + align-items: center; + margin-bottom: 1rem; +} +.spinner { + animation: rotation 1.618s ease-in-out infinite; + border-radius: 50%; + border: 3px dashed #434; + box-sizing: border-box; + display: inline-block; + height: 1.5em; + width: 1.5em; +} +@keyframes rotation { + 0% { transform: rotate(0deg) } + 100% { transform: rotate(360deg) } +} + +#user-info { + flex-grow: 1; + display: flex; + flex-direction: column; + justify-content: center; + margin-bottom: 1rem; +} +#action { + background: #eee; + display: flex; + justify-content: space-between; + padding: 0.5rem 0.25rem 0.5rem 0.5rem; + font-size: 0.8rem; + align-items: baseline; + border-radius: 0.5rem; + border: 1px solid #bbb; + cursor: pointer; +} +#action:hover { + background: #fff; +} +#allow { + background: transparent; + border: none; + border-left: 1px solid #bbb; + padding: 0 0.5rem; + color: #375; + font: inherit; + cursor: pointer; +} +#action:hover #allow { + color: #396; +} + + +.hidden { + display: none !important; +} +
-
+
🔒
who-am-i

Share your identity with {{ parent_host }}?

-
Loading…
+
+ +
+
-- 2.51.2 From 94a8ae11cf86f10602fa6008cf9ab90891d00684 Mon Sep 17 00:00:00 2001 From: phil Date: Fri, 27 Jun 2025 00:06:34 -0400 Subject: [PATCH 034/134] lil tweaks --- who-am-i/demo/index.html | 2 +- who-am-i/src/identity_resolver.rs | 1 - who-am-i/templates/prompt-known.hbs | 37 +++++++++++++++++++++++++---- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/who-am-i/demo/index.html b/who-am-i/demo/index.html index 2e6e213..e0f2fb0 100644 --- a/who-am-i/demo/index.html +++ b/who-am-i/demo/index.html @@ -13,7 +13,7 @@

hey

- + diff --git a/who-am-i/src/server.rs b/who-am-i/src/server.rs index 56a9d44..f5d1f7a 100644 --- a/who-am-i/src/server.rs +++ b/who-am-i/src/server.rs @@ -11,7 +11,7 @@ use axum_extra::extract::cookie::{Cookie, Key, SameSite, SignedCookieJar}; use axum_template::{RenderHtml, engine::Engine}; use handlebars::{Handlebars, handlebars_helper}; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; use serde_json::{Value, json}; use std::sync::Arc; use std::time::Duration; @@ -23,7 +23,6 @@ use crate::{Client, ExpiringTaskMap, authorize, client, resolve_identity}; const FAVICON: &[u8] = include_bytes!("../static/favicon.ico"); const INDEX_HTML: &str = include_str!("../static/index.html"); -const LOGIN_HTML: &str = include_str!("../static/login.html"); const DID_COOKIE_KEY: &str = "did"; @@ -83,12 +82,6 @@ pub async fn serve(shutdown: CancellationToken, app_secret: String, dev: bool) { .unwrap(); } -#[derive(Debug, Serialize)] -struct Known { - did: Value, - fetch_key: Value, - parent_host: String, -} async fn prompt( State(AppState { engine, @@ -111,7 +104,7 @@ async fn prompt( let Some(parent_host) = url.host_str() else { return "could nto get host from url".into_response(); }; - let m = if let Some(did) = jar.get(DID_COOKIE_KEY) { + if let Some(did) = jar.get(DID_COOKIE_KEY) { let did = did.value_trimmed().to_string(); let task_shutdown = shutdown.child_token(); @@ -119,16 +112,26 @@ async fn prompt( let json_did = Value::String(did); let json_fetch_key = Value::String(fetch_key); - let known = Known { - did: json_did, - fetch_key: json_fetch_key, - parent_host: parent_host.to_string(), - }; - return (jar, RenderHtml("prompt-known", engine, known)).into_response(); + RenderHtml( + "prompt-known", + engine, + json!({ + "did": json_did, + "fetch_key": json_fetch_key, + "parent_host": parent_host, + }), + ) + .into_response() } else { - LOGIN_HTML.into_response() - }; - (jar, Html(m)).into_response() + RenderHtml( + "prompt-anon", + engine, + json!({ + "parent_host": parent_host, + }), + ) + .into_response() + } } #[derive(Debug, Deserialize)] @@ -175,7 +178,7 @@ async fn complete_oauth( State(state): State, Query(params): Query, jar: SignedCookieJar, -) -> (SignedCookieJar, Html) { +) -> (SignedCookieJar, impl IntoResponse) { let Ok((oauth_session, _)) = state.client.callback(params).await else { panic!("failed to do client callback"); }; @@ -186,5 +189,8 @@ async fn complete_oauth( .same_site(SameSite::None) .max_age(std::time::Duration::from_secs(86_400).try_into().unwrap()); let jar = jar.add(cookie); - (jar, Html(format!("sup: {did:?}"))) + ( + jar, + RenderHtml("authorized", state.engine, json!({ "did": did })), + ) } diff --git a/who-am-i/static/login.html b/who-am-i/static/login.html deleted file mode 100644 index c53f701..0000000 --- a/who-am-i/static/login.html +++ /dev/null @@ -1,17 +0,0 @@ - - - - - Who-am-i - - - - -
- - -
- - diff --git a/who-am-i/static/prompt-anon.html b/who-am-i/static/prompt-anon.html deleted file mode 100644 index e69de29..0000000 diff --git a/who-am-i/templates/authorized.hbs b/who-am-i/templates/authorized.hbs new file mode 100644 index 0000000..e0a5e94 --- /dev/null +++ b/who-am-i/templates/authorized.hbs @@ -0,0 +1,9 @@ + + +

oh sick. hey {{ did }}. you can close this window now.

+ + diff --git a/who-am-i/templates/prompt-anon.hbs b/who-am-i/templates/prompt-anon.hbs new file mode 100644 index 0000000..425d3a1 --- /dev/null +++ b/who-am-i/templates/prompt-anon.hbs @@ -0,0 +1,34 @@ +{{#*inline "main"}} +

+ Share your identity with + {{ parent_host }}? +

+ +
+
+ + +
+
+ + +{{/inline}} + +{{#> prompt-base}}{{/prompt-base}} diff --git a/who-am-i/templates/prompt-base.hbs b/who-am-i/templates/prompt-base.hbs new file mode 100644 index 0000000..b7217f9 --- /dev/null +++ b/who-am-i/templates/prompt-base.hbs @@ -0,0 +1,165 @@ + + + + +
+
+
🔒
+ who-am-i + microcosm +
+ +
+ {{> main}} +
+
+ diff --git a/who-am-i/templates/prompt-known.hbs b/who-am-i/templates/prompt-known.hbs index e9444b7..f248df8 100644 --- a/who-am-i/templates/prompt-known.hbs +++ b/who-am-i/templates/prompt-known.hbs @@ -1,177 +1,20 @@ - - - - -
-
-
🔒
- who-am-i - microcosm -
- -
-

- Share your identity with - {{ parent_host }}? -

-
- -
- - -
+{{#*inline "main"}} +

+ Share your identity with + {{ parent_host }}? +

+
+ +
+ + - +{{/inline}} + +{{#> prompt-base}}{{/prompt-base}} -- 2.51.2 From 0ad8c402b9cfa221bd52b84827064480ff2c807a Mon Sep 17 00:00:00 2001 From: phil Date: Fri, 27 Jun 2025 12:08:21 -0400 Subject: [PATCH 036/134] anon auto-flow whatever --- who-am-i/src/identity_resolver.rs | 1 + who-am-i/src/server.rs | 27 +++++++++++++----- who-am-i/templates/authorized.hbs | 6 +++- who-am-i/templates/prompt-anon.hbs | 45 +++++++++++++++++++++++++++++- 4 files changed, 70 insertions(+), 9 deletions(-) diff --git a/who-am-i/src/identity_resolver.rs b/who-am-i/src/identity_resolver.rs index 57fcc9c..9990145 100644 --- a/who-am-i/src/identity_resolver.rs +++ b/who-am-i/src/identity_resolver.rs @@ -11,6 +11,7 @@ pub async fn resolve_identity(did: String) -> Option { http_client: Arc::clone(&http_client), }); let doc = resolver.resolve(&Did::new(did).unwrap()).await.unwrap(); // TODO: this is only half the resolution? or is atrium checking dns? + // tokio::time::sleep(std::time::Duration::from_secs(2)).await; doc.also_known_as.and_then(|mut aka| { if aka.is_empty() { None diff --git a/who-am-i/src/server.rs b/who-am-i/src/server.rs index f5d1f7a..3c89fb0 100644 --- a/who-am-i/src/server.rs +++ b/who-am-i/src/server.rs @@ -12,7 +12,7 @@ use axum_template::{RenderHtml, engine::Engine}; use handlebars::{Handlebars, handlebars_helper}; use serde::Deserialize; -use serde_json::{Value, json}; +use serde_json::json; use std::sync::Arc; use std::time::Duration; use tokio::net::TcpListener; @@ -49,7 +49,7 @@ pub async fn serve(shutdown: CancellationToken, app_secret: String, dev: bool) { hbs.register_templates_directory("templates", Default::default()) .unwrap(); - handlebars_helper!(json: |v: Value| serde_json::to_string(&v).unwrap()); + handlebars_helper!(json: |v: String| serde_json::to_string(&v).unwrap()); hbs.register_helper("json", Box::new(json)); // clients have to pick up their identity-resolving tasks within this period @@ -110,14 +110,12 @@ async fn prompt( let task_shutdown = shutdown.child_token(); let fetch_key = resolving.dispatch(resolve_identity(did.clone()), task_shutdown); - let json_did = Value::String(did); - let json_fetch_key = Value::String(fetch_key); RenderHtml( "prompt-known", engine, json!({ - "did": json_did, - "fetch_key": json_fetch_key, + "did": did, + "fetch_key": fetch_key, "parent_host": parent_host, }), ) @@ -183,14 +181,29 @@ async fn complete_oauth( panic!("failed to do client callback"); }; let did = oauth_session.did().await.expect("a did to be present"); + let cookie = Cookie::build((DID_COOKIE_KEY, did.to_string())) .http_only(true) .secure(true) .same_site(SameSite::None) .max_age(std::time::Duration::from_secs(86_400).try_into().unwrap()); + let jar = jar.add(cookie); + + let task_shutdown = state.shutdown.child_token(); + let fetch_key = state + .resolving + .dispatch(resolve_identity(did.to_string()), task_shutdown); + ( jar, - RenderHtml("authorized", state.engine, json!({ "did": did })), + RenderHtml( + "authorized", + state.engine, + json!({ + "did": did, + "fetch_key": fetch_key, + }), + ), ) } diff --git a/who-am-i/templates/authorized.hbs b/who-am-i/templates/authorized.hbs index e0a5e94..7c95d22 100644 --- a/who-am-i/templates/authorized.hbs +++ b/who-am-i/templates/authorized.hbs @@ -4,6 +4,10 @@ diff --git a/who-am-i/templates/prompt-anon.hbs b/who-am-i/templates/prompt-anon.hbs index 425d3a1..f883800 100644 --- a/who-am-i/templates/prompt-anon.hbs +++ b/who-am-i/templates/prompt-anon.hbs @@ -4,6 +4,10 @@ {{ parent_host }}?

+ +
{{/inline}} -- 2.51.2 From 5340be12993bcc63694dad259f6c0e168e125154 Mon Sep 17 00:00:00 2001 From: phil Date: Fri, 27 Jun 2025 12:27:50 -0400 Subject: [PATCH 037/134] referer allowlist laksdl --- who-am-i/src/main.rs | 18 ++++++++++++++++-- who-am-i/src/server.rs | 15 ++++++++++++++- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/who-am-i/src/main.rs b/who-am-i/src/main.rs index b635638..5e8e706 100644 --- a/who-am-i/src/main.rs +++ b/who-am-i/src/main.rs @@ -1,4 +1,4 @@ -use clap::Parser; +use clap::{ArgAction, Parser}; use tokio_util::sync::CancellationToken; use who_am_i::serve; @@ -18,6 +18,11 @@ struct Args { /// enables automatic template reloading #[arg(long, action)] dev: bool, + /// Hosts who are allowed to one-click auth + /// + /// Pass this argument multiple times to allow multiple hosts + #[arg(long, short = 'o', action = ArgAction::Append)] + one_click: Vec, } #[tokio::main] @@ -29,5 +34,14 @@ async fn main() { let args = Args::parse(); - serve(shutdown, args.app_secret, args.dev).await; + if args.one_click.is_empty() { + panic!("at least one --one-click host must be set"); + } + + println!("starting with allowed hosts:"); + for host in &args.one_click { + println!(" - {host}"); + } + + serve(shutdown, args.app_secret, args.one_click, args.dev).await; } diff --git a/who-am-i/src/server.rs b/who-am-i/src/server.rs index 3c89fb0..99d53e1 100644 --- a/who-am-i/src/server.rs +++ b/who-am-i/src/server.rs @@ -13,6 +13,7 @@ use handlebars::{Handlebars, handlebars_helper}; use serde::Deserialize; use serde_json::json; +use std::collections::HashSet; use std::sync::Arc; use std::time::Duration; use tokio::net::TcpListener; @@ -31,6 +32,7 @@ type AppEngine = Engine>; #[derive(Clone)] struct AppState { pub key: Key, + pub one_clicks: Arc>, pub engine: AppEngine, pub client: Arc, pub resolving: ExpiringTaskMap>, @@ -43,7 +45,12 @@ impl FromRef for Key { } } -pub async fn serve(shutdown: CancellationToken, app_secret: String, dev: bool) { +pub async fn serve( + shutdown: CancellationToken, + app_secret: String, + one_click: Vec, + dev: bool, +) { let mut hbs = Handlebars::new(); hbs.set_dev_mode(dev); hbs.register_templates_directory("templates", Default::default()) @@ -58,6 +65,7 @@ pub async fn serve(shutdown: CancellationToken, app_secret: String, dev: bool) { let state = AppState { engine: Engine::new(hbs), key: Key::from(app_secret.as_bytes()), // TODO: via config + one_clicks: Arc::new(HashSet::from_iter(one_click)), client: Arc::new(client()), resolving: ExpiringTaskMap::new(task_pickup_expiration), shutdown: shutdown.clone(), @@ -85,6 +93,7 @@ pub async fn serve(shutdown: CancellationToken, app_secret: String, dev: bool) { async fn prompt( State(AppState { engine, + one_clicks, resolving, shutdown, .. @@ -104,6 +113,10 @@ async fn prompt( let Some(parent_host) = url.host_str() else { return "could nto get host from url".into_response(); }; + if !one_clicks.contains(parent_host) { + return format!("host {parent_host:?} not in one_clicks, disallowing for now") + .into_response(); + } if let Some(did) = jar.get(DID_COOKIE_KEY) { let did = did.value_trimmed().to_string(); -- 2.51.2 From 95c60ce3e45cdcac580b7968ae75ad351e724547 Mon Sep 17 00:00:00 2001 From: phil Date: Mon, 30 Jun 2025 16:51:14 -0400 Subject: [PATCH 038/134] consolidate oauth handling --- Cargo.lock | 1 + who-am-i/Cargo.toml | 1 + who-am-i/src/dns_resolver.rs | 34 ----- who-am-i/src/expiring_task_map.rs | 19 ++- who-am-i/src/identity_resolver.rs | 22 --- who-am-i/src/lib.rs | 6 +- who-am-i/src/oauth.rs | 244 ++++++++++++++++++++++++------ who-am-i/src/server.rs | 68 +++++---- 8 files changed, 258 insertions(+), 137 deletions(-) delete mode 100644 who-am-i/src/dns_resolver.rs delete mode 100644 who-am-i/src/identity_resolver.rs diff --git a/Cargo.lock b/Cargo.lock index dccf614..192663b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5024,6 +5024,7 @@ dependencies = [ "rand 0.9.1", "serde", "serde_json", + "thiserror 2.0.12", "tokio", "tokio-util", "url", diff --git a/who-am-i/Cargo.toml b/who-am-i/Cargo.toml index 8475f0c..b59f454 100644 --- a/who-am-i/Cargo.toml +++ b/who-am-i/Cargo.toml @@ -20,6 +20,7 @@ metrics = "0.24.2" rand = "0.9.1" serde = { version = "1.0.219", features = ["derive"] } serde_json = "1.0.140" +thiserror = "2.0.12" tokio = { version = "1.45.1", features = ["full", "macros"] } tokio-util = "0.7.15" url = "2.5.4" diff --git a/who-am-i/src/dns_resolver.rs b/who-am-i/src/dns_resolver.rs deleted file mode 100644 index 2d51fc7..0000000 --- a/who-am-i/src/dns_resolver.rs +++ /dev/null @@ -1,34 +0,0 @@ -// originally from weaver: https://github.com/rsform/weaver/blob/ee08213a85e09889b9bd66beceecee92ac025801/crates/weaver-common/src/resolver.rs -// MPL 2.0: https://github.com/rsform/weaver/blob/ee08213a85e09889b9bd66beceecee92ac025801/LICENSE - -use atrium_identity::handle::DnsTxtResolver; -use hickory_resolver::TokioResolver; - -pub struct HickoryDnsTxtResolver { - resolver: TokioResolver, -} - -impl Default for HickoryDnsTxtResolver { - fn default() -> Self { - Self { - resolver: TokioResolver::builder_tokio() - .expect("failed to create resolver") - .build(), - } - } -} - -impl DnsTxtResolver for HickoryDnsTxtResolver { - async fn resolve( - &self, - query: &str, - ) -> core::result::Result, Box> { - Ok(self - .resolver - .txt_lookup(query) - .await? - .iter() - .map(|txt| txt.to_string()) - .collect()) - } -} diff --git a/who-am-i/src/expiring_task_map.rs b/who-am-i/src/expiring_task_map.rs index 050421c..f6d0899 100644 --- a/who-am-i/src/expiring_task_map.rs +++ b/who-am-i/src/expiring_task_map.rs @@ -6,9 +6,15 @@ use tokio::task::{JoinHandle, spawn}; use tokio::time::sleep; use tokio_util::sync::{CancellationToken, DropGuard}; -#[derive(Clone)] pub struct ExpiringTaskMap(TaskMap); +/// need to manually implement clone because T is allowed to not be clone +impl Clone for ExpiringTaskMap { + fn clone(&self) -> Self { + Self(self.0.clone()) + } +} + impl ExpiringTaskMap { pub fn new(expiration: Duration) -> Self { let map = TaskMap { @@ -58,8 +64,17 @@ impl ExpiringTaskMap { } } -#[derive(Clone)] struct TaskMap { map: Arc)>>, expiration: Duration, } + +/// need to manually implement clone because T is allowed to not be clone +impl Clone for TaskMap { + fn clone(&self) -> Self { + Self { + map: self.map.clone(), + expiration: self.expiration, + } + } +} diff --git a/who-am-i/src/identity_resolver.rs b/who-am-i/src/identity_resolver.rs deleted file mode 100644 index 9990145..0000000 --- a/who-am-i/src/identity_resolver.rs +++ /dev/null @@ -1,22 +0,0 @@ -use atrium_api::types::string::Did; -use atrium_common::resolver::Resolver; -use atrium_identity::did::{CommonDidResolver, CommonDidResolverConfig, DEFAULT_PLC_DIRECTORY_URL}; -use atrium_oauth::DefaultHttpClient; -use std::sync::Arc; - -pub async fn resolve_identity(did: String) -> Option { - let http_client = Arc::new(DefaultHttpClient::default()); - let resolver = CommonDidResolver::new(CommonDidResolverConfig { - plc_directory_url: DEFAULT_PLC_DIRECTORY_URL.to_string(), - http_client: Arc::clone(&http_client), - }); - let doc = resolver.resolve(&Did::new(did).unwrap()).await.unwrap(); // TODO: this is only half the resolution? or is atrium checking dns? - // tokio::time::sleep(std::time::Duration::from_secs(2)).await; - doc.also_known_as.and_then(|mut aka| { - if aka.is_empty() { - None - } else { - Some(aka.remove(0)) - } - }) -} diff --git a/who-am-i/src/lib.rs b/who-am-i/src/lib.rs index 7e7cb10..e5d3a7d 100644 --- a/who-am-i/src/lib.rs +++ b/who-am-i/src/lib.rs @@ -1,11 +1,7 @@ -mod dns_resolver; mod expiring_task_map; -mod identity_resolver; mod oauth; mod server; -pub use dns_resolver::HickoryDnsTxtResolver; pub use expiring_task_map::ExpiringTaskMap; -pub use identity_resolver::resolve_identity; -pub use oauth::{Client, authorize, client}; +pub use oauth::{OAuth, OauthCallbackParams, ResolveHandleError}; pub use server::serve; diff --git a/who-am-i/src/oauth.rs b/who-am-i/src/oauth.rs index 5bcd821..0a183b3 100644 --- a/who-am-i/src/oauth.rs +++ b/who-am-i/src/oauth.rs @@ -1,65 +1,215 @@ -use crate::HickoryDnsTxtResolver; +use atrium_api::{agent::SessionManager, types::string::Did}; +use atrium_common::resolver::Resolver; use atrium_identity::{ did::{CommonDidResolver, CommonDidResolverConfig, DEFAULT_PLC_DIRECTORY_URL}, - handle::{AtprotoHandleResolver, AtprotoHandleResolverConfig}, + handle::{AtprotoHandleResolver, AtprotoHandleResolverConfig, DnsTxtResolver}, }; use atrium_oauth::{ - AtprotoLocalhostClientMetadata, AuthorizeOptions, DefaultHttpClient, KnownScope, OAuthClient, - OAuthClientConfig, OAuthResolverConfig, Scope, + AtprotoLocalhostClientMetadata, AuthorizeOptions, CallbackParams, DefaultHttpClient, + KnownScope, OAuthClient, OAuthClientConfig, OAuthResolverConfig, Scope, store::{session::MemorySessionStore, state::MemoryStateStore}, }; +use hickory_resolver::TokioResolver; +use serde::Deserialize; use std::sync::Arc; +use thiserror::Error; -pub type Client = OAuthClient< +const READONLY_SCOPE: [Scope; 1] = [Scope::Known(KnownScope::Atproto)]; + +#[derive(Debug, Deserialize)] +pub struct CallbackErrorParams { + error: String, + error_description: Option, + #[allow(dead_code)] + state: Option, // TODO: we _should_ use state to associate the auth request but how to do that with atrium is unclear + iss: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(untagged)] +pub enum OauthCallbackParams { + Granted(CallbackParams), + Failed(CallbackErrorParams), +} + +type Client = OAuthClient< MemoryStateStore, MemorySessionStore, CommonDidResolver, AtprotoHandleResolver, >; -pub fn client() -> Client { - let http_client = Arc::new(DefaultHttpClient::default()); - let config = OAuthClientConfig { - client_metadata: AtprotoLocalhostClientMetadata { - redirect_uris: Some(vec![String::from("http://127.0.0.1:9997/authorized")]), - scopes: Some(vec![Scope::Known(KnownScope::Atproto)]), - }, - keys: None, - resolver: OAuthResolverConfig { - did_resolver: CommonDidResolver::new(CommonDidResolverConfig { - plc_directory_url: DEFAULT_PLC_DIRECTORY_URL.to_string(), - http_client: Arc::clone(&http_client), - }), - handle_resolver: AtprotoHandleResolver::new(AtprotoHandleResolverConfig { - dns_txt_resolver: HickoryDnsTxtResolver::default(), - http_client: Arc::clone(&http_client), - }), - authorization_server_metadata: Default::default(), - protected_resource_metadata: Default::default(), - }, - // A store for saving state data while the user is being redirected to the authorization server. - state_store: MemoryStateStore::default(), - // A store for saving session data. - session_store: MemorySessionStore::default(), - }; - let Ok(client) = OAuthClient::new(config) else { - panic!("failed to create oauth client"); - }; - client +#[derive(Clone)] +pub struct OAuth { + client: Arc, + did_resolver: Arc>, } -pub async fn authorize(client: &Client, handle: &str) -> String { - let Ok(url) = client - .authorize( - handle, - AuthorizeOptions { - scopes: vec![Scope::Known(KnownScope::Atproto)], - ..Default::default() +#[derive(Debug, Error)] +#[error(transparent)] +pub struct AuthSetupError(#[from] atrium_oauth::Error); + +#[derive(Debug, Error)] +#[error(transparent)] +pub struct AuthStartError(#[from] atrium_oauth::Error); + +#[derive(Debug, Error)] +pub enum AuthCompleteError { + #[error("the user denied request: {description:?} (from {issuer:?})")] + Denied { + description: Option, + issuer: Option, + }, + #[error( + "the request was denied for another reason: {error}: {description:?} (from {issuer:?})" + )] + Failed { + error: String, + description: Option, + issuer: Option, + }, + #[error("failed to complete oauth callback: {0}")] + CallbackFailed(atrium_oauth::Error), + #[error("the authorized session did not contain a DID")] + NoDid, +} + +#[derive(Debug, Error)] +pub enum ResolveHandleError { + #[error("failed to resolve: {0}")] + ResolutionFailed(#[from] atrium_identity::Error), + #[error("identity resolved but no handle found for user")] + NoHandle, + #[error("found handle {0:?} but it appears invalid: {1}")] + InvalidHandle(String, &'static str), +} + +impl OAuth { + pub fn new() -> Result { + let http_client = Arc::new(DefaultHttpClient::default()); + let did_resolver = || { + CommonDidResolver::new(CommonDidResolverConfig { + plc_directory_url: DEFAULT_PLC_DIRECTORY_URL.to_string(), + http_client: http_client.clone(), + }) + }; + let client_config = OAuthClientConfig { + client_metadata: AtprotoLocalhostClientMetadata { + redirect_uris: Some(vec![String::from("http://127.0.0.1:9997/authorized")]), + scopes: Some(READONLY_SCOPE.to_vec()), }, - ) - .await - else { - panic!("failed to authorize"); - }; - url + keys: None, + resolver: OAuthResolverConfig { + did_resolver: did_resolver(), + handle_resolver: AtprotoHandleResolver::new(AtprotoHandleResolverConfig { + dns_txt_resolver: HickoryDnsTxtResolver::default(), + http_client: Arc::clone(&http_client), + }), + authorization_server_metadata: Default::default(), + protected_resource_metadata: Default::default(), + }, + state_store: MemoryStateStore::default(), + session_store: MemorySessionStore::default(), + }; + + let client = OAuthClient::new(client_config)?; + + Ok(Self { + client: Arc::new(client), + did_resolver: Arc::new(did_resolver()), + }) + } + + pub async fn begin(&self, handle: &str) -> Result { + let auth_opts = AuthorizeOptions { + scopes: READONLY_SCOPE.to_vec(), + ..Default::default() + }; + Ok(self.client.authorize(handle, auth_opts).await?) + } + + /// Finally, resolve the oauth flow to a verified DID + pub async fn complete(&self, params: OauthCallbackParams) -> Result { + let params = match params { + OauthCallbackParams::Granted(params) => params, + OauthCallbackParams::Failed(p) if p.error == "access_denied" => { + return Err(AuthCompleteError::Denied { + description: p.error_description.clone(), + issuer: p.iss.clone(), + }); + } + OauthCallbackParams::Failed(p) => { + return Err(AuthCompleteError::Failed { + error: p.error.clone(), + description: p.error_description.clone(), + issuer: p.iss.clone(), + }); + } + }; + let (session, _) = self + .client + .callback(params) + .await + .map_err(AuthCompleteError::CallbackFailed)?; + let Some(did) = session.did().await else { + return Err(AuthCompleteError::NoDid); + }; + Ok(did) + } + + pub async fn resolve_handle(&self, did: Did) -> Result { + // TODO: this is only half the resolution? or is atrium checking dns? + let doc = self.did_resolver.resolve(&did).await?; + let Some(aka) = doc.also_known_as else { + return Err(ResolveHandleError::NoHandle); + }; + let Some(at_uri_handle) = aka.first() else { + return Err(ResolveHandleError::NoHandle); + }; + if aka.len() > 1 { + eprintln!("more than one handle found for {did:?}"); + } + let Some(bare_handle) = at_uri_handle.strip_prefix("at://") else { + return Err(ResolveHandleError::InvalidHandle( + at_uri_handle.to_string(), + "did not start with 'at://'", + )); + }; + if bare_handle.is_empty() { + return Err(ResolveHandleError::InvalidHandle( + bare_handle.to_string(), + "empty handle", + )); + } + Ok(bare_handle.to_string()) + } +} + +pub struct HickoryDnsTxtResolver { + resolver: TokioResolver, +} + +impl Default for HickoryDnsTxtResolver { + fn default() -> Self { + Self { + resolver: TokioResolver::builder_tokio() + .expect("failed to create resolver") + .build(), + } + } +} + +impl DnsTxtResolver for HickoryDnsTxtResolver { + async fn resolve( + &self, + query: &str, + ) -> core::result::Result, Box> { + Ok(self + .resolver + .txt_lookup(query) + .await? + .iter() + .map(|txt| txt.to_string()) + .collect()) + } } diff --git a/who-am-i/src/server.rs b/who-am-i/src/server.rs index 99d53e1..4e5f75b 100644 --- a/who-am-i/src/server.rs +++ b/who-am-i/src/server.rs @@ -1,5 +1,4 @@ -use atrium_api::agent::SessionManager; -use atrium_oauth::CallbackParams; +use atrium_api::types::string::Did; use axum::{ Router, extract::{FromRef, Query, State}, @@ -20,7 +19,7 @@ use tokio::net::TcpListener; use tokio_util::sync::CancellationToken; use url::Url; -use crate::{Client, ExpiringTaskMap, authorize, client, resolve_identity}; +use crate::{ExpiringTaskMap, OAuth, OauthCallbackParams, ResolveHandleError}; const FAVICON: &[u8] = include_bytes!("../static/favicon.ico"); const INDEX_HTML: &str = include_str!("../static/index.html"); @@ -34,8 +33,8 @@ struct AppState { pub key: Key, pub one_clicks: Arc>, pub engine: AppEngine, - pub client: Arc, - pub resolving: ExpiringTaskMap>, + pub oauth: Arc, + pub resolving: ExpiringTaskMap>, pub shutdown: CancellationToken, } @@ -62,11 +61,13 @@ pub async fn serve( // clients have to pick up their identity-resolving tasks within this period let task_pickup_expiration = Duration::from_secs(15); + let oauth = OAuth::new().unwrap(); + let state = AppState { engine: Engine::new(hbs), key: Key::from(app_secret.as_bytes()), // TODO: via config one_clicks: Arc::new(HashSet::from_iter(one_click)), - client: Arc::new(client()), + oauth: Arc::new(oauth), resolving: ExpiringTaskMap::new(task_pickup_expiration), shutdown: shutdown.clone(), }; @@ -92,8 +93,9 @@ pub async fn serve( async fn prompt( State(AppState { - engine, one_clicks, + engine, + oauth, resolving, shutdown, .. @@ -118,10 +120,18 @@ async fn prompt( .into_response(); } if let Some(did) = jar.get(DID_COOKIE_KEY) { - let did = did.value_trimmed().to_string(); + let Ok(did) = Did::new(did.value_trimmed().to_string()) else { + return "did from cookie failed to parse".into_response(); + }; - let task_shutdown = shutdown.child_token(); - let fetch_key = resolving.dispatch(resolve_identity(did.clone()), task_shutdown); + let fetch_key = resolving.dispatch( + { + let oauth = oauth.clone(); + let did = did.clone(); + async move { oauth.resolve_handle(did.clone()).await } + }, + shutdown.child_token(), + ); RenderHtml( "prompt-known", @@ -157,12 +167,7 @@ async fn user_info( let Some(task_handle) = resolving.take(¶ms.fetch_key) else { return "oops, task does not exist or is gone".into_response(); }; - if let Some(handle) = task_handle.await.unwrap() { - // TODO: get active state etc. - // ...but also, that's a bsky thing? - let Some(handle) = handle.strip_prefix("at://") else { - return "hmm, handle did not start with at://".into_response(); - }; + if let Ok(handle) = task_handle.await.unwrap() { Json(json!({ "handle": handle })).into_response() } else { "no handle?".into_response() @@ -174,26 +179,31 @@ struct BeginOauthParams { handle: String, } async fn start_oauth( - State(state): State, + State(AppState { oauth, .. }): State, Query(params): Query, jar: SignedCookieJar, ) -> (SignedCookieJar, Redirect) { // if any existing session was active, clear it first let jar = jar.remove(DID_COOKIE_KEY); - let auth_url = authorize(&state.client, ¶ms.handle).await; + let auth_url = oauth.begin(¶ms.handle).await.unwrap(); (jar, Redirect::to(&auth_url)) } async fn complete_oauth( - State(state): State, - Query(params): Query, + State(AppState { + engine, + resolving, + oauth, + shutdown, + .. + }): State, + Query(params): Query, jar: SignedCookieJar, ) -> (SignedCookieJar, impl IntoResponse) { - let Ok((oauth_session, _)) = state.client.callback(params).await else { + let Ok(did) = oauth.complete(params).await else { panic!("failed to do client callback"); }; - let did = oauth_session.did().await.expect("a did to be present"); let cookie = Cookie::build((DID_COOKIE_KEY, did.to_string())) .http_only(true) @@ -203,16 +213,20 @@ async fn complete_oauth( let jar = jar.add(cookie); - let task_shutdown = state.shutdown.child_token(); - let fetch_key = state - .resolving - .dispatch(resolve_identity(did.to_string()), task_shutdown); + let fetch_key = resolving.dispatch( + { + let oauth = oauth.clone(); + let did = did.clone(); + async move { oauth.resolve_handle(did.clone()).await } + }, + shutdown.child_token(), + ); ( jar, RenderHtml( "authorized", - state.engine, + engine, json!({ "did": did, "fetch_key": fetch_key, -- 2.51.2 From a6bf7d3838d5630923a809cd510eae844f888882 Mon Sep 17 00:00:00 2001 From: phil Date: Mon, 30 Jun 2025 17:04:52 -0400 Subject: [PATCH 039/134] propagate hickory init error instead of panicking --- who-am-i/src/oauth.rs | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/who-am-i/src/oauth.rs b/who-am-i/src/oauth.rs index 0a183b3..be23bc1 100644 --- a/who-am-i/src/oauth.rs +++ b/who-am-i/src/oauth.rs @@ -46,8 +46,12 @@ pub struct OAuth { } #[derive(Debug, Error)] -#[error(transparent)] -pub struct AuthSetupError(#[from] atrium_oauth::Error); +pub enum AuthSetupError { + #[error("failed to intiialize atrium client: {0}")] + AtriumClientError(atrium_oauth::Error), + #[error("failed to initialize hickory dns resolver: {0}")] + HickoryResolverError(hickory_resolver::ResolveError), +} #[derive(Debug, Error)] #[error(transparent)] @@ -93,6 +97,8 @@ impl OAuth { http_client: http_client.clone(), }) }; + let dns_txt_resolver = + HickoryDnsTxtResolver::new().map_err(AuthSetupError::HickoryResolverError)?; let client_config = OAuthClientConfig { client_metadata: AtprotoLocalhostClientMetadata { redirect_uris: Some(vec![String::from("http://127.0.0.1:9997/authorized")]), @@ -102,7 +108,7 @@ impl OAuth { resolver: OAuthResolverConfig { did_resolver: did_resolver(), handle_resolver: AtprotoHandleResolver::new(AtprotoHandleResolverConfig { - dns_txt_resolver: HickoryDnsTxtResolver::default(), + dns_txt_resolver, http_client: Arc::clone(&http_client), }), authorization_server_metadata: Default::default(), @@ -112,7 +118,7 @@ impl OAuth { session_store: MemorySessionStore::default(), }; - let client = OAuthClient::new(client_config)?; + let client = OAuthClient::new(client_config).map_err(AuthSetupError::AtriumClientError)?; Ok(Self { client: Arc::new(client), @@ -185,17 +191,12 @@ impl OAuth { } } -pub struct HickoryDnsTxtResolver { - resolver: TokioResolver, -} +pub struct HickoryDnsTxtResolver(TokioResolver); -impl Default for HickoryDnsTxtResolver { - fn default() -> Self { - Self { - resolver: TokioResolver::builder_tokio() - .expect("failed to create resolver") - .build(), - } +impl HickoryDnsTxtResolver { + fn new() -> Result { + let resolver = TokioResolver::builder_tokio()?.build(); + Ok(Self(resolver)) } } @@ -205,7 +206,7 @@ impl DnsTxtResolver for HickoryDnsTxtResolver { query: &str, ) -> core::result::Result, Box> { Ok(self - .resolver + .0 .txt_lookup(query) .await? .iter() -- 2.51.2 From db6f32732aa5b039dfb2fa90ff3abef6762f8a94 Mon Sep 17 00:00:00 2001 From: phil Date: Mon, 30 Jun 2025 17:06:48 -0400 Subject: [PATCH 040/134] import --- who-am-i/src/oauth.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/who-am-i/src/oauth.rs b/who-am-i/src/oauth.rs index be23bc1..dc26474 100644 --- a/who-am-i/src/oauth.rs +++ b/who-am-i/src/oauth.rs @@ -9,7 +9,7 @@ use atrium_oauth::{ KnownScope, OAuthClient, OAuthClientConfig, OAuthResolverConfig, Scope, store::{session::MemorySessionStore, state::MemoryStateStore}, }; -use hickory_resolver::TokioResolver; +use hickory_resolver::{ResolveError, TokioResolver}; use serde::Deserialize; use std::sync::Arc; use thiserror::Error; @@ -50,7 +50,7 @@ pub enum AuthSetupError { #[error("failed to intiialize atrium client: {0}")] AtriumClientError(atrium_oauth::Error), #[error("failed to initialize hickory dns resolver: {0}")] - HickoryResolverError(hickory_resolver::ResolveError), + HickoryResolverError(ResolveError), } #[derive(Debug, Error)] @@ -194,9 +194,8 @@ impl OAuth { pub struct HickoryDnsTxtResolver(TokioResolver); impl HickoryDnsTxtResolver { - fn new() -> Result { - let resolver = TokioResolver::builder_tokio()?.build(); - Ok(Self(resolver)) + fn new() -> Result { + Ok(Self(TokioResolver::builder_tokio()?.build())) } } -- 2.51.2 From 620ba018e44c40287acfd8fa23c3ca2581f1e4df Mon Sep 17 00:00:00 2001 From: phil Date: Tue, 1 Jul 2025 15:16:38 -0400 Subject: [PATCH 041/134] env for secret --- who-am-i/Cargo.toml | 2 +- who-am-i/src/main.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/who-am-i/Cargo.toml b/who-am-i/Cargo.toml index b59f454..c49bd9b 100644 --- a/who-am-i/Cargo.toml +++ b/who-am-i/Cargo.toml @@ -11,7 +11,7 @@ atrium-oauth = "0.1.3" axum = "0.8.4" axum-extra = { version = "0.10.1", features = ["cookie-signed", "typed-header"] } axum-template = { version = "3.0.0", features = ["handlebars"] } -clap = { version = "4.5.40", features = ["derive"] } +clap = { version = "4.5.40", features = ["derive", "env"] } ctrlc = "3.4.7" dashmap = "6.1.0" handlebars = { version = "6.3.2", features = ["dir_source"] } diff --git a/who-am-i/src/main.rs b/who-am-i/src/main.rs index 5e8e706..39d2130 100644 --- a/who-am-i/src/main.rs +++ b/who-am-i/src/main.rs @@ -11,7 +11,7 @@ struct Args { /// must have at least 512 bits (64 bytes) of randomness /// /// eg: `cat /dev/urandom | head -c 64 | base64` - #[arg(long)] + #[arg(long, env)] app_secret: String, /// Enable dev mode /// -- 2.51.2 From 1277afffdce5459c363842835ffefbd501b9cf28 Mon Sep 17 00:00:00 2001 From: phil Date: Tue, 1 Jul 2025 17:13:28 -0400 Subject: [PATCH 042/134] error handling: auth rejected sketch connecting the bits --- who-am-i/src/lib.rs | 2 +- who-am-i/src/oauth.rs | 24 ++++++++--------- who-am-i/src/server.rs | 46 +++++++++++++++++++++++++------- who-am-i/templates/auth-fail.hbs | 15 +++++++++++ 4 files changed, 64 insertions(+), 23 deletions(-) create mode 100644 who-am-i/templates/auth-fail.hbs diff --git a/who-am-i/src/lib.rs b/who-am-i/src/lib.rs index e5d3a7d..76673ad 100644 --- a/who-am-i/src/lib.rs +++ b/who-am-i/src/lib.rs @@ -3,5 +3,5 @@ mod oauth; mod server; pub use expiring_task_map::ExpiringTaskMap; -pub use oauth::{OAuth, OauthCallbackParams, ResolveHandleError}; +pub use oauth::{OAuth, OAuthCallbackParams, OAuthCompleteError, ResolveHandleError}; pub use server::serve; diff --git a/who-am-i/src/oauth.rs b/who-am-i/src/oauth.rs index dc26474..e216cf3 100644 --- a/who-am-i/src/oauth.rs +++ b/who-am-i/src/oauth.rs @@ -27,7 +27,7 @@ pub struct CallbackErrorParams { #[derive(Debug, Deserialize)] #[serde(untagged)] -pub enum OauthCallbackParams { +pub enum OAuthCallbackParams { Granted(CallbackParams), Failed(CallbackErrorParams), } @@ -58,15 +58,13 @@ pub enum AuthSetupError { pub struct AuthStartError(#[from] atrium_oauth::Error); #[derive(Debug, Error)] -pub enum AuthCompleteError { +pub enum OAuthCompleteError { #[error("the user denied request: {description:?} (from {issuer:?})")] Denied { description: Option, issuer: Option, }, - #[error( - "the request was denied for another reason: {error}: {description:?} (from {issuer:?})" - )] + #[error("the request failed: {error}: {description:?} (from {issuer:?})")] Failed { error: String, description: Option, @@ -135,17 +133,17 @@ impl OAuth { } /// Finally, resolve the oauth flow to a verified DID - pub async fn complete(&self, params: OauthCallbackParams) -> Result { + pub async fn complete(&self, params: OAuthCallbackParams) -> Result { let params = match params { - OauthCallbackParams::Granted(params) => params, - OauthCallbackParams::Failed(p) if p.error == "access_denied" => { - return Err(AuthCompleteError::Denied { + OAuthCallbackParams::Granted(params) => params, + OAuthCallbackParams::Failed(p) if p.error == "access_denied" => { + return Err(OAuthCompleteError::Denied { description: p.error_description.clone(), issuer: p.iss.clone(), }); } - OauthCallbackParams::Failed(p) => { - return Err(AuthCompleteError::Failed { + OAuthCallbackParams::Failed(p) => { + return Err(OAuthCompleteError::Failed { error: p.error.clone(), description: p.error_description.clone(), issuer: p.iss.clone(), @@ -156,9 +154,9 @@ impl OAuth { .client .callback(params) .await - .map_err(AuthCompleteError::CallbackFailed)?; + .map_err(OAuthCompleteError::CallbackFailed)?; let Some(did) = session.did().await else { - return Err(AuthCompleteError::NoDid); + return Err(OAuthCompleteError::NoDid); }; Ok(did) } diff --git a/who-am-i/src/server.rs b/who-am-i/src/server.rs index 4e5f75b..1c68128 100644 --- a/who-am-i/src/server.rs +++ b/who-am-i/src/server.rs @@ -2,8 +2,11 @@ use atrium_api::types::string::Did; use axum::{ Router, extract::{FromRef, Query, State}, - http::header::{HeaderMap, REFERER}, - response::{Html, IntoResponse, Json, Redirect}, + http::{ + StatusCode, + header::{HeaderMap, REFERER}, + }, + response::{Html, IntoResponse, Json, Redirect, Response}, routing::get, }; use axum_extra::extract::cookie::{Cookie, Key, SameSite, SignedCookieJar}; @@ -19,7 +22,7 @@ use tokio::net::TcpListener; use tokio_util::sync::CancellationToken; use url::Url; -use crate::{ExpiringTaskMap, OAuth, OauthCallbackParams, ResolveHandleError}; +use crate::{ExpiringTaskMap, OAuth, OAuthCallbackParams, OAuthCompleteError, ResolveHandleError}; const FAVICON: &[u8] = include_bytes!("../static/favicon.ico"); const INDEX_HTML: &str = include_str!("../static/index.html"); @@ -190,6 +193,30 @@ async fn start_oauth( (jar, Redirect::to(&auth_url)) } +impl OAuthCompleteError { + fn to_error_response(&self, engine: AppEngine) -> Response { + let (_level, _desc) = match self { + OAuthCompleteError::Denied { .. } => { + let status = StatusCode::FORBIDDEN; + return (status, RenderHtml("auth-fail", engine, json!({}))).into_response(); + } + OAuthCompleteError::Failed { .. } => ( + "error", + "Something went wrong while requesting permission, sorry!", + ), + OAuthCompleteError::CallbackFailed(_) => ( + "error", + "Something went wrong after permission was granted, sorry!", + ), + OAuthCompleteError::NoDid => ( + "error", + "Something went wrong when trying to confirm your identity, sorry!", + ), + }; + todo!(); + } +} + async fn complete_oauth( State(AppState { engine, @@ -198,11 +225,12 @@ async fn complete_oauth( shutdown, .. }): State, - Query(params): Query, + Query(params): Query, jar: SignedCookieJar, -) -> (SignedCookieJar, impl IntoResponse) { - let Ok(did) = oauth.complete(params).await else { - panic!("failed to do client callback"); +) -> Result<(SignedCookieJar, impl IntoResponse), Response> { + let did = match oauth.complete(params).await { + Ok(did) => did, + Err(e) => return Err(e.to_error_response(engine)), }; let cookie = Cookie::build((DID_COOKIE_KEY, did.to_string())) @@ -222,7 +250,7 @@ async fn complete_oauth( shutdown.child_token(), ); - ( + Ok(( jar, RenderHtml( "authorized", @@ -232,5 +260,5 @@ async fn complete_oauth( "fetch_key": fetch_key, }), ), - ) + )) } diff --git a/who-am-i/templates/auth-fail.hbs b/who-am-i/templates/auth-fail.hbs new file mode 100644 index 0000000..57f734b --- /dev/null +++ b/who-am-i/templates/auth-fail.hbs @@ -0,0 +1,15 @@ +{{#*inline "main"}} +

+ Share your identity with + {{ parent_host }}? +

+ +
+
+ auth failed. + +
+ +{{/inline}} + +{{#> prompt-base}}{{/prompt-base}} -- 2.51.2 From 05a7d67e8ced86171442b419a809d87528a04c45 Mon Sep 17 00:00:00 2001 From: phil Date: Wed, 2 Jul 2025 11:07:26 -0400 Subject: [PATCH 043/134] wip (more modest goals) --- who-am-i/readme.md | 46 ++++++++ who-am-i/src/expiring_task_map.rs | 1 + who-am-i/src/main.rs | 12 +-- who-am-i/src/server.rs | 81 +++++++++----- who-am-i/templates/auth-fail.hbs | 16 ++- who-am-i/templates/authorized.hbs | 1 + who-am-i/templates/prompt-anon.hbs | 25 ++++- who-am-i/templates/prompt-base.hbs | 4 +- who-am-i/templates/return-base.hbs | 168 +++++++++++++++++++++++++++++ 9 files changed, 314 insertions(+), 40 deletions(-) create mode 100644 who-am-i/readme.md create mode 100644 who-am-i/templates/return-base.hbs diff --git a/who-am-i/readme.md b/who-am-i/readme.md new file mode 100644 index 0000000..394f4cd --- /dev/null +++ b/who-am-i/readme.md @@ -0,0 +1,46 @@ +# who am i + +a little auth service for microcosm demos + +**you probably SHOULD NOT USE THIS in any serious environment** + +for now the deployment is restricted to microcosm -- expanding it for wider use likely requires solving a number of challenges that oauth exists for. + + +## a little auth service + +- you drop an iframe and a short few lines of JS on your web page, and get a nice-ish atproto login prompt. +- if the user has ever authorized this service before (and within some expiration), they will be presented with an in-frame one-click option to proceed. +- otherwise they get bounced over to the normal atproto oauth flow (in a popup or new tab) +- you get a callback containing + - a verified DID and handle + - a JWT containing the same that can be verified by public key +- **no write permissions** or any atproto permissions at all, just a verified identity + +**you probably SHOULD NOT USE THIS in any serious environment** + + +### problems + +- clickjacking: if this were allowed on arbitrary domains, malicious sites could trick users into proving their atproto identity. +- all the other problems oauth exists to solve: it's a little tricky to hook around the oauth flow so there are probably some annoying attacks. +- auth in front of auth: it's just a bit awkward to run an auth service that acts as an intermediary for a more-real auth behind it, but that's worse, less secure, and doesn't conform to any standards. + +so, **you probably SHOULD NOT USE THIS in any serious environment** + + +## why + +sometimes you want to make a thing that people can use with an atproto identity, and you might not want to let them put in any else's identity. apps that operate on public data like skircle, cred.blue, and the microcosm spacedust notifications demo don't require any special permission to operate for any user, and that's sometimes fine, but sometimes creepy/stalker-y/etc. + +to avoid building a small torment nexus for a microcosm demo (while also not wanting to get deep into oauth or operate a demo-specific auth backend), i made this little service to just get a verified identity. + +note: **you probably SHOULD NOT USE THIS in any serious environment** + +--- + +since the requirements (read-only, just verifying identity) seem modest, i was hoping that a fairly simple implementation could be Good Enough, but in the time that i was willing to spend on it, the simple version without major obvious weaknesses i was hoping for didn't emerge. + +it's still nice to have an explicit opt-in on a per-demo basis for microcosm so it will be used for that. it's allow-listed for the microcosm domain however (so not deployed on any adversarial hosting pages), so it's simultaenously overkill and restrictive. + +i will get back to oauth eventually and hopefully roll out a microcosm service to make it easy for clients, but there are a few more things in the pipeline to get to first. diff --git a/who-am-i/src/expiring_task_map.rs b/who-am-i/src/expiring_task_map.rs index f6d0899..9f0bb88 100644 --- a/who-am-i/src/expiring_task_map.rs +++ b/who-am-i/src/expiring_task_map.rs @@ -49,6 +49,7 @@ impl ExpiringTaskMap { .run_until_cancelled(sleep(expiration)) .await .is_some() + // is Some if the (sleep) task completed first { map.remove(&k); cancel.cancel(); diff --git a/who-am-i/src/main.rs b/who-am-i/src/main.rs index 39d2130..785bf97 100644 --- a/who-am-i/src/main.rs +++ b/who-am-i/src/main.rs @@ -21,8 +21,8 @@ struct Args { /// Hosts who are allowed to one-click auth /// /// Pass this argument multiple times to allow multiple hosts - #[arg(long, short = 'o', action = ArgAction::Append)] - one_click: Vec, + #[arg(long = "allow_host", short = 'a', action = ArgAction::Append)] + allowed_hosts: Vec, } #[tokio::main] @@ -34,14 +34,14 @@ async fn main() { let args = Args::parse(); - if args.one_click.is_empty() { + if args.allowed_hosts.is_empty() { panic!("at least one --one-click host must be set"); } - println!("starting with allowed hosts:"); - for host in &args.one_click { + println!("starting with allowed_hosts hosts:"); + for host in &args.allowed_hosts { println!(" - {host}"); } - serve(shutdown, args.app_secret, args.one_click, args.dev).await; + serve(shutdown, args.app_secret, args.allowed_hosts, args.dev).await; } diff --git a/who-am-i/src/server.rs b/who-am-i/src/server.rs index 1c68128..247fddf 100644 --- a/who-am-i/src/server.rs +++ b/who-am-i/src/server.rs @@ -14,7 +14,7 @@ use axum_template::{RenderHtml, engine::Engine}; use handlebars::{Handlebars, handlebars_helper}; use serde::Deserialize; -use serde_json::json; +use serde_json::{Value, json}; use std::collections::HashSet; use std::sync::Arc; use std::time::Duration; @@ -34,10 +34,10 @@ type AppEngine = Engine>; #[derive(Clone)] struct AppState { pub key: Key, - pub one_clicks: Arc>, + pub allowed_hosts: Arc>, pub engine: AppEngine, pub oauth: Arc, - pub resolving: ExpiringTaskMap>, + pub resolve_handles: ExpiringTaskMap>, pub shutdown: CancellationToken, } @@ -50,7 +50,7 @@ impl FromRef for Key { pub async fn serve( shutdown: CancellationToken, app_secret: String, - one_click: Vec, + allowed_hosts: Vec, dev: bool, ) { let mut hbs = Handlebars::new(); @@ -58,7 +58,7 @@ pub async fn serve( hbs.register_templates_directory("templates", Default::default()) .unwrap(); - handlebars_helper!(json: |v: String| serde_json::to_string(&v).unwrap()); + handlebars_helper!(json: |v: Value| serde_json::to_string(&v).unwrap()); hbs.register_helper("json", Box::new(json)); // clients have to pick up their identity-resolving tasks within this period @@ -69,9 +69,9 @@ pub async fn serve( let state = AppState { engine: Engine::new(hbs), key: Key::from(app_secret.as_bytes()), // TODO: via config - one_clicks: Arc::new(HashSet::from_iter(one_click)), + allowed_hosts: Arc::new(HashSet::from_iter(allowed_hosts)), oauth: Arc::new(oauth), - resolving: ExpiringTaskMap::new(task_pickup_expiration), + resolve_handles: ExpiringTaskMap::new(task_pickup_expiration), shutdown: shutdown.clone(), }; @@ -96,10 +96,10 @@ pub async fn serve( async fn prompt( State(AppState { - one_clicks, + allowed_hosts, engine, oauth, - resolving, + resolve_handles, shutdown, .. }): State, @@ -118,8 +118,8 @@ async fn prompt( let Some(parent_host) = url.host_str() else { return "could nto get host from url".into_response(); }; - if !one_clicks.contains(parent_host) { - return format!("host {parent_host:?} not in one_clicks, disallowing for now") + if !allowed_hosts.contains(parent_host) { + return format!("host {parent_host:?} not in allowed_hosts, disallowing for now") .into_response(); } if let Some(did) = jar.get(DID_COOKIE_KEY) { @@ -127,7 +127,7 @@ async fn prompt( return "did from cookie failed to parse".into_response(); }; - let fetch_key = resolving.dispatch( + let fetch_key = resolve_handles.dispatch( { let oauth = oauth.clone(); let did = did.clone(); @@ -164,10 +164,12 @@ struct UserInfoParams { fetch_key: String, } async fn user_info( - State(AppState { resolving, .. }): State, + State(AppState { + resolve_handles, .. + }): State, Query(params): Query, ) -> impl IntoResponse { - let Some(task_handle) = resolving.take(¶ms.fetch_key) else { + let Some(task_handle) = resolve_handles.take(¶ms.fetch_key) else { return "oops, task does not exist or is gone".into_response(); }; if let Ok(handle) = task_handle.await.unwrap() { @@ -180,47 +182,78 @@ async fn user_info( #[derive(Debug, Deserialize)] struct BeginOauthParams { handle: String, + flow: String, } async fn start_oauth( State(AppState { oauth, .. }): State, Query(params): Query, jar: SignedCookieJar, + headers: HeaderMap, ) -> (SignedCookieJar, Redirect) { // if any existing session was active, clear it first let jar = jar.remove(DID_COOKIE_KEY); + if let Some(referrer) = headers.get(REFERER) { + if let Ok(referrer) = referrer.to_str() { + println!("referrer: {referrer}"); + } else { + eprintln!("referer contained opaque bytes"); + }; + } else { + eprintln!("no referrer"); + }; + let auth_url = oauth.begin(¶ms.handle).await.unwrap(); + let flow = params.flow; + if !flow.chars().all(|c| char::is_ascii_alphanumeric(&c)) { + panic!("invalid flow (injection attempt?)"); // should probably just url-encode it instead.. + } + eprintln!("auth_url {auth_url}"); + (jar, Redirect::to(&auth_url)) } impl OAuthCompleteError { fn to_error_response(&self, engine: AppEngine) -> Response { - let (_level, _desc) = match self { - OAuthCompleteError::Denied { .. } => { - let status = StatusCode::FORBIDDEN; - return (status, RenderHtml("auth-fail", engine, json!({}))).into_response(); + let (level, desc) = match self { + OAuthCompleteError::Denied { description, .. } => { + ("warn", format!("asdf: {description:?}")) } OAuthCompleteError::Failed { .. } => ( "error", - "Something went wrong while requesting permission, sorry!", + "Something went wrong while requesting permission, sorry!".to_string(), ), OAuthCompleteError::CallbackFailed(_) => ( "error", - "Something went wrong after permission was granted, sorry!", + "Something went wrong after permission was granted, sorry!".to_string(), ), OAuthCompleteError::NoDid => ( "error", - "Something went wrong when trying to confirm your identity, sorry!", + "Something went wrong when trying to confirm your identity, sorry!".to_string(), ), }; - todo!(); + ( + if level == "warn" { + StatusCode::FORBIDDEN + } else { + StatusCode::INTERNAL_SERVER_ERROR + }, + RenderHtml( + "auth-fail", + engine, + json!({ + "reason": desc, + }), + ), + ) + .into_response() } } async fn complete_oauth( State(AppState { engine, - resolving, + resolve_handles, oauth, shutdown, .. @@ -241,7 +274,7 @@ async fn complete_oauth( let jar = jar.add(cookie); - let fetch_key = resolving.dispatch( + let fetch_key = resolve_handles.dispatch( { let oauth = oauth.clone(); let did = did.clone(); diff --git a/who-am-i/templates/auth-fail.hbs b/who-am-i/templates/auth-fail.hbs index 57f734b..5466bac 100644 --- a/who-am-i/templates/auth-fail.hbs +++ b/who-am-i/templates/auth-fail.hbs @@ -1,15 +1,21 @@ {{#*inline "main"}}

- Share your identity with - {{ parent_host }}? + Auth failed: {{ reason }}

-
auth failed. -
+ {{/inline}} -{{#> prompt-base}}{{/prompt-base}} +{{#> return-base}}{{/return-base}} diff --git a/who-am-i/templates/authorized.hbs b/who-am-i/templates/authorized.hbs index 7c95d22..38f945c 100644 --- a/who-am-i/templates/authorized.hbs +++ b/who-am-i/templates/authorized.hbs @@ -6,6 +6,7 @@ // TODO: tie this back to its source........... localStorage.setItem("who-am-i", JSON.stringify({ + result: "success", did: {{{json did}}}, fetch_key: {{{json fetch_key}}}, })); diff --git a/who-am-i/templates/prompt-anon.hbs b/who-am-i/templates/prompt-anon.hbs index f883800..a5536f5 100644 --- a/who-am-i/templates/prompt-anon.hbs +++ b/who-am-i/templates/prompt-anon.hbs @@ -1,7 +1,10 @@ {{#*inline "main"}}

- Share your identity with - {{ parent_host }}? + Connect your ATmosphere +

+ +

+ {{ parent_host }} would like to confirm your handle

- diff --git a/who-am-i/templates/return-base.hbs b/who-am-i/templates/return-base.hbs new file mode 100644 index 0000000..70f35ce --- /dev/null +++ b/who-am-i/templates/return-base.hbs @@ -0,0 +1,168 @@ + + + + +
+
+
🔒
+ who-am-i + microcosm +
+ +
+ {{> main}} +
+
-- 2.51.2 From d92e88700fb0d1906e50e8f83cf083838af7eb10 Mon Sep 17 00:00:00 2001 From: phil Date: Wed, 2 Jul 2025 12:12:26 -0400 Subject: [PATCH 044/134] wip: better pages and errors --- who-am-i/readme.md | 2 +- who-am-i/src/server.rs | 49 ++++++++++++++++++++++++++--------- who-am-i/static/index.html | 53 -------------------------------------- 3 files changed, 38 insertions(+), 66 deletions(-) delete mode 100644 who-am-i/static/index.html diff --git a/who-am-i/readme.md b/who-am-i/readme.md index 394f4cd..3df653e 100644 --- a/who-am-i/readme.md +++ b/who-am-i/readme.md @@ -43,4 +43,4 @@ since the requirements (read-only, just verifying identity) seem modest, i was h it's still nice to have an explicit opt-in on a per-demo basis for microcosm so it will be used for that. it's allow-listed for the microcosm domain however (so not deployed on any adversarial hosting pages), so it's simultaenously overkill and restrictive. -i will get back to oauth eventually and hopefully roll out a microcosm service to make it easy for clients, but there are a few more things in the pipeline to get to first. +i will get back to oauth eventually and hopefully roll out a microcosm service to make it easy for clients (and demos), but there are a few more things in the pipeline to get to first. diff --git a/who-am-i/src/server.rs b/who-am-i/src/server.rs index 247fddf..2af267f 100644 --- a/who-am-i/src/server.rs +++ b/who-am-i/src/server.rs @@ -4,9 +4,9 @@ use axum::{ extract::{FromRef, Query, State}, http::{ StatusCode, - header::{HeaderMap, REFERER}, + header::{CONTENT_TYPE, HeaderMap, REFERER}, }, - response::{Html, IntoResponse, Json, Redirect, Response}, + response::{IntoResponse, Json, Redirect, Response}, routing::get, }; use axum_extra::extract::cookie::{Cookie, Key, SameSite, SignedCookieJar}; @@ -25,11 +25,12 @@ use url::Url; use crate::{ExpiringTaskMap, OAuth, OAuthCallbackParams, OAuthCompleteError, ResolveHandleError}; const FAVICON: &[u8] = include_bytes!("../static/favicon.ico"); -const INDEX_HTML: &str = include_str!("../static/index.html"); +const STYLE_CSS: &str = include_str!("../static/style.css"); const DID_COOKIE_KEY: &str = "did"; type AppEngine = Engine>; +type Rendered = RenderHtml<&'static str, AppEngine, Value>; #[derive(Clone)] struct AppState { @@ -76,8 +77,9 @@ pub async fn serve( }; let app = Router::new() - .route("/", get(|| async { Html(INDEX_HTML) })) - .route("/favicon.ico", get(|| async { FAVICON })) // todo MIME + .route("/", get(hello)) + .route("/favicon.ico", get(favicon)) // todo MIME + .route("/style.css", get(css)) .route("/prompt", get(prompt)) .route("/user-info", get(user_info)) .route("/auth", get(start_oauth)) @@ -94,6 +96,22 @@ pub async fn serve( .unwrap(); } +async fn hello(State(AppState { engine, .. }): State) -> Rendered { + RenderHtml("hello", engine, json!({})) +} + +async fn css() -> impl IntoResponse { + let headers = [ + (CONTENT_TYPE, "text/css"), + // (CACHE_CONTROL, "") // TODO + ]; + (headers, STYLE_CSS) +} + +async fn favicon() -> impl IntoResponse { + ([(CONTENT_TYPE, "image/x-icon")], FAVICON) +} + async fn prompt( State(AppState { allowed_hosts, @@ -106,25 +124,32 @@ async fn prompt( jar: SignedCookieJar, headers: HeaderMap, ) -> impl IntoResponse { + let err = |reason, check_frame| { + let info = json!({ + "reason": reason, + "check_frame": check_frame, + }); + RenderHtml("prompt-error", engine.clone(), info).into_response() + }; + let Some(referrer) = headers.get(REFERER) else { - return Html::<&'static str>("missing referrer, sorry").into_response(); + return err("Missing referer", true); }; let Ok(referrer) = referrer.to_str() else { - return "referer contained opaque bytes".into_response(); + return err("Unreadable referer", true); }; let Ok(url) = Url::parse(referrer) else { - return "referrer was not a url".into_response(); + return err("Bad referer", true); }; let Some(parent_host) = url.host_str() else { - return "could nto get host from url".into_response(); + return err("Referer missing host", true); }; if !allowed_hosts.contains(parent_host) { - return format!("host {parent_host:?} not in allowed_hosts, disallowing for now") - .into_response(); + return err("Login is not allowed on this page", false); } if let Some(did) = jar.get(DID_COOKIE_KEY) { let Ok(did) = Did::new(did.value_trimmed().to_string()) else { - return "did from cookie failed to parse".into_response(); + return err("Bad cookie", false); }; let fetch_key = resolve_handles.dispatch( diff --git a/who-am-i/static/index.html b/who-am-i/static/index.html deleted file mode 100644 index 7e0182e..0000000 --- a/who-am-i/static/index.html +++ /dev/null @@ -1,53 +0,0 @@ - - - - - Who-am-i documentation - - - - - -
-

- Launch who-am-i [todo] -

- -
- - - - - - - - -- 2.51.2 From 867098a83040e08c02d3bfb9750b362af26f0fdd Mon Sep 17 00:00:00 2001 From: phil Date: Wed, 2 Jul 2025 12:33:57 -0400 Subject: [PATCH 045/134] wip: handle resolution failures + missed files --- who-am-i/src/oauth.rs | 2 +- who-am-i/src/server.rs | 41 ++++++-- who-am-i/static/favicon.ico | Bin 15406 -> 15406 bytes who-am-i/static/style.css | 146 ++++++++++++++++++++++++++++ who-am-i/templates/base-base.hbs | 17 ++++ who-am-i/templates/base-framed.hbs | 28 ++++++ who-am-i/templates/base-full.hbs | 26 +++++ who-am-i/templates/hello.hbs | 9 ++ who-am-i/templates/prompt-error.hbs | 17 ++++ 9 files changed, 275 insertions(+), 11 deletions(-) create mode 100644 who-am-i/static/style.css create mode 100644 who-am-i/templates/base-base.hbs create mode 100644 who-am-i/templates/base-framed.hbs create mode 100644 who-am-i/templates/base-full.hbs create mode 100644 who-am-i/templates/hello.hbs create mode 100644 who-am-i/templates/prompt-error.hbs diff --git a/who-am-i/src/oauth.rs b/who-am-i/src/oauth.rs index e216cf3..5407cb4 100644 --- a/who-am-i/src/oauth.rs +++ b/who-am-i/src/oauth.rs @@ -181,7 +181,7 @@ impl OAuth { }; if bare_handle.is_empty() { return Err(ResolveHandleError::InvalidHandle( - bare_handle.to_string(), + at_uri_handle.to_string(), "empty handle", )); } diff --git a/who-am-i/src/server.rs b/who-am-i/src/server.rs index 2af267f..d31e7da 100644 --- a/who-am-i/src/server.rs +++ b/who-am-i/src/server.rs @@ -125,11 +125,9 @@ async fn prompt( headers: HeaderMap, ) -> impl IntoResponse { let err = |reason, check_frame| { - let info = json!({ - "reason": reason, - "check_frame": check_frame, - }); - RenderHtml("prompt-error", engine.clone(), info).into_response() + let info = json!({ "reason": reason, "check_frame": check_frame }); + let html = RenderHtml("prompt-error", engine.clone(), info); + (StatusCode::BAD_REQUEST, html).into_response() }; let Some(referrer) = headers.get(REFERER) else { @@ -194,13 +192,36 @@ async fn user_info( }): State, Query(params): Query, ) -> impl IntoResponse { + let err = |status, reason| (status, Json(json!({ "reason": reason }))).into_response(); + let Some(task_handle) = resolve_handles.take(¶ms.fetch_key) else { - return "oops, task does not exist or is gone".into_response(); + return err(StatusCode::NOT_FOUND, "fetch key does not exist or expired"); }; - if let Ok(handle) = task_handle.await.unwrap() { - Json(json!({ "handle": handle })).into_response() - } else { - "no handle?".into_response() + + match task_handle.await { + Err(task_err) => { + eprintln!("task join error? {task_err:?}"); + err(StatusCode::INTERNAL_SERVER_ERROR, "server errored") + } + Ok(Err(ResolveHandleError::ResolutionFailed(atrium_identity::Error::NotFound))) => { + err(StatusCode::NOT_FOUND, "handle not found") + } + Ok(Err(ResolveHandleError::ResolutionFailed(e))) => { + eprintln!("handle resolution failed: {e:?}"); + err( + StatusCode::INTERNAL_SERVER_ERROR, + "handle resolution failed", + ) + } + Ok(Err(ResolveHandleError::NoHandle)) => err( + StatusCode::INTERNAL_SERVER_ERROR, + "resolved identity but did not find a handle", + ), + Ok(Err(ResolveHandleError::InvalidHandle(_h, reason))) => err( + StatusCode::INTERNAL_SERVER_ERROR, + &format!("handle appears invalid: {reason}"), + ), + Ok(Ok(handle)) => Json(json!({ "handle": handle })).into_response(), } } diff --git a/who-am-i/static/favicon.ico b/who-am-i/static/favicon.ico index 5db484406fbdcb26760f9c29070dfa98fc5d4463..f4fb8642604f730454cf4a51425c542d3c6af4b4 100644 GIT binary patch literal 15406 zcmZQzU}Rus5D);-3Je)63=C!r3=9ei5Wa>W1H(KP1_lEI2tPxOf#H}a1A_(w1A_oa z9Roz1fkeRR-7C!#wZ@Joa+N)&$1E8lwKH<3Y>(tFIdhJ?>DFKF&WHcGIv@V$ZoK)A ztN8RquB2VDVCP^n%%Yc*JNMWc?)Jz3cxHV1&%f~df1#znz*u16kN+SsuJ*_OxbseI zWN_`~Ml*tuEAQlL?*2Fbg%U#4yCf3Bd8|1yn-|I0QX{m&iL z`CqDGDMW6~=l>x8a8_Qu1lI36L7cPe>0gQN6aRJV=l?gHviZMZ`~3fk9ozqhp8onj z;_SEoik&AM|Rvi6ya z6|MiTvH0A7_0vE9OHMxYpFMWQf1^z&|0}ny|F7J+^}q4DBVaMfNr%8{)K^^jFP^{Z zKXdke2Zq3DVv22-{uoc$`rq>M@BiAXul^TIpYh*f?&|;UTlfEW-*)i7#jNH31v6*; z*IIS?zvY$R{|#pD`mfY__8&vSdRb5yi1oZaVH}nB-+uOn|DH#0{db(V>c7X78UK^M z{{v&M8FT+T%vt{5^T^Hrc5^oVHwaDnFW&e1B1kU?bCzEYW|KAiCnzlZUrI*izo3Z7 zf7_yx{~f>o{qOw!|G!IJ!+!xmL9m>#sOWz-S-pQ8<(FbWdSO_!>dZ2$(*FN0sTu!M zCQtdl^!=y*oB#X=^e{?D8>=f8Je!GGKGzW>4v7uF%CAFvs@+t?KszF(eq|L?#3 zzyJTg@b~}!YfubgAO8LSf5pSU|CAShT+5KQi3_X;3n4o9S%}e*FDFax{r)@e^}qk~ zU;q1Geecg-`@`SPNX~r`iD54mGeAP(-511kmOio6Ui8FDzU!m}NQ@R3RL&?cFfcfP z9R_02!WO!l$;fF|Cc_grH;^}GZ3%bGs#fmk)onacYsz>67ld$FHp_#(OKoTA=L>L0 zuPWe4+kJ+o_|z|+>dXIm>aYLjX}kf(+#ohctoZbAp7h-pLGEENt{0))8TKsP5i4?e za*n*v7U-aWY|H2<& zHb{(n))%lEkXn!)p31BLcvAKpWq0oruI?BAL1yq|A9=*>Ia!yO zaNx3Sk>t)ga*Mn1{(r9ixBq!(eS_3p%m4frU-|35#LDmgC0Bg@FS+95f2rl~!B}$T z=l_x`zk=mJYDAa+0qX(j<(lyBKX=oE|2)~p9uW=)dlsIQy}P&@?)?Yp=N&k7gvi@^g2L9(X^ZC!A=lGvP*Zx1ZedK>`r`Z48u1Wv-qMQGV z&$#nnVil-P{QaLF6c!WT{|EVpJAMB#1{Du>T;aeOxi+1<>gGSLzPJDR=l=LFw*1e3 zskQ(9%dGqJpT{%(Ka-%ue|4h6>|Cx9N|1)v%{%7Lm|IfrH{GXAVA6zFh@d$$5 zC$sh^Sg+XfzyJB?{Q!q0cg^j8T#@T4u-b2y%g3E}>KRwZb8wi8uK4p`dfos3a-0AE zSJ?XRzew)X|BS5c2+YFvpOKyKKO+bKe+Cw|{|wA5{~4HAz_dU_<$r~(f5Cb|W`NA% zoBjPiSLciWTm@&|GT2m#qPu}Jc3lQn{k?x&lRy0zUi$mL)cSw_wS-7oRK(DXb1x#oQS&(ZquKS%WHFu46`+cP-2p8n^Z z`{BP_TK*02AzTVMQFj<5Z1J7Lj({i{F!8{PQvU+4T!uwIZEDhGf4H|g8>Um>F8KkwZ4 z|Jl2r{pU#A0Z#*L={qag`(FJQnt1)cQE=ja#XaA^ZAP_Izy52S{rO*M#kv1n$us{e zFFFBshgp2)f6dt!{(GPK_TTs1_y5`pul+ZPNcwMd_1k}y<){C1CC~b=xa8D-&9guM zgUkT6KNa_V|8E?b@n5Lt!hiPu*ZOi=#A*N8lNbKyjGgk|@W%K5rag21E0)jyue4|57t9 z{Fk0}`M*+N)_;?png5M0efZB7*acUsed^%6|Lavv|8IEW$A9AszyF(F`SaiO($D|g5k3D|gL?jR2DSaS zy!GY3aaGHIkDkf@9SSS|+vivO_n0vKzj0v|SdJ^S1FV)Sv=gisWCqAA{ZkPC+h(@> z=kIy_pEZ3KJpD1)&btUki`F%d!GKcyZrmV!;QcHo$mboue0*ff1c2e z|5{5={%T>K^ndT?U;g`i{0GL~ufG4cii-VjbL7f@?UiT$^Mrxap8M~3=kI@q zn}7e?UHSdreE;+Rh7oE1jb`8e&)xadpa;1;79MHO%>Mlv(xPY2V%d z-q*kV_q+f5zyG7(|9$R!`tNq~`hSy@jQ=4Uw*8O%`tN_#xBp-aVu$WL@ZTgp>A&l- zEC0RkeEjeC=r>p|$PA|g_x~H_H~#l1@B6R4^v8dW+J~S%0;2q3U|?X)IhHIj>Dxcm zh>HK3TH623f}{RhwoLi&vTFB#*VQ}!TebH7w+IaWZx#^vKjqD*|JlF(|IdNqtY82C zr+xVL-zqxxzlC4Gf6K%V2&)c>ZTvHx{+b^psah5eV9^65WY;rVh{ z+{0=6dLEv-`;XXUbpA6kGW{135e4fpHZ}ckY+~|XM@Q#Bs2{`3&i+63%=!OyzyJNO z|NZ}e!|(t9>wf?LpLh4+e>NVT|NH_1|FyNX{~MW@{0HgP)zkYgE+O%siG}q)i;(nx z?wZ?g7y=dw!|g?4ux1}kcfXJTdj&&b3KHiMOm`#*<}_B_Q};P+IoCqOJRXsnWHeHpoGO z?a|P=qGG-8uGnn;@xSKyo&PmkcK&bLf9U_rdk_Dw`|<1l)<6IL@BH)s|DM19|L=oh z5E~@6<wf!>{`fCE`RzSezZT8Y7+j8|E%K66 z-d{D{_xFGMtAGD@{{H`e-=F{g5B>fB|M=hk|4;w@|Nq?I|NqZJF^GNY@BjbD{zBA% z)b9TM|NpeN|NdJa{QF;g#>cyCIr~9fG7P(L3GgJY5*3^DVzbJwKmWq-{rkWE`~UwJ z{`~)c_3!`xxBmYBfA{bI|M#I7#0H6h`n8w-{{O%G$N&FX5C8qw*!}09*pyfMIMX*v zRy8HKk^xePz{crsJ?|AOdKa+hwf2pr}e?oG~gBolrIE8_9 z%22eMl|j^=he1AG0F2fBco`TN$Qo~;ljoT^jML<}JSQh|hc2GU8@FW-Pr|0tJV{$F z@g#1!!jrJ+3~&6_eLP{y7w~vZNoO<3S7%@_rnvl~$Zr;foNRv6)Hy>J<#5NXKfsfH z=rd2zslPns=l}CmU;58ed*we*-BmE=uD$Y~8zfe7;XhCD>AyTVhrjZ~Z8*ghx}=2L zceWmbRy_|zZXrptJqw5bOe?O~^?f`UdvEd-pZUd8b>$y-!_EKPEqDKOx848G-SO~0 zch{r;+})2M7{msNwcm%RX}tNLr~1l&p5imVc`^??;EvxkliP2OBZEpV&M^#~+;NfHZ>A$P>UXH$ zm^l6C`SPUizsOT|=`VM~-T&MjPyciEzy8lP<^6x|nV{0cUctLN2! z?$$^Dd8%*x=}3U&Hr3&PatD;(?0*_oAdp@;KE{{`m%_%FQV_kXb!pmE#Z|0R}w`Y%5F;eWA-m;Q@(9R4p-xAnhB{g(g2^qAh#=i?$v3FV=bFzgYjd|Kihb{g+t${=fK&UtqIDmiz{r4UP-Y zxGKmGHMjn8XCAr94l0{SNkLknJlrvx4{%pp|I5|-1RN$jv%dZpT=4V1$g=HEBt4XRQb;;ul1i@ z)9OE)vf+PLC4>L0%0~a$v~BPe`Zm+|4ag6|Ct2E!DGfu0?@H!&^R)W09cG!KLe ze`XPx|9la3|7BKx`!Buj-+z$Vpm-Hp_zOH%%r)Wtf3D_7|G7)A{Ns*TJ%>R)o&&p^ z5u%(vGfla3j^F04yZfK3`}KdG>0kZ}F8ui)G+r$Ua{K!K|7AD+|1Y=k_kYpq)!=b& zMlQbpjGR3G8M#1n(A*Fy{>OjW zjsO13tpERCa_ztWVk`ba<_Tte{m<3&`af6w{r_Bfr(Sah&T~Sz6bnl$l#MffTQ66| zwclKA&;D~y`S_oI{*V8lJPI;bW&>p0Tz>2S|BBoH{+C^N`#+nC;eSSE*8hwwAj}4( zS^qP#a{g!Jk^IldBlDk;l^Z;M&cMX{AB-89z%;Y4)PKqDeg74={rRu36=F8X4v<|S zJ3;A{d+MkEps{=Is+)hg6L-vJu*coj;Pjg##hG*T3RlB}f1G`9|MSlN_Fs6}pZ^kT z|NWQQ2pM-**!J(g;!eoeyYiM#{{@0_{xh($|7T!k|Ia9*@t;}2{y(##<9`m1j{iJK z8~^j9Z2r&T-S?ka#q&Rty#0Sh5zYS$EF56>^VkRfS6cn_KWH2uWG2XL(6~IvE|4EU zc7w_~&i?oRIYDVT=fp!+|M{S90hYjLVBid#6UtS7PH`@i(YzyHNo|NAev_}71~>0kbHg623XuKnSNT2qGA@eB-190{AZ za5mig$Jzh(KkuCH|Akll`7gQd&wrWKAOA})y#HT%=EeWA(@y@En||!S!mN}3l@?$8 zueSc)f3?-u{|lDv`Y$s1@qg*55B{sod;H&W$LIfUM}PizJNEOx)$T9(p<@_)4r_x>v{x%^*Y_9?KLGSg1{m!5h4zx2Yp|7BLa|1Y)fH+W8u zci#8^oD<*u=LF5YrS3k1we4+~B*mF`@;yiUlmDDkKmO-m^z*;yiXZwo_z-T3=I1Dq9JB_ zNBma{O#UyM)%{*T!u4()UPJ1pGy-*&~R|F(;c{L%#^?&a9U;hiveg0n}rSrd~ zyYGLS2^;=v?tJ%O{SajCMeF3x|JtX2{MSA6{lC$%*Z&=7Z~SlV?e||HtNy>@lr#UW zHa`C!d-2=<{QH0Y7d-g;KmPLf{|?(<{#TxH`M-Q#>why(-~TQX7XLRo`trZtnQvgT zG*A8n&+Dik`uSgT_s9PhGxq(r^A7wkmC*EGXy$|eybHhn=a}^NKS$HO|C|Y%mVmqU z2v0Dvr*2=!(emIw=hXNA`R0H4FWk8Ozkz4Sf3pdj|10nR`CsKQbl&F#bPnR&&;RGGtB&G_Ronnp-*c3v8G2fnWcP<{bNPg2pS|tjf6mMU2N@Vtko!7p>^X-vv9~||&o%wSe}S0~|0@J${5L7=_^-14{eP81 zzrgcG>L(#{9$M#r{?|G7;=l6Jlm9g~UHNZ#`OAO9i=Y2nH%<7j>YMamWB$$m?uXw0 zkGb~if832<|GkcW_^-R{-hZW#g8$Z4?f;F=zWZ-@_1k~VE!X}lFFE;N`|#8MTIYX& z=Tks-sDk`(=+}Sc-CzEjR80D>>K*%EVCwDvTr)rXXYYLapCk9+c?OkeK1A?Ka#7_Osp1j~cTiopbys0z(8{PN@p2M<8%=oVy zU-MsW)uaEWhd%!|KlNMU(F->zfoezf2BR2|EnDR`Cs)U)a_?~{RgETiQZlR*}^CNXHQx5pDkf7cpgsg zc7giH~(ekUjMHWQT*R5Ch5P))sO!T&b|H57SR5m zEq)HfjL0efC3<%L*E;v(zs5OGJVWdR#evfPZ~yg*d;e=Y`u-PeI`N;Q`^A5@!sF)| z%#hO$10!4ZzPX(3&;QHBbo@6mxB9P_+3;Uu-v1 z?|-(W`TyBsXZ~mNZT_!!Qgj3;wf(PWUh0w&lOxh3{ZHw9i5Ena0VV|J4tC z{jXg!>%XbF<$u|ThW}h`kN>k5oIuaN3~ceMDmXh{{nKsQ|KBMn_P<_K&VRk#Z~uel z2My2t`fqgc_kWYifBu_X`Tbvh>cRhPzODaR!zcb{4eb8U8QA*Y^zz65=I5XMH;PL5 z?~$JM-!VDkzph`@e{J9B|Mu~z|Gn~y|C{&*{5Lys_rKY-&;PkX+W)f#cKv4!pY)&A zr{%xg#C`uwF8}&(a^(-$E|8sir+@v|+4teUc24Vm2k*fD`VBk&b9cV_&z5~)5nB1l zz#cTqoulQ|Z>gEz{#)0s_;2X$|KD)s<^SfVzW=wl_~*aXl|TQjul@ONbK~!SgFTP` z^MrT(XSOT(&*D(}U%GX}f4jS1|C_Db``_Hh|G&%5BmceczWN_{?85(`BWM5n+`OoRs_+M|w{r}cC{{FYV{^!5tl|NuR zP0#-LufO5ef4#uC|2BpF|7GTU`On$@`X77D>TE>bg|dv(L^!JNyb+)JHq(I-~RgVUS0p+D5vPZ{mGmE?XSK2FW)b>#Jb-KM$!t$l+3+b+KQUt;Eu{~QfZzp=SAsX|?i zOfj8H?Ud05C8u6x&QaS*X`f`-LHK8?|Ashf3um}{u>9!{`Ug; zVg2|2!jr!LXDhjUoWZ^x>$nPY!a9Gx&JW+!=KcKdyXfkFYoDPdO{)9;JFPqZ z-|OhT|GpQV|M$Q0>c9V$=l^}r-Tm*n<-mXQx|aVIzJC8hX3zg0|L*Jm#2^3vC;j~Y zKN*UXe*6cEC4Bn%KWyo$|JDJ4|4mD)|2uEk^WW$6?f?FlpMlNvz3}3{*Rcoxoi?2O zZ_+gNzmb3Ve-lU7|4tLO|JPdlEpZ}?3&iK#C z%l}_fQ|rHzyXSvB55ND$c~$>yCd~WqG^8|Rh$H*oj-@8s_OUqfI2zthwi z|1&>*`=9sg|Ns17|Nj?2F^CNk%l`K3zt@U2|J965{=2w){5N#-_-~k&``@N_+JA>x z%m3RYwQ2UW@i75Oicfqn3?^zwz2tdYG(Fd zP)G=_`l%!>;Kii|NXD|{U3~Lp>*}{|Nl!re*JG=Qud#jjqN`VFYkY2W8?oe zP%{ioO#g%C#VxI@{##mF{+E)L{?EkB0`8BpNo)QWY`pV_t?c>~2G>RCYsry)z`z>2 z+K{XI&UG&9)c?$!ybv>(nE$h|u>R-e&cny|pM{MLJSWb=!TDd) z*YAJDgU8@?$Q{4`|L^<*!5x48{|DiQ_n-corKbI7;o$hs$i(!YnVI=NH#hfxF>!IQ znUa!{;CX&lcJ}{FtZe^TIXM5b@CyEC*Z2L;S$*pzOX@y*6vtxa)HAW9Y|G^>U4@+1URxvamqr{aM+-?qK8Q{?Eb3|6f>9@qb9)g#Z1YzW$&3 z```Z=fByfU`R6~F2C=98`Tu{?_h0{$SFZjqp`r1gm5b{?D+kAaW>!}48UpaT0A?1j z9ju&O|Jix?{&R~;{^vII{?A){^dD>Pu?a+v!^wnla+O?KD_(ixkC=PLe^Cv?|3WhI z|9M5l{? z)L+h;TSpmeGLhPASOXqKkS%t(nqcemQ|dF{{56?y_`h%Cl>ct&x&QUuJpSu@c>OmI z4gK$#U+_P5+Km6*=Pv$V_T~HkwZH%U-|*-E|4qe-r!W66eD(JKiVvUvumADu|CZnX z{_ptn@Bgkpko64v|Nj4f5V~doM1wF$Z111{U^UzS{QJKd z`S$;!XK(+{d-Caj#HpYEHP`?7FFf_rd$!_JK@6d*n2B>ThAKw3tlj=%6F)xCTL0&- z>*>G$C%yUif78$Z|F{2!xCP{<1AqVjKMY-GaT2-?EnKjPBg{~8EC?(cuq&42!iO@DuxyYfN=0|Nu4ZioAUH+!p^ zaOb^svTJ_-Q9u0mzx9oO|C`?Z|9||~|Noc&K->yiYXVwx^5F0P{}2EE|NjVzL2S^P z6p-BQzyJT=fco+Lpa1_CeER=C;P$`&dWZl1mtXh$uW;|@BjZV|3cQVfY!2r z*dVcofByZy_UqsOqd)%rU-a?c|G4}A{u`b7`%iV}@842OKHU>+xzfzwQO}2IF*QZ% znfR*?8i`JS)GEE|-8rSbKfi0A`|}U9o-g^y-~Y|8|Nfu#9d9`tP5VWXfTBn#*1<)Lrv_i~gE7 zdkj{;+N-Y!U?d?JH SCQKBXIjVg$1V%#uEd&5Rgoecc literal 15406 zcmZQzU}Rus5D);-3Je)63=C!r3=9ei5Wa>W1H(KP1_lEI2tPxOf#H}a1A_(w1A_oa z9Roz10XO*1z`*>Ufr0x!1GB_`26nCg%zQ@w8MsaV>)6Hr*R_uMFKdwSpI657KLeY} ze`XGu{|t;={}~vV{xdKzK=d;5{pS_a`_I6s|6kj!_P>Bm+J6@L=>Jwxz5fk7O8<+Q zl>KLt3jQys7XF`2$mTz{sPTUWMn14!CKkc}(i*}4c@={H%iGufmvwCW&#qhjpUtr2 ze?-^W|3;Cs{<9f%|5x-~@L$xj?mw4m%74d%h5tp>LjN-`^Z#dHmi*7iWA$Istn`0S z_3{52@!S5(gzfsTmT~gG|IDZVgC|}6ub+4Ef83PE|J5S4{8#pw_n%2B>c5~$=zj(l z$^Q(js{a`T9R6!~PyR3NvHZVT$+`b#&5!=O%=qv>eCPN7ky}6hH*9|OUoi6gf7z%* z|Aky;{pT~O`Ohrn_n(1N<3FpQ&3`7Tu>XvjmH)+qw*8mMy#C*Q#<%~`+yDOWdiv*o z!@Xbsjc5M;uif$Szfi)(|17qP|Fh_~{8zB+{LjE?@Sj!0<-dSQ-G2d>dH;F*cl}qZ zefi&O&ENk$xBmUV_2cjVeP90mpK|Zt|AOOx|0|R}`!Abx=D%Ixx&N|Glm0V{IR9sr zaQV-npZ%Z9aoT^O*i-+_CVl$9`tiU2mp=dd|Mu5^Fn;>u-~Ym6fB&mDefTdNe&Ro) z@#OzZs;U1O_-+0(@LK<8kc;J)C$QT3hw{wsg{pM2}z|MN)tx4irJKYahc|7MfF z{uhWm@t@JC|38CV!hZ%noBs^l7XLXlv;Vu69{#VHd+xtl&FlZkyZ`>*_UhmN7eD{~ zfBgO5{|lf0{jWav@4w5u@BhskZu}SVTJWFOu;Rb4cG`ai4#WQp5&{3!f*1d1cUkpc zw(!RPpq0P=cU}4Szw^ew|1B5({jWd&_kZHH-~W}1ZvGbx-twQ@y!AhWMBsmBA-n$! zY?}WWgq;2}sV4ttGU@oQmU-&GZ0^nfA%J;vpb^CuV>nZ=)Jy!lVsJ`}JF6YXBp6HYR zdBP6=XSQ4RU)+E3e}nkV{|&>I{%7E_`p>{D`=4LI`9A}@?tfmT=>ME5ssDwoTK)?; zPxvqHv+zH+>)ik5`N#izl%4(0Z$I%rgM1R$uZCVT{)=j5{byj8_|L$=@t=WF^gpYR z-G3F^&i@knh5z|9)BiKb#QtZ~&Hm4*lJlQOujD_oLd1Vo8Bm%t{?8*9@SlN6_&);! z+kXZImj4V4oc|dZMgB8#X#8j3(EZQAY51Rk*Wy0|D15+iZ26yo+xWkzLHd724yFGL zOnm6p8@0! zX5s$~jNH(;0oetz6XXUE4dR3B=KIepqWYhKRSx0@PQCxK#wq_3>URGZGRphUA?y2} zLCohrlT;wM+>EQ8_dhbf`#-x((0>Mr(EldEQ~xuFh5Tm_bOO6m-YDt6fQ-j~RzB_j zT*6xa8JNZXGcfZ0XJBLpm$8s|289R2ZU$!Y|NIgL|0OkD!QsikXZ2rDBkaF|L(_kG zhlc;0DslfAI2V{|vmg|5=0_{;ODKg5wZm2Lltx4WKj%vYYik1Czji24*F2zGGn5`7fa2@n6Cy z;Xj{x+>C71rk)GYeXu3i0~!=T|m zk5R*a-SEZ#S+$D(^XX;$XIF^+&miLYpGhL*zgPD9|Ey9`{~353z;zh|6F)SqbN=TP z)c9}WUh&^Du=~HFMfrag`K14LNh|(KJNEq-wrv0JRJiv)vv$jWHlz0c%m#h`SWl;BjX_JKiOgwUs`jV0VKLd;Ge+G7=|7_y^|7ESJ{<~zX`(MBC@&D}U zkN%6hFZl1cd^=(hjN=5znEIj#DypLP1bcK((BmAhX5kDYY%zkAc={|vU9{`>dA^WK;h$2)h4g5^(y@z@`JqSD-Ker3qGL zaGnB}`68bGnU#|Nv+A|}kM6wkUnP0Rf2qJN|9yJy{bzCB{$Doz^ncI3r~j2oAN?;o z{`r6O(p&$1Cp`Mk5q9o>`r2y~uakN9zkJxP|MI~b{@dj2|IcmI`kz5E@;|3? z>VMPl8UF>;0{=5`s6x^svoxfv<~IG$An5X+Niy=kir1|FeAZL`vzW~IAJ%>2zkJ4t z|E{ff|0@;V`LA61@PESk@BcO1Km9Md`u%^*&X@lK=Dq)K(f#_rdeMXbc1;idGdS)3 z&tSFozkceG|I+@e{|ni5{O32V|1W9Z_Mb^K=s&Bh$A4x~b4VDlfbzHjIKMDT`2JUN z>-rzjcIm%Y-NpZmW()qaxor6#H|N=ZrP2rggBE@IFJAigf90va|NU40{awm+!fB!4b|NCF{{MY}|JKz4NAOHJ5Y~$bmT)AKVi|V2wyXdB*Ps3GzuT(6|Mj}R{^v`7^j|#V!hdGxP5;@Qmi%Wl?ElYaG4p?9^`-v| zlIh@b0o0~sVAuaIZj}FD#dqd^X^-ju`D`ZqXEC4hpUHaZf3CoT|JkFi{WtIZ^xu8% zpZ|XA{{2rp{_lUwjeq~2{`~iU$M?Vg=Y9S6|JbL0|HJqF`(Jwf-~aSu|NbZK`ujg) z%g_HJ$+!Nq`5gE!9=7klMgHmkwgu<^+hiR6uk1bhKd)~3e+DjnNIOH&bHe+_=O6m}f6p6GyYb)u4`6#BcEj3{Ah&?*Kk@0`|L$x5{(G$X_g|vv=YOWS zNB3!N31G zXa4<9Ir#5?$i~0_9jAT#uUK^VKY#dv|4df%{xj;f{AWD_<3g}?tN9sc(}{@B0&bDu)u9^_|a{N(5V|4U!~ z`yX}U-+z;p|Nd)E`14y-t+LkP4DafCS9NY+b#JE zZhO~X|M!3Y`+xsI_Fn(?@Be`h|Ng%R_b2}S-}L(5|KhX%{+A#B_upgXpZ{)izx)rL z{o=oO+x7nnp_~8nT2J^dV&DE>#n&%mnjpMl%#KZA(te+JpG|I+S#|9Pz^{%5h74sP!YCLjH8H|gbnui0P! z2QU5oKkLZ9{~g!<{m(xA?|;VefB!R&{`()d{onu66aW7EZu$E^YwzFxp(}p;S1h~t zpUrpge`dRt|G6Az|7Xyy{m-JA`=3cZ0+RQ+%^~$3i^_jSKGXk<;?Cf<9kXiEe-^`v z|I8+>|JmIZ{5L2*{oki6{+~g}`9Fhn(0@jiN%& zf9}6~_MZO?yx#vA*!2H1fZ82QqW`%Roc?Xm7xU-_j~y^bhJf4Opm7vd$*BL9VXOW# zaQXaaFEEKiL3uJDCYiWP)h&LppyNc-?8aGgL2V-HuJ9kI^k>oGbm;MXHZQ3&ux(LpG6*2 z$GL*rbqrjl|HbsO{|m?`Li&Z^J~GH|P+t>-LE~1S{-)@EMlO~AqDGnjb)0+ut2#~n zFJV~upMl>2QeTJ#g3Cz;$>9HtvJwBejkEqUNJoIn6gG{_{|w@R{~2Zc|1*fV{TDDz zhO}`&`HW5LKd)xMe+CxS{|rn*{}~uS_JR68pne~ypAPDqg2W(h0QZGhH2<@SyZvVp zu>a4%1sZoV|Iff@4IYyM=TlL)|4cGI{~3hcA#M|L`OhHW2=*_dpv`|3pSJ(fre*(` zcnqL%EBT*+Nf2r`$j>0VLH#cf2Dt&`9*{e@|1&TOK>FUG{K29CP8Xmt5e5z&@K_!L zsNKSC{GWlx1cE_qkQk@le+CY1a6gVi%Jo0Ldh~x5K`n@%QTo0hI~o5oGJ^WPL<}Yj zV%%=07nb+*UsCvkMMjpHWjBKE> z1I7Og%yJMjL1u&U5UB48j#rS~pfm(>4+w+Yhzp~+8#ERHG6Q55Xsm`=64D+ItJ3)4X#;!qb0gs!3!W86AbU%Q^Ko}$s z3Ukmn6==Lj2om1lF(XiYs`{T1l<#>R|1*jB{TJ1a{Le1t@Sj!04pR3DI)cY0LHStS zKJ9-}#l-(fr9J<(tm6MO3A=#XqwI=t|5KXx{g=0E`p+!t0U1*QnaQE|pPAR_zqW1k ze@1p?NFNr|hX>_t(AX7d+z`bNAoqg&0ir>119dwi$nAXpnc2nvvv7jSe%b#FY@jh1 z_5Y0A`u`>M!vEVwPW>NMx#Pd4SNne!NuU3mvOfPAL_Gd8h}Ni z;_7by!R0Ei<$nf#yZ>xbUjL;nvj5B2)c-e+p8cOgHTgfcddh!h#pM4C3TgkP?Q8xE zTebgB={@m3zkkdB^!oY#b=)ieGlBXS%31#voLl}27?l3!P|NtwC>#5qPb2m}qqzTn zMVq|;B3kkP8TcLlGw@pdXA|@O@0~XDKO>+0e+D*CdXj>Se}c#8Kz=~V*Nhzh!Q*Bi ze}c*s(6|ws`hON)!~crbG5-Zr{r(GTM*e4!^atA|XH)c_M?2@gs72X-F73Sk>>7Fh z*|duPGpLsQ7qjp9ujs$%fBx*-|BGiH`k&Fd;=hhh%YP<~%KuCnRsSXITK=A#xawn##iT{~&JO4B4b^hlv>-x{A-Tq%Cc=dntv~B+xHERDeX%_rv(<=PWsh;;=)F}Ty zt8B=BCWWm3<+E=5kE=WI-z{bFf3twz|JqI!{{^*!{xb_I|7T$0`Om-zN;hoav3+)a z?f)7UY5xN==Khbb-1OfqX~BOPtNQ;eYDNErEPMWkHJ${Ix3Q>K{CCRR_1~fN=zlJg ziT@dO+WvEz_x)!!ocN!?aK?XDi$(wSb5H({pKxpIy}EKLdw6q|E`!ub?&$csw6eJ{tUI;&=Pcqnz+x)urXX zPyUAg*?s5#7tg%=zkkD*|NeDn{);-!`mYS9JM*Er*VHE;hHEqL}{Eo}3DW-ZV-Nb!Fe$4USFD-Qo>)u{N-AR79gLBRb#1Fyq>22Rue z44}G#Mdm*PvnXUv08~eS#__@RlIDK~PUHU!ymtQ?guTH1X>R?h|JtF;{ufTW`#*L1 z|2yXH`7dhS_n$#B?mvTw&wmC%pZ}t|8UF=UBmOgT>i>s~FM`TdPhp5B|@kpZ}jpz3jiB?acp~Q*ZrutUvdk&v(s#5x?#Kvll=Auab4~ zKa=yW|B?wO{~OfZ{~tZ?&41Z~$NwW%fB2tq@Z6R113NF zZ&GvrzjDrv|MAlv|L1hy^`F&t^?zQ+mH&Aim;U#tJoR7JbJ2eeT~MDq_CKR!^nWRv zrvJ9dEB_k=wEySV^!?8On(yIOf%FYPZ4OpY{iOAufzt>wKPl(}u4@=%lK&gVEdS4I z*887Pr}ICT?V|sw)9(DYZM^iK(|6Z@v$CuI{ii?w&l`UJzhK;z|K<}v{4d-2{l8rK zoBz?rejr-(udk{{b^U{O5?i^IxOv!GBiY)BhRW_y1>h-t}KI_0)fZtP}sm1J?c* zaGLg?)2QXYjMt3+A~tRRnPn6HGs}kkXHyFL&m!#xY3qZ=h(Y}TP}>jG4&(x*du#9* z2BSp4e@=~@|GJUO{<~G4{vSK>-hT<-&HouKm;7gSUH4xg_~`$mUv;VWH z#s6o5jd6hbO`v&T(0m%RBDjCeBxv;?+)onm{Ldf}@}E&5`9H6D<9{Z-p8vegEB-rF zUHGpUb?`r%=dS-8A;3@N+ z6aQIUw*F_dTJfLVZtj0(qu&2ay7m8A^-BLUDy9EtQOf==?b!04MKbU|189s0G;a$^ z_slB)8MsXTix?$ABawsrPuz8r(OFm zUhwe0c)_dx-iv?!cb@m>fBg2p|25|R{l6GAUiRYO|Eha`|5v{G^*`^)xBtc0e*d5Q z;Ii{ zkN#Klo%f$xGYdSf2A*eO(FU^xwUYj`DW?AyG;jVd>oxbkV(`lUvY~7L3;V79&*8G_ zKbPOG|02of|BGke`OlyI{J&KF*Z-a?{{9bJ|L=eD-hcmdPW=1d_vqjMsZaj>zxoX{ z5ApB+g`a=__kI8K|Lw1T|K~mb_rLPOzyFJ0{rkV{#lQdMm;e25x%lsY&-s7Oa8t%CnNIvJ31 z2-GKK)rF)d&^#Gv97ZPoKcjNte-^#=|2+2d{!4~!`LC39;J;AR{{LK|r~ixRJ^1gp z@XLSi#lQY*F8cf5dHcWrxyS$g&pY|=|E5>}{@?lW@Bj0k|G;DCZ+`vzfA!a2@O;kh z5C8t({{HX(-Jk#duYU9I|Dm`4{#RZ4_g{VC-~TN6pZ+sOKK{=fb?ZNW^7a2V4fph%^AA~`EIQ!+_|He!I{zq^6_uq5D zpa14Ppa09`KmE@ZeEC0v>%spFwj2I4m@oX#U@+l7gI4{229<*U44`p7sgVB+Vjlk) z_-+3)a2Wh&;x_s(VjlNj!YS{+xNG@;DWB&5GC@861>C#;^Lfnt&+oV9zfk0%|5DjE z{_D5A_#e9L+yB0EfBtWL`0xL@kDxiHfB!%Hg3Nz`<~u-c1GyDMzXi{K{Qv(JG&lAe zG9UBc$N&G6pZxouaq8dyklp|O+pPHaUv=7_|9n*+{xiki`_Jfi{y&53esEir(Q@H` z7R$;1Ic+-sbJ^7X=dvjL&ta7OpIHGkR&NVl4+b7H28~UNxc_I63i{8ekno>Lqu@V_ zNz;FBx0(ONqPG24E4=jIu>Qe+&na*Jhc5sAKj+}z|DgG}4KM%wzyJN;|Cc`@b4Q@{ zS0Hy|xF0&N_TcBg|9d|D```QM-~Xzs|Ndv5`1e0@@4x>}>;C>Xn(_U=V(qK{!pXP( zbBCP#&*l!Q%NG4-G@AIIQMd6wgIXbEJrii`7t}`swJ||;0BFn`6c?a0B@qM}A6Cx& zSzU(jRmf5o_c|BXwp{C8@9^xtdR+y6cbzWxtd_xpeB?!Vx4@Hj7n zEF}LjYS;f4^I!R2E^6<8$*5!hrBW{Zm&&>IU!m&hf5qmH|7{ok{$G6jA2<)E9D&SJ z?fwWUdtmOxpnv`U|K!KN|2tp*`(JYY-+#L;|NdL7`S;&q>EHiGGk^Y9Z-4t=BKsz| zo@8-a|DO>w=U~wLpFyh@QlHAjgVz#($A&@cl{i3kupXrRgqA-{ib?;O)QbKyXg2;= ziCF(%B5eDA4!;&ag5C)|u5DmhhdGV{?|NTGt>EHiFPyhYz2hGdx{rBHy&fovKJ>UOpb-eqp(fs(o zYURcMa+&-73k9$K&t*UTKdWK$e@3;!|BRsgD<1ZrLBtO{<^rB0Wzqi602&(+bOw(* z@fa5W*A8FuUol|ifA`8$|II7T{pa-D@n0qF^ndf3JO5QHp8Qv&w4xl2%f2F+!H=9NM7+wXt<`+xciq>h_>@8AE53;+IS zAN}_~dC%Yf5vzay51#k=zxRaa|8+~Q{+Eb5{9i0|>wn3RmH(yu=lz%PnEYSDrTf3Q zUHg9}r=I^3R#o7&kPIwZ{~17YY@jiEQLq1uvI+nB&AR>vG@kvx=;F=vAO4%q|NFn__`mJ?98$J^`1K#0&mVmM_kY*BfBz$o|NCEk>fitT1ONWJt^D)fbdmkJ z>$W}qA2a{i|Kb%-{<~J6`p;`S`#-Z*-G5f~!vE|Vx&H->YyVruulcVRGVebVX#N8< zZVZ|)2jyo_z5wM9`LzGcx?TShyRZC@?7s1z$7Ab%CWp=cm2=L5*Tu3&-uN$5`}V*6 z@<0Ehw*UK|u;<_Z%G3Y;&$;*S|IrVSvS{_|fB(B4{QE!Q*1!MDpZ@!Q?eo9?TVMbC zzv%J5|4SbK`#=BQzyIEw{{9bH_4|Lt&L98vT3-Iws(<)jv+Blw;iRMgMWXlqwKfmRsY>f z_x%@kUGks7c*cJw+a>>*oj3mH4m-+unz|K6Mb{^y;B)LV6@|NhTA@b|y% zyzl>IDxUopO}+A;+56CcUjJSHC8Kxz7Y<(apUGn4e-`7W|7?0i|Cv=2!Rebp*bO{Z z%q(I9S+B+nTBD^8NlStb{~2Zd{xix&{AbrK`Y-M^<3F=O+kZC4ssCB+=lGCt zcHe)+vg`k&m%jgR-1_dn=j`wQQ`Y_dpMU7z|Cs&%{?}dj_rLYZzyA>jAnoMp6aW54 z9r*X(dfvbPB}f1LkKXz3zwOe$|0B2l{U5RN7q~w3oc8iRU&87C?B3h|Guo{9&u%~e zKd1fF|BO1IdN=<+qhj)ZMwy8J45D8D89?jIxGf;#vEX@T1@L-UMnTK}%(CwP86-Tx z{sj4#)v)+Kn?>V)Cd20cYz|ZZb9gTMFBY@!zgN$_|LVo}|GQ3k{a?TN&3~6UpZ`mA zef^)h^UwdflYjraZ2bG*7qre}{lEWVTmJsnU-0+8MAPs8VJm(0spy?|&wh=>Lo=NszTMn$`aq^m_jDIIZ|^Qg-$~pU!_G~PW|_u z_5Od zLf5D1{s*s71I-IFN&IJEQU1@!rT3pvz~Vonfc<|CrGWo@rg8t76+{0s$bkC7DgPN% z%l`A(%=m9re)PYP=j#7Vwu}BVJFfiCXush!+{{m-gd^IySd>VH;He-@PPLFGKS&7k?80W@C71R8@A1JAn& z=tceK*Ny)#X<7JR%rf&otE|_52GIPDMBsl0*~I_C)?NQiGPeE~cAEX4QNQ~?qsfH- zjHWaHGdnE(FOqoRzij&X|K5`y{ufL={h!}b3mVk_XW#>^{WXTH2?6C>@Ho2Ye+DLr|Lh_j|Go3J|2Ill@n1kQ6*6Z4+9v>7 zZ!8KL3o80=lC=6iziIn_29=Wk44ReTJk4s`|DVHk&VQk(P5%vYPW-oTzw)2WXUl&M z&+Y#e^UnPjjyv*SD)YpDq42~189n#>7tg)`u0y!o*8gWPpZlNDcq+J$#Gu>qpHZjo zzo2LTe_^Ld|24vv{^v94`tMhM&~Et8pjQ5$K{@X~qe{kq4zmKt`VR5P|E!9I|D!8T z{;!+f%Kzd9RsY$=1O78|Y5!+nkpQn>XXaP_&&+T2U)UrLJdVY{s`j4&G#ce+Fq#|H>CU4#*(j|6ku{;s3y#qyP2Y=ls`j>iI8hnDd`i%KSeA z7id0-`#)qag7AMv4rxf7fYKFc?GvZze_7kg{{k90{~7o^|1$`>{bvx3_%ChS_uo2U z)qiHt_z-A4vO&ZDEnSAVj z>ykJBd99cK=e3*jU({y$e+kcN|Ak$e{xitN{%4R31J^qYpz%LYy5+O|&&2Qb-!gpB ze=(hg{|xNT{~1_~AnU?F`<}r2kwEj?pg9-7(nGJXfB=0`ac7w<9|jz zum9ZgY5(PHC;rdty7XV%toc6!s2wXE@n6=V`@eVb?*9x*6_9XL$^OrvlJ}p_q2)iP zdF_7&)$;!g`i=j&tf&1CZ@%)MLABvOgKFM?M%AqUjPgnUg`La)vnYV-R?wOc(0nRn z{*Hmm>_4lp&wp*NN&h*7KzoGf{%2qS%_o8813>F-B>yw8 zDEw#UHUBTHnf71TbJBm?_%;6{%1{3H&f58(K{Oh&zC_gLKd)x?e?I-P{|wS`;C2&u z{6aqAKd%*NJ!j^B2KnUwjB1(xnbnK`t4A#U&!CX;pFuhbT(>Yt1pa4J4gW7>o$(*M z))Q2>g4P~_#%I~}|FcVa{uedM`_IU33`rZHH8SA2Y><1|pnDZS?g!0Bg4W4E{lF;l zpMg#GKZl~re?hIZ|3Zpc|2f2x|7*F=_^;yB@t=X$6|$EJR9A!AQzD?U*84w$IB4!V z;6G>`506R4e-`Cv$eK*h8c>O_|IEtqkntNapZ|=~pgkAv;I(mr7ODR^Ky?kbCA3ej z^`C)N{Xe^$&3`s=3-DSt@cJIG+d*yxxgF$g(A+O*?i!SiK;j@jfZ|2~GPeQB3(T7T z8CdlGGq4)}7tl-j&n_DTnac#Vxk2N=eAeK3VQ{+x)P5Cl1JA{?DhK^%lmyModqUP) zf!gLEInX$nkP~FR5GcF_ZU6IX$Ng7w?fTCl5&WMK6!xs3H6e2U88~Di)JTA|AY2%fcNUK>q6Qgpf(3+oh2yVL3IYG z{Q>Gr2-ri`27qV~AA~`2AT=PhAU#~h|Ct2M{|g$#|2K|U_+Q?s@jsU;XrGombZ!N- z#|PvNP`HBJhwOIH+&c_|`~$+EI02~v`3JPt0JLrfv<6M+KLfJ}#7xjy4$!^~Q2ouS z3~7h3Y5Zqk*MzKb;smXQ0=1>|A?wJw^!_t|FehlAl@3G=NG&_4oL2eIz^e40kwfc0 wyNv69WDG5`Po diff --git a/who-am-i/static/style.css b/who-am-i/static/style.css new file mode 100644 index 0000000..2ca5b7c --- /dev/null +++ b/who-am-i/static/style.css @@ -0,0 +1,146 @@ +body { + color: #434; + font-family: 'Iowan Old Style', 'Palatino Linotype', 'URW Palladio L', P052, serif; + margin: 0; + min-height: 100vh; + padding: 0; +} +.wrap { + border: 2px solid #221828; + border-radius: 0.5rem; + box-sizing: border-box; + overflow: hidden; + display: flex; + flex-direction: column; + height: 100vh; +} +.wrap.unframed { + border-radius: 0; + border-width: 0.4rem; +} +header { + background: #221828; + display: flex; + justify-content: space-between; + padding: 0 0.25rem; + color: #c9b; + display: flex; + gap: 0.5rem; + align-items: baseline; +} +header > * { + flex-basis: 33%; +} +header > .empty { + font-size: 0.8rem; + opacity: 0.5; +} +header > .title { + text-align: center; +} +header > a.micro { + text-decoration: none; + font-size: 0.8rem; + text-align: right; + opacity: 0.5; +} +header > a.micro:hover { + opacity: 1; +} +main { + background: #ccc; + display: flex; + flex-direction: column; + flex-grow: 1; + padding: 0.25rem 0.5rem; +} +.mini-content { + margin: 1rem auto 0; + padding: 1rem 0.5rem; + max-width: 21rem; +} + +p { + margin: 1rem 0 0; + text-align: center; +} +.parent-host { + font-weight: bold; + color: #48c; + display: inline-block; + padding: 0 0.125rem; + border-radius: 0.25rem; + border: 1px solid #aaa; + font-size: 0.8rem; +} + +#loader { + display: flex; + flex-grow: 1; + justify-content: center; + align-items: center; +} +.spinner { + animation: rotation 1.618s ease-in-out infinite; + border-radius: 50%; + border: 3px dashed #434; + box-sizing: border-box; + display: inline-block; + height: 1.5em; + width: 1.5em; +} +@keyframes rotation { + 0% { transform: rotate(0deg) } + 100% { transform: rotate(360deg) } +} + +#user-info { + flex-grow: 1; + display: flex; + flex-direction: column; + justify-content: center; +} +#action { + background: #eee; + display: flex; + justify-content: space-between; + padding: 0.5rem 0.25rem 0.5rem 0.5rem; + font-size: 0.8rem; + align-items: baseline; + border-radius: 0.5rem; + border: 1px solid #bbb; + cursor: pointer; +} +#action:hover { + background: #fff; +} +#allow { + background: transparent; + border: none; + border-left: 1px solid #bbb; + padding: 0 0.5rem; + color: #375; + font: inherit; + cursor: pointer; +} +#action:hover #allow { + color: #285; +} + +#or { + font-size: 0.8rem; + text-align: center; +} +#or p { + margin: 0 0 1rem; +} + +input#handle { + border: none; + border-bottom: 1px dashed #aaa; + background: transparent; +} + +.hidden { + display: none !important; +} diff --git a/who-am-i/templates/base-base.hbs b/who-am-i/templates/base-base.hbs new file mode 100644 index 0000000..7bd308b --- /dev/null +++ b/who-am-i/templates/base-base.hbs @@ -0,0 +1,17 @@ + + + + + who-am-i + + + + + + + + + + {{> body}} + + diff --git a/who-am-i/templates/base-framed.hbs b/who-am-i/templates/base-framed.hbs new file mode 100644 index 0000000..ba0bccd --- /dev/null +++ b/who-am-i/templates/base-framed.hbs @@ -0,0 +1,28 @@ +{{#*inline "description"}}{{/inline}} + +{{#*inline "body"}} +
+
+
🔒
+ who-am-i + microcosm +
+ +
+ {{> main}} +
+
+{{/inline}} + +{{#> base-base}}{{/base-base}} diff --git a/who-am-i/templates/base-full.hbs b/who-am-i/templates/base-full.hbs new file mode 100644 index 0000000..cd27a2b --- /dev/null +++ b/who-am-i/templates/base-full.hbs @@ -0,0 +1,26 @@ +{{#*inline "body"}} +
+
+
🔒
+ who-am-i + microcosm +
+ +
+ {{> main}} +
+
+{{/inline}} + +{{#> base-base}}{{/base-base}} diff --git a/who-am-i/templates/hello.hbs b/who-am-i/templates/hello.hbs new file mode 100644 index 0000000..779c14a --- /dev/null +++ b/who-am-i/templates/hello.hbs @@ -0,0 +1,9 @@ +{{#*inline "description"}}A little identity-verifying auth service for microcosm demos{{/inline}} + +{{#*inline "main"}} +
+ This is a little identity-verifying service for microcosm demos. +
+{{/inline}} + +{{#> base-full}}{{/base-full}} diff --git a/who-am-i/templates/prompt-error.hbs b/who-am-i/templates/prompt-error.hbs new file mode 100644 index 0000000..5b48431 --- /dev/null +++ b/who-am-i/templates/prompt-error.hbs @@ -0,0 +1,17 @@ +{{#*inline "main"}} +
+

Something went wrong :(

+

{{ reason }}

+ +
+ + +{{/inline}} + +{{#> base-framed}}{{/base-framed}} -- 2.51.2 From 3d9a9d328cdf056e28fc9fd64d6d57fbaacd0f1b Mon Sep 17 00:00:00 2001 From: phil Date: Wed, 2 Jul 2025 15:08:07 -0400 Subject: [PATCH 046/134] more error handling --- who-am-i/demo/index.html | 1 + who-am-i/src/oauth.rs | 6 +- who-am-i/src/server.rs | 56 ++++++---- who-am-i/static/style.css | 26 ++++- who-am-i/templates/auth-fail.hbs | 7 +- who-am-i/templates/prompt-anon.hbs | 94 ---------------- who-am-i/templates/prompt-error.hbs | 24 ++-- who-am-i/templates/prompt.hbs | 128 +++++++++++++++++++++ who-am-i/templates/return-base.hbs | 168 ---------------------------- 9 files changed, 204 insertions(+), 306 deletions(-) delete mode 100644 who-am-i/templates/prompt-anon.hbs create mode 100644 who-am-i/templates/prompt.hbs delete mode 100644 who-am-i/templates/return-base.hbs diff --git a/who-am-i/demo/index.html b/who-am-i/demo/index.html index 6d1ff95..06fe404 100644 --- a/who-am-i/demo/index.html +++ b/who-am-i/demo/index.html @@ -19,6 +19,7 @@ (whoami => { const handleMessage = ev => { if (ev.source !== whoami.contentWindow) { + // TODO: ALSO CHECK ev.origin!!!! console.log('nah'); return; } diff --git a/who-am-i/src/oauth.rs b/who-am-i/src/oauth.rs index 5407cb4..10fb1a2 100644 --- a/who-am-i/src/oauth.rs +++ b/who-am-i/src/oauth.rs @@ -53,10 +53,6 @@ pub enum AuthSetupError { HickoryResolverError(ResolveError), } -#[derive(Debug, Error)] -#[error(transparent)] -pub struct AuthStartError(#[from] atrium_oauth::Error); - #[derive(Debug, Error)] pub enum OAuthCompleteError { #[error("the user denied request: {description:?} (from {issuer:?})")] @@ -124,7 +120,7 @@ impl OAuth { }) } - pub async fn begin(&self, handle: &str) -> Result { + pub async fn begin(&self, handle: &str) -> Result { let auth_opts = AuthorizeOptions { scopes: READONLY_SCOPE.to_vec(), ..Default::default() diff --git a/who-am-i/src/server.rs b/who-am-i/src/server.rs index d31e7da..81a8529 100644 --- a/who-am-i/src/server.rs +++ b/who-am-i/src/server.rs @@ -145,6 +145,10 @@ async fn prompt( if !allowed_hosts.contains(parent_host) { return err("Login is not allowed on this page", false); } + let parent_origin = url.origin().ascii_serialization(); + if parent_origin == "null" { + return err("Referer origin is opaque", true); + } if let Some(did) = jar.get(DID_COOKIE_KEY) { let Ok(did) = Did::new(did.value_trimmed().to_string()) else { return err("Bad cookie", false); @@ -166,15 +170,17 @@ async fn prompt( "did": did, "fetch_key": fetch_key, "parent_host": parent_host, + "parent_origin": parent_origin, }), ) .into_response() } else { RenderHtml( - "prompt-anon", + "prompt", engine, json!({ "parent_host": parent_host, + "parent_origin": parent_origin, }), ) .into_response() @@ -228,35 +234,43 @@ async fn user_info( #[derive(Debug, Deserialize)] struct BeginOauthParams { handle: String, - flow: String, } async fn start_oauth( - State(AppState { oauth, .. }): State, + State(AppState { oauth, engine, .. }): State, Query(params): Query, jar: SignedCookieJar, - headers: HeaderMap, -) -> (SignedCookieJar, Redirect) { +) -> Response { // if any existing session was active, clear it first + // ...this might help a confusion attack w multiple sign-in flows or smth let jar = jar.remove(DID_COOKIE_KEY); - if let Some(referrer) = headers.get(REFERER) { - if let Ok(referrer) = referrer.to_str() { - println!("referrer: {referrer}"); - } else { - eprintln!("referer contained opaque bytes"); - }; - } else { - eprintln!("no referrer"); - }; + use atrium_identity::Error as IdError; + use atrium_oauth::Error as OAuthError; - let auth_url = oauth.begin(¶ms.handle).await.unwrap(); - let flow = params.flow; - if !flow.chars().all(|c| char::is_ascii_alphanumeric(&c)) { - panic!("invalid flow (injection attempt?)"); // should probably just url-encode it instead.. + match oauth.begin(¶ms.handle).await { + Ok(auth_url) => (jar, Redirect::to(&auth_url)).into_response(), + Err(OAuthError::Identity(IdError::NotFound)) => { + let info = json!({ "reason": "handle not found" }); + (StatusCode::NOT_FOUND, RenderHtml("auth-fail", engine, info)).into_response() + } + Err(OAuthError::Identity(IdError::AtIdentifier(r))) => { + let info = json!({ "reason": r }); + (StatusCode::NOT_FOUND, RenderHtml("auth-fail", engine, info)).into_response() + } + Err(OAuthError::Identity(IdError::HttpStatus(StatusCode::NOT_FOUND))) => { + let info = json!({ "reason": "handle not found" }); + (StatusCode::NOT_FOUND, RenderHtml("auth-fail", engine, info)).into_response() + } + Err(e) => { + eprintln!("begin auth failed: {e:?}"); + let info = json!({ "reason": "unknown" }); + ( + StatusCode::INTERNAL_SERVER_ERROR, + RenderHtml("auth-fail", engine, info), + ) + .into_response() + } } - eprintln!("auth_url {auth_url}"); - - (jar, Redirect::to(&auth_url)) } impl OAuthCompleteError { diff --git a/who-am-i/static/style.css b/who-am-i/static/style.css index 2ca5b7c..1036dcb 100644 --- a/who-am-i/static/style.css +++ b/who-am-i/static/style.css @@ -60,10 +60,23 @@ main { max-width: 21rem; } +#error-message { + font-size: 0.8rem; + color: #a31; +} + +#error-message:not(.hidden) + #prompt { + display: none !important; +} + +#error-message, p { margin: 1rem 0 0; text-align: center; } +p.detail { + font-size: 0.8rem; +} .parent-host { font-weight: bold; color: #48c; @@ -93,6 +106,10 @@ p { 0% { transform: rotate(0deg) } 100% { transform: rotate(360deg) } } +/* loader visibility is mutually exclusive with its immediate sibling */ +#loader:not(.hidden) + * { + display: none !important; +} #user-info { flex-grow: 1; @@ -100,7 +117,7 @@ p { flex-direction: column; justify-content: center; } -#action { +.action { background: #eee; display: flex; justify-content: space-between; @@ -111,9 +128,14 @@ p { border: 1px solid #bbb; cursor: pointer; } -#action:hover { +.action:hover { background: #fff; } +#form-action:not(.hidden) + .action { + display: none !important; +} + +#connect, #allow { background: transparent; border: none; diff --git a/who-am-i/templates/auth-fail.hbs b/who-am-i/templates/auth-fail.hbs index 5466bac..afc1696 100644 --- a/who-am-i/templates/auth-fail.hbs +++ b/who-am-i/templates/auth-fail.hbs @@ -8,14 +8,13 @@
{{/inline}} -{{#> return-base}}{{/return-base}} +{{#> base-framed}}{{/base-framed}} diff --git a/who-am-i/templates/prompt-anon.hbs b/who-am-i/templates/prompt-anon.hbs deleted file mode 100644 index a5536f5..0000000 --- a/who-am-i/templates/prompt-anon.hbs +++ /dev/null @@ -1,94 +0,0 @@ -{{#*inline "main"}} -

- Connect your ATmosphere -

- -

- {{ parent_host }} would like to confirm your handle -

- - - -
-
- - -
-
- - -{{/inline}} - -{{#> prompt-base}}{{/prompt-base}} diff --git a/who-am-i/templates/prompt-error.hbs b/who-am-i/templates/prompt-error.hbs index 5b48431..91672ad 100644 --- a/who-am-i/templates/prompt-error.hbs +++ b/who-am-i/templates/prompt-error.hbs @@ -1,17 +1,17 @@ {{#*inline "main"}} -
-

Something went wrong :(

-

{{ reason }}

- -
+
+

Something went wrong :(

+

{{ reason }}

+ +
- + {{/inline}} {{#> base-framed}}{{/base-framed}} diff --git a/who-am-i/templates/prompt.hbs b/who-am-i/templates/prompt.hbs new file mode 100644 index 0000000..2cb1e0f --- /dev/null +++ b/who-am-i/templates/prompt.hbs @@ -0,0 +1,128 @@ +{{#*inline "main"}} +

+ Connect in the ATmosphere +

+ + + +

+ {{ parent_host }} would like to confirm your handle +

+ +
+ +
+ +
+
+ + +
+ +
+ + +
+
+ + + + + +{{/inline}} + +{{#> base-framed}}{{/base-framed}} diff --git a/who-am-i/templates/return-base.hbs b/who-am-i/templates/return-base.hbs deleted file mode 100644 index 70f35ce..0000000 --- a/who-am-i/templates/return-base.hbs +++ /dev/null @@ -1,168 +0,0 @@ - - - - -
-
-
🔒
- who-am-i - microcosm -
- -
- {{> main}} -
-
-- 2.51.2 From c2463badb172c2245aff5e12c566758f9402cec2 Mon Sep 17 00:00:00 2001 From: phil Date: Wed, 2 Jul 2025 15:08:58 -0400 Subject: [PATCH 047/134] clippy is right --- who-am-i/src/oauth.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/who-am-i/src/oauth.rs b/who-am-i/src/oauth.rs index 10fb1a2..8f2b626 100644 --- a/who-am-i/src/oauth.rs +++ b/who-am-i/src/oauth.rs @@ -125,7 +125,7 @@ impl OAuth { scopes: READONLY_SCOPE.to_vec(), ..Default::default() }; - Ok(self.client.authorize(handle, auth_opts).await?) + self.client.authorize(handle, auth_opts).await } /// Finally, resolve the oauth flow to a verified DID -- 2.51.2 From 998eafedae0ee8ad443209874a63d1a74ccd1353 Mon Sep 17 00:00:00 2001 From: phil Date: Wed, 2 Jul 2025 15:37:43 -0400 Subject: [PATCH 048/134] auth rejection --- who-am-i/src/server.rs | 121 +++++++++++++------------------ who-am-i/templates/auth-fail.hbs | 2 +- who-am-i/templates/prompt.hbs | 6 +- 3 files changed, 58 insertions(+), 71 deletions(-) diff --git a/who-am-i/src/server.rs b/who-am-i/src/server.rs index 81a8529..c5436fb 100644 --- a/who-am-i/src/server.rs +++ b/who-am-i/src/server.rs @@ -247,69 +247,27 @@ async fn start_oauth( use atrium_identity::Error as IdError; use atrium_oauth::Error as OAuthError; + let err = |code, reason| { + let info = json!({ + "result": "fail", + "reason": reason, + }); + (code, RenderHtml("auth-fail", engine.clone(), info)).into_response() + }; + match oauth.begin(¶ms.handle).await { Ok(auth_url) => (jar, Redirect::to(&auth_url)).into_response(), - Err(OAuthError::Identity(IdError::NotFound)) => { - let info = json!({ "reason": "handle not found" }); - (StatusCode::NOT_FOUND, RenderHtml("auth-fail", engine, info)).into_response() - } - Err(OAuthError::Identity(IdError::AtIdentifier(r))) => { - let info = json!({ "reason": r }); - (StatusCode::NOT_FOUND, RenderHtml("auth-fail", engine, info)).into_response() - } - Err(OAuthError::Identity(IdError::HttpStatus(StatusCode::NOT_FOUND))) => { - let info = json!({ "reason": "handle not found" }); - (StatusCode::NOT_FOUND, RenderHtml("auth-fail", engine, info)).into_response() - } + Err(OAuthError::Identity( + IdError::NotFound | IdError::HttpStatus(StatusCode::NOT_FOUND), + )) => err(StatusCode::NOT_FOUND, "handle not found"), + Err(OAuthError::Identity(IdError::AtIdentifier(r))) => err(StatusCode::BAD_REQUEST, &r), Err(e) => { eprintln!("begin auth failed: {e:?}"); - let info = json!({ "reason": "unknown" }); - ( - StatusCode::INTERNAL_SERVER_ERROR, - RenderHtml("auth-fail", engine, info), - ) - .into_response() + err(StatusCode::INTERNAL_SERVER_ERROR, "unknown") } } } -impl OAuthCompleteError { - fn to_error_response(&self, engine: AppEngine) -> Response { - let (level, desc) = match self { - OAuthCompleteError::Denied { description, .. } => { - ("warn", format!("asdf: {description:?}")) - } - OAuthCompleteError::Failed { .. } => ( - "error", - "Something went wrong while requesting permission, sorry!".to_string(), - ), - OAuthCompleteError::CallbackFailed(_) => ( - "error", - "Something went wrong after permission was granted, sorry!".to_string(), - ), - OAuthCompleteError::NoDid => ( - "error", - "Something went wrong when trying to confirm your identity, sorry!".to_string(), - ), - }; - ( - if level == "warn" { - StatusCode::FORBIDDEN - } else { - StatusCode::INTERNAL_SERVER_ERROR - }, - RenderHtml( - "auth-fail", - engine, - json!({ - "reason": desc, - }), - ), - ) - .into_response() - } -} - async fn complete_oauth( State(AppState { engine, @@ -320,10 +278,42 @@ async fn complete_oauth( }): State, Query(params): Query, jar: SignedCookieJar, -) -> Result<(SignedCookieJar, impl IntoResponse), Response> { +) -> Response { + let err = |code, result, reason| { + let info = json!({ + "result": result, + "reason": reason, + }); + (code, RenderHtml("auth-fail", engine.clone(), info)).into_response() + }; + let did = match oauth.complete(params).await { Ok(did) => did, - Err(e) => return Err(e.to_error_response(engine)), + Err(e) => { + return match e { + OAuthCompleteError::Denied { description, .. } => { + let desc = description.unwrap_or("permission to share was denied".to_string()); + err(StatusCode::FORBIDDEN, "deny", desc.as_str()) + } + OAuthCompleteError::Failed { .. } => { + eprintln!("auth completion failed: {e:?}"); + err( + StatusCode::INTERNAL_SERVER_ERROR, + "fail", + "failed to complete", + ) + } + OAuthCompleteError::CallbackFailed(e) => { + eprintln!("auth callback failed: {e:?}"); + err( + StatusCode::INTERNAL_SERVER_ERROR, + "fail", + "failed to complete callback", + ) + } + OAuthCompleteError::NoDid => err(StatusCode::BAD_REQUEST, "fail", "no DID found"), + }; + } }; let cookie = Cookie::build((DID_COOKIE_KEY, did.to_string())) @@ -342,16 +332,9 @@ async fn complete_oauth( }, shutdown.child_token(), ); - - Ok(( - jar, - RenderHtml( - "authorized", - engine, - json!({ - "did": did, - "fetch_key": fetch_key, - }), - ), - )) + let info = json!({ + "did": did, + "fetch_key": fetch_key, + }); + (jar, RenderHtml("authorized", engine, info)).into_response() } diff --git a/who-am-i/templates/auth-fail.hbs b/who-am-i/templates/auth-fail.hbs index afc1696..4ba2b00 100644 --- a/who-am-i/templates/auth-fail.hbs +++ b/who-am-i/templates/auth-fail.hbs @@ -9,7 +9,7 @@ -{{/inline}} - -{{#> prompt-base}}{{/prompt-base}} diff --git a/who-am-i/templates/prompt.hbs b/who-am-i/templates/prompt.hbs index 22d9d44..5e87be9 100644 --- a/who-am-i/templates/prompt.hbs +++ b/who-am-i/templates/prompt.hbs @@ -16,13 +16,13 @@
- +
@@ -34,9 +34,10 @@ const errorEl = document.getElementById('error-message'); const promptEl = document.getElementById('prompt'); const loaderEl = document.getElementById('loader'); const infoEl = document.getElementById('user-info'); -const handleEl = document.getElementById('handle'); +const handleInputEl = document.getElementById('handle-input'); +const handleViewEl = document.getElementById('handle-view'); const formEl = document.getElementById('form-action'); // for anon -const allowEl = document.getElementById('allow'); // for known-did +const allowEl = document.getElementById('handle-action'); // for known-did const connectEl = document.getElementById('connect'); // for anon function err(e, msg) { @@ -46,15 +47,27 @@ function err(e, msg) { throw new Error(e); } -formEl && (formEl.onsubmit = e => { +// already-known user +({{{json did}}}) && (async () => { + + const handle = await lookUp({{{json fetch_key}}}); + console.log('got handle', handle); + + loaderEl.classList.add('hidden'); + handleViewEl.textContent = `@${handle}`; + allowEl.addEventListener('click', () => shareAllow(handle)); +})(); + +// anon user +formEl.onsubmit = e => { e.preventDefault(); loaderEl.classList.remove('hidden'); // TODO: include expected referer! (..this system is probably bad) // maybe a random localstorage key that we specifically listen for? const url = new URL('/auth', window.location); - url.searchParams.set('handle', handleEl.value); + url.searchParams.set('handle', handleInputEl.value); window.open(url, '_blank'); -}); +}; window.addEventListener('storage', async e => { // here's a fun minor vuln: we can't tell which flow triggers the storage event. @@ -64,8 +77,8 @@ window.addEventListener('storage', async e => { const fail = (e, msg) => { loaderEl.classList.add('hidden'); formEl.classList.remove('hidden'); - handleEl.focus(); - handleEl.select(); + handleInputEl.focus(); + handleInputEl.select(); err(e, msg); } @@ -98,7 +111,7 @@ window.addEventListener('storage', async e => { shareAllow(handle); }); -const lookUp = async fetch_key => { +async function lookUp(fetch_key) { const user_info = new URL('/user-info', window.location); user_info.searchParams.set('fetch-key', fetch_key); let info; -- 2.51.2 From 2e5e17adb9d4e5118f3ef0dcd7e225b1edf12a0a Mon Sep 17 00:00:00 2001 From: phil Date: Wed, 2 Jul 2025 16:35:51 -0400 Subject: [PATCH 050/134] refresh cookie + frame headers --- who-am-i/src/server.rs | 86 +++++++++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 31 deletions(-) diff --git a/who-am-i/src/server.rs b/who-am-i/src/server.rs index d0573e3..6950d34 100644 --- a/who-am-i/src/server.rs +++ b/who-am-i/src/server.rs @@ -4,7 +4,7 @@ use axum::{ extract::{FromRef, Query, State}, http::{ StatusCode, - header::{CONTENT_TYPE, HeaderMap, REFERER}, + header::{CONTENT_SECURITY_POLICY, CONTENT_TYPE, HeaderMap, REFERER, X_FRAME_OPTIONS}, }, response::{IntoResponse, Json, Redirect, Response}, routing::get, @@ -29,8 +29,9 @@ const STYLE_CSS: &str = include_str!("../static/style.css"); const DID_COOKIE_KEY: &str = "did"; +const COOKIE_EXPIRATION: Duration = Duration::from_secs(30 * 86_400); + type AppEngine = Engine>; -type Rendered = RenderHtml<&'static str, AppEngine, Value>; #[derive(Clone)] struct AppState { @@ -96,8 +97,23 @@ pub async fn serve( .unwrap(); } -async fn hello(State(AppState { engine, .. }): State) -> Rendered { - RenderHtml("hello", engine, json!({})) +async fn hello( + State(AppState { engine, .. }): State, + mut jar: SignedCookieJar, +) -> Response { + // push expiry (or clean up) the current cookie + if let Some(did) = jar.get(DID_COOKIE_KEY) { + if let Ok(did) = Did::new(did.value_trimmed().to_string()) { + jar = jar.add(cookie(&did)); + } else { + jar = jar.remove(DID_COOKIE_KEY); + } + } + let frame_headers = [ + (X_FRAME_OPTIONS, "deny"), + (CONTENT_SECURITY_POLICY, "frame-ancestors 'none'"), + ]; + (frame_headers, jar, RenderHtml("hello", engine, json!({}))).into_response() } async fn css() -> impl IntoResponse { @@ -112,6 +128,15 @@ async fn favicon() -> impl IntoResponse { ([(CONTENT_TYPE, "image/x-icon")], FAVICON) } +fn cookie(did: &Did) -> Cookie<'static> { + Cookie::build((DID_COOKIE_KEY, did.to_string())) + .http_only(true) + .secure(true) + .same_site(SameSite::None) + .max_age(COOKIE_EXPIRATION.try_into().unwrap()) + .into() +} + async fn prompt( State(AppState { allowed_hosts, @@ -149,11 +174,23 @@ async fn prompt( if parent_origin == "null" { return err("Referer origin is opaque", true); } + + let frame_headers = [ + (X_FRAME_OPTIONS, format!("allow-from {parent_origin}")), + ( + CONTENT_SECURITY_POLICY, + format!("frame-ancestors {parent_host}"), + ), + ]; + if let Some(did) = jar.get(DID_COOKIE_KEY) { let Ok(did) = Did::new(did.value_trimmed().to_string()) else { return err("Bad cookie", false); }; + // push cookie expiry + let jar = jar.add(cookie(&did)); + let fetch_key = resolve_handles.dispatch( { let oauth = oauth.clone(); @@ -163,27 +200,20 @@ async fn prompt( shutdown.child_token(), ); - RenderHtml( - "prompt", - engine, - json!({ - "did": did, - "fetch_key": fetch_key, - "parent_host": parent_host, - "parent_origin": parent_origin, - }), - ) - .into_response() + let info = json!({ + "did": did, + "fetch_key": fetch_key, + "parent_host": parent_host, + "parent_origin": parent_origin, + }); + + (frame_headers, jar, RenderHtml("prompt", engine, info)).into_response() } else { - RenderHtml( - "prompt", - engine, - json!({ - "parent_host": parent_host, - "parent_origin": parent_origin, - }), - ) - .into_response() + let info = json!({ + "parent_host": parent_host, + "parent_origin": parent_origin, + }); + (frame_headers, RenderHtml("prompt", engine, info)).into_response() } } @@ -316,13 +346,7 @@ async fn complete_oauth( } }; - let cookie = Cookie::build((DID_COOKIE_KEY, did.to_string())) - .http_only(true) - .secure(true) - .same_site(SameSite::None) - .max_age(std::time::Duration::from_secs(86_400).try_into().unwrap()); - - let jar = jar.add(cookie); + let jar = jar.add(cookie(&did)); let fetch_key = resolve_handles.dispatch( { -- 2.51.2 From cbd1e7fbab7e50420838fd6124c4a27392ca244a Mon Sep 17 00:00:00 2001 From: phil Date: Wed, 2 Jul 2025 17:00:13 -0400 Subject: [PATCH 051/134] show current session at hello and allow revoking do we need csrf here or... --- who-am-i/src/server.rs | 41 +++++++++++++++++---- who-am-i/static/style.css | 6 ++- who-am-i/templates/hello.hbs | 71 ++++++++++++++++++++++++++++++++++-- 3 files changed, 107 insertions(+), 11 deletions(-) diff --git a/who-am-i/src/server.rs b/who-am-i/src/server.rs index 6950d34..b5e6d3f 100644 --- a/who-am-i/src/server.rs +++ b/who-am-i/src/server.rs @@ -7,7 +7,7 @@ use axum::{ header::{CONTENT_SECURITY_POLICY, CONTENT_TYPE, HeaderMap, REFERER, X_FRAME_OPTIONS}, }, response::{IntoResponse, Json, Redirect, Response}, - routing::get, + routing::{get, post}, }; use axum_extra::extract::cookie::{Cookie, Key, SameSite, SignedCookieJar}; use axum_template::{RenderHtml, engine::Engine}; @@ -85,6 +85,7 @@ pub async fn serve( .route("/user-info", get(user_info)) .route("/auth", get(start_oauth)) .route("/authorized", get(complete_oauth)) + .route("/disconnect", post(disconnect)) .with_state(state); let listener = TcpListener::bind("0.0.0.0:9997") @@ -98,22 +99,43 @@ pub async fn serve( } async fn hello( - State(AppState { engine, .. }): State, + State(AppState { + engine, + resolve_handles, + shutdown, + oauth, + .. + }): State, mut jar: SignedCookieJar, ) -> Response { - // push expiry (or clean up) the current cookie - if let Some(did) = jar.get(DID_COOKIE_KEY) { + let info = if let Some(did) = jar.get(DID_COOKIE_KEY) { if let Ok(did) = Did::new(did.value_trimmed().to_string()) { + // push cookie expiry jar = jar.add(cookie(&did)); + let fetch_key = resolve_handles.dispatch( + { + let oauth = oauth.clone(); + let did = did.clone(); + async move { oauth.resolve_handle(did.clone()).await } + }, + shutdown.child_token(), + ); + json!({ + "did": did, + "fetch_key": fetch_key, + }) } else { jar = jar.remove(DID_COOKIE_KEY); + json!({}) } - } + } else { + json!({}) + }; let frame_headers = [ (X_FRAME_OPTIONS, "deny"), (CONTENT_SECURITY_POLICY, "frame-ancestors 'none'"), ]; - (frame_headers, jar, RenderHtml("hello", engine, json!({}))).into_response() + (frame_headers, jar, RenderHtml("hello", engine, info)).into_response() } async fn css() -> impl IntoResponse { @@ -179,7 +201,7 @@ async fn prompt( (X_FRAME_OPTIONS, format!("allow-from {parent_origin}")), ( CONTENT_SECURITY_POLICY, - format!("frame-ancestors {parent_host}"), + format!("frame-ancestors {parent_origin}"), ), ]; @@ -362,3 +384,8 @@ async fn complete_oauth( }); (jar, RenderHtml("authorized", engine, info)).into_response() } + +async fn disconnect(jar: SignedCookieJar) -> impl IntoResponse { + let jar = jar.remove(DID_COOKIE_KEY); + (jar, Json(json!({ "ok": true }))) +} diff --git a/who-am-i/static/style.css b/who-am-i/static/style.css index 5375d3e..6f42348 100644 --- a/who-am-i/static/style.css +++ b/who-am-i/static/style.css @@ -136,7 +136,8 @@ p.detail { } #connect, -#allow { +#allow, +#revoke { background: transparent; border: none; border-left: 1px solid #bbb; @@ -145,6 +146,9 @@ p.detail { font: inherit; cursor: pointer; } +#revoke { + color: #a31; +} #action:hover #allow { color: #285; } diff --git a/who-am-i/templates/hello.hbs b/who-am-i/templates/hello.hbs index 779c14a..291b866 100644 --- a/who-am-i/templates/hello.hbs +++ b/who-am-i/templates/hello.hbs @@ -1,9 +1,74 @@ {{#*inline "description"}}A little identity-verifying auth service for microcosm demos{{/inline}} {{#*inline "main"}} -
- This is a little identity-verifying service for microcosm demos. -
+
+ This is a little identity-verifying service for microcosm demos. + + {{#if did}} + + +

+ Connected identity: +

+ +
+ +
+ +
+
+ + +
+
+ + {{/if}} +
{{/inline}} {{#> base-full}}{{/base-full}} -- 2.51.2 From 6e997631dade6d7ec63f353c147c12c12f1d80fb Mon Sep 17 00:00:00 2001 From: phil Date: Wed, 2 Jul 2025 17:12:57 -0400 Subject: [PATCH 052/134] cargo blah --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 192663b..7def5a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5058,7 +5058,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] -- 2.51.2 From 930b5174357449aeece3139eb80012c7dc5f87a0 Mon Sep 17 00:00:00 2001 From: phil Date: Wed, 2 Jul 2025 17:14:00 -0400 Subject: [PATCH 053/134] note the lack of jwt --- who-am-i/readme.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/who-am-i/readme.md b/who-am-i/readme.md index 3df653e..66247d3 100644 --- a/who-am-i/readme.md +++ b/who-am-i/readme.md @@ -44,3 +44,8 @@ since the requirements (read-only, just verifying identity) seem modest, i was h it's still nice to have an explicit opt-in on a per-demo basis for microcosm so it will be used for that. it's allow-listed for the microcosm domain however (so not deployed on any adversarial hosting pages), so it's simultaenously overkill and restrictive. i will get back to oauth eventually and hopefully roll out a microcosm service to make it easy for clients (and demos), but there are a few more things in the pipeline to get to first. + + +### todo + +provide a pubkey-signed JWT of the identity (just the DID as `sub` probably). (**you probably SHOULD NOT USE THIS in any serious environment**) -- 2.51.2 From 2dbb21b08ec92b70f0b778ff0873a3ebe07855f6 Mon Sep 17 00:00:00 2001 From: phil Date: Wed, 2 Jul 2025 21:54:03 -0400 Subject: [PATCH 054/134] fix cross-compiling for original raspi model b tls it's always tls --- Cargo.lock | 160 ++++++++++++++++++-------------------------- who-am-i/Cargo.toml | 1 + who-am-i/readme.md | 15 +++++ 3 files changed, 81 insertions(+), 95 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7def5a5..9b31209 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2051,22 +2051,28 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.11" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497bbc33a26fdd4af9ed9c70d63f61cf56a938375fbb32df34db9b1cd6d643f2" +checksum = "dc2fdfdbff08affe55bb779f33b053aa1fe5dd5b54c257343c17edfa55711bdb" dependencies = [ + "base64 0.22.1", "bytes", "futures-channel", + "futures-core", "futures-util", "http", "http-body", "hyper", + "ipnet", "libc", + "percent-encoding", "pin-project-lite", "socket2", + "system-configuration", "tokio", "tower-service", "tracing", + "windows-registry", ] [[package]] @@ -2298,6 +2304,16 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +[[package]] +name = "iri-string" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "is-terminal" version = "0.4.16" @@ -3488,30 +3504,31 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" -version = "0.12.15" +version = "0.12.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb" +checksum = "cbc931937e6ca3a06e3b6c0aa7841849b160a90351d6ab467a8b9b9959767531" dependencies = [ "async-compression", "base64 0.22.1", "bytes", + "encoding_rs", "futures-core", "futures-util", + "h2", "http", "http-body", "http-body-util", "hyper", + "hyper-rustls", "hyper-tls", "hyper-util", - "ipnet", "js-sys", "log", "mime", "native-tls", - "once_cell", "percent-encoding", "pin-project-lite", - "rustls-pemfile", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", @@ -3520,12 +3537,12 @@ dependencies = [ "tokio-native-tls", "tokio-util", "tower", + "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "windows-registry", ] [[package]] @@ -4223,6 +4240,27 @@ dependencies = [ "syn", ] +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tagptr" version = "0.2.0" @@ -4551,14 +4589,18 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.2" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697" +checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" dependencies = [ "bitflags", "bytes", + "futures-util", "http", + "http-body", + "iri-string", "pin-project-lite", + "tower", "tower-layer", "tower-service", ] @@ -5022,6 +5064,7 @@ dependencies = [ "hickory-resolver", "metrics", "rand 0.9.1", + "reqwest", "serde", "serde_json", "thiserror 2.0.12", @@ -5099,8 +5142,8 @@ dependencies = [ "windows-implement 0.60.0", "windows-interface 0.59.1", "windows-link", - "windows-result 0.3.2", - "windows-strings 0.4.0", + "windows-result 0.3.4", + "windows-strings 0.4.2", ] [[package]] @@ -5155,13 +5198,13 @@ checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" [[package]] name = "windows-registry" -version = "0.4.0" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" +checksum = "b3bab093bdd303a1240bb99b8aba8ea8a69ee19d34c9e2ef9594e708a4878820" dependencies = [ - "windows-result 0.3.2", - "windows-strings 0.3.1", - "windows-targets 0.53.2", + "windows-link", + "windows-result 0.3.4", + "windows-strings 0.4.2", ] [[package]] @@ -5175,9 +5218,9 @@ dependencies = [ [[package]] name = "windows-result" -version = "0.3.2" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" dependencies = [ "windows-link", ] @@ -5194,18 +5237,9 @@ dependencies = [ [[package]] name = "windows-strings" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-strings" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" dependencies = [ "windows-link", ] @@ -5261,29 +5295,13 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", + "windows_i686_gnullvm", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] -[[package]] -name = "windows-targets" -version = "0.53.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef" -dependencies = [ - "windows_aarch64_gnullvm 0.53.0", - "windows_aarch64_msvc 0.53.0", - "windows_i686_gnu 0.53.0", - "windows_i686_gnullvm 0.53.0", - "windows_i686_msvc 0.53.0", - "windows_x86_64_gnu 0.53.0", - "windows_x86_64_gnullvm 0.53.0", - "windows_x86_64_msvc 0.53.0", -] - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -5296,12 +5314,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" - [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -5314,12 +5326,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" - [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -5332,24 +5338,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" - [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -5362,12 +5356,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" - [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -5380,12 +5368,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" - [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -5398,12 +5380,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" - [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -5416,12 +5392,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" - [[package]] name = "winnow" version = "0.7.11" diff --git a/who-am-i/Cargo.toml b/who-am-i/Cargo.toml index c49bd9b..cbf66b4 100644 --- a/who-am-i/Cargo.toml +++ b/who-am-i/Cargo.toml @@ -18,6 +18,7 @@ handlebars = { version = "6.3.2", features = ["dir_source"] } hickory-resolver = "0.25.2" metrics = "0.24.2" rand = "0.9.1" +reqwest = { version = "0.12.22", features = ["native-tls-vendored"] } serde = { version = "1.0.219", features = ["derive"] } serde_json = "1.0.140" thiserror = "2.0.12" diff --git a/who-am-i/readme.md b/who-am-i/readme.md index 66247d3..812eec5 100644 --- a/who-am-i/readme.md +++ b/who-am-i/readme.md @@ -49,3 +49,18 @@ i will get back to oauth eventually and hopefully roll out a microcosm service t ### todo provide a pubkey-signed JWT of the identity (just the DID as `sub` probably). (**you probably SHOULD NOT USE THIS in any serious environment**) + + +## building + +for raspi 1 model b: + +atrium-oauth uses reqwest with default tls config that requires openssl which `cross` doesn't have a good time getting the os deps for. + +fortunately, simply *enabling* a differnent tls feature for reqwest actually stops the default problematic one from causing problems, so we have a `reqwest` direct dependency with a feature enabled, even though it's never imported into actual code, + +it builds with + +```bash +cross build --release --target arm-unknown-linux-gnueabihf +``` -- 2.51.2 From 6e38c9c0fdc6357348a6cf3a9484fb9c025cadf9 Mon Sep 17 00:00:00 2001 From: phil Date: Wed, 2 Jul 2025 22:00:40 -0400 Subject: [PATCH 055/134] new clippies --- .github/workflows/checks.yml | 2 +- spacedust/src/server.rs | 2 +- ufos/src/consumer.rs | 2 +- ufos/src/db_types.rs | 4 ++-- ufos/src/server/collections_query.rs | 2 +- ufos/src/server/mod.rs | 10 +++++----- ufos/src/storage_fjall.rs | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 7b205f8..6641eb2 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -28,7 +28,7 @@ jobs: - name: get nightly toolchain for jetstream fmt run: rustup toolchain install nightly --allow-downgrade -c rustfmt - name: fmt - run: cargo fmt --package links --package constellation --package ufos -- --check + run: cargo fmt --package links --package constellation --package ufos --package spacedust --package who-am-i -- --check - name: fmt jetstream (nightly) run: cargo +nightly fmt --package jetstream -- --check - name: clippy diff --git a/spacedust/src/server.rs b/spacedust/src/server.rs index a74b207..9d3dcf6 100644 --- a/spacedust/src/server.rs +++ b/spacedust/src/server.rs @@ -268,7 +268,7 @@ impl SharedExtractor for MultiSubscribeQuery { ) -> Result { let raw_query = ctx.request.uri().query().unwrap_or(""); let q = serde_qs::from_str(raw_query).map_err(|e| { - HttpError::for_bad_request(None, format!("unable to parse query string: {}", e)) + HttpError::for_bad_request(None, format!("unable to parse query string: {e}")) })?; Ok(q) } diff --git a/ufos/src/consumer.rs b/ufos/src/consumer.rs index 8e2ef9a..475293d 100644 --- a/ufos/src/consumer.rs +++ b/ufos/src/consumer.rs @@ -223,7 +223,7 @@ impl Batcher { let beginning = match self.current_batch.initial_cursor.map(|c| c.elapsed()) { None => "unknown".to_string(), - Some(Ok(t)) => format!("{:?}", t), + Some(Ok(t)) => format!("{t:?}"), Some(Err(e)) => format!("+{:?}", e.duration()), }; log::trace!( diff --git a/ufos/src/db_types.rs b/ufos/src/db_types.rs index 224f171..d2f0d75 100644 --- a/ufos/src/db_types.rs +++ b/ufos/src/db_types.rs @@ -427,7 +427,7 @@ mod test { ] { let serialized = s.to_string().to_db_bytes()?; let prefixed = String::sub_prefix(pre)?; - assert_eq!(serialized.starts_with(&prefixed), is_pre, "{}", desc); + assert_eq!(serialized.starts_with(&prefixed), is_pre, "{desc}"); } Ok(()) } @@ -445,7 +445,7 @@ mod test { ] { let serialized = Nsid::new(s.to_string()).unwrap().to_db_bytes()?; let prefixed = Nsid::sub_prefix(pre)?; - assert_eq!(serialized.starts_with(&prefixed), is_pre, "{}", desc); + assert_eq!(serialized.starts_with(&prefixed), is_pre, "{desc}"); } Ok(()) } diff --git a/ufos/src/server/collections_query.rs b/ufos/src/server/collections_query.rs index d5daecb..1af927a 100644 --- a/ufos/src/server/collections_query.rs +++ b/ufos/src/server/collections_query.rs @@ -53,7 +53,7 @@ impl SharedExtractor for MultiCollectionQuery { ) -> Result { let raw_query = ctx.request.uri().query().unwrap_or(""); let q = serde_qs::from_str(raw_query).map_err(|e| { - HttpError::for_bad_request(None, format!("unable to parse query string: {}", e)) + HttpError::for_bad_request(None, format!("unable to parse query string: {e}")) })?; Ok(q) } diff --git a/ufos/src/server/mod.rs b/ufos/src/server/mod.rs index 691da14..2a054e7 100644 --- a/ufos/src/server/mod.rs +++ b/ufos/src/server/mod.rs @@ -444,7 +444,7 @@ async fn get_collections( }; if !(1..=200).contains(&limit) { - let msg = format!("limit not in 1..=200: {}", limit); + let msg = format!("limit not in 1..=200: {limit}"); return Err(HttpError::for_bad_request(None, msg)); } @@ -577,7 +577,7 @@ async fn get_prefix( }; if !(1..=200).contains(&limit) { - let msg = format!("limit not in 1..=200: {}", limit); + let msg = format!("limit not in 1..=200: {limit}"); return Err(HttpError::for_bad_request(None, msg)); } @@ -649,7 +649,7 @@ async fn get_timeseries( let step = if let Some(secs) = q.step { if secs < 3600 { - let msg = format!("step is too small: {}", secs); + let msg = format!("step is too small: {secs}"); Err(HttpError::for_bad_request(None, msg))?; } (secs / 3600) * 3600 // trucate to hour @@ -658,7 +658,7 @@ async fn get_timeseries( }; let nsid = Nsid::new(q.collection).map_err(|e| { - HttpError::for_bad_request(None, format!("collection was not a valid NSID: {:?}", e)) + HttpError::for_bad_request(None, format!("collection was not a valid NSID: {e:?}")) })?; let (range_cursors, series) = storage @@ -762,6 +762,6 @@ pub async fn serve(storage: impl StoreReader + 'static) -> Result<(), String> { ..Default::default() }) .start() - .map_err(|error| format!("failed to start server: {}", error))? + .map_err(|error| format!("failed to start server: {error}"))? .await } diff --git a/ufos/src/storage_fjall.rs b/ufos/src/storage_fjall.rs index 23a295e..4e88844 100644 --- a/ufos/src/storage_fjall.rs +++ b/ufos/src/storage_fjall.rs @@ -1615,7 +1615,7 @@ impl StoreBackground for FjallBackground { } } let dt = t0.elapsed(); - log::trace!("finished trimming {n} nsids in {:?}: {total_danglers} dangling and {total_deleted} total removed.", dt); + log::trace!("finished trimming {n} nsids in {dt:?}: {total_danglers} dangling and {total_deleted} total removed."); histogram!("storage_trim_dirty_nsids").record(completed.len() as f64); histogram!("storage_trim_duration").record(dt.as_micros() as f64); counter!("storage_trim_removed", "dangling" => "true").increment(total_danglers as u64); -- 2.51.2 From c339b68a39efa9a1de3c6114221067a33159a0a8 Mon Sep 17 00:00:00 2001 From: phil Date: Thu, 3 Jul 2025 09:58:44 -0400 Subject: [PATCH 056/134] update cardinality-estimator-safe for fp fix bumps the allowed fp error on harmonic sum to be 1.0 instead of 0.5 --- Cargo.lock | 4 ++-- ufos/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9b31209..8230ede 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -730,9 +730,9 @@ dependencies = [ [[package]] name = "cardinality-estimator-safe" -version = "4.0.1" +version = "4.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b41ec0cd313b46ba3b508377544b25aa1d56d05ce9e657e77dfb001d5e726e53" +checksum = "dc9887b4092040ea9a416fc3de9769ee7783e3cd5c168c941e6a8de69723b971" dependencies = [ "digest", "enum_dispatch", diff --git a/ufos/Cargo.toml b/ufos/Cargo.toml index a5cc415..ec22d3b 100644 --- a/ufos/Cargo.toml +++ b/ufos/Cargo.toml @@ -8,7 +8,7 @@ anyhow = "1.0.97" async-trait = "0.1.88" base64 = "0.22.1" bincode = { version = "2.0.1", features = ["serde"] } -cardinality-estimator-safe = { version = "4.0.1", features = ["with_serde", "with_digest"] } +cardinality-estimator-safe = { version = "4.0.2", features = ["with_serde", "with_digest"] } chrono = { version = "0.4.41", features = ["serde"] } clap = { version = "4.5.31", features = ["derive"] } dropshot = "0.16.0" -- 2.51.2 From bf29cd865f7a7d0d26503deb4e91ca88f9c719e5 Mon Sep 17 00:00:00 2001 From: phil Date: Thu, 3 Jul 2025 11:48:05 -0400 Subject: [PATCH 057/134] metrics --- Cargo.lock | 40 +++++++++++++++++++++++++------ who-am-i/Cargo.toml | 1 + who-am-i/src/expiring_task_map.rs | 5 +++- who-am-i/src/main.rs | 22 ++++++++++++++++- who-am-i/src/oauth.rs | 17 +++++++------ who-am-i/src/server.rs | 31 +++++++++++++++++++----- 6 files changed, 94 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8230ede..a40c951 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2726,7 +2726,7 @@ dependencies = [ "indexmap 2.9.0", "ipnet", "metrics", - "metrics-util", + "metrics-util 0.19.0", "quanta", "thiserror 1.0.69", "tokio", @@ -2735,9 +2735,9 @@ dependencies = [ [[package]] name = "metrics-exporter-prometheus" -version = "0.17.1" +version = "0.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "989903b4c7abfa6827a8d1128ef42faf83f8969d429797c5431f236f2cae8b8b" +checksum = "2b166dea96003ee2531cf14833efedced545751d800f03535801d833313f8c15" dependencies = [ "base64 0.22.1", "http-body-util", @@ -2747,7 +2747,7 @@ dependencies = [ "indexmap 2.9.0", "ipnet", "metrics", - "metrics-util", + "metrics-util 0.20.0", "quanta", "thiserror 2.0.12", "tokio", @@ -2782,7 +2782,23 @@ dependencies = [ "metrics", "quanta", "rand 0.8.5", - "rand_xoshiro", + "rand_xoshiro 0.6.0", + "sketches-ddsketch", +] + +[[package]] +name = "metrics-util" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe8db7a05415d0f919ffb905afa37784f71901c9a773188876984b4f769ab986" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.15.2", + "metrics", + "quanta", + "rand 0.9.1", + "rand_xoshiro 0.7.0", "sketches-ddsketch", ] @@ -3398,6 +3414,15 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "rand_xoshiro" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" +dependencies = [ + "rand_core 0.9.3", +] + [[package]] name = "ratelimit" version = "0.10.0" @@ -4162,7 +4187,7 @@ dependencies = [ "links", "log", "metrics", - "metrics-exporter-prometheus 0.17.1", + "metrics-exporter-prometheus 0.17.2", "rand 0.9.1", "schemars", "semver", @@ -4752,7 +4777,7 @@ dependencies = [ "log", "lsm-tree", "metrics", - "metrics-exporter-prometheus 0.17.1", + "metrics-exporter-prometheus 0.17.2", "schemars", "semver", "serde", @@ -5063,6 +5088,7 @@ dependencies = [ "handlebars", "hickory-resolver", "metrics", + "metrics-exporter-prometheus 0.17.2", "rand 0.9.1", "reqwest", "serde", diff --git a/who-am-i/Cargo.toml b/who-am-i/Cargo.toml index cbf66b4..bfa02d2 100644 --- a/who-am-i/Cargo.toml +++ b/who-am-i/Cargo.toml @@ -17,6 +17,7 @@ dashmap = "6.1.0" handlebars = { version = "6.3.2", features = ["dir_source"] } hickory-resolver = "0.25.2" metrics = "0.24.2" +metrics-exporter-prometheus = { version = "0.17.2", features = ["http-listener"] } rand = "0.9.1" reqwest = { version = "0.12.22", features = ["native-tls-vendored"] } serde = { version = "1.0.219", features = ["derive"] } diff --git a/who-am-i/src/expiring_task_map.rs b/who-am-i/src/expiring_task_map.rs index 9f0bb88..9a8f3ba 100644 --- a/who-am-i/src/expiring_task_map.rs +++ b/who-am-i/src/expiring_task_map.rs @@ -49,10 +49,12 @@ impl ExpiringTaskMap { .run_until_cancelled(sleep(expiration)) .await .is_some() - // is Some if the (sleep) task completed first { + // is Some if the (sleep) task completed first map.remove(&k); cancel.cancel(); + metrics::counter!("whoami_task_map_completions", "result" => "expired") + .increment(1); } }); @@ -60,6 +62,7 @@ impl ExpiringTaskMap { } pub fn take(&self, key: &str) -> Option> { + metrics::counter!("whoami_task_map_completions", "result" => "retrieved").increment(1); // when the _guard drops, the token gets cancelled for us self.0.map.remove(key).map(|(_, (_guard, handle))| handle) } diff --git a/who-am-i/src/main.rs b/who-am-i/src/main.rs index 785bf97..e4f5ed1 100644 --- a/who-am-i/src/main.rs +++ b/who-am-i/src/main.rs @@ -1,4 +1,5 @@ use clap::{ArgAction, Parser}; +use metrics_exporter_prometheus::PrometheusBuilder; use tokio_util::sync::CancellationToken; use who_am_i::serve; @@ -35,7 +36,7 @@ async fn main() { let args = Args::parse(); if args.allowed_hosts.is_empty() { - panic!("at least one --one-click host must be set"); + panic!("at least one --allowed-host host must be set"); } println!("starting with allowed_hosts hosts:"); @@ -43,5 +44,24 @@ async fn main() { println!(" - {host}"); } + if let Err(e) = install_metrics_server() { + eprintln!("failed to install metrics server: {e:?}"); + }; + serve(shutdown, args.app_secret, args.allowed_hosts, args.dev).await; } + +fn install_metrics_server() -> Result<(), metrics_exporter_prometheus::BuildError> { + println!("installing metrics server..."); + let host = [0, 0, 0, 0]; + let port = 8765; + PrometheusBuilder::new() + .set_enable_unit_suffix(false) + .with_http_listener((host, port)) + .install()?; + println!( + "metrics server installed! listening on http://{}.{}.{}.{}:{port}", + host[0], host[1], host[2], host[3] + ); + Ok(()) +} diff --git a/who-am-i/src/oauth.rs b/who-am-i/src/oauth.rs index 8f2b626..4b735f6 100644 --- a/who-am-i/src/oauth.rs +++ b/who-am-i/src/oauth.rs @@ -198,12 +198,15 @@ impl DnsTxtResolver for HickoryDnsTxtResolver { &self, query: &str, ) -> core::result::Result, Box> { - Ok(self - .0 - .txt_lookup(query) - .await? - .iter() - .map(|txt| txt.to_string()) - .collect()) + match self.0.txt_lookup(query).await { + Ok(r) => { + metrics::counter!("whoami_resolve_dns_txt", "success" => "true").increment(1); + Ok(r.iter().map(|r| r.to_string()).collect()) + } + Err(e) => { + metrics::counter!("whoami_resolve_dns_txt", "success" => "false").increment(1); + Err(e.into()) + } + } } } diff --git a/who-am-i/src/server.rs b/who-am-i/src/server.rs index b5e6d3f..9e15638 100644 --- a/who-am-i/src/server.rs +++ b/who-am-i/src/server.rs @@ -172,6 +172,7 @@ async fn prompt( headers: HeaderMap, ) -> impl IntoResponse { let err = |reason, check_frame| { + metrics::counter!("whoami_auth_prompt", "ok" => "false", "reason" => reason).increment(1); let info = json!({ "reason": reason, "check_frame": check_frame }); let html = RenderHtml("prompt-error", engine.clone(), info); (StatusCode::BAD_REQUEST, html).into_response() @@ -222,15 +223,16 @@ async fn prompt( shutdown.child_token(), ); + metrics::counter!("whoami_auth_prompt", "ok" => "true", "known" => "true").increment(1); let info = json!({ "did": did, "fetch_key": fetch_key, "parent_host": parent_host, "parent_origin": parent_origin, }); - (frame_headers, jar, RenderHtml("prompt", engine, info)).into_response() } else { + metrics::counter!("whoami_auth_prompt", "ok" => "true", "known" => "false").increment(1); let info = json!({ "parent_host": parent_host, "parent_origin": parent_origin, @@ -250,7 +252,11 @@ async fn user_info( }): State, Query(params): Query, ) -> impl IntoResponse { - let err = |status, reason| (status, Json(json!({ "reason": reason }))).into_response(); + let err = |status, reason: &str| { + metrics::counter!("whoami_user_info", "found" => "false", "reason" => reason.to_string()) + .increment(1); + (status, Json(json!({ "reason": reason }))).into_response() + }; let Some(task_handle) = resolve_handles.take(¶ms.fetch_key) else { return err(StatusCode::NOT_FOUND, "fetch key does not exist or expired"); @@ -279,7 +285,10 @@ async fn user_info( StatusCode::INTERNAL_SERVER_ERROR, &format!("handle appears invalid: {reason}"), ), - Ok(Ok(handle)) => Json(json!({ "handle": handle })).into_response(), + Ok(Ok(handle)) => { + metrics::counter!("whoami_user_info", "found" => "true").increment(1); + Json(json!({ "handle": handle })).into_response() + } } } @@ -299,7 +308,9 @@ async fn start_oauth( use atrium_identity::Error as IdError; use atrium_oauth::Error as OAuthError; - let err = |code, reason| { + let err = |code, reason: &str| { + metrics::counter!("whoami_auth_start", "ok" => "false", "reason" => reason.to_string()) + .increment(1); let info = json!({ "result": "fail", "reason": reason, @@ -308,7 +319,6 @@ async fn start_oauth( }; match oauth.begin(¶ms.handle).await { - Ok(auth_url) => (jar, Redirect::to(&auth_url)).into_response(), Err(OAuthError::Identity( IdError::NotFound | IdError::HttpStatus(StatusCode::NOT_FOUND), )) => err(StatusCode::NOT_FOUND, "handle not found"), @@ -317,6 +327,10 @@ async fn start_oauth( eprintln!("begin auth failed: {e:?}"); err(StatusCode::INTERNAL_SERVER_ERROR, "unknown") } + Ok(auth_url) => { + metrics::counter!("whoami_auth_start", "ok" => "true").increment(1); + (jar, Redirect::to(&auth_url)).into_response() + } } } @@ -331,7 +345,9 @@ async fn complete_oauth( Query(params): Query, jar: SignedCookieJar, ) -> Response { - let err = |code, result, reason| { + let err = |code, result, reason: &str| { + metrics::counter!("whoami_auth_complete", "ok" => "false", "reason" => reason.to_string()) + .increment(1); let info = json!({ "result": result, "reason": reason, @@ -378,6 +394,8 @@ async fn complete_oauth( }, shutdown.child_token(), ); + + metrics::counter!("whoami_auth_complete", "ok" => "true").increment(1); let info = json!({ "did": did, "fetch_key": fetch_key, @@ -386,6 +404,7 @@ async fn complete_oauth( } async fn disconnect(jar: SignedCookieJar) -> impl IntoResponse { + metrics::counter!("whoami_disconnect").increment(1); let jar = jar.remove(DID_COOKIE_KEY); (jar, Json(json!({ "ok": true }))) } -- 2.51.2 From e02c1b455ff31b8d77b67c6a514f314e1449f680 Mon Sep 17 00:00:00 2001 From: phil Date: Thu, 3 Jul 2025 11:55:39 -0400 Subject: [PATCH 058/134] minimal features so cross build works --- who-am-i/Cargo.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/who-am-i/Cargo.toml b/who-am-i/Cargo.toml index bfa02d2..ee26be6 100644 --- a/who-am-i/Cargo.toml +++ b/who-am-i/Cargo.toml @@ -17,7 +17,6 @@ dashmap = "6.1.0" handlebars = { version = "6.3.2", features = ["dir_source"] } hickory-resolver = "0.25.2" metrics = "0.24.2" -metrics-exporter-prometheus = { version = "0.17.2", features = ["http-listener"] } rand = "0.9.1" reqwest = { version = "0.12.22", features = ["native-tls-vendored"] } serde = { version = "1.0.219", features = ["derive"] } @@ -26,3 +25,8 @@ thiserror = "2.0.12" tokio = { version = "1.45.1", features = ["full", "macros"] } tokio-util = "0.7.15" url = "2.5.4" + +[dependencies.metrics-exporter-prometheus] +version = "0.17.2" +default-features = false +features = ["http-listener", "async-runtime"] -- 2.51.2 From 6d65c3f4620cc447ff9d40ceb75d3aeb878d2984 Mon Sep 17 00:00:00 2001 From: phil Date: Thu, 3 Jul 2025 12:11:30 -0400 Subject: [PATCH 059/134] single-thread tokio not actually sure if this is needed for the raspi but initially i thought metrics weren't working. (they are fine) --- who-am-i/src/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/who-am-i/src/main.rs b/who-am-i/src/main.rs index e4f5ed1..75d3bcf 100644 --- a/who-am-i/src/main.rs +++ b/who-am-i/src/main.rs @@ -1,5 +1,5 @@ use clap::{ArgAction, Parser}; -use metrics_exporter_prometheus::PrometheusBuilder; +use metrics_exporter_prometheus::{PrometheusBuilder, BuildError as PromBuildError}; use tokio_util::sync::CancellationToken; use who_am_i::serve; @@ -26,7 +26,7 @@ struct Args { allowed_hosts: Vec, } -#[tokio::main] +#[tokio::main(flavor = "current_thread")] async fn main() { let shutdown = CancellationToken::new(); @@ -51,7 +51,7 @@ async fn main() { serve(shutdown, args.app_secret, args.allowed_hosts, args.dev).await; } -fn install_metrics_server() -> Result<(), metrics_exporter_prometheus::BuildError> { +fn install_metrics_server() -> Result<(), PromBuildError> { println!("installing metrics server..."); let host = [0, 0, 0, 0]; let port = 8765; -- 2.51.2 From 65d7a109a01b6243d3bcb5dc3d108af731273609 Mon Sep 17 00:00:00 2001 From: phil Date: Thu, 3 Jul 2025 15:12:38 -0400 Subject: [PATCH 060/134] make jwts with the did in em --- Cargo.lock | 59 +++++++++++++++++++++++++++++++ who-am-i/.gitignore | 1 + who-am-i/Cargo.toml | 1 + who-am-i/demo/index.html | 2 ++ who-am-i/src/jwt.rs | 55 ++++++++++++++++++++++++++++ who-am-i/src/lib.rs | 2 ++ who-am-i/src/main.rs | 25 +++++++++++-- who-am-i/src/server.rs | 31 +++++++++++++++- who-am-i/templates/authorized.hbs | 1 + who-am-i/templates/prompt.hbs | 8 ++--- 10 files changed, 177 insertions(+), 8 deletions(-) create mode 100644 who-am-i/.gitignore create mode 100644 who-am-i/src/jwt.rs diff --git a/Cargo.lock b/Cargo.lock index a40c951..428ea70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1697,8 +1697,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", + "wasm-bindgen", ] [[package]] @@ -2454,6 +2456,21 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonwebtoken" +version = "9.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde" +dependencies = [ + "base64 0.22.1", + "js-sys", + "pem", + "ring", + "serde", + "serde_json", + "simple_asn1", +] + [[package]] name = "langtag" version = "0.3.4" @@ -2954,6 +2971,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-conv" version = "0.1.0" @@ -2970,6 +2997,15 @@ dependencies = [ "itoa", ] +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + [[package]] name = "num-modular" version = "0.6.1" @@ -3158,6 +3194,16 @@ dependencies = [ "once_cell", ] +[[package]] +name = "pem" +version = "3.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38af38e8470ac9dee3ce1bae1af9c1671fffc44ddfd8bd1d0a3445bf349a8ef3" +dependencies = [ + "base64 0.22.1", + "serde", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -4086,6 +4132,18 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simple_asn1" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb" +dependencies = [ + "num-bigint", + "num-traits", + "thiserror 2.0.12", + "time", +] + [[package]] name = "sketches-ddsketch" version = "0.3.0" @@ -5087,6 +5145,7 @@ dependencies = [ "dashmap", "handlebars", "hickory-resolver", + "jsonwebtoken", "metrics", "metrics-exporter-prometheus 0.17.2", "rand 0.9.1", diff --git a/who-am-i/.gitignore b/who-am-i/.gitignore new file mode 100644 index 0000000..9ff601a --- /dev/null +++ b/who-am-i/.gitignore @@ -0,0 +1 @@ +jwt-key.pem diff --git a/who-am-i/Cargo.toml b/who-am-i/Cargo.toml index ee26be6..e194c3a 100644 --- a/who-am-i/Cargo.toml +++ b/who-am-i/Cargo.toml @@ -16,6 +16,7 @@ ctrlc = "3.4.7" dashmap = "6.1.0" handlebars = { version = "6.3.2", features = ["dir_source"] } hickory-resolver = "0.25.2" +jsonwebtoken = "9.3.1" metrics = "0.24.2" rand = "0.9.1" reqwest = { version = "0.12.22", features = ["native-tls-vendored"] } diff --git a/who-am-i/demo/index.html b/who-am-i/demo/index.html index 06fe404..801a226 100644 --- a/who-am-i/demo/index.html +++ b/who-am-i/demo/index.html @@ -12,6 +12,7 @@

hey

+

@@ -27,6 +28,7 @@ window.removeEventListener('message', handleMessage); document.getElementById('who').textContent = ev.data.handle; + document.getElementById('jwt').textContent = ev.data.token; } window.addEventListener('message', handleMessage); })(document.getElementById('whoami')); diff --git a/who-am-i/src/jwt.rs b/who-am-i/src/jwt.rs new file mode 100644 index 0000000..f56a040 --- /dev/null +++ b/who-am-i/src/jwt.rs @@ -0,0 +1,55 @@ +use jsonwebtoken::{Algorithm, EncodingKey, Header, encode, errors::Error as JWTError}; +use serde::Serialize; +use std::fs; +use std::io::Error as IOError; +use std::path::Path; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum TokensSetupError { + #[error(transparent)] + Io(#[from] IOError), + #[error("failed to retrieve ec key: {0}")] + FromEc(JWTError), +} + +#[derive(Debug, Error)] +pub enum TokenMintingError { + #[error("failed to mint: {0}")] + FromEc(#[from] JWTError), +} + +pub struct Tokens { + encoding_key: EncodingKey, +} + +impl Tokens { + pub fn from_file(f: impl AsRef) -> Result { + let data: Vec = fs::read(f)?; + let encoding_key = EncodingKey::from_ec_pem(&data).map_err(TokensSetupError::FromEc)?; + Ok(Self { encoding_key }) + } + + pub fn mint(&self, t: impl ToString) -> Result { + let sub = t.to_string(); + + let dt_now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("unix epoch is in the past"); + let dt_exp = dt_now + Duration::from_secs(30 * 86_400); + let exp = dt_exp.as_secs(); + + Ok(encode( + &Header::new(Algorithm::ES256), + &Claims { sub, exp }, + &self.encoding_key, + )?) + } +} + +#[derive(Debug, Serialize)] +struct Claims { + sub: String, + exp: u64, +} diff --git a/who-am-i/src/lib.rs b/who-am-i/src/lib.rs index 76673ad..975fb4b 100644 --- a/who-am-i/src/lib.rs +++ b/who-am-i/src/lib.rs @@ -1,7 +1,9 @@ mod expiring_task_map; +mod jwt; mod oauth; mod server; pub use expiring_task_map::ExpiringTaskMap; +pub use jwt::Tokens; pub use oauth::{OAuth, OAuthCallbackParams, OAuthCompleteError, ResolveHandleError}; pub use server::serve; diff --git a/who-am-i/src/main.rs b/who-am-i/src/main.rs index 75d3bcf..7f9f866 100644 --- a/who-am-i/src/main.rs +++ b/who-am-i/src/main.rs @@ -1,7 +1,8 @@ use clap::{ArgAction, Parser}; -use metrics_exporter_prometheus::{PrometheusBuilder, BuildError as PromBuildError}; +use metrics_exporter_prometheus::{BuildError as PromBuildError, PrometheusBuilder}; +use std::path::PathBuf; use tokio_util::sync::CancellationToken; -use who_am_i::serve; +use who_am_i::{Tokens, serve}; /// Aggregate links in the at-mosphere #[derive(Parser, Debug, Clone)] @@ -14,6 +15,15 @@ struct Args { /// eg: `cat /dev/urandom | head -c 64 | base64` #[arg(long, env)] app_secret: String, + /// path to jwt key (PEM format) + /// + /// generate with: + /// ```bash + /// openssl ecparam -genkey -noout -name prime256v1 \ + /// | openssl pkcs8 -topk8 -nocrypt -out .pem + /// ``` + #[arg(long)] + jwt_key: PathBuf, /// Enable dev mode /// /// enables automatic template reloading @@ -44,11 +54,20 @@ async fn main() { println!(" - {host}"); } + let tokens = Tokens::from_file(args.jwt_key).unwrap(); + if let Err(e) = install_metrics_server() { eprintln!("failed to install metrics server: {e:?}"); }; - serve(shutdown, args.app_secret, args.allowed_hosts, args.dev).await; + serve( + shutdown, + args.app_secret, + tokens, + args.allowed_hosts, + args.dev, + ) + .await; } fn install_metrics_server() -> Result<(), PromBuildError> { diff --git a/who-am-i/src/server.rs b/who-am-i/src/server.rs index 9e15638..51d088b 100644 --- a/who-am-i/src/server.rs +++ b/who-am-i/src/server.rs @@ -22,7 +22,9 @@ use tokio::net::TcpListener; use tokio_util::sync::CancellationToken; use url::Url; -use crate::{ExpiringTaskMap, OAuth, OAuthCallbackParams, OAuthCompleteError, ResolveHandleError}; +use crate::{ + ExpiringTaskMap, OAuth, OAuthCallbackParams, OAuthCompleteError, ResolveHandleError, Tokens, +}; const FAVICON: &[u8] = include_bytes!("../static/favicon.ico"); const STYLE_CSS: &str = include_str!("../static/style.css"); @@ -41,6 +43,7 @@ struct AppState { pub oauth: Arc, pub resolve_handles: ExpiringTaskMap>, pub shutdown: CancellationToken, + pub tokens: Arc, } impl FromRef for Key { @@ -52,6 +55,7 @@ impl FromRef for Key { pub async fn serve( shutdown: CancellationToken, app_secret: String, + tokens: Tokens, allowed_hosts: Vec, dev: bool, ) { @@ -75,6 +79,7 @@ pub async fn serve( oauth: Arc::new(oauth), resolve_handles: ExpiringTaskMap::new(task_pickup_expiration), shutdown: shutdown.clone(), + tokens: Arc::new(tokens), }; let app = Router::new() @@ -166,6 +171,7 @@ async fn prompt( oauth, resolve_handles, shutdown, + tokens, .. }): State, jar: SignedCookieJar, @@ -214,6 +220,14 @@ async fn prompt( // push cookie expiry let jar = jar.add(cookie(&did)); + let token = match tokens.mint(&*did) { + Ok(t) => t, + Err(e) => { + eprintln!("failed to create JWT: {e:?}"); + return err("failed to create JWT", false); + } + }; + let fetch_key = resolve_handles.dispatch( { let oauth = oauth.clone(); @@ -226,6 +240,7 @@ async fn prompt( metrics::counter!("whoami_auth_prompt", "ok" => "true", "known" => "true").increment(1); let info = json!({ "did": did, + "token": token, "fetch_key": fetch_key, "parent_host": parent_host, "parent_origin": parent_origin, @@ -340,6 +355,7 @@ async fn complete_oauth( resolve_handles, oauth, shutdown, + tokens, .. }): State, Query(params): Query, @@ -386,6 +402,18 @@ async fn complete_oauth( let jar = jar.add(cookie(&did)); + let token = match tokens.mint(&*did) { + Ok(t) => t, + Err(e) => { + eprintln!("failed to create JWT: {e:?}"); + return err( + StatusCode::INTERNAL_SERVER_ERROR, + "fail", + "failed to create JWT", + ); + } + }; + let fetch_key = resolve_handles.dispatch( { let oauth = oauth.clone(); @@ -398,6 +426,7 @@ async fn complete_oauth( metrics::counter!("whoami_auth_complete", "ok" => "true").increment(1); let info = json!({ "did": did, + "token": token, "fetch_key": fetch_key, }); (jar, RenderHtml("authorized", engine, info)).into_response() diff --git a/who-am-i/templates/authorized.hbs b/who-am-i/templates/authorized.hbs index 38f945c..7edacbc 100644 --- a/who-am-i/templates/authorized.hbs +++ b/who-am-i/templates/authorized.hbs @@ -8,6 +8,7 @@ localStorage.setItem("who-am-i", JSON.stringify({ result: "success", did: {{{json did}}}, + token: {{{json token}}}, fetch_key: {{{json fetch_key}}}, })); window.close(); diff --git a/who-am-i/templates/prompt.hbs b/who-am-i/templates/prompt.hbs index 5e87be9..05b6f59 100644 --- a/who-am-i/templates/prompt.hbs +++ b/who-am-i/templates/prompt.hbs @@ -55,7 +55,7 @@ function err(e, msg) { loaderEl.classList.add('hidden'); handleViewEl.textContent = `@${handle}`; - allowEl.addEventListener('click', () => shareAllow(handle)); + allowEl.addEventListener('click', () => shareAllow(handle, {{{json token}}})); })(); // anon user @@ -108,7 +108,7 @@ window.addEventListener('storage', async e => { const handle = await lookUp(parsed.fetch_key); - shareAllow(handle); + shareAllow(handle, token); }); async function lookUp(fetch_key) { @@ -125,9 +125,9 @@ async function lookUp(fetch_key) { return info.handle; } -const shareAllow = handle => { +const shareAllow = (handle, token) => { top.postMessage( - { action: "allow", handle }, + { action: "allow", handle, token }, {{{json parent_origin}}}, ); } -- 2.51.2 From bf9a8fe53eb74c443477ab3ff6a982aec6da9e7d Mon Sep 17 00:00:00 2001 From: phil Date: Thu, 3 Jul 2025 15:48:13 -0400 Subject: [PATCH 061/134] serve jwks for token validation --- who-am-i/.gitignore | 3 ++- who-am-i/src/jwt.rs | 37 ++++++++++++++++++++++++++++--------- who-am-i/src/main.rs | 26 +++++++++++++++++++------- who-am-i/src/server.rs | 9 +++++++++ 4 files changed, 58 insertions(+), 17 deletions(-) diff --git a/who-am-i/.gitignore b/who-am-i/.gitignore index 9ff601a..9dab1bc 100644 --- a/who-am-i/.gitignore +++ b/who-am-i/.gitignore @@ -1 +1,2 @@ -jwt-key.pem +*.pem +jwks.json diff --git a/who-am-i/src/jwt.rs b/who-am-i/src/jwt.rs index f56a040..5f89fcc 100644 --- a/who-am-i/src/jwt.rs +++ b/who-am-i/src/jwt.rs @@ -3,32 +3,47 @@ use serde::Serialize; use std::fs; use std::io::Error as IOError; use std::path::Path; +use std::string::FromUtf8Error; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use thiserror::Error; #[derive(Debug, Error)] pub enum TokensSetupError { - #[error(transparent)] - Io(#[from] IOError), - #[error("failed to retrieve ec key: {0}")] - FromEc(JWTError), + #[error("failed to read private key")] + ReadPrivateKey(IOError), + #[error("failed to retrieve private key: {0}")] + PrivateKey(JWTError), + #[error("failed to read private key")] + ReadJwks(IOError), + #[error("failed to retrieve jwks: {0}")] + DecodeJwks(FromUtf8Error), } #[derive(Debug, Error)] pub enum TokenMintingError { #[error("failed to mint: {0}")] - FromEc(#[from] JWTError), + EncodingError(#[from] JWTError), } pub struct Tokens { encoding_key: EncodingKey, + jwks: String, } impl Tokens { - pub fn from_file(f: impl AsRef) -> Result { - let data: Vec = fs::read(f)?; - let encoding_key = EncodingKey::from_ec_pem(&data).map_err(TokensSetupError::FromEc)?; - Ok(Self { encoding_key }) + pub fn from_files( + priv_f: impl AsRef, + jwks_f: impl AsRef, + ) -> Result { + let private_key_data: Vec = + fs::read(priv_f).map_err(TokensSetupError::ReadPrivateKey)?; + let encoding_key = + EncodingKey::from_ec_pem(&private_key_data).map_err(TokensSetupError::PrivateKey)?; + + let jwks_data: Vec = fs::read(jwks_f).map_err(TokensSetupError::ReadJwks)?; + let jwks = String::from_utf8(jwks_data).map_err(TokensSetupError::DecodeJwks)?; + + Ok(Self { encoding_key, jwks }) } pub fn mint(&self, t: impl ToString) -> Result { @@ -46,6 +61,10 @@ impl Tokens { &self.encoding_key, )?) } + + pub fn jwks(&self) -> String { + self.jwks.clone() + } } #[derive(Debug, Serialize)] diff --git a/who-am-i/src/main.rs b/who-am-i/src/main.rs index 7f9f866..cc911f5 100644 --- a/who-am-i/src/main.rs +++ b/who-am-i/src/main.rs @@ -15,15 +15,27 @@ struct Args { /// eg: `cat /dev/urandom | head -c 64 | base64` #[arg(long, env)] app_secret: String, - /// path to jwt key (PEM format) + /// path to jwt private key (PEM pk8 format) /// /// generate with: - /// ```bash - /// openssl ecparam -genkey -noout -name prime256v1 \ - /// | openssl pkcs8 -topk8 -nocrypt -out .pem - /// ``` + /// + /// openssl ecparam -genkey -noout -name prime256v1 \ + /// | openssl pkcs8 -topk8 -nocrypt -out .pem + #[arg(long)] + jwt_private_key: PathBuf, + /// path to pubkeys file (jwks format) + /// + /// get pem of pubkey from private key with: + /// + /// openssl ec -in .pem -pubout + /// + /// then convert to a jwk, probably with something less sketchy than an [online tool](https://jwkset.com/generate) + /// + /// wrap the jwk in an array, then in an object under "keys": + /// + /// { "keys": [] } #[arg(long)] - jwt_key: PathBuf, + jwks: PathBuf, /// Enable dev mode /// /// enables automatic template reloading @@ -54,7 +66,7 @@ async fn main() { println!(" - {host}"); } - let tokens = Tokens::from_file(args.jwt_key).unwrap(); + let tokens = Tokens::from_files(args.jwt_private_key, args.jwks).unwrap(); if let Err(e) = install_metrics_server() { eprintln!("failed to install metrics server: {e:?}"); diff --git a/who-am-i/src/server.rs b/who-am-i/src/server.rs index 51d088b..eb5bbbe 100644 --- a/who-am-i/src/server.rs +++ b/who-am-i/src/server.rs @@ -91,6 +91,7 @@ pub async fn serve( .route("/auth", get(start_oauth)) .route("/authorized", get(complete_oauth)) .route("/disconnect", post(disconnect)) + .route("/.well-known/jwks.json", get(jwks)) .with_state(state); let listener = TcpListener::bind("0.0.0.0:9997") @@ -437,3 +438,11 @@ async fn disconnect(jar: SignedCookieJar) -> impl IntoResponse { let jar = jar.remove(DID_COOKIE_KEY); (jar, Json(json!({ "ok": true }))) } + +async fn jwks(State(AppState { tokens, .. }): State) -> impl IntoResponse { + let headers = [ + (CONTENT_TYPE, "application/json"), + // (CACHE_CONTROL, "") // TODO + ]; + (headers, tokens.jwks()) +} -- 2.51.2 From f3ef9dacde4d8bf8f5e0a3b816a06667581ab1a2 Mon Sep 17 00:00:00 2001 From: phil Date: Fri, 4 Jul 2025 13:15:33 -0400 Subject: [PATCH 062/134] post for user info follow-up instead of get saw some spurious double-requests that led to taking the expiring task map key early which is maybe an indication that the fragility of only having once chance to retrieve it by key is kind of a problem, but hey we can paper over that by POSTing for now for a slightly higher chance that nothing in the stack will try to re-request somehow. --- who-am-i/src/expiring_task_map.rs | 13 +++++++--- who-am-i/src/server.rs | 23 ++++++----------- who-am-i/templates/hello.hbs | 9 ++++--- who-am-i/templates/prompt.hbs | 41 +++++++++++++++++-------------- 4 files changed, 43 insertions(+), 43 deletions(-) diff --git a/who-am-i/src/expiring_task_map.rs b/who-am-i/src/expiring_task_map.rs index 9a8f3ba..7277b71 100644 --- a/who-am-i/src/expiring_task_map.rs +++ b/who-am-i/src/expiring_task_map.rs @@ -49,8 +49,8 @@ impl ExpiringTaskMap { .run_until_cancelled(sleep(expiration)) .await .is_some() + // the (sleep) task completed first { - // is Some if the (sleep) task completed first map.remove(&k); cancel.cancel(); metrics::counter!("whoami_task_map_completions", "result" => "expired") @@ -62,9 +62,14 @@ impl ExpiringTaskMap { } pub fn take(&self, key: &str) -> Option> { - metrics::counter!("whoami_task_map_completions", "result" => "retrieved").increment(1); - // when the _guard drops, the token gets cancelled for us - self.0.map.remove(key).map(|(_, (_guard, handle))| handle) + if let Some((_key, (_guard, handle))) = self.0.map.remove(key) { + // when the _guard drops, it cancels the token for us + metrics::counter!("whoami_task_map_completions", "result" => "retrieved").increment(1); + Some(handle) + } else { + metrics::counter!("whoami_task_map_gones").increment(1); + None + } } } diff --git a/who-am-i/src/server.rs b/who-am-i/src/server.rs index eb5bbbe..96c1dd5 100644 --- a/who-am-i/src/server.rs +++ b/who-am-i/src/server.rs @@ -1,10 +1,10 @@ use atrium_api::types::string::Did; use axum::{ Router, - extract::{FromRef, Query, State}, + extract::{FromRef, Json as ExtractJson, Query, State}, http::{ StatusCode, - header::{CONTENT_SECURITY_POLICY, CONTENT_TYPE, HeaderMap, REFERER, X_FRAME_OPTIONS}, + header::{CONTENT_SECURITY_POLICY, CONTENT_TYPE, HeaderMap, REFERER}, }, response::{IntoResponse, Json, Redirect, Response}, routing::{get, post}, @@ -87,7 +87,7 @@ pub async fn serve( .route("/favicon.ico", get(favicon)) // todo MIME .route("/style.css", get(css)) .route("/prompt", get(prompt)) - .route("/user-info", get(user_info)) + .route("/user-info", post(user_info)) .route("/auth", get(start_oauth)) .route("/authorized", get(complete_oauth)) .route("/disconnect", post(disconnect)) @@ -137,10 +137,7 @@ async fn hello( } else { json!({}) }; - let frame_headers = [ - (X_FRAME_OPTIONS, "deny"), - (CONTENT_SECURITY_POLICY, "frame-ancestors 'none'"), - ]; + let frame_headers = [(CONTENT_SECURITY_POLICY, "frame-ancestors 'none'")]; (frame_headers, jar, RenderHtml("hello", engine, info)).into_response() } @@ -205,13 +202,8 @@ async fn prompt( return err("Referer origin is opaque", true); } - let frame_headers = [ - (X_FRAME_OPTIONS, format!("allow-from {parent_origin}")), - ( - CONTENT_SECURITY_POLICY, - format!("frame-ancestors {parent_origin}"), - ), - ]; + let csp = format!("frame-ancestors {parent_origin}"); + let frame_headers = [(CONTENT_SECURITY_POLICY, &csp)]; if let Some(did) = jar.get(DID_COOKIE_KEY) { let Ok(did) = Did::new(did.value_trimmed().to_string()) else { @@ -258,7 +250,6 @@ async fn prompt( } #[derive(Debug, Deserialize)] -#[serde(rename_all = "kebab-case")] struct UserInfoParams { fetch_key: String, } @@ -266,7 +257,7 @@ async fn user_info( State(AppState { resolve_handles, .. }): State, - Query(params): Query, + ExtractJson(params): ExtractJson, ) -> impl IntoResponse { let err = |status, reason: &str| { metrics::counter!("whoami_user_info", "found" => "false", "reason" => reason.to_string()) diff --git a/who-am-i/templates/hello.hbs b/who-am-i/templates/hello.hbs index 291b866..1009eeb 100644 --- a/who-am-i/templates/hello.hbs +++ b/who-am-i/templates/hello.hbs @@ -38,7 +38,6 @@ ({{{json did}}}) && (async () => { const handle = await lookUp({{{json fetch_key}}}); - console.log('got handle', handle); loaderEl.classList.add('hidden'); handleViewEl.textContent = `@${handle}`; @@ -54,11 +53,13 @@ })(); async function lookUp(fetch_key) { - const user_info = new URL('/user-info', window.location); - user_info.searchParams.set('fetch-key', fetch_key); let info; try { - const resp = await fetch(user_info); + const resp = await fetch('/user-info', { + method: 'POST', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify({ fetch_key }), + }); if (!resp.ok) throw resp; info = await resp.json(); } catch (e) { diff --git a/who-am-i/templates/prompt.hbs b/who-am-i/templates/prompt.hbs index 05b6f59..9e0278e 100644 --- a/who-am-i/templates/prompt.hbs +++ b/who-am-i/templates/prompt.hbs @@ -49,10 +49,7 @@ function err(e, msg) { // already-known user ({{{json did}}}) && (async () => { - const handle = await lookUp({{{json fetch_key}}}); - console.log('got handle', handle); - loaderEl.classList.add('hidden'); handleViewEl.textContent = `@${handle}`; allowEl.addEventListener('click', () => shareAllow(handle, {{{json token}}})); @@ -74,20 +71,15 @@ window.addEventListener('storage', async e => { // so if you have two flows going, it grants for both (or the first responder?) if you grant for either. // (letting this slide while parent pages are allowlisted to microcosm only) - const fail = (e, msg) => { - loaderEl.classList.add('hidden'); - formEl.classList.remove('hidden'); - handleInputEl.focus(); - handleInputEl.select(); - err(e, msg); - } + if (e.key !== 'who-am-i') return; + if (e.newValue === null) return; - const details = localStorage.getItem("who-am-i"); + const details = e.newValue; if (!details) { - console.error("hmm, heard from localstorage but did not get DID"); - return; + console.error("hmm, heard from localstorage but did not get DID", details, e); + err('sorry, something went wrong getting your details'); } - localStorage.removeItem("who-am-i"); + localStorage.removeItem(e.key); let parsed; try { @@ -96,6 +88,14 @@ window.addEventListener('storage', async e => { err(e, "something went wrong getting the details back"); } + const fail = (e, msg) => { + loaderEl.classList.add('hidden'); + formEl.classList.remove('hidden'); + handleInputEl.focus(); + handleInputEl.select(); + err(e, msg); + } + if (parsed.result === "fail") { fail(`uh oh: ${parsed.reason}`); } @@ -108,19 +108,21 @@ window.addEventListener('storage', async e => { const handle = await lookUp(parsed.fetch_key); - shareAllow(handle, token); + shareAllow(handle, parsed.token); }); async function lookUp(fetch_key) { - const user_info = new URL('/user-info', window.location); - user_info.searchParams.set('fetch-key', fetch_key); let info; try { - const resp = await fetch(user_info); + const resp = await fetch('/user-info', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ fetch_key }), + }); if (!resp.ok) throw resp; info = await resp.json(); } catch (e) { - err(e, 'failed to resolve handle from DID') + err(e, `failed to resolve handle from DID with ${fetch_key}`); } return info.handle; } @@ -130,6 +132,7 @@ const shareAllow = (handle, token) => { { action: "allow", handle, token }, {{{json parent_origin}}}, ); + promptEl.textContent = '✔️ shared'; } const shareDeny = reason => { -- 2.51.2 From 8304c4de85f0a70f38a2fe400ce5e8d29cbfe82d Mon Sep 17 00:00:00 2001 From: phil Date: Fri, 4 Jul 2025 13:47:43 -0400 Subject: [PATCH 063/134] explain --- who-am-i/static/style.css | 12 ++++++++++++ who-am-i/templates/hello.hbs | 8 +++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/who-am-i/static/style.css b/who-am-i/static/style.css index 6f42348..89f0fa4 100644 --- a/who-am-i/static/style.css +++ b/who-am-i/static/style.css @@ -60,6 +60,15 @@ main { max-width: 21rem; } +.explain { + border-bottom: 1px dashed #888; + margin-bottom: 1rem; + padding-bottom: 2rem; +} +.explain p { + text-align: left; +} + #error-message { font-size: 0.8rem; color: #a31; @@ -77,6 +86,9 @@ p { p.detail { font-size: 0.8rem; } +p.detail.no { + font-style: italic; +} .parent-host { font-weight: bold; color: #48c; diff --git a/who-am-i/templates/hello.hbs b/who-am-i/templates/hello.hbs index 1009eeb..5d62822 100644 --- a/who-am-i/templates/hello.hbs +++ b/who-am-i/templates/hello.hbs @@ -2,7 +2,9 @@ {{#*inline "main"}}
- This is a little identity-verifying service for microcosm demos. +
+

This is a little identity-verifying service for microcosm demos.

+
{{#if did}} @@ -68,6 +70,10 @@ return info.handle; } + {{else}} +

+ No identity connected. +

{{/if}}
{{/inline}} -- 2.51.2 From c99e3c33cb3a0a31745b02db324deb89d1c7b14b Mon Sep 17 00:00:00 2001 From: phil Date: Fri, 11 Jul 2025 11:07:02 -0400 Subject: [PATCH 064/134] dev and prod with all the oauth joy --- Cargo.lock | 113 +++++++++++++++++++++++++++++++++++++++++ who-am-i/Cargo.toml | 4 ++ who-am-i/src/main.rs | 39 +++++++++++++- who-am-i/src/oauth.rs | 98 +++++++++++++++++++++++++++-------- who-am-i/src/server.rs | 24 ++++++++- 5 files changed, 254 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 428ea70..4da0c78 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1162,6 +1162,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ "const-oid", + "pem-rfc7468", "zeroize", ] @@ -1344,6 +1345,7 @@ dependencies = [ "elliptic-curve", "rfc6979", "signature", + "spki", ] [[package]] @@ -1364,6 +1366,8 @@ dependencies = [ "ff", "generic-array", "group", + "pem-rfc7468", + "pkcs8", "rand_core 0.6.4", "sec1", "subtle", @@ -2442,6 +2446,8 @@ dependencies = [ "jose-b64", "jose-jwa", "p256", + "p384", + "rsa", "serde", "zeroize", ] @@ -2485,6 +2491,9 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] [[package]] name = "lazycell" @@ -2981,6 +2990,23 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +dependencies = [ + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.5", + "smallvec", + "zeroize", +] + [[package]] name = "num-conv" version = "0.1.0" @@ -3006,6 +3032,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-modular" version = "0.6.1" @@ -3028,6 +3065,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -3141,6 +3179,16 @@ dependencies = [ "sha2", ] +[[package]] +name = "p384" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" +dependencies = [ + "elliptic-curve", + "primeorder", +] + [[package]] name = "parking" version = "2.2.1" @@ -3204,6 +3252,15 @@ dependencies = [ "serde", ] +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -3266,6 +3323,27 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.32" @@ -3665,6 +3743,26 @@ dependencies = [ "librocksdb-sys", ] +[[package]] +name = "rsa" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78928ac1ed176a5ca1d17e578a1825f3d81ca54cf41053a592584b020cfd691b" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "signature", + "spki", + "subtle", + "zeroize", +] + [[package]] name = "rustc-demangle" version = "0.1.24" @@ -3873,6 +3971,7 @@ dependencies = [ "base16ct", "der", "generic-array", + "pkcs8", "subtle", "zeroize", ] @@ -4268,6 +4367,16 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -5143,11 +5252,15 @@ dependencies = [ "clap", "ctrlc", "dashmap", + "elliptic-curve", "handlebars", "hickory-resolver", + "jose-jwk", "jsonwebtoken", "metrics", "metrics-exporter-prometheus 0.17.2", + "p256", + "pkcs8", "rand 0.9.1", "reqwest", "serde", diff --git a/who-am-i/Cargo.toml b/who-am-i/Cargo.toml index e194c3a..29f097f 100644 --- a/who-am-i/Cargo.toml +++ b/who-am-i/Cargo.toml @@ -14,10 +14,14 @@ axum-template = { version = "3.0.0", features = ["handlebars"] } clap = { version = "4.5.40", features = ["derive", "env"] } ctrlc = "3.4.7" dashmap = "6.1.0" +elliptic-curve = "0.13.8" handlebars = { version = "6.3.2", features = ["dir_source"] } hickory-resolver = "0.25.2" +jose-jwk = "0.1.2" jsonwebtoken = "9.3.1" metrics = "0.24.2" +p256 = "0.13.2" +pkcs8 = "0.10.2" rand = "0.9.1" reqwest = { version = "0.12.22", features = ["native-tls-vendored"] } serde = { version = "1.0.219", features = ["derive"] } diff --git a/who-am-i/src/main.rs b/who-am-i/src/main.rs index cc911f5..d06b436 100644 --- a/who-am-i/src/main.rs +++ b/who-am-i/src/main.rs @@ -15,6 +15,14 @@ struct Args { /// eg: `cat /dev/urandom | head -c 64 | base64` #[arg(long, env)] app_secret: String, + /// path to at-oauth private key (PEM pk8 format) + /// + /// generate with: + /// + /// openssl ecparam -genkey -noout -name prime256v1 \ + /// | openssl pkcs8 -topk8 -nocrypt -out .pem + #[arg(long, env)] + oauth_private_key: Option, /// path to jwt private key (PEM pk8 format) /// /// generate with: @@ -34,11 +42,21 @@ struct Args { /// wrap the jwk in an array, then in an object under "keys": /// /// { "keys": [] } + /// + /// TODO: remove this, serve automatically #[arg(long)] jwks: PathBuf, + /// this server's client-reachable base url, for oauth redirect + jwt check + /// + /// required unless running in localhost mode with --dev + #[arg(long, env)] + base_url: Option, + /// host:port to bind to on startup + #[arg(long, env, default_value = "127.0.0.1:9997")] + bind: String, /// Enable dev mode /// - /// enables automatic template reloading + /// enables automatic template reloading, uses localhost oauth config, etc #[arg(long, action)] dev: bool, /// Hosts who are allowed to one-click auth @@ -57,6 +75,22 @@ async fn main() { let args = Args::parse(); + // let bind = args.bind.to_socket_addrs().expect("--bind must be ToSocketAddrs"); + + let base = args.base_url.unwrap_or_else(|| { + if args.dev { + format!("http://{}", args.bind) + } else { + panic!("not in --dev mode so --base-url is required") + } + }); + + if !args.dev && args.oauth_private_key.is_none() { + panic!("--at-oauth-key is required except in --dev"); + } else if args.dev && args.oauth_private_key.is_some() { + eprintln!("warn: --at-oauth-key is ignored in dev (localhost config)"); + } + if args.allowed_hosts.is_empty() { panic!("at least one --allowed-host host must be set"); } @@ -75,7 +109,10 @@ async fn main() { serve( shutdown, args.app_secret, + args.oauth_private_key, tokens, + base, + args.bind, args.allowed_hosts, args.dev, ) diff --git a/who-am-i/src/oauth.rs b/who-am-i/src/oauth.rs index 4b735f6..9968ef1 100644 --- a/who-am-i/src/oauth.rs +++ b/who-am-i/src/oauth.rs @@ -1,3 +1,10 @@ +use jose_jwk::Class; +use jose_jwk::Jwk; +use jose_jwk::Key; +use jose_jwk::Parameters; +use std::fs; +use std::path::PathBuf; +// use p256::SecretKey; use atrium_api::{agent::SessionManager, types::string::Did}; use atrium_common::resolver::Resolver; use atrium_identity::{ @@ -5,11 +12,15 @@ use atrium_identity::{ handle::{AtprotoHandleResolver, AtprotoHandleResolverConfig, DnsTxtResolver}, }; use atrium_oauth::{ - AtprotoLocalhostClientMetadata, AuthorizeOptions, CallbackParams, DefaultHttpClient, - KnownScope, OAuthClient, OAuthClientConfig, OAuthResolverConfig, Scope, + AtprotoClientMetadata, AtprotoLocalhostClientMetadata, AuthMethod, AuthorizeOptions, + CallbackParams, DefaultHttpClient, GrantType, KnownScope, OAuthClient, OAuthClientConfig, + OAuthClientMetadata, OAuthResolverConfig, Scope, store::{session::MemorySessionStore, state::MemoryStateStore}, }; +use elliptic_curve::SecretKey; use hickory_resolver::{ResolveError, TokioResolver}; +use jose_jwk::JwkSet; +use pkcs8::DecodePrivateKey; use serde::Deserialize; use std::sync::Arc; use thiserror::Error; @@ -83,7 +94,7 @@ pub enum ResolveHandleError { } impl OAuth { - pub fn new() -> Result { + pub fn new(oauth_private_key: Option, base: String) -> Result { let http_client = Arc::new(DefaultHttpClient::default()); let did_resolver = || { CommonDidResolver::new(CommonDidResolverConfig { @@ -93,26 +104,63 @@ impl OAuth { }; let dns_txt_resolver = HickoryDnsTxtResolver::new().map_err(AuthSetupError::HickoryResolverError)?; - let client_config = OAuthClientConfig { - client_metadata: AtprotoLocalhostClientMetadata { - redirect_uris: Some(vec![String::from("http://127.0.0.1:9997/authorized")]), - scopes: Some(READONLY_SCOPE.to_vec()), - }, - keys: None, - resolver: OAuthResolverConfig { - did_resolver: did_resolver(), - handle_resolver: AtprotoHandleResolver::new(AtprotoHandleResolverConfig { - dns_txt_resolver, - http_client: Arc::clone(&http_client), - }), - authorization_server_metadata: Default::default(), - protected_resource_metadata: Default::default(), - }, - state_store: MemoryStateStore::default(), - session_store: MemorySessionStore::default(), + + let resolver = OAuthResolverConfig { + did_resolver: did_resolver(), + handle_resolver: AtprotoHandleResolver::new(AtprotoHandleResolverConfig { + dns_txt_resolver, + http_client: Arc::clone(&http_client), + }), + authorization_server_metadata: Default::default(), + protected_resource_metadata: Default::default(), }; - let client = OAuthClient::new(client_config).map_err(AuthSetupError::AtriumClientError)?; + let state_store = MemoryStateStore::default(); + let session_store = MemorySessionStore::default(); + + let client = if let Some(path) = oauth_private_key { + let key_contents: Vec = fs::read(path).unwrap(); + let key_string = String::from_utf8(key_contents).unwrap(); + let key = SecretKey::::from_pkcs8_pem(&key_string) + .map(|secret_key| Jwk { + key: Key::from(&secret_key.into()), + prm: Parameters { + kid: Some("at-oauth-00".to_string()), + cls: Some(Class::Signing), + ..Default::default() + }, + }) + .expect("to get private key"); + OAuthClient::new(OAuthClientConfig { + client_metadata: AtprotoClientMetadata { + client_id: format!("{base}/client-metadata.json"), + client_uri: Some(base.clone()), + redirect_uris: vec![format!("{base}/authorized")], + token_endpoint_auth_method: AuthMethod::PrivateKeyJwt, + grant_types: vec![GrantType::AuthorizationCode, GrantType::RefreshToken], + scopes: READONLY_SCOPE.to_vec(), + jwks_uri: Some(format!("{base}/.well-known/at-jwks.json")), + token_endpoint_auth_signing_alg: Some(String::from("ES256")), + }, + keys: Some(vec![key]), + resolver, + state_store, + session_store, + }) + .map_err(AuthSetupError::AtriumClientError)? + } else { + OAuthClient::new(OAuthClientConfig { + client_metadata: AtprotoLocalhostClientMetadata { + redirect_uris: Some(vec![String::from("http://127.0.0.1:9997/authorized")]), + scopes: Some(READONLY_SCOPE.to_vec()), + }, + keys: None, + resolver, + state_store, + session_store, + }) + .map_err(AuthSetupError::AtriumClientError)? + }; Ok(Self { client: Arc::new(client), @@ -120,6 +168,14 @@ impl OAuth { }) } + pub fn client_metadata(&self) -> OAuthClientMetadata { + self.client.client_metadata.clone() + } + + pub fn jwks(&self) -> JwkSet { + self.client.jwks() + } + pub async fn begin(&self, handle: &str) -> Result { let auth_opts = AuthorizeOptions { scopes: READONLY_SCOPE.to_vec(), diff --git a/who-am-i/src/server.rs b/who-am-i/src/server.rs index 96c1dd5..bfb06bc 100644 --- a/who-am-i/src/server.rs +++ b/who-am-i/src/server.rs @@ -1,4 +1,5 @@ use atrium_api::types::string::Did; +use atrium_oauth::OAuthClientMetadata; use axum::{ Router, extract::{FromRef, Json as ExtractJson, Query, State}, @@ -12,6 +13,8 @@ use axum::{ use axum_extra::extract::cookie::{Cookie, Key, SameSite, SignedCookieJar}; use axum_template::{RenderHtml, engine::Engine}; use handlebars::{Handlebars, handlebars_helper}; +use jose_jwk::JwkSet; +use std::path::PathBuf; use serde::Deserialize; use serde_json::{Value, json}; @@ -52,10 +55,14 @@ impl FromRef for Key { } } +#[allow(clippy::too_many_arguments)] pub async fn serve( shutdown: CancellationToken, app_secret: String, + oauth_private_key: Option, tokens: Tokens, + base: String, + bind: String, allowed_hosts: Vec, dev: bool, ) { @@ -70,7 +77,7 @@ pub async fn serve( // clients have to pick up their identity-resolving tasks within this period let task_pickup_expiration = Duration::from_secs(15); - let oauth = OAuth::new().unwrap(); + let oauth = OAuth::new(oauth_private_key, base).unwrap(); let state = AppState { engine: Engine::new(hbs), @@ -88,13 +95,16 @@ pub async fn serve( .route("/style.css", get(css)) .route("/prompt", get(prompt)) .route("/user-info", post(user_info)) + .route("/client-metadata.json", get(client_metadata)) .route("/auth", get(start_oauth)) .route("/authorized", get(complete_oauth)) .route("/disconnect", post(disconnect)) + .route("/.well-known/at-jwks.json", get(at_jwks)) // todo combine jwks eps (key id is enough?) .route("/.well-known/jwks.json", get(jwks)) .with_state(state); - let listener = TcpListener::bind("0.0.0.0:9997") + eprintln!("starting server at http://{bind}"); + let listener = TcpListener::bind(bind) .await .expect("listener binding to work"); @@ -299,6 +309,16 @@ async fn user_info( } } +async fn client_metadata( + State(AppState { oauth, .. }): State, +) -> Json { + Json(oauth.client_metadata()) +} + +async fn at_jwks(State(AppState { oauth, .. }): State) -> Json { + Json(oauth.jwks()) +} + #[derive(Debug, Deserialize)] struct BeginOauthParams { handle: String, -- 2.51.2 From 1bf861a9f3162f01c3ac477836499151fd46f5f5 Mon Sep 17 00:00:00 2001 From: phil Date: Fri, 11 Jul 2025 12:07:21 -0400 Subject: [PATCH 065/134] slightly okay-er jwks treatment bleh --- who-am-i/src/jwt.rs | 39 +++++++++++++++++++++++++++++---------- who-am-i/src/main.rs | 17 +---------------- who-am-i/src/server.rs | 15 ++++----------- 3 files changed, 34 insertions(+), 37 deletions(-) diff --git a/who-am-i/src/jwt.rs b/who-am-i/src/jwt.rs index 5f89fcc..625caed 100644 --- a/who-am-i/src/jwt.rs +++ b/who-am-i/src/jwt.rs @@ -1,4 +1,7 @@ +use elliptic_curve::SecretKey; +use jose_jwk::{Class, Jwk, Key, Parameters}; use jsonwebtoken::{Algorithm, EncodingKey, Header, encode, errors::Error as JWTError}; +use pkcs8::DecodePrivateKey; use serde::Serialize; use std::fs; use std::io::Error as IOError; @@ -27,23 +30,39 @@ pub enum TokenMintingError { pub struct Tokens { encoding_key: EncodingKey, - jwks: String, + jwk: Jwk, } impl Tokens { - pub fn from_files( - priv_f: impl AsRef, - jwks_f: impl AsRef, - ) -> Result { + pub fn from_files(priv_f: impl AsRef) -> Result { let private_key_data: Vec = fs::read(priv_f).map_err(TokensSetupError::ReadPrivateKey)?; let encoding_key = EncodingKey::from_ec_pem(&private_key_data).map_err(TokensSetupError::PrivateKey)?; - let jwks_data: Vec = fs::read(jwks_f).map_err(TokensSetupError::ReadJwks)?; - let jwks = String::from_utf8(jwks_data).map_err(TokensSetupError::DecodeJwks)?; + let jwk_key_string = String::from_utf8(private_key_data).unwrap(); + let mut jwk = SecretKey::::from_pkcs8_pem(&jwk_key_string) + .map(|secret_key| Jwk { + key: Key::from(&secret_key.into()), + prm: Parameters { + kid: Some("who-am-i-00".to_string()), + cls: Some(Class::Signing), + ..Default::default() + }, + }) + .expect("to get private key"); - Ok(Self { encoding_key, jwks }) + // CRITICAL: this is what turns the private jwk into a public one: the + // `d` parameter is the secret for an EC key; a pubkey just has no `d`. + // + // this feels baaaadd but hey we're just copying atrium + // https://github.com/atrium-rs/atrium/blob/b48810f84d83d037ee89b79b8566df9e0f2a6dae/atrium-oauth/src/keyset.rs#L41 + let Key::Ec(ref mut ec) = jwk.key else { + unimplemented!() + }; + ec.d = None; // CRITICAL + + Ok(Self { encoding_key, jwk }) } pub fn mint(&self, t: impl ToString) -> Result { @@ -62,8 +81,8 @@ impl Tokens { )?) } - pub fn jwks(&self) -> String { - self.jwks.clone() + pub fn jwk(&self) -> Jwk { + self.jwk.clone() } } diff --git a/who-am-i/src/main.rs b/who-am-i/src/main.rs index d06b436..33282d9 100644 --- a/who-am-i/src/main.rs +++ b/who-am-i/src/main.rs @@ -31,21 +31,6 @@ struct Args { /// | openssl pkcs8 -topk8 -nocrypt -out .pem #[arg(long)] jwt_private_key: PathBuf, - /// path to pubkeys file (jwks format) - /// - /// get pem of pubkey from private key with: - /// - /// openssl ec -in .pem -pubout - /// - /// then convert to a jwk, probably with something less sketchy than an [online tool](https://jwkset.com/generate) - /// - /// wrap the jwk in an array, then in an object under "keys": - /// - /// { "keys": [] } - /// - /// TODO: remove this, serve automatically - #[arg(long)] - jwks: PathBuf, /// this server's client-reachable base url, for oauth redirect + jwt check /// /// required unless running in localhost mode with --dev @@ -100,7 +85,7 @@ async fn main() { println!(" - {host}"); } - let tokens = Tokens::from_files(args.jwt_private_key, args.jwks).unwrap(); + let tokens = Tokens::from_files(args.jwt_private_key).unwrap(); if let Err(e) = install_metrics_server() { eprintln!("failed to install metrics server: {e:?}"); diff --git a/who-am-i/src/server.rs b/who-am-i/src/server.rs index bfb06bc..aedbffa 100644 --- a/who-am-i/src/server.rs +++ b/who-am-i/src/server.rs @@ -99,7 +99,6 @@ pub async fn serve( .route("/auth", get(start_oauth)) .route("/authorized", get(complete_oauth)) .route("/disconnect", post(disconnect)) - .route("/.well-known/at-jwks.json", get(at_jwks)) // todo combine jwks eps (key id is enough?) .route("/.well-known/jwks.json", get(jwks)) .with_state(state); @@ -315,10 +314,6 @@ async fn client_metadata( Json(oauth.client_metadata()) } -async fn at_jwks(State(AppState { oauth, .. }): State) -> Json { - Json(oauth.jwks()) -} - #[derive(Debug, Deserialize)] struct BeginOauthParams { handle: String, @@ -450,10 +445,8 @@ async fn disconnect(jar: SignedCookieJar) -> impl IntoResponse { (jar, Json(json!({ "ok": true }))) } -async fn jwks(State(AppState { tokens, .. }): State) -> impl IntoResponse { - let headers = [ - (CONTENT_TYPE, "application/json"), - // (CACHE_CONTROL, "") // TODO - ]; - (headers, tokens.jwks()) +async fn jwks(State(AppState { oauth, tokens, .. }): State) -> Json { + let mut jwks = oauth.jwks(); + jwks.keys.push(tokens.jwk()); + Json(jwks) } -- 2.51.2 From 986819f99167efccc8c734c057289d5a4c27441d Mon Sep 17 00:00:00 2001 From: phil Date: Fri, 11 Jul 2025 14:31:34 -0400 Subject: [PATCH 066/134] put the kid in the header so that later jose knows which key to try (it will not just try them all) --- who-am-i/src/jwt.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/who-am-i/src/jwt.rs b/who-am-i/src/jwt.rs index 625caed..70ad46a 100644 --- a/who-am-i/src/jwt.rs +++ b/who-am-i/src/jwt.rs @@ -74,11 +74,11 @@ impl Tokens { let dt_exp = dt_now + Duration::from_secs(30 * 86_400); let exp = dt_exp.as_secs(); - Ok(encode( - &Header::new(Algorithm::ES256), - &Claims { sub, exp }, - &self.encoding_key, - )?) + let mut header = Header::new(Algorithm::ES256); + header.kid = Some("who-am-i-00".to_string()); + // todo: consider setting jku? + + Ok(encode(&header, &Claims { sub, exp }, &self.encoding_key)?) } pub fn jwk(&self) -> Jwk { -- 2.51.2 From 0a250c4505a56486b68766e82b9273df0bb8dd48 Mon Sep 17 00:00:00 2001 From: phil Date: Fri, 11 Jul 2025 14:35:40 -0400 Subject: [PATCH 067/134] one jwks.json to rule them all --- who-am-i/src/oauth.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/who-am-i/src/oauth.rs b/who-am-i/src/oauth.rs index 9968ef1..bdd87e1 100644 --- a/who-am-i/src/oauth.rs +++ b/who-am-i/src/oauth.rs @@ -139,7 +139,7 @@ impl OAuth { token_endpoint_auth_method: AuthMethod::PrivateKeyJwt, grant_types: vec![GrantType::AuthorizationCode, GrantType::RefreshToken], scopes: READONLY_SCOPE.to_vec(), - jwks_uri: Some(format!("{base}/.well-known/at-jwks.json")), + jwks_uri: Some(format!("{base}/.well-known/jwks.json")), token_endpoint_auth_signing_alg: Some(String::from("ES256")), }, keys: Some(vec![key]), -- 2.51.2 From f751a2bac11fdc5ca85e8eec75e6f1bc7cdf42ba Mon Sep 17 00:00:00 2001 From: phil Date: Mon, 14 Jul 2025 12:04:27 -0400 Subject: [PATCH 068/134] desparate attempts for cross-browser compat safari might be a lost cause who-am-i may have been a mistake --- who-am-i/src/server.rs | 48 +++++++++++++++++++++++++++--- who-am-i/static/style.css | 11 +++++++ who-am-i/templates/authorized.hbs | 6 +++- who-am-i/templates/hello.hbs | 49 ++++++++++++++++++++++++++++--- who-am-i/templates/prompt.hbs | 19 +++++++++++- 5 files changed, 123 insertions(+), 10 deletions(-) diff --git a/who-am-i/src/server.rs b/who-am-i/src/server.rs index aedbffa..64f682e 100644 --- a/who-am-i/src/server.rs +++ b/who-am-i/src/server.rs @@ -10,7 +10,7 @@ use axum::{ response::{IntoResponse, Json, Redirect, Response}, routing::{get, post}, }; -use axum_extra::extract::cookie::{Cookie, Key, SameSite, SignedCookieJar}; +use axum_extra::extract::cookie::{Cookie, Expiration, Key, SameSite, SignedCookieJar}; use axum_template::{RenderHtml, engine::Engine}; use handlebars::{Handlebars, handlebars_helper}; use jose_jwk::JwkSet; @@ -20,7 +20,7 @@ use serde::Deserialize; use serde_json::{Value, json}; use std::collections::HashSet; use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, SystemTime}; use tokio::net::TcpListener; use tokio_util::sync::CancellationToken; use url::Url; @@ -32,6 +32,7 @@ use crate::{ const FAVICON: &[u8] = include_bytes!("../static/favicon.ico"); const STYLE_CSS: &str = include_str!("../static/style.css"); +const HELLO_COOKIE_KEY: &str = "hello-who-am-i"; const DID_COOKIE_KEY: &str = "did"; const COOKIE_EXPIRATION: Duration = Duration::from_secs(30 * 86_400); @@ -113,6 +114,11 @@ pub async fn serve( .unwrap(); } +#[derive(Debug, Deserialize)] +struct HelloQuery { + auth_reload: Option, + auth_failed: Option, +} async fn hello( State(AppState { engine, @@ -121,8 +127,14 @@ async fn hello( oauth, .. }): State, + Query(params): Query, mut jar: SignedCookieJar, ) -> Response { + let is_auth_reload = params.auth_reload.is_some(); + let auth_failed = params.auth_failed.is_some(); + let no_cookie = jar.get(HELLO_COOKIE_KEY).is_none(); + jar = jar.add(hello_cookie()); + let info = if let Some(did) = jar.get(DID_COOKIE_KEY) { if let Ok(did) = Did::new(did.value_trimmed().to_string()) { // push cookie expiry @@ -138,13 +150,24 @@ async fn hello( json!({ "did": did, "fetch_key": fetch_key, + "is_auth_reload": is_auth_reload, + "auth_failed": auth_failed, + "no_cookie": no_cookie, }) } else { jar = jar.remove(DID_COOKIE_KEY); - json!({}) + json!({ + "is_auth_reload": is_auth_reload, + "auth_failed": auth_failed, + "no_cookie": no_cookie, + }) } } else { - json!({}) + json!({ + "is_auth_reload": is_auth_reload, + "auth_failed": auth_failed, + "no_cookie": no_cookie, + }) }; let frame_headers = [(CONTENT_SECURITY_POLICY, "frame-ancestors 'none'")]; (frame_headers, jar, RenderHtml("hello", engine, info)).into_response() @@ -162,12 +185,29 @@ async fn favicon() -> impl IntoResponse { ([(CONTENT_TYPE, "image/x-icon")], FAVICON) } +fn hello_cookie() -> Cookie<'static> { + Cookie::build((HELLO_COOKIE_KEY, "hiiii")) + .http_only(true) + .secure(true) + .same_site(SameSite::None) + .expires(Expiration::DateTime( + (SystemTime::now() + COOKIE_EXPIRATION).into(), + )) // wtf safari needs this to not be a session cookie?? + .max_age(COOKIE_EXPIRATION.try_into().unwrap()) + .path("/") + .into() +} + fn cookie(did: &Did) -> Cookie<'static> { Cookie::build((DID_COOKIE_KEY, did.to_string())) .http_only(true) .secure(true) .same_site(SameSite::None) + .expires(Expiration::DateTime( + (SystemTime::now() + COOKIE_EXPIRATION).into(), + )) // wtf safari needs this to not be a session cookie?? .max_age(COOKIE_EXPIRATION.try_into().unwrap()) + .path("/") .into() } diff --git a/who-am-i/static/style.css b/who-am-i/static/style.css index 89f0fa4..6d732dc 100644 --- a/who-am-i/static/style.css +++ b/who-am-i/static/style.css @@ -165,6 +165,13 @@ p.detail.no { color: #285; } +#need-storage { + font-size: 0.8rem; +} +.problem { + color: #a31; +} + #or { font-size: 0.8rem; text-align: center; @@ -182,3 +189,7 @@ input.handle { .hidden { display: none !important; } + +.hello-connect-plz { + margin: 1.667rem 0 0.667rem; +} diff --git a/who-am-i/templates/authorized.hbs b/who-am-i/templates/authorized.hbs index 7edacbc..9138f4c 100644 --- a/who-am-i/templates/authorized.hbs +++ b/who-am-i/templates/authorized.hbs @@ -1,6 +1,9 @@ + +great job! -

oh sick. hey {{ did }}. you can close this window now.

+

oauth success!

+

this window should automatically close itself (probably a bug if it hasn't)

diff --git a/who-am-i/templates/hello.hbs b/who-am-i/templates/hello.hbs index 5d62822..24cbad2 100644 --- a/who-am-i/templates/hello.hbs +++ b/who-am-i/templates/hello.hbs @@ -4,6 +4,7 @@

This is a little identity-verifying service for microcosm demos.

+

Only read access to your public data is required to connect: connecting does not grant any ability to modify your account or data.

{{#if did}} @@ -50,7 +51,8 @@ } catch (e) { err(e, 'failed to clear session, sorry'); } - window.location.reload(); + window.location.replace(location.pathname); + window.location.reload(); // backup, in case there is no query? }); })(); @@ -71,11 +73,50 @@ } {{else}} -

- No identity connected. -

+ +

Connect your handle

+ + {{#if is_auth_reload}} + {{#if no_cookie}} +

+ No identity connected. Your browser may be blocking access for connecting. +

+ {{else}} + {{#if auth_failed}} +

+ No identity connected. Connecting failed or was denied. +

+ {{else}} +

+ No identity connected. +

+ {{/if}} + {{/if}} + {{/if}} + +
+
+ + +
+
{{/if}} +
+ {{/inline}} {{#> base-full}}{{/base-full}} diff --git a/who-am-i/templates/prompt.hbs b/who-am-i/templates/prompt.hbs index 9e0278e..afdb713 100644 --- a/who-am-i/templates/prompt.hbs +++ b/who-am-i/templates/prompt.hbs @@ -27,6 +27,11 @@
+ + -- 2.51.2 From 084d69fdc0750491c180764a02779aa1a90ad92f Mon Sep 17 00:00:00 2001 From: phil Date: Tue, 15 Jul 2025 14:12:29 -0400 Subject: [PATCH 072/134] restrict app param --- who-am-i/src/server.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/who-am-i/src/server.rs b/who-am-i/src/server.rs index 9e24733..f6a2008 100644 --- a/who-am-i/src/server.rs +++ b/who-am-i/src/server.rs @@ -268,6 +268,11 @@ async fn prompt( Some(parent_host), ); } + if let Some(ref app) = params.app { + if !allowed_hosts.contains(app) { + return err("Login is not allowed for this app", false, Some(app)); + } + } let parent_origin = url.origin().ascii_serialization(); if parent_origin == "null" { return err("Origin or referrer header value is opaque", true, None); -- 2.51.2 From edd0c2e237d61ef59ae044ae657ab46cda18f7e8 Mon Sep 17 00:00:00 2001 From: phil Date: Mon, 28 Jul 2025 20:26:03 -0400 Subject: [PATCH 073/134] cachey --- Cargo.lock | 584 ++++++++++++++++++++++++++++---- Cargo.toml | 1 + slingshot/.gitignore | 1 + slingshot/Cargo.toml | 19 ++ slingshot/src/consumer.rs | 76 +++++ slingshot/src/error.rs | 21 ++ slingshot/src/firehose_cache.rs | 17 + slingshot/src/lib.rs | 8 + slingshot/src/main.rs | 107 ++++++ slingshot/src/record.rs | 24 ++ 10 files changed, 790 insertions(+), 68 deletions(-) create mode 100644 slingshot/.gitignore create mode 100644 slingshot/Cargo.toml create mode 100644 slingshot/src/consumer.rs create mode 100644 slingshot/src/error.rs create mode 100644 slingshot/src/firehose_cache.rs create mode 100644 slingshot/src/lib.rs create mode 100644 slingshot/src/main.rs create mode 100644 slingshot/src/record.rs diff --git a/Cargo.lock b/Cargo.lock index 4da0c78..2197e61 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,6 +24,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", + "getrandom 0.2.15", "once_cell", "version_check", "zerocopy 0.7.35", @@ -121,6 +122,12 @@ version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + [[package]] name = "arrayvec" version = "0.7.6" @@ -155,7 +162,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn", + "syn 2.0.103", ] [[package]] @@ -173,6 +180,18 @@ dependencies = [ "nom", ] +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + [[package]] name = "async-compression" version = "0.4.25" @@ -216,9 +235,15 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + [[package]] name = "async-trait" version = "0.1.88" @@ -227,7 +252,7 @@ checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -377,6 +402,18 @@ dependencies = [ "trait-variant", ] +[[package]] +name = "auto_enums" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c170965892137a3a9aeb000b4524aa3cc022a310e709d848b6e1cdce4ab4781" +dependencies = [ + "derive_utils", + "proc-macro2", + "quote", + "syn 2.0.103", +] + [[package]] name = "autocfg" version = "1.4.0" @@ -613,7 +650,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn", + "syn 2.0.103", "which", ] @@ -632,7 +669,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn", + "syn 2.0.103", ] [[package]] @@ -650,7 +687,7 @@ dependencies = [ "regex", "rustc-hash 2.1.1", "shlex", - "syn", + "syn 2.0.103", ] [[package]] @@ -813,9 +850,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.40" +version = "4.5.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40b6887a1d8685cebccf115538db5c0efe625ccac9696ad45c409d96566e910f" +checksum = "be92d32e80243a54711e5d7ce823c35c41c9d929dc4ab58e1276f625841aadf9" dependencies = [ "clap_builder", "clap_derive", @@ -823,26 +860,26 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.40" +version = "4.5.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0c66c08ce9f0c698cbce5c0279d0bb6ac936d8674174fe48f736533b964f59e" +checksum = "707eab41e9622f9139419d573eca0900137718000c517d47da73045f54331c3d" dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim", + "strsim 0.11.1", ] [[package]] name = "clap_derive" -version = "4.5.40" +version = "4.5.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2c7947ae4cc3d851207c1adb5b5e260ff0cca11446b1d6d1423788e442257ce" +checksum = "ef4f52386a59ca4c860f7393bcf8abd8dfd91ecccc0f774635ff68e92eeef491" dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -871,6 +908,15 @@ dependencies = [ "cc", ] +[[package]] +name = "cmsketch" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "553c840ee51da812c6cd621f9f7e07dfb00a49f91283a8e6380c78cba4f61aba" +dependencies = [ + "paste", +] + [[package]] name = "colorchoice" version = "1.0.3" @@ -911,7 +957,7 @@ dependencies = [ "clap", "ctrlc", "flume", - "fs4", + "fs4 0.12.0", "headers-accept", "links", "mediatype", @@ -1074,14 +1120,38 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core 0.14.4", + "darling_macro 0.14.4", +] + [[package]] name = "darling" version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.10.0", + "syn 1.0.109", ] [[package]] @@ -1094,8 +1164,19 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim", - "syn", + "strsim 0.11.1", + "syn 2.0.103", +] + +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core 0.14.4", + "quote", + "syn 1.0.109", ] [[package]] @@ -1104,9 +1185,9 @@ version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ - "darling_core", + "darling_core 0.20.11", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -1146,7 +1227,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18e4fdb82bd54a12e42fb58a800dcae6b9e13982238ce2296dc3570b92148e1f" dependencies = [ "data-encoding", - "syn", + "syn 2.0.103", ] [[package]] @@ -1191,10 +1272,10 @@ version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" dependencies = [ - "darling", + "darling 0.20.11", "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -1204,7 +1285,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn", + "syn 2.0.103", +] + +[[package]] +name = "derive_utils" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccfae181bab5ab6c5478b2ccb69e4c68a02f8c3ec72f6616bfec9dbc599d2ee0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.103", ] [[package]] @@ -1248,7 +1340,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -1257,6 +1349,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57" +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + [[package]] name = "dropshot" version = "0.16.2" @@ -1319,7 +1417,7 @@ dependencies = [ "semver", "serde", "serde_tokenstream", - "syn", + "syn 2.0.103", ] [[package]] @@ -1392,7 +1490,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -1404,7 +1502,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -1526,6 +1624,9 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" dependencies = [ + "futures-core", + "futures-sink", + "nanorand", "spin", ] @@ -1565,6 +1666,112 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "foyer" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b4d8e96374206ff1b4265f2e2e6e1f80bc3048957b2a1e7fdeef929d68f318f" +dependencies = [ + "equivalent", + "foyer-common", + "foyer-memory", + "foyer-storage", + "madsim-tokio", + "mixtrics", + "pin-project", + "serde", + "thiserror 2.0.12", + "tokio", + "tracing", +] + +[[package]] +name = "foyer-common" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "911b8e3f23d5fe55b0b240f75af1d2fa5cb7261d3f9b38ef1c57bbc9f0449317" +dependencies = [ + "bincode 1.3.3", + "bytes", + "cfg-if", + "itertools 0.14.0", + "madsim-tokio", + "mixtrics", + "parking_lot", + "pin-project", + "serde", + "thiserror 2.0.12", + "tokio", + "twox-hash", +] + +[[package]] +name = "foyer-intrusive-collections" +version = "0.10.0-dev" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4fee46bea69e0596130e3210e65d3424e0ac1e6df3bde6636304bdf1ca4a3b" +dependencies = [ + "memoffset", +] + +[[package]] +name = "foyer-memory" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "506883d5a8500dea1b1662f7180f3534bdcbfa718d3253db7179552ef83612fa" +dependencies = [ + "arc-swap", + "bitflags", + "cmsketch", + "equivalent", + "foyer-common", + "foyer-intrusive-collections", + "hashbrown 0.15.2", + "itertools 0.14.0", + "madsim-tokio", + "mixtrics", + "parking_lot", + "pin-project", + "serde", + "thiserror 2.0.12", + "tokio", + "tracing", +] + +[[package]] +name = "foyer-storage" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ba8403a54a2f2032fb647e49c442e5feeb33f3989f7024f1b178341a016f06d" +dependencies = [ + "allocator-api2", + "anyhow", + "auto_enums", + "bytes", + "equivalent", + "flume", + "foyer-common", + "foyer-memory", + "fs4 0.13.1", + "futures-core", + "futures-util", + "itertools 0.14.0", + "libc", + "lz4", + "madsim-tokio", + "ordered_hash_map", + "parking_lot", + "paste", + "pin-project", + "rand 0.9.1", + "serde", + "thiserror 2.0.12", + "tokio", + "tracing", + "twox-hash", + "zstd", +] + [[package]] name = "fs4" version = "0.12.0" @@ -1575,6 +1782,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "fs4" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4" +dependencies = [ + "rustix 1.0.5", + "windows-sys 0.59.0", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -1637,7 +1854,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -1790,6 +2007,15 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" +dependencies = [ + "ahash", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -2073,7 +2299,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2", + "socket2 0.5.9", "system-configuration", "tokio", "tower-service", @@ -2220,7 +2446,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -2281,13 +2507,24 @@ dependencies = [ "compare", ] +[[package]] +name = "io-uring" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4" +dependencies = [ + "bitflags", + "cfg-if", + "libc", +] + [[package]] name = "ipconfig" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" dependencies = [ - "socket2", + "socket2 0.5.9", "widestring", "windows-sys 0.48.0", "winreg", @@ -2355,6 +2592,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.15" @@ -2403,7 +2649,7 @@ checksum = "43ce13c40ec6956157a3635d97a1ee2df323b263f09ea14165131289cb0f5c19" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -2503,9 +2749,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.171" +version = "0.2.174" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" +checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" [[package]] name = "libfuzzer-sys" @@ -2671,6 +2917,15 @@ dependencies = [ "xxhash-rust", ] +[[package]] +name = "lz4" +version = "1.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" +dependencies = [ + "lz4-sys", +] + [[package]] name = "lz4-sys" version = "1.11.1+lz4-1.10.0" @@ -2696,6 +2951,60 @@ dependencies = [ "libc", ] +[[package]] +name = "madsim" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db6694555643da293dfb89e33c2880a13b62711d64b6588bc7df6ce4110b27f1" +dependencies = [ + "ahash", + "async-channel", + "async-stream", + "async-task", + "bincode 1.3.3", + "bytes", + "downcast-rs", + "futures-util", + "lazy_static", + "libc", + "madsim-macros", + "naive-timer", + "panic-message", + "rand 0.8.5", + "rand_xoshiro 0.6.0", + "rustversion", + "serde", + "spin", + "tokio", + "tokio-util", + "toml", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "madsim-macros" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d248e97b1a48826a12c3828d921e8548e714394bf17274dd0a93910dc946e1" +dependencies = [ + "darling 0.14.4", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "madsim-tokio" +version = "0.2.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d3eb2acc57c82d21d699119b859e2df70a91dbdb84734885a1e72be83bdecb5" +dependencies = [ + "madsim", + "spin", + "tokio", +] + [[package]] name = "match_cfg" version = "0.1.0" @@ -2729,6 +3038,15 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + [[package]] name = "metrics" version = "0.24.2" @@ -2870,6 +3188,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "mixtrics" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adbcddf5a90b959eea97ae505e0391f5c6dd411fbf546d43b9c59ad1c3bd4391" +dependencies = [ + "itertools 0.14.0", + "parking_lot", +] + [[package]] name = "moka" version = "0.12.10" @@ -2931,6 +3259,21 @@ dependencies = [ "unsigned-varint", ] +[[package]] +name = "naive-timer" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "034a0ad7deebf0c2abcf2435950a6666c3c15ea9d8fad0c0f48efa8a7f843fed" + +[[package]] +name = "nanorand" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" +dependencies = [ + "getrandom 0.2.15", +] + [[package]] name = "native-tls" version = "0.2.14" @@ -3130,7 +3473,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -3161,6 +3504,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "ordered_hash_map" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab0e5f22bf6dd04abd854a8874247813a8fa2c8c1260eba6fbb150270ce7c176" +dependencies = [ + "hashbrown 0.13.2", +] + [[package]] name = "overload" version = "0.1.1" @@ -3189,6 +3541,12 @@ dependencies = [ "primeorder", ] +[[package]] +name = "panic-message" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384e52fd8fbd4cbe3c317e8216260c21a0f9134de108cea8a4dd4e7e152c472d" + [[package]] name = "parking" version = "2.2.1" @@ -3298,7 +3656,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -3311,6 +3669,26 @@ dependencies = [ "sha2", ] +[[package]] +name = "pin-project" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.103", +] + [[package]] name = "pin-project-lite" version = "0.2.16" @@ -3387,7 +3765,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6837b9e10d61f45f987d50808f83d1ee3d206c66acf650c3e4ae2e1f6ddedf55" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.103", ] [[package]] @@ -3604,7 +3982,7 @@ checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -3947,7 +4325,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn", + "syn 2.0.103", ] [[package]] @@ -4050,7 +4428,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -4061,7 +4439,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -4079,9 +4457,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.140" +version = "1.0.141" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +checksum = "30b9eff21ebe718216c6ec64e1d9ac57087aad11efc64e32002bce4a0d4c03d3" dependencies = [ "itoa", "memchr", @@ -4130,7 +4508,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn", + "syn 2.0.103", ] [[package]] @@ -4169,10 +4547,10 @@ version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e" dependencies = [ - "darling", + "darling 0.20.11", "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -4258,6 +4636,26 @@ dependencies = [ "autocfg", ] +[[package]] +name = "slingshot" +version = "0.1.0" +dependencies = [ + "axum", + "clap", + "ctrlc", + "env_logger", + "foyer", + "jetstream", + "log", + "metrics", + "metrics-exporter-prometheus 0.17.2", + "serde", + "serde_json", + "thiserror 2.0.12", + "tokio", + "tokio-util", +] + [[package]] name = "slog" version = "2.7.0" @@ -4329,6 +4727,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "socket2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "spacedust" version = "0.1.0" @@ -4389,6 +4797,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ae9eec00137a8eed469fb4148acd9fc6ac8c3f9b110f52cd34698c8b5bfa0e" +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "strsim" version = "0.11.1" @@ -4401,6 +4815,17 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.103" @@ -4429,7 +4854,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -4515,7 +4940,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -4526,7 +4951,7 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -4625,20 +5050,22 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.45.1" +version = "1.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779" +checksum = "43864ed400b6043a4757a25c7a64a8efde741aed79a056a2fb348a406701bb35" dependencies = [ "backtrace", "bytes", + "io-uring", "libc", "mio", "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "slab", + "socket2 0.6.0", "tokio-macros", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4649,7 +5076,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -4817,9 +5244,21 @@ checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ "log", "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.103", +] + [[package]] name = "tracing-core" version = "0.1.33" @@ -4867,7 +5306,7 @@ checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -4912,6 +5351,15 @@ dependencies = [ "utf-8", ] +[[package]] +name = "twox-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b907da542cbced5261bd3256de1b3a1bf340a3d37f93425a07362a1d687de56" +dependencies = [ + "rand 0.9.1", +] + [[package]] name = "typenum" version = "1.18.0" @@ -5157,7 +5605,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn", + "syn 2.0.103", "wasm-bindgen-shared", ] @@ -5192,7 +5640,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -5352,7 +5800,7 @@ checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -5363,7 +5811,7 @@ checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -5374,7 +5822,7 @@ checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -5385,7 +5833,7 @@ checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -5665,7 +6113,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", "synstructure", ] @@ -5695,7 +6143,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -5706,7 +6154,7 @@ checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -5726,7 +6174,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", "synstructure", ] @@ -5758,7 +6206,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index f0a3b13..6604a84 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,4 +8,5 @@ members = [ "ufos/fuzz", "spacedust", "who-am-i", + "slingshot", ] diff --git a/slingshot/.gitignore b/slingshot/.gitignore new file mode 100644 index 0000000..01c6be7 --- /dev/null +++ b/slingshot/.gitignore @@ -0,0 +1 @@ +foyer diff --git a/slingshot/Cargo.toml b/slingshot/Cargo.toml new file mode 100644 index 0000000..7d76e31 --- /dev/null +++ b/slingshot/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "slingshot" +version = "0.1.0" +edition = "2024" + +[dependencies] +clap = { version = "4.5.41", features = ["derive"] } +ctrlc = "3.4.7" +env_logger = "0.11.8" +foyer = { version = "0.18.0", features = ["serde"] } +jetstream = { path = "../jetstream", features = ["metrics"] } +log = "0.4.27" +metrics = "0.24.2" +metrics-exporter-prometheus = { version = "0.17.1", features = ["http-listener"] } +serde = { version = "1.0.219", features = ["derive"] } +serde_json = { version = "1.0.141", features = ["raw_value"] } +thiserror = "2.0.12" +tokio = { version = "1.47.0", features = ["full"] } +tokio-util = "0.7.15" diff --git a/slingshot/src/consumer.rs b/slingshot/src/consumer.rs new file mode 100644 index 0000000..84f2cc6 --- /dev/null +++ b/slingshot/src/consumer.rs @@ -0,0 +1,76 @@ +use crate::CachedRecord; +use foyer::HybridCache; +use crate::error::ConsumerError; +use jetstream::{ + DefaultJetstreamEndpoints, JetstreamCompression, JetstreamConfig, JetstreamConnector, + events::{CommitOp, Cursor, EventKind}, +}; +use tokio_util::sync::CancellationToken; + +pub async fn consume( + jetstream_endpoint: String, + cursor: Option, + no_zstd: bool, + shutdown: CancellationToken, + cache: HybridCache, +) -> Result<(), ConsumerError> { + let endpoint = DefaultJetstreamEndpoints::endpoint_or_shortcut(&jetstream_endpoint); + if endpoint == jetstream_endpoint { + log::info!("consumer: connecting jetstream at {endpoint}"); + } else { + log::info!("consumer: connecting jetstream at {jetstream_endpoint} => {endpoint}"); + } + let config: JetstreamConfig = JetstreamConfig { + endpoint, + compression: if no_zstd { + JetstreamCompression::None + } else { + JetstreamCompression::Zstd + }, + replay_on_reconnect: true, + channel_size: 1024, // buffer up to ~1s of jetstream events + ..Default::default() + }; + let mut receiver = JetstreamConnector::new(config)? + .connect_cursor(cursor) + .await?; + + log::info!("consumer: receiving messages.."); + loop { + if shutdown.is_cancelled() { + log::info!("consumer: exiting for shutdown"); + return Ok(()); + } + let Some(mut event) = receiver.recv().await else { + log::error!("consumer: could not receive event, bailing"); + break; + }; + + if event.kind != EventKind::Commit { + continue; + } + let Some(ref mut commit) = event.commit else { + log::warn!("consumer: commit event missing commit data, ignoring"); + continue; + }; + + // TODO: something a bit more robust + let at_uri = format!( + "at://{}/{}/{}", + &*event.did, &*commit.collection, &*commit.rkey + ); + + if commit.operation == CommitOp::Delete { + cache.insert(at_uri, CachedRecord::Deleted); + } else { + let Some(record) = commit.record.take() else { + log::warn!("consumer: commit update/delete missing record, ignoring"); + continue; + }; + + cache.insert(at_uri, CachedRecord::Found(record.into())); + } + } + + Err(ConsumerError::JetstreamEnded) +} diff --git a/slingshot/src/error.rs b/slingshot/src/error.rs new file mode 100644 index 0000000..b427c42 --- /dev/null +++ b/slingshot/src/error.rs @@ -0,0 +1,21 @@ +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum ConsumerError { + #[error(transparent)] + JetstreamConnectionError(#[from] jetstream::error::ConnectionError), + #[error(transparent)] + JetstreamConfigValidationError(#[from] jetstream::error::ConfigValidationError), + #[error("jetstream ended")] + JetstreamEnded, + #[error("delay queue output dropped")] + DelayQueueOutputDropped, +} + +#[derive(Debug, Error)] +pub enum MainTaskError { + #[error(transparent)] + ConsumerTaskError(#[from] ConsumerError), + // #[error(transparent)] + // ServerTaskError(#[from] ServerError), +} diff --git a/slingshot/src/firehose_cache.rs b/slingshot/src/firehose_cache.rs new file mode 100644 index 0000000..d3b9a6d --- /dev/null +++ b/slingshot/src/firehose_cache.rs @@ -0,0 +1,17 @@ +use std::path::Path; +use crate::CachedRecord; +use foyer::{HybridCache, DirectFsDeviceOptions, Engine, HybridCacheBuilder}; + + +pub async fn firehose_cache(dir: impl AsRef) -> Result, String> { + let cache = HybridCacheBuilder::new() + .with_name("firehose") + .memory(64 * 2_usize.pow(20)) + .with_weighter(|k: &String, v| k.len() + std::mem::size_of_val(v)) + .storage(Engine::large()) + .with_device_options(DirectFsDeviceOptions::new(dir)) + .build() + .await + .map_err(|e| format!("foyer setup error: {e:?}"))?; + Ok(cache) +} diff --git a/slingshot/src/lib.rs b/slingshot/src/lib.rs new file mode 100644 index 0000000..75dd162 --- /dev/null +++ b/slingshot/src/lib.rs @@ -0,0 +1,8 @@ +mod consumer; +pub mod error; +mod firehose_cache; +mod record; + +pub use consumer::consume; +pub use firehose_cache::firehose_cache; +pub use record::CachedRecord; diff --git a/slingshot/src/main.rs b/slingshot/src/main.rs new file mode 100644 index 0000000..4c46187 --- /dev/null +++ b/slingshot/src/main.rs @@ -0,0 +1,107 @@ +// use foyer::HybridCache; +// use foyer::{Engine, DirectFsDeviceOptions, HybridCacheBuilder}; +use metrics_exporter_prometheus::PrometheusBuilder; +use slingshot::{consume, error::MainTaskError, firehose_cache}; + +use clap::Parser; +use tokio_util::sync::CancellationToken; + + +/// Slingshot record edge cache +#[derive(Parser, Debug, Clone)] +#[command(version, about, long_about = None)] +struct Args { + /// Jetstream server to connect to (exclusive with --fixture). Provide either a wss:// URL, or a shorhand value: + /// 'us-east-1', 'us-east-2', 'us-west-1', or 'us-west-2' + #[arg(long)] + jetstream: String, + /// don't request zstd-compressed jetstream events + /// + /// reduces CPU at the expense of more ingress bandwidth + #[arg(long, action)] + jetstream_no_zstd: bool, +} + +#[tokio::main] +async fn main() -> Result<(), String> { + env_logger::init(); + + let shutdown = CancellationToken::new(); + + let ctrlc_shutdown = shutdown.clone(); + ctrlc::set_handler(move || ctrlc_shutdown.cancel()).expect("failed to set ctrl-c handler"); + + let args = Args::parse(); + + if let Err(e) = install_metrics_server() { + log::error!("failed to install metrics server: {e:?}"); + } else { + log::info!("metrics listening at http://0.0.0.0:8765"); + } + + log::info!("setting up firehose cache..."); + let cache = firehose_cache("./foyer").await?; + log::info!("firehose cache ready."); + + let mut tasks: tokio::task::JoinSet> = tokio::task::JoinSet::new(); + + let consumer_shutdown = shutdown.clone(); + tasks.spawn(async move { + consume( + args.jetstream, + None, + args.jetstream_no_zstd, + consumer_shutdown, + cache, + ) + .await?; + Ok(()) + }); + + + tokio::select! { + _ = shutdown.cancelled() => log::warn!("shutdown requested"), + Some(r) = tasks.join_next() => { + log::warn!("a task exited, shutting down: {r:?}"); + shutdown.cancel(); + } + } + + tokio::select! { + _ = async { + while let Some(completed) = tasks.join_next().await { + log::info!("shutdown: task completed: {completed:?}"); + } + } => {}, + _ = tokio::time::sleep(std::time::Duration::from_secs(3)) => { + log::info!("shutdown: not all tasks completed on time. aborting..."); + tasks.shutdown().await; + }, + } + + log::info!("bye!"); + + Ok(()) +} + +fn install_metrics_server() -> Result<(), metrics_exporter_prometheus::BuildError> { + log::info!("installing metrics server..."); + let host = [0, 0, 0, 0]; + let port = 8765; + PrometheusBuilder::new() + .set_quantiles(&[0.5, 0.9, 0.99, 1.0])? + .set_bucket_duration(std::time::Duration::from_secs(300))? + .set_bucket_count(std::num::NonZero::new(12).unwrap()) // count * duration = 60 mins. stuff doesn't happen that fast here. + .set_enable_unit_suffix(false) // this seemed buggy for constellation (sometimes wouldn't engage) + .with_http_listener((host, port)) + .install()?; + log::info!( + "metrics server installed! listening on http://{}.{}.{}.{}:{port}", + host[0], + host[1], + host[2], + host[3] + ); + Ok(()) +} + diff --git a/slingshot/src/record.rs b/slingshot/src/record.rs new file mode 100644 index 0000000..e6d7d7f --- /dev/null +++ b/slingshot/src/record.rs @@ -0,0 +1,24 @@ +use serde_json::value::RawValue; +use serde::{Serialize, Deserialize}; + +#[derive(Debug, Serialize, Deserialize)] +pub struct RawRecord(String); + +impl From> for RawRecord { + fn from(rv: Box) -> Self { + Self(rv.get().to_string()) + } +} + +/// only for use with stored (validated) values, not general strings +impl From for Box { + fn from(RawRecord(s): RawRecord) -> Self { + RawValue::from_string(s).expect("stored string from RawValue to be valid") + } +} + +#[derive(Debug, Serialize, Deserialize)] +pub enum CachedRecord { + Found(RawRecord), + Deleted, +} -- 2.51.2 From 1df7007f3955fc74f8068118936eeb076cbcf046 Mon Sep 17 00:00:00 2001 From: phil Date: Tue, 29 Jul 2025 12:19:56 -0400 Subject: [PATCH 074/134] scaffold dropshot + cache lookup (no actual fetch) the response almost matches but dropshot uses "error_code: string" instead of "error: string" in its error responses --- Cargo.lock | 5 +- slingshot/Cargo.toml | 4 + slingshot/src/consumer.rs | 8 +- slingshot/src/error.rs | 20 ++- slingshot/src/lib.rs | 2 + slingshot/src/main.rs | 9 +- slingshot/src/record.rs | 25 ++- slingshot/src/server.rs | 308 +++++++++++++++++++++++++++++++++++ slingshot/static/favicon.ico | 0 slingshot/static/index.html | 53 ++++++ 10 files changed, 421 insertions(+), 13 deletions(-) create mode 100644 slingshot/src/server.rs create mode 100644 slingshot/static/favicon.ico create mode 100644 slingshot/static/index.html diff --git a/Cargo.lock b/Cargo.lock index 2197e61..c1c36b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4640,15 +4640,18 @@ dependencies = [ name = "slingshot" version = "0.1.0" dependencies = [ - "axum", "clap", "ctrlc", + "dropshot", "env_logger", "foyer", + "http", "jetstream", "log", "metrics", "metrics-exporter-prometheus 0.17.2", + "schemars", + "semver", "serde", "serde_json", "thiserror 2.0.12", diff --git a/slingshot/Cargo.toml b/slingshot/Cargo.toml index 7d76e31..d034716 100644 --- a/slingshot/Cargo.toml +++ b/slingshot/Cargo.toml @@ -6,12 +6,16 @@ edition = "2024" [dependencies] clap = { version = "4.5.41", features = ["derive"] } ctrlc = "3.4.7" +dropshot = "0.16.2" env_logger = "0.11.8" foyer = { version = "0.18.0", features = ["serde"] } +http = "1.3.1" jetstream = { path = "../jetstream", features = ["metrics"] } log = "0.4.27" metrics = "0.24.2" metrics-exporter-prometheus = { version = "0.17.1", features = ["http-listener"] } +schemars = { version = "0.8.22", features = ["raw_value"] } +semver = "1.0.26" serde = { version = "1.0.219", features = ["derive"] } serde_json = { version = "1.0.141", features = ["raw_value"] } thiserror = "2.0.12" diff --git a/slingshot/src/consumer.rs b/slingshot/src/consumer.rs index 84f2cc6..51fb3ec 100644 --- a/slingshot/src/consumer.rs +++ b/slingshot/src/consumer.rs @@ -64,11 +64,15 @@ pub async fn consume( cache.insert(at_uri, CachedRecord::Deleted); } else { let Some(record) = commit.record.take() else { - log::warn!("consumer: commit update/delete missing record, ignoring"); + log::warn!("consumer: commit insert or update missing record, ignoring"); + continue; + }; + let Some(cid) = commit.cid.take() else { + log::warn!("consumer: commit insert or update missing CID, ignoring"); continue; }; - cache.insert(at_uri, CachedRecord::Found(record.into())); + cache.insert(at_uri, CachedRecord::Found((cid, record).into())); } } diff --git a/slingshot/src/error.rs b/slingshot/src/error.rs index b427c42..ba90399 100644 --- a/slingshot/src/error.rs +++ b/slingshot/src/error.rs @@ -12,10 +12,26 @@ pub enum ConsumerError { DelayQueueOutputDropped, } +#[derive(Debug, Error)] +pub enum ServerError { + #[error("failed to configure server logger: {0}")] + ConfigLogError(std::io::Error), + #[error("failed to render json for openapi: {0}")] + OpenApiJsonFail(serde_json::Error), + #[error(transparent)] + FailedToBuildServer(#[from] dropshot::BuildError), + #[error("server exited: {0}")] + ServerExited(String), + #[error("server closed badly: {0}")] + BadClose(String), + #[error("blahhhahhahha")] + OhNo(String), +} + #[derive(Debug, Error)] pub enum MainTaskError { #[error(transparent)] ConsumerTaskError(#[from] ConsumerError), - // #[error(transparent)] - // ServerTaskError(#[from] ServerError), + #[error(transparent)] + ServerTaskError(#[from] ServerError), } diff --git a/slingshot/src/lib.rs b/slingshot/src/lib.rs index 75dd162..240b4a7 100644 --- a/slingshot/src/lib.rs +++ b/slingshot/src/lib.rs @@ -2,7 +2,9 @@ mod consumer; pub mod error; mod firehose_cache; mod record; +mod server; pub use consumer::consume; pub use firehose_cache::firehose_cache; pub use record::CachedRecord; +pub use server::serve; diff --git a/slingshot/src/main.rs b/slingshot/src/main.rs index 4c46187..d7ce143 100644 --- a/slingshot/src/main.rs +++ b/slingshot/src/main.rs @@ -1,7 +1,7 @@ // use foyer::HybridCache; // use foyer::{Engine, DirectFsDeviceOptions, HybridCacheBuilder}; use metrics_exporter_prometheus::PrometheusBuilder; -use slingshot::{consume, error::MainTaskError, firehose_cache}; +use slingshot::{consume, error::MainTaskError, firehose_cache, serve}; use clap::Parser; use tokio_util::sync::CancellationToken; @@ -45,6 +45,13 @@ async fn main() -> Result<(), String> { let mut tasks: tokio::task::JoinSet> = tokio::task::JoinSet::new(); + let server_shutdown = shutdown.clone(); + let server_cache_handle = cache.clone(); + tasks.spawn(async move { + serve(server_cache_handle, server_shutdown).await?; + Ok(()) + }); + let consumer_shutdown = shutdown.clone(); tasks.spawn(async move { consume( diff --git a/slingshot/src/record.rs b/slingshot/src/record.rs index e6d7d7f..9df588b 100644 --- a/slingshot/src/record.rs +++ b/slingshot/src/record.rs @@ -1,19 +1,30 @@ use serde_json::value::RawValue; use serde::{Serialize, Deserialize}; +use jetstream::exports::Cid; #[derive(Debug, Serialize, Deserialize)] -pub struct RawRecord(String); +pub struct RawRecord { + cid: Cid, + record: String, +} -impl From> for RawRecord { - fn from(rv: Box) -> Self { - Self(rv.get().to_string()) +// TODO: should be able to do typed CID +impl From<(Cid, Box)> for RawRecord { + fn from((cid, rv): (Cid, Box)) -> Self { + Self { + cid, + record: rv.get().to_string(), + } } } /// only for use with stored (validated) values, not general strings -impl From for Box { - fn from(RawRecord(s): RawRecord) -> Self { - RawValue::from_string(s).expect("stored string from RawValue to be valid") +impl From<&RawRecord> for (Cid, Box) { + fn from(RawRecord { cid, record }: &RawRecord) -> Self { + ( + cid.clone(), + RawValue::from_string(record.to_string()).expect("stored string from RawValue to be valid"), + ) } } diff --git a/slingshot/src/server.rs b/slingshot/src/server.rs new file mode 100644 index 0000000..b071fee --- /dev/null +++ b/slingshot/src/server.rs @@ -0,0 +1,308 @@ +use serde_json::value::RawValue; +use crate::CachedRecord; +use foyer::HybridCache; +use crate::error::ServerError; +use dropshot::{ + ApiDescription, Body, ConfigDropshot, ConfigLogging, + ConfigLoggingLevel, HttpError, HttpResponse, Query, RequestContext, + ServerBuilder, ServerContext, endpoint, + ClientErrorStatusCode, +}; +use http::{ + Response, StatusCode, + header::{ORIGIN, USER_AGENT}, +}; +use metrics::{counter, histogram}; +use std::sync::Arc; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; + +const INDEX_HTML: &str = include_str!("../static/index.html"); +const FAVICON: &[u8] = include_bytes!("../static/favicon.ico"); + +pub async fn serve( + cache: HybridCache, + shutdown: CancellationToken, +) -> Result<(), ServerError> { + let config_logging = ConfigLogging::StderrTerminal { + level: ConfigLoggingLevel::Info, + }; + + let log = config_logging + .to_logger("example-basic") + .map_err(ServerError::ConfigLogError)?; + + let mut api = ApiDescription::new(); + api.register(index).unwrap(); + api.register(favicon).unwrap(); + api.register(openapi).unwrap(); + api.register(get_record).unwrap(); + + // TODO: put spec in a once cell / lazy lock thing? + let spec = Arc::new( + api.openapi( + "Slingshot", + env!("CARGO_PKG_VERSION") + .parse() + .inspect_err(|e| { + eprintln!("failed to parse cargo package version for openapi: {e:?}") + }) + .unwrap_or(semver::Version::new(0, 0, 1)), + ) + .description("A fast edge cache for getRecord") + .contact_name("part of @microcosm.blue") + .contact_url("https://microcosm.blue") + .json() + .map_err(ServerError::OpenApiJsonFail)?, + ); + + let sub_shutdown = shutdown.clone(); + let ctx = Context { + cache, + spec, + shutdown: sub_shutdown, + }; + + let server = ServerBuilder::new(api, ctx, log) + .config(ConfigDropshot { + bind_address: "0.0.0.0:9996".parse().unwrap(), + ..Default::default() + }) + .start()?; + + tokio::select! { + s = server.wait_for_shutdown() => { + s.map_err(ServerError::ServerExited)?; + log::info!("server shut down normally."); + }, + _ = shutdown.cancelled() => { + log::info!("shutting down: closing server"); + server.close().await.map_err(ServerError::BadClose)?; + }, + } + Ok(()) +} + +#[derive(Debug, Clone)] +struct Context { + pub cache: HybridCache, + pub spec: Arc, + pub shutdown: CancellationToken, +} + +async fn instrument_handler(ctx: &RequestContext, handler: H) -> Result +where + R: HttpResponse, + H: Future>, + T: ServerContext, +{ + let start = Instant::now(); + let result = handler.await; + let latency = start.elapsed(); + let status_code = match &result { + Ok(response) => response.status_code(), + Err(e) => e.status_code.as_status(), + } + .as_str() // just the number (.to_string()'s Display does eg `200 OK`) + .to_string(); + let endpoint = ctx.endpoint.operation_id.clone(); + let headers = ctx.request.headers(); + let origin = headers + .get(ORIGIN) + .and_then(|v| v.to_str().ok()) + .unwrap_or("") + .to_string(); + let ua = headers + .get(USER_AGENT) + .and_then(|v| v.to_str().ok()) + .map(|ua| { + if ua.starts_with("Mozilla/5.0 ") { + "browser" + } else { + ua + } + }) + .unwrap_or("") + .to_string(); + counter!("server_requests_total", + "endpoint" => endpoint.clone(), + "origin" => origin, + "ua" => ua, + "status_code" => status_code, + ) + .increment(1); + histogram!("server_handler_latency", "endpoint" => endpoint).record(latency.as_micros() as f64); + result +} + +use dropshot::{HttpResponseHeaders, HttpResponseOk}; + +pub type OkCorsResponse = Result>, HttpError>; + +/// Helper for constructing Ok responses: return OkCors(T).into() +/// (not happy with this yet) +pub struct OkCors(pub T); + +impl From> for OkCorsResponse +where + T: Serialize + JsonSchema + Send + Sync, +{ + fn from(ok: OkCors) -> OkCorsResponse { + let mut res = HttpResponseHeaders::new_unnamed(HttpResponseOk(ok.0)); + res.headers_mut() + .insert("access-control-allow-origin", "*".parse().unwrap()); + Ok(res) + } +} + +pub fn cors_err(e: HttpError) -> HttpError { + e.with_header("access-control-allow-origin", "*").unwrap() +} + + +// TODO: cors for HttpError + +/// Serve index page as html +#[endpoint { + method = GET, + path = "/", + /* + * not useful to have this in openapi + */ + unpublished = true, +}] +async fn index(ctx: RequestContext) -> Result, HttpError> { + instrument_handler(&ctx, async { + Ok(Response::builder() + .status(StatusCode::OK) + .header(http::header::CONTENT_TYPE, "text/html") + .body(INDEX_HTML.into())?) + }) + .await +} + +/// Serve index page as html +#[endpoint { + method = GET, + path = "/favicon.ico", + /* + * not useful to have this in openapi + */ + unpublished = true, +}] +async fn favicon(ctx: RequestContext) -> Result, HttpError> { + instrument_handler(&ctx, async { + Ok(Response::builder() + .status(StatusCode::OK) + .header(http::header::CONTENT_TYPE, "image/x-icon") + .body(FAVICON.to_vec().into())?) + }) + .await +} + +/// Meta: get the openapi spec for this api +#[endpoint { + method = GET, + path = "/openapi", + /* + * not useful to have this in openapi + */ + unpublished = true, +}] +async fn openapi(ctx: RequestContext) -> OkCorsResponse { + instrument_handler(&ctx, async { + let spec = (*ctx.context().spec).clone(); + OkCors(spec).into() + }) + .await +} + + +#[derive(Debug, Deserialize, JsonSchema)] +struct GetRecordQuery { + /// The DID of the repo + /// + /// NOTE: handles should be accepted here but this is still TODO in slingshot + pub repo: String, + /// The NSID of the record collection + pub collection: String, + /// The Record key + pub rkey: String, + /// Optional: the CID of the version of the record. + /// + /// If not specified, then return the most recent version. + /// + /// If specified and a newer version of the record exists, returns 404 not + /// found. That is: slingshot only retains the most recent version of a + /// record. + #[serde(default)] + pub cid: Option, +} + +#[derive(Debug, Serialize, JsonSchema)] +struct GetRecordResponse { + pub uri: String, + pub cid: String, + pub value: Box, +} + +/// com.atproto.repo.getRecord +/// +/// Get a single record from a repository. Does not require auth. +/// +/// See https://docs.bsky.app/docs/api/com-atproto-repo-get-record for the +/// canonical XRPC documentation that this endpoint aims to be compatible with. +#[endpoint { + method = GET, + path = "/xrpc/com.atproto.repo.getRecord", +}] +async fn get_record( + ctx: RequestContext, + query: Query, +) -> OkCorsResponse { + + let Context { cache, .. } = ctx.context(); + let GetRecordQuery { repo, collection, rkey, cid } = query.into_inner(); + + // TODO: yeah yeah + let at_uri = format!( + "at://{}/{}/{}", + &*repo, &*collection, &*rkey + ); + + instrument_handler(&ctx, async { + let entry = cache + .fetch(at_uri.clone(), || async move { + Err(foyer::Error::Other(Box::new(ServerError::OhNo("booo".to_string())))) + }) + .await + .unwrap(); + + match *entry { + CachedRecord::Found(ref raw) => { + let (found_cid, raw_value) = raw.into(); + let found_cid = found_cid.as_ref().to_string(); + if cid.map(|c| c != found_cid).unwrap_or(false) { + Err(HttpError::for_not_found(None, "CID mismatch".to_string())) + .map_err(cors_err)?; + } + OkCors(GetRecordResponse { + uri: at_uri, + cid: found_cid, + value: raw_value, + }).into() + }, + CachedRecord::Deleted => { + Err(HttpError::for_client_error_with_status( + Some("Gone".to_string()), + ClientErrorStatusCode::GONE, + )).map_err(cors_err) + } + } + }) + .await + +} diff --git a/slingshot/static/favicon.ico b/slingshot/static/favicon.ico new file mode 100644 index 0000000..e69de29 diff --git a/slingshot/static/index.html b/slingshot/static/index.html new file mode 100644 index 0000000..d598eba --- /dev/null +++ b/slingshot/static/index.html @@ -0,0 +1,53 @@ + + + + + Slingshot documentation + + + + + +
+

+ todo: what link goes here?: blah +

+ +
+ + + + + + + + -- 2.51.2 From 0800d4ba784f200e8b36e4e144a55b22879974f2 Mon Sep 17 00:00:00 2001 From: phil Date: Tue, 29 Jul 2025 16:40:42 -0400 Subject: [PATCH 075/134] swap dropshot for poem_openapi it's... nice here? --- Cargo.lock | 207 +++++++++++++++++++- slingshot/Cargo.toml | 6 +- slingshot/src/error.rs | 10 - slingshot/src/server.rs | 423 +++++++++++++++------------------------- 4 files changed, 356 insertions(+), 290 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c1c36b9..0d49226 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1288,6 +1288,27 @@ dependencies = [ "syn 2.0.103", ] +[[package]] +name = "derive_more" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "093242cf7570c207c83073cf82f79706fe7b8317e98620a47d5be7c3d8497678" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.103", + "unicode-xid", +] + [[package]] name = "derive_utils" version = "0.15.0" @@ -2283,9 +2304,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.14" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc2fdfdbff08affe55bb779f33b053aa1fe5dd5b54c257343c17edfa55711bdb" +checksum = "8d9b05277c7e8da2c93a568989bb6207bef0112e8d17df7a6eda4a3cf143bc5e" dependencies = [ "base64 0.22.1", "bytes", @@ -2299,7 +2320,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.9", + "socket2 0.6.0", "system-configuration", "tokio", "tower-service", @@ -3234,6 +3255,7 @@ dependencies = [ "memchr", "mime", "spin", + "tokio", "version_check", ] @@ -3728,6 +3750,100 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "poem" +version = "3.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f977080932c87287147dca052951c3e2696f8759863f6b4e4c0c9ffe7a4cc8b" +dependencies = [ + "bytes", + "futures-util", + "headers", + "http", + "http-body-util", + "hyper", + "hyper-util", + "mime", + "multer", + "nix", + "parking_lot", + "percent-encoding", + "pin-project-lite", + "poem-derive", + "quick-xml", + "regex", + "rfc7239", + "serde", + "serde_json", + "serde_urlencoded", + "serde_yaml", + "smallvec", + "sync_wrapper", + "tempfile", + "thiserror 2.0.12", + "tokio", + "tokio-stream", + "tokio-util", + "tracing", + "wildmatch", +] + +[[package]] +name = "poem-derive" +version = "3.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "056e2fea6de1cb240ffe23cfc4fc370b629f8be83b5f27e16b7acd5231a72de4" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.103", +] + +[[package]] +name = "poem-openapi" +version = "5.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ccbcc395bf4dd03df1da32da351b6b6732e4074ce27ddec315650e52a2be44c" +dependencies = [ + "base64 0.22.1", + "bytes", + "derive_more", + "futures-util", + "indexmap 2.9.0", + "itertools 0.14.0", + "mime", + "num-traits", + "poem", + "poem-openapi-derive", + "quick-xml", + "regex", + "serde", + "serde_json", + "serde_urlencoded", + "serde_yaml", + "thiserror 2.0.12", + "tokio", +] + +[[package]] +name = "poem-openapi-derive" +version = "5.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41273b691a3d467a8c44d05506afba9f7b6bd56c9cdf80123de13fe52d7ec587" +dependencies = [ + "darling 0.20.11", + "http", + "indexmap 2.9.0", + "mime", + "proc-macro-crate", + "proc-macro2", + "quote", + "regex", + "syn 2.0.103", + "thiserror 2.0.12", +] + [[package]] name = "portable-atomic" version = "1.11.0" @@ -3777,6 +3893,15 @@ dependencies = [ "elliptic-curve", ] +[[package]] +name = "proc-macro-crate" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro2" version = "1.0.94" @@ -3823,6 +3948,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "quick-xml" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quick_cache" version = "0.6.12" @@ -4088,6 +4223,15 @@ dependencies = [ "subtle", ] +[[package]] +name = "rfc7239" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a82f1d1e38e9a85bb58ffcfadf22ed6f2c94e8cd8581ec2b0f80a2a6858350f" +dependencies = [ + "uncased", +] + [[package]] name = "ring" version = "0.17.14" @@ -4553,6 +4697,19 @@ dependencies = [ "syn 2.0.103", ] +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap 2.9.0", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "sha1" version = "0.10.6" @@ -4642,16 +4799,14 @@ version = "0.1.0" dependencies = [ "clap", "ctrlc", - "dropshot", "env_logger", "foyer", - "http", "jetstream", "log", "metrics", "metrics-exporter-prometheus 0.17.2", - "schemars", - "semver", + "poem", + "poem-openapi", "serde", "serde_json", "thiserror 2.0.12", @@ -5113,6 +5268,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-tungstenite" version = "0.26.2" @@ -5421,6 +5587,15 @@ dependencies = [ "ufos", ] +[[package]] +name = "uncased" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" +dependencies = [ + "version_check", +] + [[package]] name = "unicase" version = "2.8.1" @@ -5433,6 +5608,18 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "unsigned-varint" version = "0.8.0" @@ -5728,6 +5915,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd7cf3379ca1aac9eea11fba24fd7e315d621f8dfe35c8d7d2be8b793726e07d" +[[package]] +name = "wildmatch" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ce1ab1f8c62655ebe1350f589c61e505cf94d385bc6a12899442d9081e71fd" + [[package]] name = "winapi" version = "0.3.9" diff --git a/slingshot/Cargo.toml b/slingshot/Cargo.toml index d034716..6b70c8b 100644 --- a/slingshot/Cargo.toml +++ b/slingshot/Cargo.toml @@ -6,16 +6,14 @@ edition = "2024" [dependencies] clap = { version = "4.5.41", features = ["derive"] } ctrlc = "3.4.7" -dropshot = "0.16.2" env_logger = "0.11.8" foyer = { version = "0.18.0", features = ["serde"] } -http = "1.3.1" jetstream = { path = "../jetstream", features = ["metrics"] } log = "0.4.27" metrics = "0.24.2" metrics-exporter-prometheus = { version = "0.17.1", features = ["http-listener"] } -schemars = { version = "0.8.22", features = ["raw_value"] } -semver = "1.0.26" +poem = "3.1.12" +poem-openapi = { version = "5.1.16", features = ["scalar"] } serde = { version = "1.0.219", features = ["derive"] } serde_json = { version = "1.0.141", features = ["raw_value"] } thiserror = "2.0.12" diff --git a/slingshot/src/error.rs b/slingshot/src/error.rs index ba90399..dd6bde6 100644 --- a/slingshot/src/error.rs +++ b/slingshot/src/error.rs @@ -14,18 +14,8 @@ pub enum ConsumerError { #[derive(Debug, Error)] pub enum ServerError { - #[error("failed to configure server logger: {0}")] - ConfigLogError(std::io::Error), - #[error("failed to render json for openapi: {0}")] - OpenApiJsonFail(serde_json::Error), - #[error(transparent)] - FailedToBuildServer(#[from] dropshot::BuildError), #[error("server exited: {0}")] ServerExited(String), - #[error("server closed badly: {0}")] - BadClose(String), - #[error("blahhhahhahha")] - OhNo(String), } #[derive(Debug, Error)] diff --git a/slingshot/src/server.rs b/slingshot/src/server.rs index b071fee..4f0e111 100644 --- a/slingshot/src/server.rs +++ b/slingshot/src/server.rs @@ -1,308 +1,193 @@ -use serde_json::value::RawValue; -use crate::CachedRecord; use foyer::HybridCache; -use crate::error::ServerError; -use dropshot::{ - ApiDescription, Body, ConfigDropshot, ConfigLogging, - ConfigLoggingLevel, HttpError, HttpResponse, Query, RequestContext, - ServerBuilder, ServerContext, endpoint, - ClientErrorStatusCode, -}; -use http::{ - Response, StatusCode, - header::{ORIGIN, USER_AGENT}, -}; -use metrics::{counter, histogram}; -use std::sync::Arc; - -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; -use tokio::time::Instant; +use crate::{error::ServerError, CachedRecord}; use tokio_util::sync::CancellationToken; -const INDEX_HTML: &str = include_str!("../static/index.html"); -const FAVICON: &[u8] = include_bytes!("../static/favicon.ico"); - -pub async fn serve( - cache: HybridCache, - shutdown: CancellationToken, -) -> Result<(), ServerError> { - let config_logging = ConfigLogging::StderrTerminal { - level: ConfigLoggingLevel::Info, - }; - - let log = config_logging - .to_logger("example-basic") - .map_err(ServerError::ConfigLogError)?; - - let mut api = ApiDescription::new(); - api.register(index).unwrap(); - api.register(favicon).unwrap(); - api.register(openapi).unwrap(); - api.register(get_record).unwrap(); - - // TODO: put spec in a once cell / lazy lock thing? - let spec = Arc::new( - api.openapi( - "Slingshot", - env!("CARGO_PKG_VERSION") - .parse() - .inspect_err(|e| { - eprintln!("failed to parse cargo package version for openapi: {e:?}") - }) - .unwrap_or(semver::Version::new(0, 0, 1)), - ) - .description("A fast edge cache for getRecord") - .contact_name("part of @microcosm.blue") - .contact_url("https://microcosm.blue") - .json() - .map_err(ServerError::OpenApiJsonFail)?, - ); - - let sub_shutdown = shutdown.clone(); - let ctx = Context { - cache, - spec, - shutdown: sub_shutdown, - }; - - let server = ServerBuilder::new(api, ctx, log) - .config(ConfigDropshot { - bind_address: "0.0.0.0:9996".parse().unwrap(), - ..Default::default() - }) - .start()?; +use poem::{listener::TcpListener, Route, Server}; +use poem_openapi::{ + payload::Json, + param::Query, + OpenApi, OpenApiService, + ApiResponse, + Object, + types::Example, +}; - tokio::select! { - s = server.wait_for_shutdown() => { - s.map_err(ServerError::ServerExited)?; - log::info!("server shut down normally."); - }, - _ = shutdown.cancelled() => { - log::info!("shutting down: closing server"); - server.close().await.map_err(ServerError::BadClose)?; - }, - } - Ok(()) +fn example_did() -> String { + "did:plc:hdhoaan3xa3jiuq4fg4mefid".to_string() } - -#[derive(Debug, Clone)] -struct Context { - pub cache: HybridCache, - pub spec: Arc, - pub shutdown: CancellationToken, +fn example_collection() -> String { + "app.bsky.feed.like".to_string() } - -async fn instrument_handler(ctx: &RequestContext, handler: H) -> Result -where - R: HttpResponse, - H: Future>, - T: ServerContext, -{ - let start = Instant::now(); - let result = handler.await; - let latency = start.elapsed(); - let status_code = match &result { - Ok(response) => response.status_code(), - Err(e) => e.status_code.as_status(), - } - .as_str() // just the number (.to_string()'s Display does eg `200 OK`) - .to_string(); - let endpoint = ctx.endpoint.operation_id.clone(); - let headers = ctx.request.headers(); - let origin = headers - .get(ORIGIN) - .and_then(|v| v.to_str().ok()) - .unwrap_or("") - .to_string(); - let ua = headers - .get(USER_AGENT) - .and_then(|v| v.to_str().ok()) - .map(|ua| { - if ua.starts_with("Mozilla/5.0 ") { - "browser" - } else { - ua - } - }) - .unwrap_or("") - .to_string(); - counter!("server_requests_total", - "endpoint" => endpoint.clone(), - "origin" => origin, - "ua" => ua, - "status_code" => status_code, - ) - .increment(1); - histogram!("server_handler_latency", "endpoint" => endpoint).record(latency.as_micros() as f64); - result +fn example_rkey() -> String { + "3lv4ouczo2b2a".to_string() } -use dropshot::{HttpResponseHeaders, HttpResponseOk}; - -pub type OkCorsResponse = Result>, HttpError>; - -/// Helper for constructing Ok responses: return OkCors(T).into() -/// (not happy with this yet) -pub struct OkCors(pub T); - -impl From> for OkCorsResponse -where - T: Serialize + JsonSchema + Send + Sync, -{ - fn from(ok: OkCors) -> OkCorsResponse { - let mut res = HttpResponseHeaders::new_unnamed(HttpResponseOk(ok.0)); - res.headers_mut() - .insert("access-control-allow-origin", "*".parse().unwrap()); - Ok(res) - } +#[derive(Object)] +#[oai(example = true)] +struct XrpcErrorResponseObject { + /// Should correspond an error `name` in the lexicon errors array + error: String, + /// Human-readable description and possibly additonal context + message: String, } - -pub fn cors_err(e: HttpError) -> HttpError { - e.with_header("access-control-allow-origin", "*").unwrap() +impl Example for XrpcErrorResponseObject { + fn example() -> Self { + Self { + error: "RecordNotFound".to_string(), + message: "This record was deleted".to_string(), + } + } } -// TODO: cors for HttpError - -/// Serve index page as html -#[endpoint { - method = GET, - path = "/", - /* - * not useful to have this in openapi - */ - unpublished = true, -}] -async fn index(ctx: RequestContext) -> Result, HttpError> { - instrument_handler(&ctx, async { - Ok(Response::builder() - .status(StatusCode::OK) - .header(http::header::CONTENT_TYPE, "text/html") - .body(INDEX_HTML.into())?) - }) - .await +fn bad_request_handler(err: poem::Error) -> GetRecordResponse { + GetRecordResponse::BadRequest(Json(XrpcErrorResponseObject { + error: "InvalidRequest".to_string(), + message: format!("Bad request, here's some info that maybe should not be exposed: {err}"), + })) } -/// Serve index page as html -#[endpoint { - method = GET, - path = "/favicon.ico", - /* - * not useful to have this in openapi - */ - unpublished = true, -}] -async fn favicon(ctx: RequestContext) -> Result, HttpError> { - instrument_handler(&ctx, async { - Ok(Response::builder() - .status(StatusCode::OK) - .header(http::header::CONTENT_TYPE, "image/x-icon") - .body(FAVICON.to_vec().into())?) - }) - .await +#[derive(Object)] +#[oai(example = true)] +struct FoundRecordResponseObject { + /// at-uri for this record + uri: String, + /// CID for this exact version of the record + /// + /// Slingshot will always return the CID, despite it not being a required + /// response property in the official lexicon. + cid: Option, + /// the record itself as JSON + value: serde_json::Value, } - -/// Meta: get the openapi spec for this api -#[endpoint { - method = GET, - path = "/openapi", - /* - * not useful to have this in openapi - */ - unpublished = true, -}] -async fn openapi(ctx: RequestContext) -> OkCorsResponse { - instrument_handler(&ctx, async { - let spec = (*ctx.context().spec).clone(); - OkCors(spec).into() - }) - .await +impl Example for FoundRecordResponseObject { + fn example() -> Self { + Self { + uri: format!("at://{}/{}/{}", example_did(), example_collection(), example_rkey()), + cid: Some("bafyreialv3mzvvxaoyrfrwoer3xmabbmdchvrbyhayd7bga47qjbycy74e".to_string()), + value: serde_json::json!({ + "$type": "app.bsky.feed.like", + "createdAt": "2025-07-29T18:02:02.327Z", + "subject": { + "cid": "bafyreia2gy6eyk5qfetgahvshpq35vtbwy6negpy3gnuulcdi723mi7vxy", + "uri": "at://did:plc:vwzwgnygau7ed7b7wt5ux7y2/app.bsky.feed.post/3lv4lkb4vgs2k" + } + }), + } + } } - -#[derive(Debug, Deserialize, JsonSchema)] -struct GetRecordQuery { - /// The DID of the repo +#[derive(ApiResponse)] +#[oai(bad_request_handler = "bad_request_handler")] +enum GetRecordResponse { + /// Record found + #[oai(status = 200)] + Ok(Json), + /// Bad request or no record to return /// - /// NOTE: handles should be accepted here but this is still TODO in slingshot - pub repo: String, - /// The NSID of the record collection - pub collection: String, - /// The Record key - pub rkey: String, - /// Optional: the CID of the version of the record. - /// - /// If not specified, then return the most recent version. - /// - /// If specified and a newer version of the record exists, returns 404 not - /// found. That is: slingshot only retains the most recent version of a - /// record. - #[serde(default)] - pub cid: Option, + /// The only error name in the repo.getRecord lexicon is `RecordNotFound`, + /// but the [canonical api docs](https://docs.bsky.app/docs/api/com-atproto-repo-get-record) + /// also list `InvalidRequest`, `ExpiredToken`, and `InvalidToken`. Of + /// these, slingshot will only return `RecordNotFound` or `InvalidRequest`. + #[oai(status = 400)] + BadRequest(Json), } -#[derive(Debug, Serialize, JsonSchema)] -struct GetRecordResponse { - pub uri: String, - pub cid: String, - pub value: Box, +struct Xrpc { + cache: HybridCache, } -/// com.atproto.repo.getRecord -/// -/// Get a single record from a repository. Does not require auth. -/// -/// See https://docs.bsky.app/docs/api/com-atproto-repo-get-record for the -/// canonical XRPC documentation that this endpoint aims to be compatible with. -#[endpoint { - method = GET, - path = "/xrpc/com.atproto.repo.getRecord", -}] -async fn get_record( - ctx: RequestContext, - query: Query, -) -> OkCorsResponse { - - let Context { cache, .. } = ctx.context(); - let GetRecordQuery { repo, collection, rkey, cid } = query.into_inner(); - - // TODO: yeah yeah - let at_uri = format!( - "at://{}/{}/{}", - &*repo, &*collection, &*rkey - ); - - instrument_handler(&ctx, async { - let entry = cache +#[OpenApi] +impl Xrpc { + /// com.atproto.repo.getRecord + /// + /// Get a single record from a repository. Does not require auth. + /// + /// See https://docs.bsky.app/docs/api/com-atproto-repo-get-record for the + /// canonical XRPC documentation that this endpoint aims to be compatible + /// with. + #[oai(path = "/com.atproto.repo.getRecord", method = "get")] + async fn get_record( + &self, + /// The DID of the repo + /// + /// NOTE: handles should be accepted here but this is still TODO in slingshot + #[oai(example = "example_did")] + repo: Query, + /// The NSID of the record collection + #[oai(example = "example_collection")] + collection: Query, + /// The Record key + #[oai(example = "example_rkey")] + rkey: Query, + /// Optional: the CID of the version of the record. + /// + /// If not specified, then return the most recent version. + /// + /// If specified and a newer version of the record exists, returns 404 not + /// found. That is: slingshot only retains the most recent version of a + /// record. + cid: Query>, + ) -> GetRecordResponse { + // TODO: yeah yeah + let at_uri = format!( + "at://{}/{}/{}", + &*repo, &*collection, &*rkey + ); + + let entry = self.cache .fetch(at_uri.clone(), || async move { - Err(foyer::Error::Other(Box::new(ServerError::OhNo("booo".to_string())))) + todo!() }) .await .unwrap(); + // TODO: actual 404 + match *entry { CachedRecord::Found(ref raw) => { let (found_cid, raw_value) = raw.into(); let found_cid = found_cid.as_ref().to_string(); - if cid.map(|c| c != found_cid).unwrap_or(false) { - Err(HttpError::for_not_found(None, "CID mismatch".to_string())) - .map_err(cors_err)?; + if cid.clone().map(|c| c != found_cid).unwrap_or(false) { + return GetRecordResponse::BadRequest(Json(XrpcErrorResponseObject { + error: "RecordNotFound".to_string(), + message: "A record was found but its CID did not match that requested".to_string(), + })); } - OkCors(GetRecordResponse { + // TODO: thank u stellz: https://gist.github.com/stella3d/51e679e55b264adff89d00a1e58d0272 + let value = serde_json::from_str(raw_value.get()).expect("RawValue to be valid json"); + GetRecordResponse::Ok(Json(FoundRecordResponseObject { uri: at_uri, - cid: found_cid, - value: raw_value, - }).into() + cid: Some(found_cid), + value, + })) }, CachedRecord::Deleted => { - Err(HttpError::for_client_error_with_status( - Some("Gone".to_string()), - ClientErrorStatusCode::GONE, - )).map_err(cors_err) + GetRecordResponse::BadRequest(Json(XrpcErrorResponseObject { + error: "RecordNotFound".to_string(), + message: "This record was deleted".to_string(), + })) } } - }) - .await + } +} +pub async fn serve( + cache: HybridCache, + _shutdown: CancellationToken, +) -> Result<(), ServerError> { + let api_service = + OpenApiService::new(Xrpc { cache }, "Slingshot", env!("CARGO_PKG_VERSION")) + .server("http://localhost:3000") + .url_prefix("/xrpc"); + + let app = Route::new() + .nest("/", api_service.scalar()) + .nest("/openapi.json", api_service.spec_endpoint()) + .nest("/xrpc/", api_service); + + Server::new(TcpListener::bind("127.0.0.1:3000")) + .run(app) + .await + .map_err(|e| ServerError::ServerExited(format!("uh oh: {e:?}"))) } -- 2.51.2 From 1de5a1b55882d4056be4552e8252c16943944c82 Mon Sep 17 00:00:00 2001 From: phil Date: Tue, 29 Jul 2025 16:53:29 -0400 Subject: [PATCH 076/134] slingshot fmt +ci --- .github/workflows/checks.yml | 2 +- Makefile | 2 +- slingshot/src/consumer.rs | 2 +- slingshot/src/firehose_cache.rs | 9 +++--- slingshot/src/main.rs | 3 -- slingshot/src/record.rs | 7 +++-- slingshot/src/server.rs | 54 +++++++++++++++------------------ 7 files changed, 36 insertions(+), 43 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 6641eb2..f26327e 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -28,7 +28,7 @@ jobs: - name: get nightly toolchain for jetstream fmt run: rustup toolchain install nightly --allow-downgrade -c rustfmt - name: fmt - run: cargo fmt --package links --package constellation --package ufos --package spacedust --package who-am-i -- --check + run: cargo fmt --package links --package constellation --package ufos --package spacedust --package who-am-i --package slingshot -- --check - name: fmt jetstream (nightly) run: cargo +nightly fmt --package jetstream -- --check - name: clippy diff --git a/Makefile b/Makefile index 4803c3c..eb29af2 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ test: cargo test --all-features fmt: - cargo fmt --package links --package constellation --package ufos --package spacedust --package who-am-i + cargo fmt --package links --package constellation --package ufos --package spacedust --package who-am-i --package slingshot cargo +nightly fmt --package jetstream clippy: diff --git a/slingshot/src/consumer.rs b/slingshot/src/consumer.rs index 51fb3ec..ee270d2 100644 --- a/slingshot/src/consumer.rs +++ b/slingshot/src/consumer.rs @@ -1,6 +1,6 @@ use crate::CachedRecord; -use foyer::HybridCache; use crate::error::ConsumerError; +use foyer::HybridCache; use jetstream::{ DefaultJetstreamEndpoints, JetstreamCompression, JetstreamConfig, JetstreamConnector, events::{CommitOp, Cursor, EventKind}, diff --git a/slingshot/src/firehose_cache.rs b/slingshot/src/firehose_cache.rs index d3b9a6d..3c92955 100644 --- a/slingshot/src/firehose_cache.rs +++ b/slingshot/src/firehose_cache.rs @@ -1,9 +1,10 @@ -use std::path::Path; use crate::CachedRecord; -use foyer::{HybridCache, DirectFsDeviceOptions, Engine, HybridCacheBuilder}; - +use foyer::{DirectFsDeviceOptions, Engine, HybridCache, HybridCacheBuilder}; +use std::path::Path; -pub async fn firehose_cache(dir: impl AsRef) -> Result, String> { +pub async fn firehose_cache( + dir: impl AsRef, +) -> Result, String> { let cache = HybridCacheBuilder::new() .with_name("firehose") .memory(64 * 2_usize.pow(20)) diff --git a/slingshot/src/main.rs b/slingshot/src/main.rs index d7ce143..b64902f 100644 --- a/slingshot/src/main.rs +++ b/slingshot/src/main.rs @@ -6,7 +6,6 @@ use slingshot::{consume, error::MainTaskError, firehose_cache, serve}; use clap::Parser; use tokio_util::sync::CancellationToken; - /// Slingshot record edge cache #[derive(Parser, Debug, Clone)] #[command(version, about, long_about = None)] @@ -65,7 +64,6 @@ async fn main() -> Result<(), String> { Ok(()) }); - tokio::select! { _ = shutdown.cancelled() => log::warn!("shutdown requested"), Some(r) = tasks.join_next() => { @@ -111,4 +109,3 @@ fn install_metrics_server() -> Result<(), metrics_exporter_prometheus::BuildErro ); Ok(()) } - diff --git a/slingshot/src/record.rs b/slingshot/src/record.rs index 9df588b..d3ea281 100644 --- a/slingshot/src/record.rs +++ b/slingshot/src/record.rs @@ -1,6 +1,6 @@ -use serde_json::value::RawValue; -use serde::{Serialize, Deserialize}; use jetstream::exports::Cid; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; #[derive(Debug, Serialize, Deserialize)] pub struct RawRecord { @@ -23,7 +23,8 @@ impl From<&RawRecord> for (Cid, Box) { fn from(RawRecord { cid, record }: &RawRecord) -> Self { ( cid.clone(), - RawValue::from_string(record.to_string()).expect("stored string from RawValue to be valid"), + RawValue::from_string(record.to_string()) + .expect("stored string from RawValue to be valid"), ) } } diff --git a/slingshot/src/server.rs b/slingshot/src/server.rs index 4f0e111..3962215 100644 --- a/slingshot/src/server.rs +++ b/slingshot/src/server.rs @@ -1,15 +1,10 @@ +use crate::{CachedRecord, error::ServerError}; use foyer::HybridCache; -use crate::{error::ServerError, CachedRecord}; use tokio_util::sync::CancellationToken; -use poem::{listener::TcpListener, Route, Server}; +use poem::{Route, Server, listener::TcpListener}; use poem_openapi::{ - payload::Json, - param::Query, - OpenApi, OpenApiService, - ApiResponse, - Object, - types::Example, + ApiResponse, Object, OpenApi, OpenApiService, param::Query, payload::Json, types::Example, }; fn example_did() -> String { @@ -39,7 +34,6 @@ impl Example for XrpcErrorResponseObject { } } - fn bad_request_handler(err: poem::Error) -> GetRecordResponse { GetRecordResponse::BadRequest(Json(XrpcErrorResponseObject { error: "InvalidRequest".to_string(), @@ -63,7 +57,12 @@ struct FoundRecordResponseObject { impl Example for FoundRecordResponseObject { fn example() -> Self { Self { - uri: format!("at://{}/{}/{}", example_did(), example_collection(), example_rkey()), + uri: format!( + "at://{}/{}/{}", + example_did(), + example_collection(), + example_rkey() + ), cid: Some("bafyreialv3mzvvxaoyrfrwoer3xmabbmdchvrbyhayd7bga47qjbycy74e".to_string()), value: serde_json::json!({ "$type": "app.bsky.feed.like", @@ -130,15 +129,11 @@ impl Xrpc { cid: Query>, ) -> GetRecordResponse { // TODO: yeah yeah - let at_uri = format!( - "at://{}/{}/{}", - &*repo, &*collection, &*rkey - ); + let at_uri = format!("at://{}/{}/{}", &*repo, &*collection, &*rkey); - let entry = self.cache - .fetch(at_uri.clone(), || async move { - todo!() - }) + let entry = self + .cache + .fetch(at_uri.clone(), || async move { todo!() }) .await .unwrap(); @@ -151,23 +146,23 @@ impl Xrpc { if cid.clone().map(|c| c != found_cid).unwrap_or(false) { return GetRecordResponse::BadRequest(Json(XrpcErrorResponseObject { error: "RecordNotFound".to_string(), - message: "A record was found but its CID did not match that requested".to_string(), + message: "A record was found but its CID did not match that requested" + .to_string(), })); } // TODO: thank u stellz: https://gist.github.com/stella3d/51e679e55b264adff89d00a1e58d0272 - let value = serde_json::from_str(raw_value.get()).expect("RawValue to be valid json"); + let value = + serde_json::from_str(raw_value.get()).expect("RawValue to be valid json"); GetRecordResponse::Ok(Json(FoundRecordResponseObject { uri: at_uri, cid: Some(found_cid), value, })) - }, - CachedRecord::Deleted => { - GetRecordResponse::BadRequest(Json(XrpcErrorResponseObject { - error: "RecordNotFound".to_string(), - message: "This record was deleted".to_string(), - })) } + CachedRecord::Deleted => GetRecordResponse::BadRequest(Json(XrpcErrorResponseObject { + error: "RecordNotFound".to_string(), + message: "This record was deleted".to_string(), + })), } } } @@ -176,10 +171,9 @@ pub async fn serve( cache: HybridCache, _shutdown: CancellationToken, ) -> Result<(), ServerError> { - let api_service = - OpenApiService::new(Xrpc { cache }, "Slingshot", env!("CARGO_PKG_VERSION")) - .server("http://localhost:3000") - .url_prefix("/xrpc"); + let api_service = OpenApiService::new(Xrpc { cache }, "Slingshot", env!("CARGO_PKG_VERSION")) + .server("http://localhost:3000") + .url_prefix("/xrpc"); let app = Route::new() .nest("/", api_service.scalar()) -- 2.51.2 From 1111ae5502fc9ab0ab04837a25e62b705e6bf88c Mon Sep 17 00:00:00 2001 From: phil Date: Tue, 29 Jul 2025 19:06:48 -0400 Subject: [PATCH 077/134] old file cleanup --- slingshot/static/index.html | 53 ------------------------------------- 1 file changed, 53 deletions(-) delete mode 100644 slingshot/static/index.html diff --git a/slingshot/static/index.html b/slingshot/static/index.html deleted file mode 100644 index d598eba..0000000 --- a/slingshot/static/index.html +++ /dev/null @@ -1,53 +0,0 @@ - - - - - Slingshot documentation - - - - - -
-

- todo: what link goes here?: blah -

- -
- - - - - - - - -- 2.51.2 From f87935905a965ec9ce903b4b466861bbf8847357 Mon Sep 17 00:00:00 2001 From: phil Date: Wed, 30 Jul 2025 18:10:08 -0400 Subject: [PATCH 078/134] upstream getrecord --- Cargo.lock | 76 ++--- jetstream/Cargo.toml | 2 +- slingshot/Cargo.toml | 8 + slingshot/readme.md | 7 + slingshot/src/error.rs | 49 +++ slingshot/src/firehose_cache.rs | 4 +- slingshot/src/identity.rs | 510 ++++++++++++++++++++++++++++++++ slingshot/src/lib.rs | 4 +- slingshot/src/main.rs | 37 ++- slingshot/src/record.rs | 101 ++++++- slingshot/src/server.rs | 49 ++- 11 files changed, 771 insertions(+), 76 deletions(-) create mode 100644 slingshot/readme.md create mode 100644 slingshot/src/identity.rs diff --git a/Cargo.lock b/Cargo.lock index 0d49226..2c1d3ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -261,33 +261,14 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" -[[package]] -name = "atrium-api" -version = "0.25.3" -source = "git+https://github.com/uniphil/atrium?branch=fix%2Fnsid-allow-nonleading-name-digits#c4364f318d337bbc3e3e3aaf97c9f971e95f5f7e" -dependencies = [ - "atrium-common 0.1.2 (git+https://github.com/uniphil/atrium?branch=fix%2Fnsid-allow-nonleading-name-digits)", - "atrium-xrpc 0.12.3 (git+https://github.com/uniphil/atrium?branch=fix%2Fnsid-allow-nonleading-name-digits)", - "chrono", - "http", - "ipld-core", - "langtag", - "regex", - "serde", - "serde_bytes", - "serde_json", - "thiserror 1.0.69", - "trait-variant", -] - [[package]] name = "atrium-api" version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46355d3245edc7b3160b2a45fe55d09a6963ebd3eee0252feb6b72fb0eb71463" dependencies = [ - "atrium-common 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "atrium-xrpc 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", + "atrium-common", + "atrium-xrpc", "chrono", "http", "ipld-core", @@ -316,29 +297,15 @@ dependencies = [ "web-time", ] -[[package]] -name = "atrium-common" -version = "0.1.2" -source = "git+https://github.com/uniphil/atrium?branch=fix%2Fnsid-allow-nonleading-name-digits#c4364f318d337bbc3e3e3aaf97c9f971e95f5f7e" -dependencies = [ - "dashmap", - "lru", - "moka", - "thiserror 1.0.69", - "tokio", - "trait-variant", - "web-time", -] - [[package]] name = "atrium-identity" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9e2d42bb4dbea038f4f5f45e3af2a89d61a9894a75f06aa550b74a60d2be380" dependencies = [ - "atrium-api 0.25.4", - "atrium-common 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "atrium-xrpc 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", + "atrium-api", + "atrium-common", + "atrium-xrpc", "serde", "serde_html_form", "serde_json", @@ -352,10 +319,10 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca22dc4eaf77fd9bf050b21192ac58cd654a437d28e000ec114ebd93a51d36f5" dependencies = [ - "atrium-api 0.25.4", - "atrium-common 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "atrium-api", + "atrium-common", "atrium-identity", - "atrium-xrpc 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", + "atrium-xrpc", "base64 0.22.1", "chrono", "dashmap", @@ -389,19 +356,6 @@ dependencies = [ "trait-variant", ] -[[package]] -name = "atrium-xrpc" -version = "0.12.3" -source = "git+https://github.com/uniphil/atrium?branch=fix%2Fnsid-allow-nonleading-name-digits#c4364f318d337bbc3e3e3aaf97c9f971e95f5f7e" -dependencies = [ - "http", - "serde", - "serde_html_form", - "serde_json", - "thiserror 1.0.69", - "trait-variant", -] - [[package]] name = "auto_enums" version = "0.8.7" @@ -2634,7 +2588,7 @@ version = "0.1.1" dependencies = [ "anyhow", "async-trait", - "atrium-api 0.25.3", + "atrium-api", "chrono", "clap", "futures-util", @@ -4797,21 +4751,29 @@ dependencies = [ name = "slingshot" version = "0.1.0" dependencies = [ + "atrium-api", + "atrium-common", + "atrium-identity", + "atrium-oauth", "clap", "ctrlc", "env_logger", "foyer", + "hickory-resolver", "jetstream", "log", "metrics", "metrics-exporter-prometheus 0.17.2", "poem", "poem-openapi", + "reqwest", "serde", "serde_json", "thiserror 2.0.12", + "time", "tokio", "tokio-util", + "url", ] [[package]] @@ -5880,8 +5842,8 @@ dependencies = [ name = "who-am-i" version = "0.1.0" dependencies = [ - "atrium-api 0.25.4", - "atrium-common 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "atrium-api", + "atrium-common", "atrium-identity", "atrium-oauth", "axum", diff --git a/jetstream/Cargo.toml b/jetstream/Cargo.toml index 72f56d3..96bd42d 100644 --- a/jetstream/Cargo.toml +++ b/jetstream/Cargo.toml @@ -10,7 +10,7 @@ readme = "README.md" [dependencies] async-trait = "0.1.83" -atrium-api = { git = "https://github.com/uniphil/atrium", branch = "fix/nsid-allow-nonleading-name-digits", default-features = false, features = [ +atrium-api = { version = "0.25.4", default-features = false, features = [ "namespace-appbsky", ] } tokio = { version = "1.44.2", features = ["full", "sync", "time"] } diff --git a/slingshot/Cargo.toml b/slingshot/Cargo.toml index 6b70c8b..f69de9d 100644 --- a/slingshot/Cargo.toml +++ b/slingshot/Cargo.toml @@ -4,18 +4,26 @@ version = "0.1.0" edition = "2024" [dependencies] +atrium-api = { version = "0.25.4", default-features = false } +atrium-common = "0.1.2" +atrium-identity = "0.1.5" +atrium-oauth = "0.1.3" clap = { version = "4.5.41", features = ["derive"] } ctrlc = "3.4.7" env_logger = "0.11.8" foyer = { version = "0.18.0", features = ["serde"] } +hickory-resolver = "0.25.2" jetstream = { path = "../jetstream", features = ["metrics"] } log = "0.4.27" metrics = "0.24.2" metrics-exporter-prometheus = { version = "0.17.1", features = ["http-listener"] } poem = "3.1.12" poem-openapi = { version = "5.1.16", features = ["scalar"] } +reqwest = { version = "0.12.22", features = ["json"] } serde = { version = "1.0.219", features = ["derive"] } serde_json = { version = "1.0.141", features = ["raw_value"] } thiserror = "2.0.12" +time = { version = "0.3.41", features = ["serde"] } tokio = { version = "1.47.0", features = ["full"] } tokio-util = "0.7.15" +url = "2.5.4" diff --git a/slingshot/readme.md b/slingshot/readme.md new file mode 100644 index 0000000..beb4de8 --- /dev/null +++ b/slingshot/readme.md @@ -0,0 +1,7 @@ +# slingshot: atproto record edge cache + +local dev running: + +```bash +RUST_LOG=info,slingshot=trace ulimit -n 4096 && RUST_LOG=info cargo run -- --jetstream us-east-1 --cache-dir ./foyer +``` diff --git a/slingshot/src/error.rs b/slingshot/src/error.rs index dd6bde6..b2888a7 100644 --- a/slingshot/src/error.rs +++ b/slingshot/src/error.rs @@ -18,10 +18,59 @@ pub enum ServerError { ServerExited(String), } +#[derive(Debug, Error)] +pub enum IdentityError { + #[error("whatever: {0}")] + WhateverError(String), + #[error("bad DID: {0}")] + BadDid(&'static str), + #[error("identity types got mixed up: {0}")] + IdentityValTypeMixup(String), + #[error("foyer error: {0}")] + FoyerError(#[from] foyer::Error), + + #[error("failed to resolve: {0}")] + ResolutionFailed(#[from] atrium_identity::Error), + // #[error("identity resolved but no handle found for user")] + // NoHandle, + #[error("found handle {0:?} but it appears invalid: {1}")] + InvalidHandle(String, &'static str), + + #[error("could not convert atrium did doc to partial mini doc: {0}")] + BadDidDoc(String), + + #[error("wrong key for clearing refresh queue: {0}")] + RefreshQueueKeyError(&'static str), +} + #[derive(Debug, Error)] pub enum MainTaskError { #[error(transparent)] ConsumerTaskError(#[from] ConsumerError), #[error(transparent)] ServerTaskError(#[from] ServerError), + #[error(transparent)] + IdentityTaskError(#[from] IdentityError), +} + +#[derive(Debug, Error)] +pub enum RecordError { + #[error("identity error: {0}")] + IdentityError(#[from] IdentityError), + #[error("repo could not be validated as either a DID or an atproto handle")] + BadRepo, + #[error("could not get record: {0}")] + NotFound(&'static str), + #[error("could nto parse pds url: {0}")] + UrlParseError(#[from] url::ParseError), + #[error("reqwest send failed: {0}")] + SendError(reqwest::Error), + #[error("reqwest raised for status: {0}")] + StatusError(reqwest::Error), + #[error("reqwest failed to parse json: {0}")] + ParseJsonError(reqwest::Error), + #[error("upstream getRecord did not include a CID")] + MissingUpstreamCid, + #[error("upstream CID was not valid: {0}")] + BadUpstreamCid(String), } diff --git a/slingshot/src/firehose_cache.rs b/slingshot/src/firehose_cache.rs index 3c92955..0163dca 100644 --- a/slingshot/src/firehose_cache.rs +++ b/slingshot/src/firehose_cache.rs @@ -3,14 +3,14 @@ use foyer::{DirectFsDeviceOptions, Engine, HybridCache, HybridCacheBuilder}; use std::path::Path; pub async fn firehose_cache( - dir: impl AsRef, + cache_dir: impl AsRef, ) -> Result, String> { let cache = HybridCacheBuilder::new() .with_name("firehose") .memory(64 * 2_usize.pow(20)) .with_weighter(|k: &String, v| k.len() + std::mem::size_of_val(v)) .storage(Engine::large()) - .with_device_options(DirectFsDeviceOptions::new(dir)) + .with_device_options(DirectFsDeviceOptions::new(cache_dir)) .build() .await .map_err(|e| format!("foyer setup error: {e:?}"))?; diff --git a/slingshot/src/identity.rs b/slingshot/src/identity.rs new file mode 100644 index 0000000..4550ce7 --- /dev/null +++ b/slingshot/src/identity.rs @@ -0,0 +1,510 @@ +use hickory_resolver::{ResolveError, TokioResolver}; +use std::collections::{HashSet, VecDeque}; +use std::path::Path; +use std::sync::Arc; +/// for now we're gonna just keep doing more cache +/// +/// plc.director x foyer, ttl kept with data, refresh deferred to background on fetch +/// +/// things we need: +/// +/// 1. handle -> DID resolution: getRecord must accept a handle for `repo` param +/// 2. DID -> PDS resolution: so we know where to getRecord +/// 3. DID -> handle resolution: for bidirectional handle validation and in case we want to offer this +use std::time::Duration; +use tokio::sync::Mutex; + +use crate::error::IdentityError; +use atrium_api::{ + did_doc::DidDocument, + types::string::{Did, Handle}, +}; +use atrium_common::resolver::Resolver; +use atrium_identity::{ + did::{CommonDidResolver, CommonDidResolverConfig, DEFAULT_PLC_DIRECTORY_URL}, + handle::{AtprotoHandleResolver, AtprotoHandleResolverConfig, DnsTxtResolver}, +}; +use atrium_oauth::DefaultHttpClient; // it's probably not worth bringing all of atrium_oauth for this but +use foyer::{DirectFsDeviceOptions, Engine, HybridCache, HybridCacheBuilder}; +use serde::{Deserialize, Serialize}; +use time::UtcDateTime; + +/// once we have something resolved, don't re-resolve until after this period +const MIN_TTL: Duration = Duration::from_secs(4 * 3600); // probably shoudl have a max ttl +const MIN_NOT_FOUND_TTL: Duration = Duration::from_secs(60); + +#[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] +enum IdentityKey { + Handle(Handle), + Did(Did), +} + +#[derive(Debug, Serialize, Deserialize)] +struct IdentityVal(UtcDateTime, IdentityData); + +#[derive(Debug, Serialize, Deserialize)] +enum IdentityData { + NotFound, + Did(Did), + Doc(PartialMiniDoc), +} + +/// partial representation of a com.bad-example.identity mini atproto doc +/// +/// partial because the handle is not verified +#[derive(Debug, Clone, Serialize, Deserialize)] +struct PartialMiniDoc { + /// an atproto handle (**unverified**) + /// + /// the first valid atproto handle from the did doc's aka + unverified_handle: Handle, + /// the did's atproto pds url (TODO: type this?) + /// + /// note: atrium *does* actually parse it into a URI, it just doesn't return + /// that for some reason + pds: String, + /// for now we're just pulling this straight from the did doc + /// + /// would be nice to type and validate it + /// + /// this is the publicKeyMultibase from the did doc. + /// legacy key encoding not supported. + /// `id`, `type`, and `controller` must be checked, but aren't stored. + signing_key: String, +} + +impl TryFrom for PartialMiniDoc { + type Error = String; + fn try_from(did_doc: DidDocument) -> Result { + // must use the first valid handle + let mut unverified_handle = None; + let Some(ref doc_akas) = did_doc.also_known_as else { + return Err("did doc missing `also_known_as`".to_string()); + }; + for aka in doc_akas { + let Some(maybe_handle) = aka.strip_prefix("at://") else { + continue; + }; + let Ok(valid_handle) = Handle::new(maybe_handle.to_string()) else { + continue; + }; + unverified_handle = Some(valid_handle); + break; + } + let Some(unverified_handle) = unverified_handle else { + return Err("no valid atproto handles in `also_known_as`".to_string()); + }; + + // atrium seems to get service endpoint getters + let Some(pds) = did_doc.get_pds_endpoint() else { + return Err("no valid pds service found".to_string()); + }; + + // TODO can't use atrium's get_signing_key() becuase it fails to check type and controller + // so if we check those and reject it, we might miss a later valid key in the array + // (todo is to fix atrium) + // actually: atrium might be flexible for legacy reps. for now we're rejecting legacy rep. + + // must use the first valid signing key + let mut signing_key = None; + let Some(verification_methods) = did_doc.verification_method else { + return Err("no verification methods found".to_string()); + }; + for method in verification_methods { + if method.id != format!("{}#atproto", did_doc.id) { + continue; + } + if method.r#type != "Multikey" { + continue; + } + if method.controller != did_doc.id { + continue; + } + let Some(key) = method.public_key_multibase else { + continue; + }; + signing_key = Some(key); + break; + } + let Some(signing_key) = signing_key else { + return Err("no valid atproto signing key found in verification methods".to_string()); + }; + + Ok(PartialMiniDoc { + unverified_handle, + pds, + signing_key, + }) + } +} + +/// multi-producer *single-consumer* queue structures (wrap in arc-mutex plz) +/// +/// the hashset allows testing for presense of items in the queue. +/// this has absolutely no support for multiple queue consumers. +#[derive(Debug, Default)] +struct RefreshQueue { + queue: VecDeque, + items: HashSet, +} + +#[derive(Clone)] +pub struct Identity { + handle_resolver: Arc>, + did_resolver: Arc>, + cache: HybridCache, + /// multi-producer *single consumer* queue + refresh_queue: Arc>, + /// just a lock to ensure only one refresher (queue consumer) is running (to be improved with a better refresher) + refresher: Arc>, +} + +impl Identity { + pub async fn new(cache_dir: impl AsRef) -> Result { + let http_client = Arc::new(DefaultHttpClient::default()); + let handle_resolver = AtprotoHandleResolver::new(AtprotoHandleResolverConfig { + dns_txt_resolver: HickoryDnsTxtResolver::new().unwrap(), + http_client: http_client.clone(), + }); + let did_resolver = CommonDidResolver::new(CommonDidResolverConfig { + plc_directory_url: DEFAULT_PLC_DIRECTORY_URL.to_string(), + http_client: http_client.clone(), + }); + + let cache = HybridCacheBuilder::new() + .with_name("identity") + .memory(16 * 2_usize.pow(20)) + .with_weighter(|k, v| std::mem::size_of_val(k) + std::mem::size_of_val(v)) + .storage(Engine::large()) + .with_device_options(DirectFsDeviceOptions::new(cache_dir)) + .build() + .await?; + + Ok(Self { + handle_resolver: Arc::new(handle_resolver), + did_resolver: Arc::new(did_resolver), + cache, + refresh_queue: Default::default(), + refresher: Default::default(), + }) + } + + /// Resolve (and verify!) an atproto handle to a DID + /// + /// The result can be stale + /// + /// `None` if the handle can't be found or verification fails + pub async fn handle_to_did(&self, handle: Handle) -> Result, IdentityError> { + let Some(did) = self.handle_to_unverified_did(&handle).await? else { + return Ok(None); + }; + let Some(doc) = self.did_to_partial_mini_doc(&did).await? else { + return Ok(None); + }; + if doc.unverified_handle != handle { + return Ok(None); + } + Ok(Some(did)) + } + + /// Resolve (and verify!) a DID to a pds url + /// + /// This *also* incidentally resolves and verifies the handle, which might + /// make it slower than expected + pub async fn did_to_pds(&self, did: Did) -> Result, IdentityError> { + let Some(mini_doc) = self.did_to_partial_mini_doc(&did).await? else { + return Ok(None); + }; + Ok(Some(mini_doc.pds)) + } + + /// Resolve (and cache but **not verify**) a handle to a DID + async fn handle_to_unverified_did( + &self, + handle: &Handle, + ) -> Result, IdentityError> { + let key = IdentityKey::Handle(handle.clone()); + let entry = self + .cache + .fetch(key.clone(), { + let handle = handle.clone(); + let resolver = self.handle_resolver.clone(); + || async move { + match resolver.resolve(&handle).await { + Ok(did) => Ok(IdentityVal(UtcDateTime::now(), IdentityData::Did(did))), + Err(atrium_identity::Error::NotFound) => { + Ok(IdentityVal(UtcDateTime::now(), IdentityData::NotFound)) + } + Err(other) => Err(foyer::Error::Other(Box::new( + IdentityError::ResolutionFailed(other), + ))), + } + } + }) + .await?; + + let now = UtcDateTime::now(); + let IdentityVal(last_fetch, data) = entry.value(); + match data { + IdentityData::Doc(_) => { + log::error!("identity value mixup: got a doc from a handle key (should be a did)"); + Err(IdentityError::IdentityValTypeMixup(handle.to_string())) + } + IdentityData::NotFound => { + if (now - *last_fetch) >= MIN_NOT_FOUND_TTL { + self.queue_refresh(key).await; + } + Ok(None) + } + IdentityData::Did(did) => { + if (now - *last_fetch) >= MIN_TTL { + self.queue_refresh(key).await; + } + Ok(Some(did.clone())) + } + } + } + + /// Fetch (and cache) a partial mini doc from a did + async fn did_to_partial_mini_doc( + &self, + did: &Did, + ) -> Result, IdentityError> { + let key = IdentityKey::Did(did.clone()); + let entry = self + .cache + .fetch(key.clone(), { + let did = did.clone(); + let resolver = self.did_resolver.clone(); + || async move { + match resolver.resolve(&did).await { + Ok(did_doc) => { + // TODO: fix in atrium: should verify id is did + if did_doc.id != did.to_string() { + return Err(foyer::Error::other(Box::new( + IdentityError::BadDidDoc( + "did doc's id did not match did".to_string(), + ), + ))); + } + let mini_doc = did_doc.try_into().map_err(|e| { + foyer::Error::Other(Box::new(IdentityError::BadDidDoc(e))) + })?; + Ok(IdentityVal(UtcDateTime::now(), IdentityData::Doc(mini_doc))) + } + Err(atrium_identity::Error::NotFound) => { + Ok(IdentityVal(UtcDateTime::now(), IdentityData::NotFound)) + } + Err(other) => Err(foyer::Error::Other(Box::new( + IdentityError::ResolutionFailed(other), + ))), + } + } + }) + .await?; + + let now = UtcDateTime::now(); + let IdentityVal(last_fetch, data) = entry.value(); + match data { + IdentityData::Did(_) => { + log::error!("identity value mixup: got a did from a did key (should be a doc)"); + Err(IdentityError::IdentityValTypeMixup(did.to_string())) + } + IdentityData::NotFound => { + if (now - *last_fetch) >= MIN_NOT_FOUND_TTL { + self.queue_refresh(key).await; + } + Ok(None) + } + IdentityData::Doc(mini_did) => { + if (now - *last_fetch) >= MIN_TTL { + self.queue_refresh(key).await; + } + Ok(Some(mini_did.clone())) + } + } + } + + /// put a refresh task on the queue + /// + /// this can be safely called from multiple concurrent tasks + async fn queue_refresh(&self, key: IdentityKey) { + // todo: max queue size + let mut q = self.refresh_queue.lock().await; + if !q.items.contains(&key) { + q.items.insert(key.clone()); + q.queue.push_back(key); + } + } + + /// find out what's next in the queue. concurrent consumers are not allowed. + /// + /// intent is to leave the item in the queue while refreshing, so that a + /// producer will not re-add it if it's in progress. there's definitely + /// better ways to do this, but this is ~simple for as far as a single + /// consumer can take us. + /// + /// we could take it from the queue but leave it in the set and remove from + /// set later, but splitting them apart feels more bug-prone. + async fn peek_refresh(&self) -> Option { + let q = self.refresh_queue.lock().await; + q.queue.front().cloned() + } + + /// call to clear the latest key from the refresh queue. concurrent consumers not allowed. + /// + /// must provide the last peeked refresh queue item as a small safety check + async fn complete_refresh(&self, key: &IdentityKey) -> Result<(), IdentityError> { + let mut q = self.refresh_queue.lock().await; + + let Some(queue_key) = q.queue.pop_front() else { + // gone from queue + since we're in an error condition, make sure it's not stuck in items + // (not toctou because we have the lock) + // bolder here than below and removing from items because if the queue is *empty*, then we + // know it hasn't been re-added since losing sync. + if q.items.remove(key) { + log::error!("identity refresh: queue de-sync: not in "); + } else { + log::warn!( + "identity refresh: tried to complete with wrong key. are multiple queue consumers running?" + ); + } + return Err(IdentityError::RefreshQueueKeyError("no key in queue")); + }; + + if queue_key != *key { + // extra weird case here, what's the most defensive behaviour? + // we have two keys: ours should have been first but isn't. this shouldn't happen, so let's + // just leave items alone for it. risks unbounded growth but we're in a bad place already. + // the other key is the one we just popped. we didn't want it, so maybe we should put it + // back, BUT if we somehow ended up with concurrent consumers, we have bigger problems. take + // responsibility for taking it instead: remove it from items as well, and just drop it. + // + // hope that whoever calls us takes this error seriously. + if q.items.remove(&queue_key) { + log::warn!( + "identity refresh: queue de-sync + dropping a bystander key without refreshing it!" + ); + } else { + // you thought things couldn't get weirder? (i mean hopefully they can't) + log::error!("identity refresh: queue de-sync + bystander key also de-sync!?"); + } + return Err(IdentityError::RefreshQueueKeyError( + "wrong key at front of queue", + )); + } + + if q.items.remove(key) { + Ok(()) + } else { + log::error!("identity refresh: queue de-sync: key not in items"); + Err(IdentityError::RefreshQueueKeyError("key not in items")) + } + } + + /// run the refresh queue consumer + pub async fn run_refresher(&self) -> Result<(), IdentityError> { + let _guard = self + .refresher + .try_lock() + .expect("there to only be one refresher running"); + loop { + let Some(task_key) = self.peek_refresh().await else { + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + continue; + }; + match task_key { + IdentityKey::Handle(ref handle) => { + log::trace!("refreshing handle {handle:?}"); + match self.handle_resolver.resolve(handle).await { + Ok(did) => { + self.cache.insert( + task_key.clone(), + IdentityVal(UtcDateTime::now(), IdentityData::Did(did)), + ); + } + Err(atrium_identity::Error::NotFound) => { + self.cache.insert( + task_key.clone(), + IdentityVal(UtcDateTime::now(), IdentityData::NotFound), + ); + } + Err(err) => { + log::warn!( + "failed to refresh handle: {err:?}. leaving stale (should we eventually do something?)" + ); + } + } + self.complete_refresh(&task_key).await?; // failures are bugs, so break loop + } + IdentityKey::Did(ref did) => { + log::trace!("refreshing did doc: {did:?}"); + + match self.did_resolver.resolve(did).await { + Ok(did_doc) => { + // TODO: fix in atrium: should verify id is did + if did_doc.id != did.to_string() { + log::warn!( + "refreshed did doc failed: wrong did doc id. dropping refresh." + ); + continue; + } + let mini_doc = match did_doc.try_into() { + Ok(md) => md, + Err(e) => { + log::warn!( + "converting mini doc failed: {e:?}. dropping refresh." + ); + continue; + } + }; + self.cache.insert( + task_key.clone(), + IdentityVal(UtcDateTime::now(), IdentityData::Doc(mini_doc)), + ); + } + Err(atrium_identity::Error::NotFound) => { + self.cache.insert( + task_key.clone(), + IdentityVal(UtcDateTime::now(), IdentityData::NotFound), + ); + } + Err(err) => { + log::warn!( + "failed to refresh did doc: {err:?}. leaving stale (should we eventually do something?)" + ); + } + } + + self.complete_refresh(&task_key).await?; // failures are bugs, so break loop + } + } + } + } +} + +pub struct HickoryDnsTxtResolver(TokioResolver); + +impl HickoryDnsTxtResolver { + fn new() -> Result { + Ok(Self(TokioResolver::builder_tokio()?.build())) + } +} + +impl DnsTxtResolver for HickoryDnsTxtResolver { + async fn resolve( + &self, + query: &str, + ) -> core::result::Result, Box> { + match self.0.txt_lookup(query).await { + Ok(r) => { + metrics::counter!("whoami_resolve_dns_txt", "success" => "true").increment(1); + Ok(r.iter().map(|r| r.to_string()).collect()) + } + Err(e) => { + metrics::counter!("whoami_resolve_dns_txt", "success" => "false").increment(1); + Err(e.into()) + } + } + } +} diff --git a/slingshot/src/lib.rs b/slingshot/src/lib.rs index 240b4a7..a5f2496 100644 --- a/slingshot/src/lib.rs +++ b/slingshot/src/lib.rs @@ -1,10 +1,12 @@ mod consumer; pub mod error; mod firehose_cache; +mod identity; mod record; mod server; pub use consumer::consume; pub use firehose_cache::firehose_cache; -pub use record::CachedRecord; +pub use identity::Identity; +pub use record::{CachedRecord, Repo}; pub use server::serve; diff --git a/slingshot/src/main.rs b/slingshot/src/main.rs index b64902f..843f3e7 100644 --- a/slingshot/src/main.rs +++ b/slingshot/src/main.rs @@ -1,7 +1,8 @@ // use foyer::HybridCache; // use foyer::{Engine, DirectFsDeviceOptions, HybridCacheBuilder}; use metrics_exporter_prometheus::PrometheusBuilder; -use slingshot::{consume, error::MainTaskError, firehose_cache, serve}; +use slingshot::{Identity, Repo, consume, error::MainTaskError, firehose_cache, serve}; +use std::path::PathBuf; use clap::Parser; use tokio_util::sync::CancellationToken; @@ -19,6 +20,9 @@ struct Args { /// reduces CPU at the expense of more ingress bandwidth #[arg(long, action)] jetstream_no_zstd: bool, + /// where to keep disk caches + #[arg(long)] + cache_dir: PathBuf, } #[tokio::main] @@ -38,16 +42,43 @@ async fn main() -> Result<(), String> { log::info!("metrics listening at http://0.0.0.0:8765"); } + std::fs::create_dir_all(&args.cache_dir).map_err(|e| { + format!( + "failed to ensure cache parent dir: {e:?} (dir: {:?})", + args.cache_dir + ) + })?; + let cache_dir = args.cache_dir.canonicalize().map_err(|e| { + format!( + "failed to canonicalize cache_dir: {e:?} (dir: {:?})", + args.cache_dir + ) + })?; + log::info!("cache dir ready at at {cache_dir:?}."); + log::info!("setting up firehose cache..."); - let cache = firehose_cache("./foyer").await?; + let cache = firehose_cache(cache_dir.join("./firehose")).await?; log::info!("firehose cache ready."); let mut tasks: tokio::task::JoinSet> = tokio::task::JoinSet::new(); + log::info!("starting identity service..."); + let identity = Identity::new(cache_dir.join("./identity")) + .await + .map_err(|e| format!("identity setup failed: {e:?}"))?; + log::info!("identity service ready."); + let identity_refresher = identity.clone(); + tasks.spawn(async move { + identity_refresher.run_refresher().await?; + Ok(()) + }); + + let repo = Repo::new(identity); + let server_shutdown = shutdown.clone(); let server_cache_handle = cache.clone(); tasks.spawn(async move { - serve(server_cache_handle, server_shutdown).await?; + serve(server_cache_handle, repo, server_shutdown).await?; Ok(()) }); diff --git a/slingshot/src/record.rs b/slingshot/src/record.rs index d3ea281..853abf9 100644 --- a/slingshot/src/record.rs +++ b/slingshot/src/record.rs @@ -1,6 +1,13 @@ -use jetstream::exports::Cid; +//! cached record storage + +use crate::{Identity, error::RecordError}; +use atrium_api::types::string::{Cid, Did, Handle}; +use reqwest::Client; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; +use std::str::FromStr; +use std::time::Duration; +use url::Url; #[derive(Debug, Serialize, Deserialize)] pub struct RawRecord { @@ -34,3 +41,95 @@ pub enum CachedRecord { Found(RawRecord), Deleted, } + +//////// upstream record fetching + +#[derive(Deserialize)] +struct RecordResponseObject { + #[allow(dead_code)] // expect it to be there but we ignore it + uri: String, + /// CID for this exact version of the record + /// + /// this is optional in the spec and that's potentially TODO for slingshot + cid: Option, + /// the record itself as JSON + value: Box, +} + +#[derive(Clone)] +pub struct Repo { + identity: Identity, + client: Client, +} + +impl Repo { + pub fn new(identity: Identity) -> Self { + let client = Client::builder() + .user_agent(format!( + "microcosm slingshot v{} (dev: @bad-example.com)", + env!("CARGO_PKG_VERSION") + )) + .no_proxy() + .timeout(Duration::from_secs(10)) + .build() + .unwrap(); + Repo { identity, client } + } + + pub async fn get_record( + &self, + did_or_handle: String, + collection: String, + rkey: String, + cid: Option, + ) -> Result { + let did = match Did::new(did_or_handle.clone()) { + Ok(did) => did, + Err(_) => { + let handle = Handle::new(did_or_handle).map_err(|_| RecordError::BadRepo)?; + let Some(did) = self.identity.handle_to_did(handle).await? else { + return Err(RecordError::NotFound("could not resolve and verify handle")); + }; + did + } + }; + let Some(pds) = self.identity.did_to_pds(did.clone()).await? else { + return Err(RecordError::NotFound("could not get pds for DID")); + }; + + // TODO: throttle by host probably, generally guard against outgoing requests + + let mut params = vec![ + ("repo", did.to_string()), + ("collection", collection), + ("rkey", rkey), + ]; + if let Some(cid) = cid { + params.push(("cid", cid)); + } + let mut url = Url::parse_with_params(&pds, ¶ms)?; + url.set_path("/xrpc/com.atproto.repo.getRecord"); + + let res = self + .client + .get(url) + .send() + .await + .map_err(RecordError::SendError)? + .error_for_status() + .map_err(RecordError::StatusError)? // TODO atproto error handling (think about handling not found) + .json::() + .await + .map_err(RecordError::ParseJsonError)?; // todo... + + let Some(cid) = res.cid else { + return Err(RecordError::MissingUpstreamCid); + }; + let cid = Cid::from_str(&cid).map_err(|e| RecordError::BadUpstreamCid(e.to_string()))?; + + Ok(CachedRecord::Found(RawRecord { + cid, + record: res.value.to_string(), + })) + } +} diff --git a/slingshot/src/server.rs b/slingshot/src/server.rs index 3962215..a27121b 100644 --- a/slingshot/src/server.rs +++ b/slingshot/src/server.rs @@ -1,5 +1,6 @@ -use crate::{CachedRecord, error::ServerError}; +use crate::{CachedRecord, Repo, error::ServerError}; use foyer::HybridCache; +use std::sync::Arc; use tokio_util::sync::CancellationToken; use poem::{Route, Server, listener::TcpListener}; @@ -94,6 +95,7 @@ enum GetRecordResponse { struct Xrpc { cache: HybridCache, + repo: Arc, } #[OpenApi] @@ -112,13 +114,13 @@ impl Xrpc { /// /// NOTE: handles should be accepted here but this is still TODO in slingshot #[oai(example = "example_did")] - repo: Query, + Query(repo): Query, /// The NSID of the record collection #[oai(example = "example_collection")] - collection: Query, + Query(collection): Query, /// The Record key #[oai(example = "example_rkey")] - rkey: Query, + Query(rkey): Query, /// Optional: the CID of the version of the record. /// /// If not specified, then return the most recent version. @@ -126,16 +128,25 @@ impl Xrpc { /// If specified and a newer version of the record exists, returns 404 not /// found. That is: slingshot only retains the most recent version of a /// record. - cid: Query>, + Query(cid): Query>, ) -> GetRecordResponse { // TODO: yeah yeah - let at_uri = format!("at://{}/{}/{}", &*repo, &*collection, &*rkey); + let at_uri = format!("at://{repo}/{collection}/{rkey}"); let entry = self .cache - .fetch(at_uri.clone(), || async move { todo!() }) + .fetch(at_uri.clone(), { + let cid = cid.clone(); + let repo_api = self.repo.clone(); + || async move { + repo_api + .get_record(repo, collection, rkey, cid) + .await + .map_err(|e| foyer::Error::Other(Box::new(e))) + } + }) .await - .unwrap(); + .unwrap(); // todo // TODO: actual 404 @@ -165,15 +176,31 @@ impl Xrpc { })), } } + + // TODO + // #[oai(path = "/com.atproto.identity.resolveHandle", method = "get")] + // #[oai(path = "/com.atproto.identity.resolveDid", method = "get")] + // but these are both not specified to do bidirectional validation, which is what we want to offer + // com.atproto.identity.resolveIdentity seems right, but requires returning the full did-doc + // would be nice if there were two queries: + // did -> verified handle + pds url + // handle -> verified did + pds url + // + // we could do horrible things and implement resolveIdentity with only a stripped-down fake did doc + // but this will *definitely* cause problems because eg. we're not currently storing pubkeys and + // those are a little bit important } pub async fn serve( cache: HybridCache, + repo: Repo, _shutdown: CancellationToken, ) -> Result<(), ServerError> { - let api_service = OpenApiService::new(Xrpc { cache }, "Slingshot", env!("CARGO_PKG_VERSION")) - .server("http://localhost:3000") - .url_prefix("/xrpc"); + let repo = Arc::new(repo); + let api_service = + OpenApiService::new(Xrpc { cache, repo }, "Slingshot", env!("CARGO_PKG_VERSION")) + .server("http://localhost:3000") + .url_prefix("/xrpc"); let app = Route::new() .nest("/", api_service.scalar()) -- 2.51.2 From 62eafdee74cefda1a2e37fc8bb2bb6be21026cce Mon Sep 17 00:00:00 2001 From: phil Date: Thu, 31 Jul 2025 12:09:46 -0400 Subject: [PATCH 079/134] serve service did doc --- slingshot/src/main.rs | 10 +++++++++- slingshot/src/server.rs | 41 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/slingshot/src/main.rs b/slingshot/src/main.rs index 843f3e7..63a2fa0 100644 --- a/slingshot/src/main.rs +++ b/slingshot/src/main.rs @@ -23,6 +23,14 @@ struct Args { /// where to keep disk caches #[arg(long)] cache_dir: PathBuf, + /// the domain pointing to this server + /// + /// if present: + /// - a did:web document will be served at /.well-known/did.json + /// - TODO: HTTPS certs will be automatically configured with Acme/letsencrypt + /// - TODO: a rate-limiter will be installed + #[arg(long)] + host: Option, } #[tokio::main] @@ -78,7 +86,7 @@ async fn main() -> Result<(), String> { let server_shutdown = shutdown.clone(); let server_cache_handle = cache.clone(); tasks.spawn(async move { - serve(server_cache_handle, repo, server_shutdown).await?; + serve(server_cache_handle, repo, args.host, server_shutdown).await?; Ok(()) }); diff --git a/slingshot/src/server.rs b/slingshot/src/server.rs index a27121b..7e8619b 100644 --- a/slingshot/src/server.rs +++ b/slingshot/src/server.rs @@ -1,9 +1,10 @@ use crate::{CachedRecord, Repo, error::ServerError}; use foyer::HybridCache; +use serde::Serialize; use std::sync::Arc; use tokio_util::sync::CancellationToken; -use poem::{Route, Server, listener::TcpListener}; +use poem::{Endpoint, Route, Server, endpoint::make_sync, listener::TcpListener}; use poem_openapi::{ ApiResponse, Object, OpenApi, OpenApiService, param::Query, payload::Json, types::Example, }; @@ -191,9 +192,41 @@ impl Xrpc { // those are a little bit important } +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct AppViewService { + id: String, + r#type: String, + service_endpoint: String, +} +#[derive(Debug, Clone, Serialize)] +struct AppViewDoc { + id: String, + service: [AppViewService; 1], +} +/// Serve a did document for did:web for this to be an xrpc appview +/// +/// No slingshot endpoints currently require auth, so it's not necessary to do +/// service proxying, however clients may wish to: +/// +/// - PDS proxying offers a level of client IP anonymity from slingshot +/// - slingshot *may* implement more generous per-user rate-limits for proxied requests in the future +fn get_did_doc(host: String) -> impl Endpoint { + let doc = poem::web::Json(AppViewDoc { + id: format!("did:web:{host}"), + service: [AppViewService { + id: "#slingshot".to_string(), + r#type: "SlingshotRecordProxy".to_string(), + service_endpoint: format!("https://{host}"), + }], + }); + make_sync(move |_| doc.clone()) +} + pub async fn serve( cache: HybridCache, repo: Repo, + host: Option, _shutdown: CancellationToken, ) -> Result<(), ServerError> { let repo = Arc::new(repo); @@ -202,11 +235,15 @@ pub async fn serve( .server("http://localhost:3000") .url_prefix("/xrpc"); - let app = Route::new() + let mut app = Route::new() .nest("/", api_service.scalar()) .nest("/openapi.json", api_service.spec_endpoint()) .nest("/xrpc/", api_service); + if let Some(host) = host { + app = app.at("/.well-known/did.json", get_did_doc(host)); + }; + Server::new(TcpListener::bind("127.0.0.1:3000")) .run(app) .await -- 2.51.2 From 7b07cb8fac6a971a771e36bd4b8e1c281af47f10 Mon Sep 17 00:00:00 2001 From: phil Date: Thu, 31 Jul 2025 12:17:59 -0400 Subject: [PATCH 080/134] acme???? --- Cargo.lock | 189 ++++++++++++++++++++++++++++++++++++++++ slingshot/Cargo.toml | 2 +- slingshot/src/error.rs | 4 +- slingshot/src/server.rs | 38 ++++++-- 4 files changed, 222 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2c1d3ca..17ef55f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -180,6 +180,45 @@ dependencies = [ "nom", ] +[[package]] +name = "asn1-rs" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56624a96882bb8c26d61312ae18cb45868e5a9992ea73c58e45c3101e56a1e60" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 2.0.12", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.103", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.103", +] + [[package]] name = "async-channel" version = "2.5.0" @@ -1201,6 +1240,20 @@ dependencies = [ "zeroize", ] +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + [[package]] name = "deranged" version = "0.4.0" @@ -1906,9 +1959,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", "wasi 0.14.2+wasi-0.2.4", + "wasm-bindgen", ] [[package]] @@ -2868,6 +2923,12 @@ dependencies = [ "hashbrown 0.15.2", ] +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "lsm-tree" version = "2.8.0" @@ -3405,6 +3466,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -3710,7 +3780,9 @@ version = "3.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f977080932c87287147dca052951c3e2696f8759863f6b4e4c0c9ffe7a4cc8b" dependencies = [ + "base64 0.22.1", "bytes", + "chrono", "futures-util", "headers", "http", @@ -3725,8 +3797,12 @@ dependencies = [ "pin-project-lite", "poem-derive", "quick-xml", + "rcgen", "regex", + "reqwest", "rfc7239", + "ring", + "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", @@ -3736,10 +3812,12 @@ dependencies = [ "tempfile", "thiserror 2.0.12", "tokio", + "tokio-rustls 0.26.2", "tokio-stream", "tokio-util", "tracing", "wildmatch", + "x509-parser", ] [[package]] @@ -3922,6 +4000,61 @@ dependencies = [ "hashbrown 0.15.2", ] +[[package]] +name = "quinn" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626214629cda6781b6dc1d316ba307189c85ba657213ce642d9c77670f8202c8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash 2.1.1", + "rustls 0.23.28", + "socket2 0.5.9", + "thiserror 2.0.12", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49df843a9161c85bb8aae55f101bc0bac8bcafd637a620d9122fd7e0b2f7422e" +dependencies = [ + "bytes", + "getrandom 0.3.3", + "lru-slab", + "rand 0.9.1", + "ring", + "rustc-hash 2.1.1", + "rustls 0.23.28", + "rustls-pki-types", + "slab", + "thiserror 2.0.12", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcebb1209ee276352ef14ff8732e24cc2b02bbac986cd74a4c81bcb2f9881970" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2 0.5.9", + "tracing", + "windows-sys 0.52.0", +] + [[package]] name = "quote" version = "1.0.40" @@ -4034,6 +4167,18 @@ dependencies = [ "bitflags", ] +[[package]] +name = "rcgen" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48406db8ac1f3cbc7dcdb56ec355343817958a356ff430259bb07baf7607e1e1" +dependencies = [ + "pem", + "ring", + "time", + "yasna", +] + [[package]] name = "redox_syscall" version = "0.5.11" @@ -4144,6 +4289,9 @@ dependencies = [ "native-tls", "percent-encoding", "pin-project-lite", + "quinn", + "rustls 0.23.28", + "rustls-native-certs", "rustls-pki-types", "serde", "serde_json", @@ -4151,6 +4299,7 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-native-tls", + "tokio-rustls 0.26.2", "tokio-util", "tower", "tower-http", @@ -4266,6 +4415,15 @@ dependencies = [ "semver", ] +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + [[package]] name = "rustix" version = "0.38.44" @@ -4313,7 +4471,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7160e3e10bf4535308537f3c4e1641468cd0e485175d6163087c0393c7d46643" dependencies = [ "aws-lc-rs", + "log", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki 0.103.3", "subtle", @@ -4346,6 +4506,9 @@ name = "rustls-pki-types" version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" +dependencies = [ + "web-time", +] [[package]] name = "rustls-webpki" @@ -6245,12 +6408,38 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "x509-parser" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4569f339c0c402346d4a75a9e39cf8dad310e287eef1ff56d4c68e5067f53460" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "rusticata-macros", + "thiserror 2.0.12", + "time", +] + [[package]] name = "xxhash-rust" version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", +] + [[package]] name = "yoke" version = "0.7.5" diff --git a/slingshot/Cargo.toml b/slingshot/Cargo.toml index f69de9d..ca24da4 100644 --- a/slingshot/Cargo.toml +++ b/slingshot/Cargo.toml @@ -17,7 +17,7 @@ jetstream = { path = "../jetstream", features = ["metrics"] } log = "0.4.27" metrics = "0.24.2" metrics-exporter-prometheus = { version = "0.17.1", features = ["http-listener"] } -poem = "3.1.12" +poem = { version = "3.1.12", features = ["acme"] } poem-openapi = { version = "5.1.16", features = ["scalar"] } reqwest = { version = "0.12.22", features = ["json"] } serde = { version = "1.0.219", features = ["derive"] } diff --git a/slingshot/src/error.rs b/slingshot/src/error.rs index b2888a7..a062bfb 100644 --- a/slingshot/src/error.rs +++ b/slingshot/src/error.rs @@ -14,8 +14,10 @@ pub enum ConsumerError { #[derive(Debug, Error)] pub enum ServerError { + #[error("server build error: {0}")] + AcmeBuildError(std::io::Error), #[error("server exited: {0}")] - ServerExited(String), + ServerExited(std::io::Error), } #[derive(Debug, Error)] diff --git a/slingshot/src/server.rs b/slingshot/src/server.rs index 7e8619b..0f4b866 100644 --- a/slingshot/src/server.rs +++ b/slingshot/src/server.rs @@ -4,7 +4,14 @@ use serde::Serialize; use std::sync::Arc; use tokio_util::sync::CancellationToken; -use poem::{Endpoint, Route, Server, endpoint::make_sync, listener::TcpListener}; +use poem::{ + Endpoint, Route, Server, + endpoint::make_sync, + listener::{ + Listener, TcpListener, + acme::{AutoCert, LETS_ENCRYPT_PRODUCTION}, + }, +}; use poem_openapi::{ ApiResponse, Object, OpenApi, OpenApiService, param::Query, payload::Json, types::Example, }; @@ -211,7 +218,7 @@ struct AppViewDoc { /// /// - PDS proxying offers a level of client IP anonymity from slingshot /// - slingshot *may* implement more generous per-user rate-limits for proxied requests in the future -fn get_did_doc(host: String) -> impl Endpoint { +fn get_did_doc(host: &str) -> impl Endpoint + use<> { let doc = poem::web::Json(AppViewDoc { id: format!("did:web:{host}"), service: [AppViewService { @@ -235,17 +242,30 @@ pub async fn serve( .server("http://localhost:3000") .url_prefix("/xrpc"); - let mut app = Route::new() + let app = Route::new() .nest("/", api_service.scalar()) .nest("/openapi.json", api_service.spec_endpoint()) .nest("/xrpc/", api_service); if let Some(host) = host { - app = app.at("/.well-known/did.json", get_did_doc(host)); - }; + let app = app.at("/.well-known/did.json", get_did_doc(&host)); + + let auto_cert = AutoCert::builder() + .directory_url(LETS_ENCRYPT_PRODUCTION) + .domain(&host) + .build() + .map_err(ServerError::AcmeBuildError)?; - Server::new(TcpListener::bind("127.0.0.1:3000")) - .run(app) - .await - .map_err(|e| ServerError::ServerExited(format!("uh oh: {e:?}"))) + Server::new(TcpListener::bind("0.0.0.0:443").acme(auto_cert)) + .name("slingshot") + .run(app) + .await + .map_err(ServerError::ServerExited) + } else { + Server::new(TcpListener::bind("127.0.0.1:3000")) + .name("slingshot") + .run(app) + .await + .map_err(ServerError::ServerExited) + } } -- 2.51.2 From 9608af65c6b94b2c74d783680023530636f5c89f Mon Sep 17 00:00:00 2001 From: phil Date: Thu, 31 Jul 2025 13:25:03 -0400 Subject: [PATCH 081/134] crypto provider alksdfjlaksjdflskj --- Cargo.lock | 28 +++++++++++++++------------- slingshot/Cargo.toml | 1 + slingshot/src/server.rs | 4 ++++ 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 17ef55f..e8ffb2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2287,7 +2287,7 @@ dependencies = [ "http", "hyper", "hyper-util", - "rustls 0.23.28", + "rustls 0.23.31", "rustls-native-certs", "rustls-pki-types", "tokio", @@ -4012,7 +4012,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.1", - "rustls 0.23.28", + "rustls 0.23.31", "socket2 0.5.9", "thiserror 2.0.12", "tokio", @@ -4032,7 +4032,7 @@ dependencies = [ "rand 0.9.1", "ring", "rustc-hash 2.1.1", - "rustls 0.23.28", + "rustls 0.23.31", "rustls-pki-types", "slab", "thiserror 2.0.12", @@ -4052,7 +4052,7 @@ dependencies = [ "once_cell", "socket2 0.5.9", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4290,7 +4290,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.28", + "rustls 0.23.31", "rustls-native-certs", "rustls-pki-types", "serde", @@ -4466,16 +4466,16 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.28" +version = "0.23.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7160e3e10bf4535308537f3c4e1641468cd0e485175d6163087c0393c7d46643" +checksum = "c0ebcbd2f03de0fc1122ad9bb24b127a5a6cd51d72604a3f3c50ac459762b6cc" dependencies = [ "aws-lc-rs", "log", "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.103.3", + "rustls-webpki 0.103.4", "subtle", "zeroize", ] @@ -4503,11 +4503,12 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" +checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" dependencies = [ "web-time", + "zeroize", ] [[package]] @@ -4523,9 +4524,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.3" +version = "0.103.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435" +checksum = "0a17884ae0c1b773f1ccd2bd4a8c72f16da897310a98b0e84bf349ad5ead92fc" dependencies = [ "aws-lc-rs", "ring", @@ -4930,6 +4931,7 @@ dependencies = [ "poem", "poem-openapi", "reqwest", + "rustls 0.23.31", "serde", "serde_json", "thiserror 2.0.12", @@ -5389,7 +5391,7 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ - "rustls 0.23.28", + "rustls 0.23.31", "tokio", ] diff --git a/slingshot/Cargo.toml b/slingshot/Cargo.toml index ca24da4..b492950 100644 --- a/slingshot/Cargo.toml +++ b/slingshot/Cargo.toml @@ -20,6 +20,7 @@ metrics-exporter-prometheus = { version = "0.17.1", features = ["http-listener"] poem = { version = "3.1.12", features = ["acme"] } poem-openapi = { version = "5.1.16", features = ["scalar"] } reqwest = { version = "0.12.22", features = ["json"] } +rustls = "0.23.31" serde = { version = "1.0.219", features = ["derive"] } serde_json = { version = "1.0.141", features = ["raw_value"] } thiserror = "2.0.12" diff --git a/slingshot/src/server.rs b/slingshot/src/server.rs index 0f4b866..2bdd705 100644 --- a/slingshot/src/server.rs +++ b/slingshot/src/server.rs @@ -248,6 +248,10 @@ pub async fn serve( .nest("/xrpc/", api_service); if let Some(host) = host { + rustls::crypto::aws_lc_rs::default_provider() + .install_default() + .expect("alskfjalksdjf"); + let app = app.at("/.well-known/did.json", get_did_doc(&host)); let auto_cert = AutoCert::builder() -- 2.51.2 From f3559cb1abbab268f57d017847cbacb0b02edf56 Mon Sep 17 00:00:00 2001 From: phil Date: Thu, 31 Jul 2025 13:37:26 -0400 Subject: [PATCH 082/134] todo done --- slingshot/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slingshot/src/main.rs b/slingshot/src/main.rs index 63a2fa0..053c11d 100644 --- a/slingshot/src/main.rs +++ b/slingshot/src/main.rs @@ -27,7 +27,7 @@ struct Args { /// /// if present: /// - a did:web document will be served at /.well-known/did.json - /// - TODO: HTTPS certs will be automatically configured with Acme/letsencrypt + /// - an HTTPS certs will be automatically configured with Acme/letsencrypt /// - TODO: a rate-limiter will be installed #[arg(long)] host: Option, -- 2.51.2 From c622bf48dc349049f73f2865c149ea6d7d9c807d Mon Sep 17 00:00:00 2001 From: phil Date: Thu, 31 Jul 2025 14:05:36 -0400 Subject: [PATCH 083/134] cors and tracing --- Cargo.lock | 2 +- slingshot/Cargo.toml | 2 +- slingshot/src/main.rs | 2 +- slingshot/src/server.rs | 37 ++++++++++++++++++++++++------------- 4 files changed, 27 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e8ffb2b..633aa52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4921,7 +4921,6 @@ dependencies = [ "atrium-oauth", "clap", "ctrlc", - "env_logger", "foyer", "hickory-resolver", "jetstream", @@ -4938,6 +4937,7 @@ dependencies = [ "time", "tokio", "tokio-util", + "tracing-subscriber", "url", ] diff --git a/slingshot/Cargo.toml b/slingshot/Cargo.toml index b492950..e8b075b 100644 --- a/slingshot/Cargo.toml +++ b/slingshot/Cargo.toml @@ -10,7 +10,6 @@ atrium-identity = "0.1.5" atrium-oauth = "0.1.3" clap = { version = "4.5.41", features = ["derive"] } ctrlc = "3.4.7" -env_logger = "0.11.8" foyer = { version = "0.18.0", features = ["serde"] } hickory-resolver = "0.25.2" jetstream = { path = "../jetstream", features = ["metrics"] } @@ -27,4 +26,5 @@ thiserror = "2.0.12" time = { version = "0.3.41", features = ["serde"] } tokio = { version = "1.47.0", features = ["full"] } tokio-util = "0.7.15" +tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } url = "2.5.4" diff --git a/slingshot/src/main.rs b/slingshot/src/main.rs index 053c11d..19d247e 100644 --- a/slingshot/src/main.rs +++ b/slingshot/src/main.rs @@ -35,7 +35,7 @@ struct Args { #[tokio::main] async fn main() -> Result<(), String> { - env_logger::init(); + tracing_subscriber::fmt::init(); let shutdown = CancellationToken::new(); diff --git a/slingshot/src/server.rs b/slingshot/src/server.rs index 2bdd705..ee87459 100644 --- a/slingshot/src/server.rs +++ b/slingshot/src/server.rs @@ -5,12 +5,14 @@ use std::sync::Arc; use tokio_util::sync::CancellationToken; use poem::{ - Endpoint, Route, Server, + Endpoint, EndpointExt, Route, Server, endpoint::make_sync, + http::Method, listener::{ Listener, TcpListener, acme::{AutoCert, LETS_ENCRYPT_PRODUCTION}, }, + middleware::{Cors, Tracing}, }; use poem_openapi::{ ApiResponse, Object, OpenApi, OpenApiService, param::Query, payload::Json, types::Example, @@ -242,7 +244,7 @@ pub async fn serve( .server("http://localhost:3000") .url_prefix("/xrpc"); - let app = Route::new() + let mut app = Route::new() .nest("/", api_service.scalar()) .nest("/openapi.json", api_service.spec_endpoint()) .nest("/xrpc/", api_service); @@ -252,7 +254,8 @@ pub async fn serve( .install_default() .expect("alskfjalksdjf"); - let app = app.at("/.well-known/did.json", get_did_doc(&host)); + app = app + .at("/.well-known/did.json", get_did_doc(&host)); let auto_cert = AutoCert::builder() .directory_url(LETS_ENCRYPT_PRODUCTION) @@ -260,16 +263,24 @@ pub async fn serve( .build() .map_err(ServerError::AcmeBuildError)?; - Server::new(TcpListener::bind("0.0.0.0:443").acme(auto_cert)) - .name("slingshot") - .run(app) - .await - .map_err(ServerError::ServerExited) + run(TcpListener::bind("0.0.0.0:443").acme(auto_cert), app).await } else { - Server::new(TcpListener::bind("127.0.0.1:3000")) - .name("slingshot") - .run(app) - .await - .map_err(ServerError::ServerExited) + run(TcpListener::bind("127.0.0.1:3000"), app).await } } + +async fn run(listener: L, app: Route) -> Result<(), ServerError> +where + L: Listener + 'static +{ + let app = app + .with(Cors::new() + .allow_method(Method::GET) + .allow_credentials(false)) + .with(Tracing); + Server::new(listener) + .name("slingshot") + .run(app) + .await + .map_err(ServerError::ServerExited) +} -- 2.51.2 From 0a8c89f70f43aba92cb4e6b821b3b41f1720c956 Mon Sep 17 00:00:00 2001 From: phil Date: Thu, 31 Jul 2025 15:28:41 -0400 Subject: [PATCH 084/134] resolve handle early and return uri with did --- slingshot/src/main.rs | 11 ++++- slingshot/src/record.rs | 26 ++++------ slingshot/src/server.rs | 102 +++++++++++++++++++++++++++++++++------- 3 files changed, 102 insertions(+), 37 deletions(-) diff --git a/slingshot/src/main.rs b/slingshot/src/main.rs index 19d247e..f6c5056 100644 --- a/slingshot/src/main.rs +++ b/slingshot/src/main.rs @@ -81,12 +81,19 @@ async fn main() -> Result<(), String> { Ok(()) }); - let repo = Repo::new(identity); + let repo = Repo::new(identity.clone()); let server_shutdown = shutdown.clone(); let server_cache_handle = cache.clone(); tasks.spawn(async move { - serve(server_cache_handle, repo, args.host, server_shutdown).await?; + serve( + server_cache_handle, + identity, + repo, + args.host, + server_shutdown, + ) + .await?; Ok(()) }); diff --git a/slingshot/src/record.rs b/slingshot/src/record.rs index 853abf9..963437e 100644 --- a/slingshot/src/record.rs +++ b/slingshot/src/record.rs @@ -1,7 +1,7 @@ //! cached record storage use crate::{Identity, error::RecordError}; -use atrium_api::types::string::{Cid, Did, Handle}; +use atrium_api::types::string::{Cid, Did, Nsid, RecordKey}; use reqwest::Client; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; @@ -78,21 +78,11 @@ impl Repo { pub async fn get_record( &self, - did_or_handle: String, - collection: String, - rkey: String, - cid: Option, + did: &Did, + collection: &Nsid, + rkey: &RecordKey, + cid: &Option, ) -> Result { - let did = match Did::new(did_or_handle.clone()) { - Ok(did) => did, - Err(_) => { - let handle = Handle::new(did_or_handle).map_err(|_| RecordError::BadRepo)?; - let Some(did) = self.identity.handle_to_did(handle).await? else { - return Err(RecordError::NotFound("could not resolve and verify handle")); - }; - did - } - }; let Some(pds) = self.identity.did_to_pds(did.clone()).await? else { return Err(RecordError::NotFound("could not get pds for DID")); }; @@ -101,11 +91,11 @@ impl Repo { let mut params = vec![ ("repo", did.to_string()), - ("collection", collection), - ("rkey", rkey), + ("collection", collection.to_string()), + ("rkey", rkey.to_string()), ]; if let Some(cid) = cid { - params.push(("cid", cid)); + params.push(("cid", cid.as_ref().to_string())); } let mut url = Url::parse_with_params(&pds, ¶ms)?; url.set_path("/xrpc/com.atproto.repo.getRecord"); diff --git a/slingshot/src/server.rs b/slingshot/src/server.rs index ee87459..248bfd9 100644 --- a/slingshot/src/server.rs +++ b/slingshot/src/server.rs @@ -1,6 +1,8 @@ -use crate::{CachedRecord, Repo, error::ServerError}; +use crate::{CachedRecord, Identity, Repo, error::ServerError}; +use atrium_api::types::string::{Cid, Did, Handle, Nsid, RecordKey}; use foyer::HybridCache; use serde::Serialize; +use std::str::FromStr; use std::sync::Arc; use tokio_util::sync::CancellationToken; @@ -44,6 +46,13 @@ impl Example for XrpcErrorResponseObject { } } } +type XrpcError = Json; +fn xrpc_error(error: impl AsRef, message: impl AsRef) -> XrpcError { + Json(XrpcErrorResponseObject { + error: error.as_ref().to_string(), + message: message.as_ref().to_string(), + }) +} fn bad_request_handler(err: poem::Error) -> GetRecordResponse { GetRecordResponse::BadRequest(Json(XrpcErrorResponseObject { @@ -100,11 +109,15 @@ enum GetRecordResponse { /// also list `InvalidRequest`, `ExpiredToken`, and `InvalidToken`. Of /// these, slingshot will only return `RecordNotFound` or `InvalidRequest`. #[oai(status = 400)] - BadRequest(Json), + BadRequest(XrpcError), + /// Just using 500 for potentially upstream errors for now + #[oai(status = 500)] + ServerError(XrpcError), } struct Xrpc { cache: HybridCache, + identity: Identity, repo: Arc, } @@ -140,8 +153,54 @@ impl Xrpc { /// record. Query(cid): Query>, ) -> GetRecordResponse { - // TODO: yeah yeah - let at_uri = format!("at://{repo}/{collection}/{rkey}"); + let did = match Did::new(repo.clone()) { + Ok(did) => did, + Err(_) => { + let Ok(handle) = Handle::new(repo) else { + return GetRecordResponse::BadRequest(xrpc_error( + "InvalidRequest", + "repo was not a valid DID or handle", + )); + }; + if let Ok(res) = self.identity.handle_to_did(handle).await { + if let Some(did) = res { + did + } else { + return GetRecordResponse::BadRequest(xrpc_error( + "InvalidRequest", + "Could not resolve handle repo to a DID", + )); + } + } else { + return GetRecordResponse::ServerError(xrpc_error( + "ResolutionFailed", + "errored while trying to resolve handle to DID", + )); + } + } + }; + + let Ok(collection) = Nsid::new(collection) else { + return GetRecordResponse::BadRequest(xrpc_error( + "InvalidRequest", + "invalid NSID for collection", + )); + }; + + let Ok(rkey) = RecordKey::new(rkey) else { + return GetRecordResponse::BadRequest(xrpc_error("InvalidRequest", "invalid rkey")); + }; + + let cid: Option = if let Some(cid) = cid { + let Ok(cid) = Cid::from_str(&cid) else { + return GetRecordResponse::BadRequest(xrpc_error("InvalidRequest", "invalid CID")); + }; + Some(cid) + } else { + None + }; + + let at_uri = format!("at://{}/{}/{}", &*did, &*collection, &*rkey); let entry = self .cache @@ -150,7 +209,7 @@ impl Xrpc { let repo_api = self.repo.clone(); || async move { repo_api - .get_record(repo, collection, rkey, cid) + .get_record(&did, &collection, &rkey, &cid) .await .map_err(|e| foyer::Error::Other(Box::new(e))) } @@ -163,7 +222,6 @@ impl Xrpc { match *entry { CachedRecord::Found(ref raw) => { let (found_cid, raw_value) = raw.into(); - let found_cid = found_cid.as_ref().to_string(); if cid.clone().map(|c| c != found_cid).unwrap_or(false) { return GetRecordResponse::BadRequest(Json(XrpcErrorResponseObject { error: "RecordNotFound".to_string(), @@ -176,7 +234,7 @@ impl Xrpc { serde_json::from_str(raw_value.get()).expect("RawValue to be valid json"); GetRecordResponse::Ok(Json(FoundRecordResponseObject { uri: at_uri, - cid: Some(found_cid), + cid: Some(found_cid.as_ref().to_string()), value, })) } @@ -234,15 +292,23 @@ fn get_did_doc(host: &str) -> impl Endpoint + use<> { pub async fn serve( cache: HybridCache, + identity: Identity, repo: Repo, host: Option, _shutdown: CancellationToken, ) -> Result<(), ServerError> { let repo = Arc::new(repo); - let api_service = - OpenApiService::new(Xrpc { cache, repo }, "Slingshot", env!("CARGO_PKG_VERSION")) - .server("http://localhost:3000") - .url_prefix("/xrpc"); + let api_service = OpenApiService::new( + Xrpc { + cache, + identity, + repo, + }, + "Slingshot", + env!("CARGO_PKG_VERSION"), + ) + .server("http://localhost:3000") + .url_prefix("/xrpc"); let mut app = Route::new() .nest("/", api_service.scalar()) @@ -254,8 +320,7 @@ pub async fn serve( .install_default() .expect("alskfjalksdjf"); - app = app - .at("/.well-known/did.json", get_did_doc(&host)); + app = app.at("/.well-known/did.json", get_did_doc(&host)); let auto_cert = AutoCert::builder() .directory_url(LETS_ENCRYPT_PRODUCTION) @@ -271,12 +336,15 @@ pub async fn serve( async fn run(listener: L, app: Route) -> Result<(), ServerError> where - L: Listener + 'static + L: Listener + 'static, { let app = app - .with(Cors::new() - .allow_method(Method::GET) - .allow_credentials(false)) + .with( + Cors::new() + .allow_origin("*") + .allow_methods([Method::GET]) + .allow_credentials(false), + ) .with(Tracing); Server::new(listener) .name("slingshot") -- 2.51.2 From 6b45c16c991f50044427df79a933f79254469ca1 Mon Sep 17 00:00:00 2001 From: phil Date: Thu, 31 Jul 2025 15:37:31 -0400 Subject: [PATCH 085/134] update docs to match --- slingshot/src/server.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/slingshot/src/server.rs b/slingshot/src/server.rs index 248bfd9..9ef27b3 100644 --- a/slingshot/src/server.rs +++ b/slingshot/src/server.rs @@ -133,9 +133,7 @@ impl Xrpc { #[oai(path = "/com.atproto.repo.getRecord", method = "get")] async fn get_record( &self, - /// The DID of the repo - /// - /// NOTE: handles should be accepted here but this is still TODO in slingshot + /// The DID or handle of the repo #[oai(example = "example_did")] Query(repo): Query, /// The NSID of the record collection @@ -150,7 +148,7 @@ impl Xrpc { /// /// If specified and a newer version of the record exists, returns 404 not /// found. That is: slingshot only retains the most recent version of a - /// record. + /// record. (TODO: verify bsky behaviour for mismatched/old CID) Query(cid): Query>, ) -> GetRecordResponse { let did = match Did::new(repo.clone()) { -- 2.51.2 From 8e74aecf80b8d78f3582e8982c9c5bc865516452 Mon Sep 17 00:00:00 2001 From: phil Date: Thu, 31 Jul 2025 15:44:00 -0400 Subject: [PATCH 086/134] does this pass the wildcard through? doesn't *really* matter, just want to reply star to clients please --- slingshot/src/server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slingshot/src/server.rs b/slingshot/src/server.rs index 9ef27b3..ed38cd3 100644 --- a/slingshot/src/server.rs +++ b/slingshot/src/server.rs @@ -339,7 +339,7 @@ where let app = app .with( Cors::new() - .allow_origin("*") + .allow_origin_regex("*") .allow_methods([Method::GET]) .allow_credentials(false), ) -- 2.51.2 From aa3f28bcd2b6565b28a0c24ae59800171c4df3dd Mon Sep 17 00:00:00 2001 From: phil Date: Thu, 31 Jul 2025 15:52:25 -0400 Subject: [PATCH 087/134] cache certs --- slingshot/src/main.rs | 9 +++++++++ slingshot/src/server.rs | 12 ++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/slingshot/src/main.rs b/slingshot/src/main.rs index f6c5056..c14862c 100644 --- a/slingshot/src/main.rs +++ b/slingshot/src/main.rs @@ -31,6 +31,14 @@ struct Args { /// - TODO: a rate-limiter will be installed #[arg(long)] host: Option, + /// a location to cache acme https certs + /// + /// only used if --host is specified. omitting requires re-requesting certs + /// on every restart, and letsencrypt has rate limits that are easy to hit. + /// + /// recommended in production, but mind the file permissions. + #[arg(long)] + certs: Option, } #[tokio::main] @@ -91,6 +99,7 @@ async fn main() -> Result<(), String> { identity, repo, args.host, + args.certs, server_shutdown, ) .await?; diff --git a/slingshot/src/server.rs b/slingshot/src/server.rs index ed38cd3..460863e 100644 --- a/slingshot/src/server.rs +++ b/slingshot/src/server.rs @@ -2,6 +2,7 @@ use crate::{CachedRecord, Identity, Repo, error::ServerError}; use atrium_api::types::string::{Cid, Did, Handle, Nsid, RecordKey}; use foyer::HybridCache; use serde::Serialize; +use std::path::PathBuf; use std::str::FromStr; use std::sync::Arc; use tokio_util::sync::CancellationToken; @@ -293,6 +294,7 @@ pub async fn serve( identity: Identity, repo: Repo, host: Option, + certs: Option, _shutdown: CancellationToken, ) -> Result<(), ServerError> { let repo = Arc::new(repo); @@ -320,11 +322,13 @@ pub async fn serve( app = app.at("/.well-known/did.json", get_did_doc(&host)); - let auto_cert = AutoCert::builder() + let mut auto_cert = AutoCert::builder() .directory_url(LETS_ENCRYPT_PRODUCTION) - .domain(&host) - .build() - .map_err(ServerError::AcmeBuildError)?; + .domain(&host); + if let Some(certs) = certs { + auto_cert = auto_cert.cache_path(certs) + } + let auto_cert = auto_cert.build().map_err(ServerError::AcmeBuildError)?; run(TcpListener::bind("0.0.0.0:443").acme(auto_cert), app).await } else { -- 2.51.2 From f427071ee5168973e8f2ad0c7558eae3cb171462 Mon Sep 17 00:00:00 2001 From: phil Date: Thu, 31 Jul 2025 16:54:04 -0400 Subject: [PATCH 088/134] retry upstream getRecord without CID if NotFound --- slingshot/src/error.rs | 5 +++ slingshot/src/lib.rs | 2 +- slingshot/src/record.rs | 71 +++++++++++++++++++++++++++++------------ 3 files changed, 57 insertions(+), 21 deletions(-) diff --git a/slingshot/src/error.rs b/slingshot/src/error.rs index a062bfb..1412142 100644 --- a/slingshot/src/error.rs +++ b/slingshot/src/error.rs @@ -1,3 +1,4 @@ +use crate::ErrorResponseObject; use thiserror::Error; #[derive(Debug, Error)] @@ -75,4 +76,8 @@ pub enum RecordError { MissingUpstreamCid, #[error("upstream CID was not valid: {0}")] BadUpstreamCid(String), + #[error("upstream atproto-looking bad request")] + UpstreamBadRequest(ErrorResponseObject), + #[error("upstream non-atproto bad request")] + UpstreamBadBadNotGoodRequest(reqwest::Error), } diff --git a/slingshot/src/lib.rs b/slingshot/src/lib.rs index a5f2496..2e5e687 100644 --- a/slingshot/src/lib.rs +++ b/slingshot/src/lib.rs @@ -8,5 +8,5 @@ mod server; pub use consumer::consume; pub use firehose_cache::firehose_cache; pub use identity::Identity; -pub use record::{CachedRecord, Repo}; +pub use record::{CachedRecord, ErrorResponseObject, Repo}; pub use server::serve; diff --git a/slingshot/src/record.rs b/slingshot/src/record.rs index 963437e..9b3ed60 100644 --- a/slingshot/src/record.rs +++ b/slingshot/src/record.rs @@ -2,7 +2,7 @@ use crate::{Identity, error::RecordError}; use atrium_api::types::string::{Cid, Did, Nsid, RecordKey}; -use reqwest::Client; +use reqwest::{Client, StatusCode}; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use std::str::FromStr; @@ -56,6 +56,13 @@ struct RecordResponseObject { value: Box, } +#[derive(Debug, Deserialize)] +pub struct ErrorResponseObject { + error: String, + #[allow(dead_code)] + message: String, +} + #[derive(Clone)] pub struct Repo { identity: Identity, @@ -87,39 +94,63 @@ impl Repo { return Err(RecordError::NotFound("could not get pds for DID")); }; - // TODO: throttle by host probably, generally guard against outgoing requests + // cid gets set to None for a retry, if it's Some and we got NotFound + let mut cid = cid; - let mut params = vec![ - ("repo", did.to_string()), - ("collection", collection.to_string()), - ("rkey", rkey.to_string()), - ]; - if let Some(cid) = cid { - params.push(("cid", cid.as_ref().to_string())); - } - let mut url = Url::parse_with_params(&pds, ¶ms)?; - url.set_path("/xrpc/com.atproto.repo.getRecord"); + let res = loop { + // TODO: throttle outgoing requests by host probably, generally guard against outgoing requests + let mut params = vec![ + ("repo", did.to_string()), + ("collection", collection.to_string()), + ("rkey", rkey.to_string()), + ]; + if let Some(cid) = cid { + params.push(("cid", cid.as_ref().to_string())); + } + let mut url = Url::parse_with_params(&pds, ¶ms)?; + url.set_path("/xrpc/com.atproto.repo.getRecord"); - let res = self - .client - .get(url) - .send() - .await - .map_err(RecordError::SendError)? + let res = self + .client + .get(url.clone()) + .send() + .await + .map_err(RecordError::SendError)?; + + if res.status() == StatusCode::BAD_REQUEST { + // 1. if we're not able to parse json, it's not something we can handle + let err = res + .json::() + .await + .map_err(RecordError::UpstreamBadBadNotGoodRequest)?; + // 2. if we are, is it a NotFound? and if so, did we try with a CID? + // if so, retry with no CID (api handler will reject for mismatch but + // with a nice error + warm cache) + if err.error == "NotFound" && cid.is_some() { + cid = &None; + continue; + } else { + return Err(RecordError::UpstreamBadRequest(err)); + } + } + break res; + }; + + let data = res .error_for_status() .map_err(RecordError::StatusError)? // TODO atproto error handling (think about handling not found) .json::() .await .map_err(RecordError::ParseJsonError)?; // todo... - let Some(cid) = res.cid else { + let Some(cid) = data.cid else { return Err(RecordError::MissingUpstreamCid); }; let cid = Cid::from_str(&cid).map_err(|e| RecordError::BadUpstreamCid(e.to_string()))?; Ok(CachedRecord::Found(RawRecord { cid, - record: res.value.to_string(), + record: data.value.to_string(), })) } } -- 2.51.2 From fa9bc65682e4c817117fcee9ea1195778245da3c Mon Sep 17 00:00:00 2001 From: phil Date: Thu, 31 Jul 2025 17:25:08 -0400 Subject: [PATCH 089/134] proxy back upstream BadRequests --- slingshot/src/record.rs | 5 ++--- slingshot/src/server.rs | 46 ++++++++++++++++++++++++++++++++++++----- 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/slingshot/src/record.rs b/slingshot/src/record.rs index 9b3ed60..971c648 100644 --- a/slingshot/src/record.rs +++ b/slingshot/src/record.rs @@ -58,9 +58,8 @@ struct RecordResponseObject { #[derive(Debug, Deserialize)] pub struct ErrorResponseObject { - error: String, - #[allow(dead_code)] - message: String, + pub error: String, + pub message: String, } #[derive(Clone)] diff --git a/slingshot/src/server.rs b/slingshot/src/server.rs index 460863e..1407e90 100644 --- a/slingshot/src/server.rs +++ b/slingshot/src/server.rs @@ -1,4 +1,7 @@ -use crate::{CachedRecord, Identity, Repo, error::ServerError}; +use crate::{ + CachedRecord, ErrorResponseObject, Identity, Repo, + error::{RecordError, ServerError}, +}; use atrium_api::types::string::{Cid, Did, Handle, Nsid, RecordKey}; use foyer::HybridCache; use serde::Serialize; @@ -201,7 +204,7 @@ impl Xrpc { let at_uri = format!("at://{}/{}/{}", &*did, &*collection, &*rkey); - let entry = self + let fr = self .cache .fetch(at_uri.clone(), { let cid = cid.clone(); @@ -213,10 +216,43 @@ impl Xrpc { .map_err(|e| foyer::Error::Other(Box::new(e))) } }) - .await - .unwrap(); // todo + .await; - // TODO: actual 404 + let entry = match fr { + Ok(e) => e, + Err(foyer::Error::Other(e)) => { + let record_error = match e.downcast::() { + Ok(e) => e, + Err(e) => { + log::error!("error (foyer other) getting cache entry, {e:?}"); + return GetRecordResponse::ServerError(xrpc_error( + "ServerError", + "sorry, something went wrong", + )); + } + }; + let RecordError::UpstreamBadRequest(ErrorResponseObject { error, message }) = + *record_error + else { + log::error!("RecordError getting cache entry, {record_error:?}"); + return GetRecordResponse::ServerError(xrpc_error( + "ServerError", + "sorry, something went wrong", + )); + }; + return GetRecordResponse::BadRequest(xrpc_error( + error, + format!("Upstream bad request: {message}"), + )); + } + Err(e) => { + log::error!("error (foyer) getting cache entry, {e:?}"); + return GetRecordResponse::ServerError(xrpc_error( + "ServerError", + "sorry, something went wrong", + )); + } + }; match *entry { CachedRecord::Found(ref raw) => { -- 2.51.2 From 27e28435df28023da13716e860c6527988f357b4 Mon Sep 17 00:00:00 2001 From: phil Date: Thu, 31 Jul 2025 17:29:42 -0400 Subject: [PATCH 090/134] probably less than the minimal comment warranted --- slingshot/src/server.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/slingshot/src/server.rs b/slingshot/src/server.rs index 1407e90..4d23998 100644 --- a/slingshot/src/server.rs +++ b/slingshot/src/server.rs @@ -240,6 +240,9 @@ impl Xrpc { "sorry, something went wrong", )); }; + + // all of the noise around here is so that we can ultimately reach this: + // upstream BadRequest extracted from the foyer result which we can proxy back return GetRecordResponse::BadRequest(xrpc_error( error, format!("Upstream bad request: {message}"), -- 2.51.2 From 6aa09d1522d86cf77eb49acd4c0aba478d2eee9d Mon Sep 17 00:00:00 2001 From: phil Date: Thu, 31 Jul 2025 17:44:25 -0400 Subject: [PATCH 091/134] more doc tweaks --- slingshot/src/server.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/slingshot/src/server.rs b/slingshot/src/server.rs index 4d23998..7021576 100644 --- a/slingshot/src/server.rs +++ b/slingshot/src/server.rs @@ -74,6 +74,9 @@ struct FoundRecordResponseObject { /// /// Slingshot will always return the CID, despite it not being a required /// response property in the official lexicon. + /// + /// TODO: probably actually let it be optional, idk are some pds's weirdly + /// not returning it? cid: Option, /// the record itself as JSON value: serde_json::Value, @@ -111,10 +114,11 @@ enum GetRecordResponse { /// The only error name in the repo.getRecord lexicon is `RecordNotFound`, /// but the [canonical api docs](https://docs.bsky.app/docs/api/com-atproto-repo-get-record) /// also list `InvalidRequest`, `ExpiredToken`, and `InvalidToken`. Of - /// these, slingshot will only return `RecordNotFound` or `InvalidRequest`. + /// these, slingshot will only generate `RecordNotFound` or `InvalidRequest`, + /// but may return any proxied error code from the upstream repo. #[oai(status = 400)] BadRequest(XrpcError), - /// Just using 500 for potentially upstream errors for now + /// Server errors #[oai(status = 500)] ServerError(XrpcError), } @@ -131,9 +135,8 @@ impl Xrpc { /// /// Get a single record from a repository. Does not require auth. /// - /// See https://docs.bsky.app/docs/api/com-atproto-repo-get-record for the - /// canonical XRPC documentation that this endpoint aims to be compatible - /// with. + /// See also the [canonical `com.atproto` XRPC documentation](https://docs.bsky.app/docs/api/com-atproto-repo-get-record) + /// that this endpoint aims to be compatible with. #[oai(path = "/com.atproto.repo.getRecord", method = "get")] async fn get_record( &self, -- 2.51.2 From 5bc7fd285388184a0993a195a6ee18e13af7e1fc Mon Sep 17 00:00:00 2001 From: phil Date: Thu, 31 Jul 2025 17:46:43 -0400 Subject: [PATCH 092/134] use the host val if set for docs server --- slingshot/src/server.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/slingshot/src/server.rs b/slingshot/src/server.rs index 7021576..ae22afd 100644 --- a/slingshot/src/server.rs +++ b/slingshot/src/server.rs @@ -349,7 +349,11 @@ pub async fn serve( "Slingshot", env!("CARGO_PKG_VERSION"), ) - .server("http://localhost:3000") + .server(if let Some(ref h) = host { + format!("https://{h}") + } else { + "http://localhost:3000".to_string() + }) .url_prefix("/xrpc"); let mut app = Route::new() -- 2.51.2 From b894085dbe60d870f91787906483b7dd9ef8c00f Mon Sep 17 00:00:00 2001 From: phil Date: Fri, 1 Aug 2025 10:54:08 -0400 Subject: [PATCH 093/134] add acme contact idk if this helps rate-limiting, prob not --- slingshot/src/main.rs | 6 ++++++ slingshot/src/server.rs | 6 +++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/slingshot/src/main.rs b/slingshot/src/main.rs index c14862c..4cbd0a7 100644 --- a/slingshot/src/main.rs +++ b/slingshot/src/main.rs @@ -31,6 +31,11 @@ struct Args { /// - TODO: a rate-limiter will be installed #[arg(long)] host: Option, + /// email address for letsencrypt contact + /// + /// recommended in production, i guess? + #[arg(long)] + acme_contact: Option, /// a location to cache acme https certs /// /// only used if --host is specified. omitting requires re-requesting certs @@ -99,6 +104,7 @@ async fn main() -> Result<(), String> { identity, repo, args.host, + args.acme_contact, args.certs, server_shutdown, ) diff --git a/slingshot/src/server.rs b/slingshot/src/server.rs index ae22afd..9fa723e 100644 --- a/slingshot/src/server.rs +++ b/slingshot/src/server.rs @@ -336,6 +336,7 @@ pub async fn serve( identity: Identity, repo: Repo, host: Option, + acme_contact: Option, certs: Option, _shutdown: CancellationToken, ) -> Result<(), ServerError> { @@ -371,8 +372,11 @@ pub async fn serve( let mut auto_cert = AutoCert::builder() .directory_url(LETS_ENCRYPT_PRODUCTION) .domain(&host); + if let Some(contact) = acme_contact { + auto_cert = auto_cert.contact(contact); + } if let Some(certs) = certs { - auto_cert = auto_cert.cache_path(certs) + auto_cert = auto_cert.cache_path(certs); } let auto_cert = auto_cert.build().map_err(ServerError::AcmeBuildError)?; -- 2.51.2 From 0b1ba939dc544b51ca9f01d6854a2affbf8b60d4 Mon Sep 17 00:00:00 2001 From: phil Date: Fri, 1 Aug 2025 11:20:58 -0400 Subject: [PATCH 094/134] com.bad-example.repo.getUriRecord --- Cargo.lock | 1 + slingshot/Cargo.toml | 1 + slingshot/src/server.rs | 84 ++++++++++++++++++++++++++++++++++++++--- 3 files changed, 80 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 633aa52..c1bb9ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4924,6 +4924,7 @@ dependencies = [ "foyer", "hickory-resolver", "jetstream", + "links", "log", "metrics", "metrics-exporter-prometheus 0.17.2", diff --git a/slingshot/Cargo.toml b/slingshot/Cargo.toml index e8b075b..0e0cdfa 100644 --- a/slingshot/Cargo.toml +++ b/slingshot/Cargo.toml @@ -13,6 +13,7 @@ ctrlc = "3.4.7" foyer = { version = "0.18.0", features = ["serde"] } hickory-resolver = "0.25.2" jetstream = { path = "../jetstream", features = ["metrics"] } +links = { path = "../links" } log = "0.4.27" metrics = "0.24.2" metrics-exporter-prometheus = { version = "0.17.1", features = ["http-listener"] } diff --git a/slingshot/src/server.rs b/slingshot/src/server.rs index 9fa723e..8607474 100644 --- a/slingshot/src/server.rs +++ b/slingshot/src/server.rs @@ -4,6 +4,7 @@ use crate::{ }; use atrium_api::types::string::{Cid, Did, Handle, Nsid, RecordKey}; use foyer::HybridCache; +use links::at_uri::parse_at_uri as normalize_at_uri; use serde::Serialize; use std::path::PathBuf; use std::str::FromStr; @@ -33,6 +34,14 @@ fn example_collection() -> String { fn example_rkey() -> String { "3lv4ouczo2b2a".to_string() } +fn example_uri() -> String { + format!( + "at://{}/{}/{}", + example_did(), + example_collection(), + example_rkey() + ) +} #[derive(Object)] #[oai(example = true)] @@ -84,12 +93,7 @@ struct FoundRecordResponseObject { impl Example for FoundRecordResponseObject { fn example() -> Self { Self { - uri: format!( - "at://{}/{}/{}", - example_did(), - example_collection(), - example_rkey() - ), + uri: example_uri(), cid: Some("bafyreialv3mzvvxaoyrfrwoer3xmabbmdchvrbyhayd7bga47qjbycy74e".to_string()), value: serde_json::json!({ "$type": "app.bsky.feed.like", @@ -157,6 +161,74 @@ impl Xrpc { /// found. That is: slingshot only retains the most recent version of a /// record. (TODO: verify bsky behaviour for mismatched/old CID) Query(cid): Query>, + ) -> GetRecordResponse { + self.get_record_impl(repo, collection, rkey, cid).await + } + + /// com.bad-example.repo.getUriRecord + /// + /// Ergonomic complement to [`com.atproto.repo.getRecord`](https://docs.bsky.app/docs/api/com-atproto-repo-get-record) + /// which accepts an at-uri instead of individual rep/collection/rkey params + #[oai(path = "/com.bad-example.repo.getUriRecord", method = "get")] + async fn get_uri_record( + &self, + /// The at-uri of the record + /// + /// The identifier can be a DID or an atproto handle, and the collection + /// and rkey segments must be present. + #[oai(example = "example_uri")] + Query(at_uri): Query, + /// Optional: the CID of the version of the record. + /// + /// If not specified, then return the most recent version. + /// + /// If specified and a newer version of the record exists, returns 404 not + /// found. That is: slingshot only retains the most recent version of a + /// record. + Query(cid): Query>, + ) -> GetRecordResponse { + let bad_at_uri = || { + GetRecordResponse::BadRequest(xrpc_error( + "InvalidRequest", + "at-uri does not appear to be valid", + )) + }; + + let Some(normalized) = normalize_at_uri(&at_uri) else { + return bad_at_uri(); + }; + + // TODO: move this to links + let Some(rest) = normalized.strip_prefix("at://") else { + return bad_at_uri(); + }; + let Some((repo, rest)) = rest.split_once('/') else { + return bad_at_uri(); + }; + let Some((collection, rest)) = rest.split_once('/') else { + return bad_at_uri(); + }; + let rkey = if let Some((rkey, _rest)) = rest.split_once('?') { + rkey + } else { + rest + }; + + self.get_record_impl( + repo.to_string(), + collection.to_string(), + rkey.to_string(), + cid, + ) + .await + } + + async fn get_record_impl( + &self, + repo: String, + collection: String, + rkey: String, + cid: Option, ) -> GetRecordResponse { let did = match Did::new(repo.clone()) { Ok(did) => did, -- 2.51.2 From 80421fe66d9dc289ca8cb4083e8623da24f41c1f Mon Sep 17 00:00:00 2001 From: phil Date: Fri, 1 Aug 2025 11:23:16 -0400 Subject: [PATCH 095/134] typo --- slingshot/src/server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slingshot/src/server.rs b/slingshot/src/server.rs index 8607474..3c52bb6 100644 --- a/slingshot/src/server.rs +++ b/slingshot/src/server.rs @@ -168,7 +168,7 @@ impl Xrpc { /// com.bad-example.repo.getUriRecord /// /// Ergonomic complement to [`com.atproto.repo.getRecord`](https://docs.bsky.app/docs/api/com-atproto-repo-get-record) - /// which accepts an at-uri instead of individual rep/collection/rkey params + /// which accepts an at-uri instead of individual repo/collection/rkey params #[oai(path = "/com.bad-example.repo.getUriRecord", method = "get")] async fn get_uri_record( &self, -- 2.51.2 From efc93039d5beb58259ff5650fa73b7f77713b824 Mon Sep 17 00:00:00 2001 From: phil Date: Fri, 1 Aug 2025 17:05:46 -0400 Subject: [PATCH 096/134] handle shutdown in all tasks --- slingshot/src/error.rs | 2 ++ slingshot/src/firehose_cache.rs | 6 +++++- slingshot/src/identity.rs | 20 +++++++++++++++++--- slingshot/src/main.rs | 15 ++++++++++++--- slingshot/src/server.rs | 16 +++++++++++----- 5 files changed, 47 insertions(+), 12 deletions(-) diff --git a/slingshot/src/error.rs b/slingshot/src/error.rs index 1412142..17eb0ee 100644 --- a/slingshot/src/error.rs +++ b/slingshot/src/error.rs @@ -54,6 +54,8 @@ pub enum MainTaskError { ServerTaskError(#[from] ServerError), #[error(transparent)] IdentityTaskError(#[from] IdentityError), + #[error("firehose cache failed to close: {0}")] + FirehoseCacheCloseError(foyer::Error), } #[derive(Debug, Error)] diff --git a/slingshot/src/firehose_cache.rs b/slingshot/src/firehose_cache.rs index 0163dca..1ff33da 100644 --- a/slingshot/src/firehose_cache.rs +++ b/slingshot/src/firehose_cache.rs @@ -10,7 +10,11 @@ pub async fn firehose_cache( .memory(64 * 2_usize.pow(20)) .with_weighter(|k: &String, v| k.len() + std::mem::size_of_val(v)) .storage(Engine::large()) - .with_device_options(DirectFsDeviceOptions::new(cache_dir)) + .with_device_options( + DirectFsDeviceOptions::new(cache_dir) + .with_capacity(2_usize.pow(30)) // TODO: configurable (1GB to have something) + .with_file_size(16 * 2_usize.pow(20)), // note: this does limit the max cached item size, warning jumbo records + ) .build() .await .map_err(|e| format!("foyer setup error: {e:?}"))?; diff --git a/slingshot/src/identity.rs b/slingshot/src/identity.rs index 4550ce7..a55821c 100644 --- a/slingshot/src/identity.rs +++ b/slingshot/src/identity.rs @@ -13,6 +13,7 @@ use std::sync::Arc; /// 3. DID -> handle resolution: for bidirectional handle validation and in case we want to offer this use std::time::Duration; use tokio::sync::Mutex; +use tokio_util::sync::CancellationToken; use crate::error::IdentityError; use atrium_api::{ @@ -175,8 +176,12 @@ impl Identity { .with_name("identity") .memory(16 * 2_usize.pow(20)) .with_weighter(|k, v| std::mem::size_of_val(k) + std::mem::size_of_val(v)) - .storage(Engine::large()) - .with_device_options(DirectFsDeviceOptions::new(cache_dir)) + .storage(Engine::small()) + .with_device_options( + DirectFsDeviceOptions::new(cache_dir) + .with_capacity(2_usize.pow(30)) // TODO: configurable (1GB to have something) + .with_file_size(2_usize.pow(20)), // note: this does limit the max cached item size, warning jumbo records + ) .build() .await?; @@ -403,12 +408,21 @@ impl Identity { } /// run the refresh queue consumer - pub async fn run_refresher(&self) -> Result<(), IdentityError> { + pub async fn run_refresher(&self, shutdown: CancellationToken) -> Result<(), IdentityError> { let _guard = self .refresher .try_lock() .expect("there to only be one refresher running"); loop { + if shutdown.is_cancelled() { + log::info!("identity refresher: exiting for shutdown: closing cache..."); + if let Err(e) = self.cache.close().await { + log::error!("cache close errored: {e}"); + } else { + log::info!("identity cache closed.") + } + return Ok(()); + } let Some(task_key) = self.peek_refresh().await else { tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; continue; diff --git a/slingshot/src/main.rs b/slingshot/src/main.rs index 4cbd0a7..c427e2a 100644 --- a/slingshot/src/main.rs +++ b/slingshot/src/main.rs @@ -89,8 +89,9 @@ async fn main() -> Result<(), String> { .map_err(|e| format!("identity setup failed: {e:?}"))?; log::info!("identity service ready."); let identity_refresher = identity.clone(); + let identity_shutdown = shutdown.clone(); tasks.spawn(async move { - identity_refresher.run_refresher().await?; + identity_refresher.run_refresher(identity_shutdown).await?; Ok(()) }); @@ -113,13 +114,14 @@ async fn main() -> Result<(), String> { }); let consumer_shutdown = shutdown.clone(); + let consumer_cache = cache.clone(); tasks.spawn(async move { consume( args.jetstream, None, args.jetstream_no_zstd, consumer_shutdown, - cache, + consumer_cache, ) .await?; Ok(()) @@ -133,13 +135,20 @@ async fn main() -> Result<(), String> { } } + tasks.spawn(async move { + cache + .close() + .await + .map_err(MainTaskError::FirehoseCacheCloseError) + }); + tokio::select! { _ = async { while let Some(completed) = tasks.join_next().await { log::info!("shutdown: task completed: {completed:?}"); } } => {}, - _ = tokio::time::sleep(std::time::Duration::from_secs(3)) => { + _ = tokio::time::sleep(std::time::Duration::from_secs(30)) => { log::info!("shutdown: not all tasks completed on time. aborting..."); tasks.shutdown().await; }, diff --git a/slingshot/src/server.rs b/slingshot/src/server.rs index 3c52bb6..26612d6 100644 --- a/slingshot/src/server.rs +++ b/slingshot/src/server.rs @@ -410,7 +410,7 @@ pub async fn serve( host: Option, acme_contact: Option, certs: Option, - _shutdown: CancellationToken, + shutdown: CancellationToken, ) -> Result<(), ServerError> { let repo = Arc::new(repo); let api_service = OpenApiService::new( @@ -452,13 +452,18 @@ pub async fn serve( } let auto_cert = auto_cert.build().map_err(ServerError::AcmeBuildError)?; - run(TcpListener::bind("0.0.0.0:443").acme(auto_cert), app).await + run( + TcpListener::bind("0.0.0.0:443").acme(auto_cert), + app, + shutdown, + ) + .await } else { - run(TcpListener::bind("127.0.0.1:3000"), app).await + run(TcpListener::bind("127.0.0.1:3000"), app, shutdown).await } } -async fn run(listener: L, app: Route) -> Result<(), ServerError> +async fn run(listener: L, app: Route, shutdown: CancellationToken) -> Result<(), ServerError> where L: Listener + 'static, { @@ -472,7 +477,8 @@ where .with(Tracing); Server::new(listener) .name("slingshot") - .run(app) + .run_with_graceful_shutdown(app, shutdown.cancelled(), None) .await .map_err(ServerError::ServerExited) + .inspect(|()| log::info!("server ended. goodbye.")) } -- 2.51.2 From 166550d15e2957301f6ad442d697c6f45847f9fa Mon Sep 17 00:00:00 2001 From: phil Date: Fri, 1 Aug 2025 18:18:47 -0400 Subject: [PATCH 097/134] com.bad-example.identity.resolveMiniDoc --- slingshot/src/identity.rs | 12 ++-- slingshot/src/server.rs | 139 +++++++++++++++++++++++++++++++++++++- 2 files changed, 143 insertions(+), 8 deletions(-) diff --git a/slingshot/src/identity.rs b/slingshot/src/identity.rs index a55821c..1abd0d0 100644 --- a/slingshot/src/identity.rs +++ b/slingshot/src/identity.rs @@ -54,16 +54,16 @@ enum IdentityData { /// /// partial because the handle is not verified #[derive(Debug, Clone, Serialize, Deserialize)] -struct PartialMiniDoc { +pub struct PartialMiniDoc { /// an atproto handle (**unverified**) /// /// the first valid atproto handle from the did doc's aka - unverified_handle: Handle, + pub unverified_handle: Handle, /// the did's atproto pds url (TODO: type this?) /// /// note: atrium *does* actually parse it into a URI, it just doesn't return /// that for some reason - pds: String, + pub pds: String, /// for now we're just pulling this straight from the did doc /// /// would be nice to type and validate it @@ -71,7 +71,7 @@ struct PartialMiniDoc { /// this is the publicKeyMultibase from the did doc. /// legacy key encoding not supported. /// `id`, `type`, and `controller` must be checked, but aren't stored. - signing_key: String, + pub signing_key: String, } impl TryFrom for PartialMiniDoc { @@ -212,7 +212,7 @@ impl Identity { Ok(Some(did)) } - /// Resolve (and verify!) a DID to a pds url + /// Resolve a DID to a pds url /// /// This *also* incidentally resolves and verifies the handle, which might /// make it slower than expected @@ -271,7 +271,7 @@ impl Identity { } /// Fetch (and cache) a partial mini doc from a did - async fn did_to_partial_mini_doc( + pub async fn did_to_partial_mini_doc( &self, did: &Did, ) -> Result, IdentityError> { diff --git a/slingshot/src/server.rs b/slingshot/src/server.rs index 26612d6..595d364 100644 --- a/slingshot/src/server.rs +++ b/slingshot/src/server.rs @@ -25,6 +25,9 @@ use poem_openapi::{ ApiResponse, Object, OpenApi, OpenApiService, param::Query, payload::Json, types::Example, }; +fn example_handle() -> String { + "bad-example.com".to_string() +} fn example_did() -> String { "did:plc:hdhoaan3xa3jiuq4fg4mefid".to_string() } @@ -42,6 +45,12 @@ fn example_uri() -> String { example_rkey() ) } +fn example_pds() -> String { + "https://porcini.us-east.host.bsky.network".to_string() +} +fn example_signing_key() -> String { + "zQ3shpq1g134o7HGDb86CtQFxnHqzx5pZWknrVX2Waum3fF6j".to_string() +} #[derive(Object)] #[oai(example = true)] @@ -67,13 +76,20 @@ fn xrpc_error(error: impl AsRef, message: impl AsRef) -> XrpcError { }) } -fn bad_request_handler(err: poem::Error) -> GetRecordResponse { +fn bad_request_handler_get_record(err: poem::Error) -> GetRecordResponse { GetRecordResponse::BadRequest(Json(XrpcErrorResponseObject { error: "InvalidRequest".to_string(), message: format!("Bad request, here's some info that maybe should not be exposed: {err}"), })) } +fn bad_request_handler_resolve_mini(err: poem::Error) -> ResolveMiniIDResponse { + ResolveMiniIDResponse::BadRequest(Json(XrpcErrorResponseObject { + error: "InvalidRequest".to_string(), + message: format!("Bad request, here's some info that maybe should not be exposed: {err}"), + })) +} + #[derive(Object)] #[oai(example = true)] struct FoundRecordResponseObject { @@ -108,7 +124,7 @@ impl Example for FoundRecordResponseObject { } #[derive(ApiResponse)] -#[oai(bad_request_handler = "bad_request_handler")] +#[oai(bad_request_handler = "bad_request_handler_get_record")] enum GetRecordResponse { /// Record found #[oai(status = 200)] @@ -127,6 +143,44 @@ enum GetRecordResponse { ServerError(XrpcError), } +#[derive(Object)] +#[oai(example = true)] +struct MiniDocResponseObject { + /// DID, bi-directionally verified if a handle was provided in the query. + did: String, + /// The validated handle of the account or `handle.invalid` if the handle + /// did not bi-directionally match the DID document. + handle: String, + /// The identity's PDS URL + pds: String, + /// The atproto signing key publicKeyMultibase + /// + /// Legacy key encoding not supported. the key is returned directly; `id`, + /// `type`, and `controller` are omitted. + signing_key: String, +} +impl Example for MiniDocResponseObject { + fn example() -> Self { + Self { + did: example_did(), + handle: example_handle(), + pds: example_pds(), + signing_key: example_signing_key(), + } + } +} + +#[derive(ApiResponse)] +#[oai(bad_request_handler = "bad_request_handler_resolve_mini")] +enum ResolveMiniIDResponse { + /// Identity resolved + #[oai(status = 200)] + Ok(Json), + /// Bad request or identity not resolved + #[oai(status = 400)] + BadRequest(XrpcError), +} + struct Xrpc { cache: HybridCache, identity: Identity, @@ -223,6 +277,87 @@ impl Xrpc { .await } + /// com.bad-example.identity.resolveMiniDoc + /// + /// Like [com.atproto.identity.resolveIdentity](https://docs.bsky.app/docs/api/com-atproto-identity-resolve-identity) + /// but instead of the full `didDoc` it returns an atproto-relevant subset. + #[oai(path = "/com.bad-example.identity.resolveMiniDoc", method = "get")] + async fn resolve_mini_id( + &self, + /// Handle or DID to resolve + #[oai(example = "example_handle")] + Query(identifier): Query, + ) -> ResolveMiniIDResponse { + let invalid = |reason: &'static str| { + ResolveMiniIDResponse::BadRequest(xrpc_error("InvalidRequest", reason)) + }; + + let mut unverified_handle = None; + let did = match Did::new(identifier.clone()) { + Ok(did) => did, + Err(_) => { + let Ok(alleged_handle) = Handle::new(identifier) else { + return invalid("identifier was not a valid DID or handle"); + }; + if let Ok(res) = self.identity.handle_to_did(alleged_handle.clone()).await { + if let Some(did) = res { + // we did it joe + unverified_handle = Some(alleged_handle); + did + } else { + return invalid("Could not resolve handle identifier to a DID"); + } + } else { + // TODO: ServerError not BadRequest + return invalid("errored while trying to resolve handle to DID"); + } + } + }; + let Ok(partial_doc) = self.identity.did_to_partial_mini_doc(&did).await else { + return invalid("failed to get DID doc"); + }; + let Some(partial_doc) = partial_doc else { + return invalid("failed to find DID doc"); + }; + + // ok so here's where we're at: + // ✅ we have a DID + // ✅ we have a partial doc + // 🔶 if we have a handle, it's from the `identifier` (user-input) + // -> then we just need to compare to the partial doc to confirm + // -> else we need to resolve the DID doc's to a handle and check + let handle = if let Some(h) = unverified_handle { + if h == partial_doc.unverified_handle { + h.to_string() + } else { + "handle.invalid".to_string() + } + } else { + let Ok(handle_did) = self + .identity + .handle_to_did(partial_doc.unverified_handle.clone()) + .await + else { + return invalid("failed to get did doc's handle"); + }; + let Some(handle_did) = handle_did else { + return invalid("failed to resolve did doc's handle"); + }; + if handle_did == did { + partial_doc.unverified_handle.to_string() + } else { + "handle.invalid".to_string() + } + }; + + ResolveMiniIDResponse::Ok(Json(MiniDocResponseObject { + did: did.to_string(), + handle, + pds: partial_doc.pds, + signing_key: partial_doc.signing_key, + })) + } + async fn get_record_impl( &self, repo: String, -- 2.51.2 From ba8518a52358891aff7d1309dd861fd4241468e7 Mon Sep 17 00:00:00 2001 From: phil Date: Sat, 2 Aug 2025 11:34:53 -0400 Subject: [PATCH 098/134] add stoplight elements because why not --- slingshot/Cargo.toml | 2 +- slingshot/src/server.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/slingshot/Cargo.toml b/slingshot/Cargo.toml index 0e0cdfa..3e9a8dc 100644 --- a/slingshot/Cargo.toml +++ b/slingshot/Cargo.toml @@ -18,7 +18,7 @@ log = "0.4.27" metrics = "0.24.2" metrics-exporter-prometheus = { version = "0.17.1", features = ["http-listener"] } poem = { version = "3.1.12", features = ["acme"] } -poem-openapi = { version = "5.1.16", features = ["scalar"] } +poem-openapi = { version = "5.1.16", features = ["scalar", "stoplight-elements"] } reqwest = { version = "0.12.22", features = ["json"] } rustls = "0.23.31" serde = { version = "1.0.219", features = ["derive"] } diff --git a/slingshot/src/server.rs b/slingshot/src/server.rs index 595d364..952e545 100644 --- a/slingshot/src/server.rs +++ b/slingshot/src/server.rs @@ -566,6 +566,7 @@ pub async fn serve( let mut app = Route::new() .nest("/", api_service.scalar()) + .nest("/se", api_service.stoplight_elements()) .nest("/openapi.json", api_service.spec_endpoint()) .nest("/xrpc/", api_service); -- 2.51.2 From 1d646ef2251dade1cc50379af6c035b6956405cd Mon Sep 17 00:00:00 2001 From: phil Date: Mon, 4 Aug 2025 18:19:25 -0400 Subject: [PATCH 099/134] more documentation text --- slingshot/Cargo.toml | 2 +- slingshot/api-description.md | 73 ++++++++++++++++++++++++++++++++++++ slingshot/src/server.rs | 63 +++++++++++++++++++++++++++---- 3 files changed, 130 insertions(+), 8 deletions(-) create mode 100644 slingshot/api-description.md diff --git a/slingshot/Cargo.toml b/slingshot/Cargo.toml index 3e9a8dc..0e0cdfa 100644 --- a/slingshot/Cargo.toml +++ b/slingshot/Cargo.toml @@ -18,7 +18,7 @@ log = "0.4.27" metrics = "0.24.2" metrics-exporter-prometheus = { version = "0.17.1", features = ["http-listener"] } poem = { version = "3.1.12", features = ["acme"] } -poem-openapi = { version = "5.1.16", features = ["scalar", "stoplight-elements"] } +poem-openapi = { version = "5.1.16", features = ["scalar"] } reqwest = { version = "0.12.22", features = ["json"] } rustls = "0.23.31" serde = { version = "1.0.219", features = ["derive"] } diff --git a/slingshot/api-description.md b/slingshot/api-description.md new file mode 100644 index 0000000..e923421 --- /dev/null +++ b/slingshot/api-description.md @@ -0,0 +1,73 @@ +_A [gravitational slingshot](https://en.wikipedia.org/wiki/Gravity_assist) makes use of the gravity and relative movements of celestial bodies to accelerate a spacecraft and change its trajectory._ + + +# Slingshot: edge record cache + +Applications in [ATProtocol](https://atproto.com/) store data in users' own [PDS](https://atproto.com/guides/self-hosting) (Personal Data Server), which are distributed across thousands of independently-run servers all over the world. Trying to access this data poses challenges for client applications: + +- A PDS might be far away with long network latency +- or may be on an unreliable connection +- or overloaded when you need it, or offline, or… + +Large projects like [Bluesky](https://bsky.app/) control their performance and reliability by syncing all app-relevant data from PDSs into first-party databases. But for new apps, building out this additional data infrastructure adds significant effort and complexity up front. + +**Slingshot is a fast, eager, production-grade cache of data in the [ATmosphere](https://atproto.com/)**, offering performance and reliability without custom infrastructure. + + +### Current status + +Slingshot is currently in a **v0, pre-release state**. There is one production instance and you can use it! Expect short downtimes for restarts as development progresses and lower cache hit-rates as the internal storage caches are adjusted and reset. + +The core APIs will not change, since they are standard third-party `com.atproto` query APIs from ATProtocol. + + +## Eager caching + +In many cases, Slingshot can cache the data you need *before* first request! + +Slingshot subscribes to the global [Firehose](https://atproto.com/specs/sync#firehose) of data updates. It keeps a short-term rolling indexed window of *all* data, and automatically promotes content likely to be requested to its longer-term main cache. _(automatic promotion is still a work in progress)_ + +When there is a cache miss, Slingshot can often still accelerate record fetching, since it keeps a large cache of resolved identities: it can usually request from the correct PDS without extra lookups. + + +## Precise invalidation + +The fireshose includes **update** and **delete** events, which Slingshot uses to ensure stale and deleted data is removed within a very short window. Additonally, identity and account-level events can trigger rapid cleanup of data for deactivated and deleted accounts. _(some of this is still a work in progress)_ + + +## Low-trust + +The "AT" in ATProtocol [stands for _Authenticated Transfer_](https://atproto.com/guides/glossary#at-protocol): all data is cryptographically signed, which makes it possible to broadcast data through third parties and trust that it's real _without_ having to directly contact the originating server. + +Two core standard query APIs are supported to balance convenience and trust. They both fetch [records](https://atproto.com/guides/glossary#record): + +### [`com.atproto.repo.getRecord`](#tag/comatproto-queries/GET/xrpc/com.atproto.repo.getRecord) + +- convenient `JSON` response format +- cannot be proven authentic + +### [`com.atproto.sync.getRecord`](#tag/comatproto-queries/GET/xrpc/com.atproto.sync.getRecord) + +- [`DAG-CBOR`](https://atproto.com/specs/data-model)-encoded response requires extra libraries to decode, but +- includes a cryptographic proof of authenticity! + +_(work on this endpoint is in progress)_ + + +## Ergonomic APIs + +- Slingshot also offers variants of the `getRecord` endpoints that accept a full `at-uri` as a parameter, to save clients from needing to parse and validate all parts of a record location. + +- Bi-directionally verifying identity endpoints, so you can directly exchange atproto [`handle`](https://atproto.com/guides/glossary#handle)s for [`DID`](https://atproto.com/guides/glossary#did-decentralized-id)s without extra steps, plus a convenient [Mini-Doc](#tag/slingshot-specific-queries/GET/xrpc/com.bad-example.identity.resolveMiniDoc) verified identity summary. + + +## Part of microcosm + +[Microcosm](https://www.microcosm.blue/) is a collection of services and independent community-run infrastructure for ATProtocol. + +Slingshot excels when combined with _shallow indexing_ services, which offer fast queries of global data relationships but with only references to the data records. Microcosm has a few! + +- [🌌 Constellation](https://constellation.microcosm.blue/), a global backlink index (all social interactions in atproto are links!) +- [🎇 Spacedust](https://spacedust.microcosm.blue/), a firehose of all social interactions + +All microcosm projects are [open source](https://tangled.sh/@bad-example.com/microcosm-links). **You can help sustain Slingshot** and all of microcosm by becoming a [Github sponsor](https://github.com/sponsors/uniphil/) or a [Ko-fi supporter](https://ko-fi.com/bad_example)! diff --git a/slingshot/src/server.rs b/slingshot/src/server.rs index 952e545..c672b42 100644 --- a/slingshot/src/server.rs +++ b/slingshot/src/server.rs @@ -22,7 +22,8 @@ use poem::{ middleware::{Cors, Tracing}, }; use poem_openapi::{ - ApiResponse, Object, OpenApi, OpenApiService, param::Query, payload::Json, types::Example, + ApiResponse, ContactObject, ExternalDocumentObject, Object, OpenApi, OpenApiService, Tags, + param::Query, payload::Json, types::Example, }; fn example_handle() -> String { @@ -187,6 +188,34 @@ struct Xrpc { repo: Arc, } +#[derive(Tags)] +enum ApiTags { + /// Core ATProtocol-compatible APIs. + /// + /// Upstream documentation is available at + /// https://docs.bsky.app/docs/category/http-reference + /// + /// These queries are usually executed directly against the PDS containing + /// the data being requested. Slingshot offers a caching view of the same + /// contents with better expected performance and reliability. + #[oai(rename = "com.atproto.* queries")] + ComAtproto, + /// Additional and improved APIs. + /// + /// These APIs offer small tweaks to the core ATProtocol APIs, with more + /// more convenient [request parameters](#tag/slingshot-specific-queries/GET/xrpc/com.bad-example.repo.getUriRecord) + /// or [response formats](#tag/slingshot-specific-queries/GET/xrpc/com.bad-example.identity.resolveMiniDoc). + /// + /// At the moment, these are namespaced under the `com.bad-example.*` NSID + /// prefix, but as they stabilize they will likely be moved to either + /// `blue.microcosm.*` or a slingshot-instance-specific lexicon under its + /// `did:web` (ie., `blue.microcosm.slingshot.*`). Maybe one day they can + /// be promoted to the [Lexicon Community](https://discourse.lexicon.community/) + /// namespace. + #[oai(rename = "slingshot-specific queries")] + Custom, +} + #[OpenApi] impl Xrpc { /// com.atproto.repo.getRecord @@ -195,7 +224,11 @@ impl Xrpc { /// /// See also the [canonical `com.atproto` XRPC documentation](https://docs.bsky.app/docs/api/com-atproto-repo-get-record) /// that this endpoint aims to be compatible with. - #[oai(path = "/com.atproto.repo.getRecord", method = "get")] + #[oai( + path = "/com.atproto.repo.getRecord", + method = "get", + tag = "ApiTags::ComAtproto" + )] async fn get_record( &self, /// The DID or handle of the repo @@ -222,8 +255,12 @@ impl Xrpc { /// com.bad-example.repo.getUriRecord /// /// Ergonomic complement to [`com.atproto.repo.getRecord`](https://docs.bsky.app/docs/api/com-atproto-repo-get-record) - /// which accepts an at-uri instead of individual repo/collection/rkey params - #[oai(path = "/com.bad-example.repo.getUriRecord", method = "get")] + /// which accepts an `at-uri` instead of individual repo/collection/rkey params + #[oai( + path = "/com.bad-example.repo.getUriRecord", + method = "get", + tag = "ApiTags::Custom" + )] async fn get_uri_record( &self, /// The at-uri of the record @@ -281,7 +318,11 @@ impl Xrpc { /// /// Like [com.atproto.identity.resolveIdentity](https://docs.bsky.app/docs/api/com-atproto-identity-resolve-identity) /// but instead of the full `didDoc` it returns an atproto-relevant subset. - #[oai(path = "/com.bad-example.identity.resolveMiniDoc", method = "get")] + #[oai( + path = "/com.bad-example.identity.resolveMiniDoc", + method = "get", + tag = "ApiTags::Custom" + )] async fn resolve_mini_id( &self, /// Handle or DID to resolve @@ -562,11 +603,19 @@ pub async fn serve( } else { "http://localhost:3000".to_string() }) - .url_prefix("/xrpc"); + .url_prefix("/xrpc") + .contact( + ContactObject::new() + .name("@microcosm.blue") + .url("https://bsky.app/profile/microcosm.blue"), + ) + .description(include_str!("../api-description.md")) + .external_document(ExternalDocumentObject::new( + "https://microcosm.blue/slingshot", + )); let mut app = Route::new() .nest("/", api_service.scalar()) - .nest("/se", api_service.stoplight_elements()) .nest("/openapi.json", api_service.spec_endpoint()) .nest("/xrpc/", api_service); -- 2.51.2 From 2afb053ee5382502d0f283f2d3b97b9d7ba740ae Mon Sep 17 00:00:00 2001 From: phil Date: Mon, 4 Aug 2025 19:05:34 -0400 Subject: [PATCH 100/134] com.atproto.identity.resolveHandle --- slingshot/src/server.rs | 97 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/slingshot/src/server.rs b/slingshot/src/server.rs index c672b42..049c021 100644 --- a/slingshot/src/server.rs +++ b/slingshot/src/server.rs @@ -91,6 +91,13 @@ fn bad_request_handler_resolve_mini(err: poem::Error) -> ResolveMiniIDResponse { })) } +fn bad_request_handler_resolve_handle(err: poem::Error) -> JustDidResponse { + JustDidResponse::BadRequest(Json(XrpcErrorResponseObject { + error: "InvalidRequest".to_string(), + message: format!("Bad request, here's some info that maybe should not be exposed: {err}"), + })) +} + #[derive(Object)] #[oai(example = true)] struct FoundRecordResponseObject { @@ -182,6 +189,34 @@ enum ResolveMiniIDResponse { BadRequest(XrpcError), } +#[derive(Object)] +#[oai(example = true)] +struct FoundDidResponseObject { + /// the DID, bi-directionally verified if using Slingshot + did: String, +} +impl Example for FoundDidResponseObject { + fn example() -> Self { + Self { did: example_did() } + } +} + +#[derive(ApiResponse)] +#[oai(bad_request_handler = "bad_request_handler_resolve_handle")] +enum JustDidResponse { + /// Resolution succeeded + #[oai(status = 200)] + Ok(Json), + /// Bad request, failed to resolve, or failed to verify + /// + /// `error` will be one of `InvalidRequest`, `HandleNotFound`. + #[oai(status = 400)] + BadRequest(XrpcError), + /// Something went wrong trying to complete the request + #[oai(status = 500)] + ServerError(XrpcError), +} + struct Xrpc { cache: HybridCache, identity: Identity, @@ -314,6 +349,68 @@ impl Xrpc { .await } + /// com.atproto.identity.resolveHandle + /// + /// Resolves an atproto [`handle`](https://atproto.com/guides/glossary#handle) + /// (hostname) to a [`DID`](https://atproto.com/guides/glossary#did-decentralized-id). + /// + /// Compatibility note: **Slingshot will _always_ bi-directionally verify + /// against the DID document**, which is optional by the authoritative + /// lexicon. You can trust the `DID` returned from this endpoint without + /// further checks, but this may not hold if you switch from Slingshot to a + /// different service offering this query. + /// + /// See also the [canonical `com.atproto` XRPC documentation](https://docs.bsky.app/docs/api/com-atproto-identity-resolve-handle) + /// that this endpoint aims to be compatible with. + #[oai( + path = "/com.atproto.identity.resolveHandle", + method = "get", + tag = "ApiTags::ComAtproto" + )] + async fn resolve_handle( + &self, + /// The handle to resolve. + #[oai(example = "example_handle")] + Query(handle): Query, + ) -> JustDidResponse { + let Ok(handle) = Handle::new(handle) else { + return JustDidResponse::BadRequest(xrpc_error("InvalidRequest", "not a valid handle")); + }; + + let Ok(alleged_did) = self.identity.handle_to_did(handle.clone()).await else { + return JustDidResponse::ServerError(xrpc_error("Failed", "Could not resolve handle")); + }; + + let Some(alleged_did) = alleged_did else { + return JustDidResponse::BadRequest(xrpc_error( + "HandleNotFound", + "Could not resolve handle to a DID", + )); + }; + + let Ok(partial_doc) = self.identity.did_to_partial_mini_doc(&alleged_did).await else { + return JustDidResponse::ServerError(xrpc_error("Failed", "Could not fetch DID doc")); + }; + + let Some(partial_doc) = partial_doc else { + return JustDidResponse::BadRequest(xrpc_error( + "HandleNotFound", + "Resolved handle but could not find DID doc for the DID", + )); + }; + + if partial_doc.unverified_handle != handle { + return JustDidResponse::BadRequest(xrpc_error( + "HandleNotFound", + "Resolved handle failed bi-directional validation", + )); + } + + JustDidResponse::Ok(Json(FoundDidResponseObject { + did: alleged_did.to_string(), + })) + } + /// com.bad-example.identity.resolveMiniDoc /// /// Like [com.atproto.identity.resolveIdentity](https://docs.bsky.app/docs/api/com-atproto-identity-resolve-identity) -- 2.51.2 From 59bf7b7124ff21a4124be282a4628d8e01c60774 Mon Sep 17 00:00:00 2001 From: phil Date: Tue, 5 Aug 2025 11:26:29 -0400 Subject: [PATCH 101/134] custom scalar template --- Cargo.lock | 2 ++ slingshot/Cargo.toml | 2 +- slingshot/src/server.rs | 6 ++--- slingshot/static/index.html | 53 +++++++++++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 slingshot/static/index.html diff --git a/Cargo.lock b/Cargo.lock index c1bb9ec..3f51793 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3787,9 +3787,11 @@ dependencies = [ "headers", "http", "http-body-util", + "httpdate", "hyper", "hyper-util", "mime", + "mime_guess", "multer", "nix", "parking_lot", diff --git a/slingshot/Cargo.toml b/slingshot/Cargo.toml index 0e0cdfa..06a97a5 100644 --- a/slingshot/Cargo.toml +++ b/slingshot/Cargo.toml @@ -17,7 +17,7 @@ links = { path = "../links" } log = "0.4.27" metrics = "0.24.2" metrics-exporter-prometheus = { version = "0.17.1", features = ["http-listener"] } -poem = { version = "3.1.12", features = ["acme"] } +poem = { version = "3.1.12", features = ["acme", "static-files"] } poem-openapi = { version = "5.1.16", features = ["scalar"] } reqwest = { version = "0.12.22", features = ["json"] } rustls = "0.23.31" diff --git a/slingshot/src/server.rs b/slingshot/src/server.rs index 049c021..247cbe5 100644 --- a/slingshot/src/server.rs +++ b/slingshot/src/server.rs @@ -13,7 +13,7 @@ use tokio_util::sync::CancellationToken; use poem::{ Endpoint, EndpointExt, Route, Server, - endpoint::make_sync, + endpoint::{StaticFileEndpoint, make_sync}, http::Method, listener::{ Listener, TcpListener, @@ -712,8 +712,8 @@ pub async fn serve( )); let mut app = Route::new() - .nest("/", api_service.scalar()) - .nest("/openapi.json", api_service.spec_endpoint()) + .at("/", StaticFileEndpoint::new("./static/index.html")) + .nest("/openapi", api_service.spec_endpoint()) .nest("/xrpc/", api_service); if let Some(host) = host { diff --git a/slingshot/static/index.html b/slingshot/static/index.html new file mode 100644 index 0000000..38ad65f --- /dev/null +++ b/slingshot/static/index.html @@ -0,0 +1,53 @@ + + + + + Slingshot: atproto edge record cache + + + + + +
+

+ TODO: thing +

+ +
+ + + + + + + + -- 2.51.2 From 9322d0a48ef0545226643e504f110c21a2f10dca Mon Sep 17 00:00:00 2001 From: phil Date: Tue, 5 Aug 2025 11:28:05 -0400 Subject: [PATCH 102/134] shorter caveat was a bit redundant --- slingshot/src/server.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/slingshot/src/server.rs b/slingshot/src/server.rs index 247cbe5..478df5a 100644 --- a/slingshot/src/server.rs +++ b/slingshot/src/server.rs @@ -355,12 +355,10 @@ impl Xrpc { /// (hostname) to a [`DID`](https://atproto.com/guides/glossary#did-decentralized-id). /// /// Compatibility note: **Slingshot will _always_ bi-directionally verify - /// against the DID document**, which is optional by the authoritative - /// lexicon. You can trust the `DID` returned from this endpoint without - /// further checks, but this may not hold if you switch from Slingshot to a - /// different service offering this query. + /// against the DID document**, which is optional according to the + /// authoritative lexicon. /// - /// See also the [canonical `com.atproto` XRPC documentation](https://docs.bsky.app/docs/api/com-atproto-identity-resolve-handle) + /// See the [canonical `com.atproto` XRPC documentation](https://docs.bsky.app/docs/api/com-atproto-identity-resolve-handle) /// that this endpoint aims to be compatible with. #[oai( path = "/com.atproto.identity.resolveHandle", -- 2.51.2 From c04b055e89bda944bbec396c3530d5e2dced60d9 Mon Sep 17 00:00:00 2001 From: phil Date: Tue, 5 Aug 2025 11:31:57 -0400 Subject: [PATCH 103/134] slight link gen diff from built-in scalar should probably pin (er, bundle) the scalar version actually used --- slingshot/api-description.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/slingshot/api-description.md b/slingshot/api-description.md index e923421..851a9d4 100644 --- a/slingshot/api-description.md +++ b/slingshot/api-description.md @@ -41,12 +41,12 @@ The "AT" in ATProtocol [stands for _Authenticated Transfer_](https://atproto.com Two core standard query APIs are supported to balance convenience and trust. They both fetch [records](https://atproto.com/guides/glossary#record): -### [`com.atproto.repo.getRecord`](#tag/comatproto-queries/GET/xrpc/com.atproto.repo.getRecord) +### [`com.atproto.repo.getRecord`](#tag/comatproto-queries/get/xrpc/com.atproto.repo.getRecord) - convenient `JSON` response format - cannot be proven authentic -### [`com.atproto.sync.getRecord`](#tag/comatproto-queries/GET/xrpc/com.atproto.sync.getRecord) +### [`com.atproto.sync.getRecord`](#tag/comatproto-queries/get/xrpc/com.atproto.sync.getRecord) - [`DAG-CBOR`](https://atproto.com/specs/data-model)-encoded response requires extra libraries to decode, but - includes a cryptographic proof of authenticity! @@ -58,7 +58,7 @@ _(work on this endpoint is in progress)_ - Slingshot also offers variants of the `getRecord` endpoints that accept a full `at-uri` as a parameter, to save clients from needing to parse and validate all parts of a record location. -- Bi-directionally verifying identity endpoints, so you can directly exchange atproto [`handle`](https://atproto.com/guides/glossary#handle)s for [`DID`](https://atproto.com/guides/glossary#did-decentralized-id)s without extra steps, plus a convenient [Mini-Doc](#tag/slingshot-specific-queries/GET/xrpc/com.bad-example.identity.resolveMiniDoc) verified identity summary. +- Bi-directionally verifying identity endpoints, so you can directly exchange atproto [`handle`](https://atproto.com/guides/glossary#handle)s for [`DID`](https://atproto.com/guides/glossary#did-decentralized-id)s without extra steps, plus a convenient [Mini-Doc](#tag/slingshot-specific-queries/get/xrpc/com.bad-example.identity.resolveMiniDoc) verified identity summary. ## Part of microcosm -- 2.51.2 From b24c68c6b9851f103ca647c45dcfeff255d00afa Mon Sep 17 00:00:00 2001 From: phil Date: Tue, 5 Aug 2025 12:15:55 -0400 Subject: [PATCH 104/134] note boxes in docs! --- slingshot/api-description.md | 24 ++++++++++++-- slingshot/src/main.rs | 4 +-- slingshot/src/server.rs | 62 +++++++++++++++++++----------------- 3 files changed, 57 insertions(+), 33 deletions(-) diff --git a/slingshot/api-description.md b/slingshot/api-description.md index 851a9d4..e029693 100644 --- a/slingshot/api-description.md +++ b/slingshot/api-description.md @@ -16,7 +16,8 @@ Large projects like [Bluesky](https://bsky.app/) control their performance and r ### Current status -Slingshot is currently in a **v0, pre-release state**. There is one production instance and you can use it! Expect short downtimes for restarts as development progresses and lower cache hit-rates as the internal storage caches are adjusted and reset. +> [!important] +> Slingshot is currently in a **v0, pre-release state**. There is one production instance and you can use it! Expect short downtimes for restarts as development progresses and lower cache hit-rates as the internal storage caches are adjusted and reset. The core APIs will not change, since they are standard third-party `com.atproto` query APIs from ATProtocol. @@ -54,6 +55,24 @@ Two core standard query APIs are supported to balance convenience and trust. The _(work on this endpoint is in progress)_ +## Service proxying + +Clients can proxy atproto queries through their own PDS with [Service Proxying](https://atproto.com/specs/xrpc#service-proxying), and this is supported by Slingshot. The Slingshot instance must be started the `--domain` argument specified. + +Service-proxied requests can specify a Slingshot instance via the `atproto-proxy` header: + +```http +GET /xrpc/com.bad-example.identity.resolveMiniDoc?identifier=bad-example.com +Host: +atproto-proxy: did:web:#slingshot +``` + +Where `` is the user's own PDS host, and `` is the domain that the slingshot instance is deployed at (eg. `slingshot.microcosm.blue`). See the [Service Proxying](https://atproto.com/specs/xrpc#service-proxying) docs for more. + +> [!tip] +> Service proxying is supported but completely optional. All APIs are directly accessible over the public internet, and GeoDNS helps route users to the closest instance to them for the lowest possible latency. (_note: deploying multiple slingshot instances with GeoDNS is still TODO_) + + ## Ergonomic APIs - Slingshot also offers variants of the `getRecord` endpoints that accept a full `at-uri` as a parameter, to save clients from needing to parse and validate all parts of a record location. @@ -70,4 +89,5 @@ Slingshot excels when combined with _shallow indexing_ services, which offer fas - [🌌 Constellation](https://constellation.microcosm.blue/), a global backlink index (all social interactions in atproto are links!) - [🎇 Spacedust](https://spacedust.microcosm.blue/), a firehose of all social interactions -All microcosm projects are [open source](https://tangled.sh/@bad-example.com/microcosm-links). **You can help sustain Slingshot** and all of microcosm by becoming a [Github sponsor](https://github.com/sponsors/uniphil/) or a [Ko-fi supporter](https://ko-fi.com/bad_example)! +> [!success] +> All microcosm projects are [open source](https://tangled.sh/@bad-example.com/microcosm-links). **You can help sustain Slingshot** and all of microcosm by becoming a [Github sponsor](https://github.com/sponsors/uniphil/) or a [Ko-fi supporter](https://ko-fi.com/bad_example)! diff --git a/slingshot/src/main.rs b/slingshot/src/main.rs index c427e2a..a02d02e 100644 --- a/slingshot/src/main.rs +++ b/slingshot/src/main.rs @@ -30,7 +30,7 @@ struct Args { /// - an HTTPS certs will be automatically configured with Acme/letsencrypt /// - TODO: a rate-limiter will be installed #[arg(long)] - host: Option, + domain: Option, /// email address for letsencrypt contact /// /// recommended in production, i guess? @@ -104,7 +104,7 @@ async fn main() -> Result<(), String> { server_cache_handle, identity, repo, - args.host, + args.domain, args.acme_contact, args.certs, server_shutdown, diff --git a/slingshot/src/server.rs b/slingshot/src/server.rs index 478df5a..db198a4 100644 --- a/slingshot/src/server.rs +++ b/slingshot/src/server.rs @@ -227,8 +227,9 @@ struct Xrpc { enum ApiTags { /// Core ATProtocol-compatible APIs. /// - /// Upstream documentation is available at - /// https://docs.bsky.app/docs/category/http-reference + /// > [!tip] + /// > Upstream documentation is available at + /// > https://docs.bsky.app/docs/category/http-reference /// /// These queries are usually executed directly against the PDS containing /// the data being requested. Slingshot offers a caching view of the same @@ -241,12 +242,11 @@ enum ApiTags { /// more convenient [request parameters](#tag/slingshot-specific-queries/GET/xrpc/com.bad-example.repo.getUriRecord) /// or [response formats](#tag/slingshot-specific-queries/GET/xrpc/com.bad-example.identity.resolveMiniDoc). /// - /// At the moment, these are namespaced under the `com.bad-example.*` NSID - /// prefix, but as they stabilize they will likely be moved to either - /// `blue.microcosm.*` or a slingshot-instance-specific lexicon under its - /// `did:web` (ie., `blue.microcosm.slingshot.*`). Maybe one day they can - /// be promoted to the [Lexicon Community](https://discourse.lexicon.community/) - /// namespace. + /// > [!important] + /// > At the moment, these are namespaced under the `com.bad-example.*` NSID + /// > prefix, but as they stabilize they may be migrated to an org namespace + /// > like `blue.microcosm.*`. Support for asliasing to `com.bad-example.*` + /// > will be maintained as long as it's in use. #[oai(rename = "slingshot-specific queries")] Custom, } @@ -257,8 +257,9 @@ impl Xrpc { /// /// Get a single record from a repository. Does not require auth. /// - /// See also the [canonical `com.atproto` XRPC documentation](https://docs.bsky.app/docs/api/com-atproto-repo-get-record) - /// that this endpoint aims to be compatible with. + /// > [!tip] + /// > See also the [canonical `com.atproto` XRPC documentation](https://docs.bsky.app/docs/api/com-atproto-repo-get-record) + /// > that this endpoint aims to be compatible with. #[oai( path = "/com.atproto.repo.getRecord", method = "get", @@ -279,9 +280,9 @@ impl Xrpc { /// /// If not specified, then return the most recent version. /// - /// If specified and a newer version of the record exists, returns 404 not - /// found. That is: slingshot only retains the most recent version of a - /// record. (TODO: verify bsky behaviour for mismatched/old CID) + /// If a stale `CID` is specified and a newer version of the record + /// exists, Slingshot returns a `NotFound` error. That is: Slingshot + /// only retains the most recent version of a record. Query(cid): Query>, ) -> GetRecordResponse { self.get_record_impl(repo, collection, rkey, cid).await @@ -308,9 +309,10 @@ impl Xrpc { /// /// If not specified, then return the most recent version. /// - /// If specified and a newer version of the record exists, returns 404 not - /// found. That is: slingshot only retains the most recent version of a - /// record. + /// > [!tip] + /// > If specified and a newer version of the record exists, returns 404 not + /// > found. That is: slingshot only retains the most recent version of a + /// > record. Query(cid): Query>, ) -> GetRecordResponse { let bad_at_uri = || { @@ -354,12 +356,14 @@ impl Xrpc { /// Resolves an atproto [`handle`](https://atproto.com/guides/glossary#handle) /// (hostname) to a [`DID`](https://atproto.com/guides/glossary#did-decentralized-id). /// - /// Compatibility note: **Slingshot will _always_ bi-directionally verify - /// against the DID document**, which is optional according to the - /// authoritative lexicon. + /// > [!tip] + /// > Compatibility note: Slingshot will **always bi-directionally verify + /// > against the DID document**, which is optional according to the + /// > authoritative lexicon. /// - /// See the [canonical `com.atproto` XRPC documentation](https://docs.bsky.app/docs/api/com-atproto-identity-resolve-handle) - /// that this endpoint aims to be compatible with. + /// > [!tip] + /// > See the [canonical `com.atproto` XRPC documentation](https://docs.bsky.app/docs/api/com-atproto-identity-resolve-handle) + /// > that this endpoint aims to be compatible with. #[oai( path = "/com.atproto.identity.resolveHandle", method = "get", @@ -662,13 +666,13 @@ struct AppViewDoc { /// /// - PDS proxying offers a level of client IP anonymity from slingshot /// - slingshot *may* implement more generous per-user rate-limits for proxied requests in the future -fn get_did_doc(host: &str) -> impl Endpoint + use<> { +fn get_did_doc(domain: &str) -> impl Endpoint + use<> { let doc = poem::web::Json(AppViewDoc { - id: format!("did:web:{host}"), + id: format!("did:web:{domain}"), service: [AppViewService { id: "#slingshot".to_string(), r#type: "SlingshotRecordProxy".to_string(), - service_endpoint: format!("https://{host}"), + service_endpoint: format!("https://{domain}"), }], }); make_sync(move |_| doc.clone()) @@ -678,7 +682,7 @@ pub async fn serve( cache: HybridCache, identity: Identity, repo: Repo, - host: Option, + domain: Option, acme_contact: Option, certs: Option, shutdown: CancellationToken, @@ -693,7 +697,7 @@ pub async fn serve( "Slingshot", env!("CARGO_PKG_VERSION"), ) - .server(if let Some(ref h) = host { + .server(if let Some(ref h) = domain { format!("https://{h}") } else { "http://localhost:3000".to_string() @@ -714,16 +718,16 @@ pub async fn serve( .nest("/openapi", api_service.spec_endpoint()) .nest("/xrpc/", api_service); - if let Some(host) = host { + if let Some(domain) = domain { rustls::crypto::aws_lc_rs::default_provider() .install_default() .expect("alskfjalksdjf"); - app = app.at("/.well-known/did.json", get_did_doc(&host)); + app = app.at("/.well-known/did.json", get_did_doc(&domain)); let mut auto_cert = AutoCert::builder() .directory_url(LETS_ENCRYPT_PRODUCTION) - .domain(&host); + .domain(&domain); if let Some(contact) = acme_contact { auto_cert = auto_cert.contact(contact); } -- 2.51.2 From e9891b23a61b931b061cd36d56f3184327936ac3 Mon Sep 17 00:00:00 2001 From: phil Date: Tue, 5 Aug 2025 12:35:31 -0400 Subject: [PATCH 105/134] basic style tweaks and light-mode fix --- slingshot/static/index.html | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/slingshot/static/index.html b/slingshot/static/index.html index 38ad65f..307bf07 100644 --- a/slingshot/static/index.html +++ b/slingshot/static/index.html @@ -6,6 +6,12 @@ -- 2.51.2 From 8a15ab251d9bc576f78c66f0aebcaf898b85a80f Mon Sep 17 00:00:00 2001 From: phil Date: Tue, 5 Aug 2025 12:39:02 -0400 Subject: [PATCH 106/134] smaller font size in alerts --- slingshot/static/index.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/slingshot/static/index.html b/slingshot/static/index.html index 307bf07..a08e0b5 100644 --- a/slingshot/static/index.html +++ b/slingshot/static/index.html @@ -9,6 +9,9 @@ :root { --scalar-small: 13px; } + .scalar-app .markdown .markdown-alert { + font-size: var(--scalar-small); + } .sidebar-heading-link-title { line-height: 1.2; } -- 2.51.2 From aaf84cd09cc8e2f0ef81ef29d72b778d1e765334 Mon Sep 17 00:00:00 2001 From: phil Date: Tue, 5 Aug 2025 12:54:47 -0400 Subject: [PATCH 107/134] hide models idk do people use this? cleaner sidebar without --- slingshot/static/index.html | 1 + 1 file changed, 1 insertion(+) diff --git a/slingshot/static/index.html b/slingshot/static/index.html index a08e0b5..74b0a3d 100644 --- a/slingshot/static/index.html +++ b/slingshot/static/index.html @@ -57,6 +57,7 @@ -- 2.51.2 From c30958a32dbdc85244ddac563245b06ed0af8e71 Mon Sep 17 00:00:00 2001 From: phil Date: Tue, 5 Aug 2025 16:11:44 -0400 Subject: [PATCH 108/134] healthcheck --- slingshot/src/error.rs | 8 ++++++++ slingshot/src/healthcheck.rs | 32 ++++++++++++++++++++++++++++++++ slingshot/src/lib.rs | 2 ++ slingshot/src/main.rs | 15 ++++++++++++++- slingshot/src/server.rs | 3 ++- 5 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 slingshot/src/healthcheck.rs diff --git a/slingshot/src/error.rs b/slingshot/src/error.rs index 17eb0ee..67b3e07 100644 --- a/slingshot/src/error.rs +++ b/slingshot/src/error.rs @@ -46,6 +46,12 @@ pub enum IdentityError { RefreshQueueKeyError(&'static str), } +#[derive(Debug, Error)] +pub enum HealthCheckError { + #[error("failed to send checkin: {0}")] + HealthCheckError(#[from] reqwest::Error), +} + #[derive(Debug, Error)] pub enum MainTaskError { #[error(transparent)] @@ -54,6 +60,8 @@ pub enum MainTaskError { ServerTaskError(#[from] ServerError), #[error(transparent)] IdentityTaskError(#[from] IdentityError), + #[error(transparent)] + HealthCheckError(#[from] HealthCheckError), #[error("firehose cache failed to close: {0}")] FirehoseCacheCloseError(foyer::Error), } diff --git a/slingshot/src/healthcheck.rs b/slingshot/src/healthcheck.rs new file mode 100644 index 0000000..fd60758 --- /dev/null +++ b/slingshot/src/healthcheck.rs @@ -0,0 +1,32 @@ +use crate::error::HealthCheckError; +use reqwest::Client; +use std::time::Duration; +use tokio::time::sleep; +use tokio_util::sync::CancellationToken; + +pub async fn healthcheck( + endpoint: String, + shutdown: CancellationToken, +) -> Result<(), HealthCheckError> { + let client = Client::builder() + .user_agent(format!( + "microcosm slingshot v{} (dev: @bad-example.com)", + env!("CARGO_PKG_VERSION") + )) + .no_proxy() + .timeout(Duration::from_secs(10)) + .build()?; + + loop { + tokio::select! { + res = client.get(&endpoint).send() => { + let _ = res + .and_then(|r| r.error_for_status()) + .inspect_err(|e| log::error!("failed to send healthcheck: {e}")); + }, + _ = shutdown.cancelled() => break, + } + sleep(Duration::from_secs(51)).await; + } + Ok(()) +} diff --git a/slingshot/src/lib.rs b/slingshot/src/lib.rs index 2e5e687..7737374 100644 --- a/slingshot/src/lib.rs +++ b/slingshot/src/lib.rs @@ -1,12 +1,14 @@ mod consumer; pub mod error; mod firehose_cache; +mod healthcheck; mod identity; mod record; mod server; pub use consumer::consume; pub use firehose_cache::firehose_cache; +pub use healthcheck::healthcheck; pub use identity::Identity; pub use record::{CachedRecord, ErrorResponseObject, Repo}; pub use server::serve; diff --git a/slingshot/src/main.rs b/slingshot/src/main.rs index a02d02e..ae5fdbb 100644 --- a/slingshot/src/main.rs +++ b/slingshot/src/main.rs @@ -1,7 +1,9 @@ // use foyer::HybridCache; // use foyer::{Engine, DirectFsDeviceOptions, HybridCacheBuilder}; use metrics_exporter_prometheus::PrometheusBuilder; -use slingshot::{Identity, Repo, consume, error::MainTaskError, firehose_cache, serve}; +use slingshot::{ + Identity, Repo, consume, error::MainTaskError, firehose_cache, healthcheck, serve, +}; use std::path::PathBuf; use clap::Parser; @@ -44,6 +46,9 @@ struct Args { /// recommended in production, but mind the file permissions. #[arg(long)] certs: Option, + /// an web address to send healtcheck pings to every ~51s or so + #[arg(long)] + healthcheck: Option, } #[tokio::main] @@ -127,6 +132,14 @@ async fn main() -> Result<(), String> { Ok(()) }); + if let Some(hc) = args.healthcheck { + let healthcheck_shutdown = shutdown.clone(); + tasks.spawn(async move { + healthcheck(hc, healthcheck_shutdown).await?; + Ok(()) + }); + } + tokio::select! { _ = shutdown.cancelled() => log::warn!("shutdown requested"), Some(r) = tasks.join_next() => { diff --git a/slingshot/src/server.rs b/slingshot/src/server.rs index db198a4..9066da1 100644 --- a/slingshot/src/server.rs +++ b/slingshot/src/server.rs @@ -19,7 +19,7 @@ use poem::{ Listener, TcpListener, acme::{AutoCert, LETS_ENCRYPT_PRODUCTION}, }, - middleware::{Cors, Tracing}, + middleware::{CatchPanic, Cors, Tracing}, }; use poem_openapi::{ ApiResponse, ContactObject, ExternalDocumentObject, Object, OpenApi, OpenApiService, Tags, @@ -758,6 +758,7 @@ where .allow_methods([Method::GET]) .allow_credentials(false), ) + .with(CatchPanic::new()) .with(Tracing); Server::new(listener) .name("slingshot") -- 2.51.2 From 3516df8abcfe80c5c79cee3b651b3e822000a06b Mon Sep 17 00:00:00 2001 From: phil Date: Wed, 6 Aug 2025 23:40:14 -0400 Subject: [PATCH 109/134] actually update the main readme a bit --- .../cozy-setup (move to another repo).md | 0 legacy/old-readme-details.md | 35 ++++ legacy/original-notes.md | 123 ++++++++++++ .../ufos ops (move to micro-ops).md | 0 readme.md | 178 ++++-------------- 5 files changed, 199 insertions(+), 137 deletions(-) rename cozy-setup (move to another repo).md => legacy/cozy-setup (move to another repo).md (100%) create mode 100644 legacy/old-readme-details.md create mode 100644 legacy/original-notes.md rename ufos ops (move to micro-ops).md => legacy/ufos ops (move to micro-ops).md (100%) diff --git a/cozy-setup (move to another repo).md b/legacy/cozy-setup (move to another repo).md similarity index 100% rename from cozy-setup (move to another repo).md rename to legacy/cozy-setup (move to another repo).md diff --git a/legacy/old-readme-details.md b/legacy/old-readme-details.md new file mode 100644 index 0000000..77a3b1b --- /dev/null +++ b/legacy/old-readme-details.md @@ -0,0 +1,35 @@ +[Constellation](./constellation/) +-------------------------------------------- + +A global atproto backlink index ✨ + +- Self hostable: handles the full write throughput of the global atproto firehose on a raspberry pi 4b + single SSD +- Storage efficient: less than 2GB/day disk consumption indexing all references in all lexicons and all non-atproto URLs +- Handles record deletion, account de/re-activation, and account deletion, ensuring accurate link counts and respecting users data choices +- Simple JSON API + +All social interactions in atproto tend to be represented by links (or references) between PDS records. This index can answer questions like "how many likes does a bsky post have", "who follows an account", "what are all the comments on a [frontpage](https://frontpage.fyi/) post", and more. + +- **status**: works! api is unstable and likely to change, and no known instances have a full network backfill yet. +- source: [./constellation/](./constellation/) +- public instance: [constellation.microcosm.blue](https://constellation.microcosm.blue/) + +_note: the public instance currently runs on a little raspberry pi in my house, feel free to use it! it comes with only with best-effort uptime, no commitment to not breaking the api for now, and possible rate-limiting. if you want to be nice you can put your project name and bsky username (or email) in your user-agent header for api requests._ + + +App: Spacedust +-------------- + +A notification subscription service 💫 + +using the same "link source" concept as [constellation](./constellation/), offer webhook notifications for new references created to records + +- **status**: in design + + +Library: [links](./links/) +------------------------------------ + +A rust crate (not published on crates.io yet) for optimistically parsing links out of arbitrary atproto PDS records, and potentially canonicalizing them + +- **status**: unstable, might remain an internal lib for constellation (and spacedust, soon) diff --git a/legacy/original-notes.md b/legacy/original-notes.md new file mode 100644 index 0000000..b321ea5 --- /dev/null +++ b/legacy/original-notes.md @@ -0,0 +1,123 @@ +--- + + +old notes follow, ignore +------------------------ + + +as far as i can tell, atproto lexicons today don't follow much of a convention for referencing across documents: sometimes it's a StrongRef, sometimes it's a DID, sometimes it's a bare at-uri. lexicon authors choose any old link-sounding key name for the key in their document. + +it's pretty messy so embrace the mess: atproto wants to be part of the web, so this library will also extract URLs and other URIs if you want it to. all the links. + + +why +--- + +the atproto firehose that bluesky sprays at you will contain raw _contents_ from peoples' pdses. these are isolated, decontextualized updates. it's very easy to build some kinds of interesting downstream apps off of this feed. + +- bluesky posts (firesky, deletions, ) +- blueksy post stats (emojis, ) +- trending keywords () + +but bringing almost kind of _context_ into your project requires a big step up in complexity and potentially cost: you're entering "appview" territory. _how many likes does a post have? who follows this account?_ + +you own your atproto data: it's kept in your personal data repository (PDS) and noone else can write to it. when someone likes your post, they create a "like" record in their _own_ pds, and that like belongs to _them_, not to you/your post. + +in the firehose you'll see a `app.bsky.feed.post` record created, with no details about who has liked it. then you'll see separate `app.bsky.feed.like` records show up for each like that comes in on that post, with no context about the post except a random-looking reference to it. storing these in order to do so is up to you! + +**so, why** + +everything is links, and they're a mess, but they all kinda work the same, so maybe some tooling can bring down that big step in complexity from firehose raw-content apps -> apps requiring any social context. + +everything is links: + +- likes +- follows +- blocks +- reposts +- quotes + +some low-level things you could make from links: + +- notification streams (part of ucosm) +- a global reverse index (part of ucosm) + +i think that making these low-level services as easy to use as jetstream could open up pathways for building more atproto apps that operate at full scale with interesting features for reasonable effort at low cost to operate. + + +extracting links +--------------- + + +- low-level: pass a &str of a field value and get a parsed link back + +- med-level: pass a &str of record in json form and get a list of parsed links + json paths back. (todo: should also handle dag-cbor prob?) + +- high-ish level: pass the json record and maybe apply some pre-loaded rules based on known lexicons to get the best result. + +for now, a link is only considered if it matches for the entire value of the record's field -- links embedded in text content are not included. note that urls in bluesky posts _will_ still be extracted, since they are broken out into facets. + + +resolving / canonicalizing links +-------------------------------- + + +### at-uris + +every at-uri has at least two equivalent forms, one with a `DID`, and one with an account handle. the at-uri spec [illustrates this by example](https://atproto.com/specs/at-uri-scheme): + +- `at://did:plc:44ybard66vv44zksje25o7dz/app.bsky.feed.post/3jwdwj2ctlk26` +- `at://bnewbold.bsky.team/app.bsky.feed.post/3jwdwj2ctlk26` + +some applications, like a reverse link index, may wish to canonicalize at-uris to a single form. the `DID`-form is stable as an account changes its handle and probably the right choice to canonicalize to, but maybe some apps would actually perfer to canonicalise to handles? + +hopefully atrium will make it easy to resolve at-uris. + + +### urls + +canonicalizing URLs is more annoying but also a bit more established. lots of details. + +- do we have to deal with punycode? +- follow redirects (todo: only permanent ones, or all?) +- check for rel=canonical http header and possibly follow it +- check link rel=canonical meta tag and possibly follow it +- do we need to check site maps?? +- do we have to care at all about AMP? +- do we want anything to do with url shorteners?? +- how do multilingual sites affect this? +- do we have to care about `script type="application/ld+json"` ??? + +ugh. is there a crate for this. + + +### relative uris? + +links might be relative, in which case they might need to be made absolute before being useful. is that a concern for this library, or up to the user? (seems like we might not have context here to determine its absolute) + + +### canonicalizing + +there should be a few async functions available to canonicalize already-parsed links. + +- what happens if a link can't be resolved? + + +--- + +- using `tinyjson` because it's nice -- maybe should switch to serde_json to share deps with atrium? + +- would use atrium for parsing at-uris, but it's not in there. there's a did-only version in the non-lib commands.rs. its identifier parser is strict to did + handle, which makes sense, but for our purposes we might want to allow unknown methods too? + + - rsky-syntax has an aturi + - adenosyne also + - might come back to these + + +------- + +rocks + +```bash +ROCKSDB_LIB_DIR=/nix/store/z2chn0hsik0clridr8mlprx1cngh1g3c-rocksdb-9.7.3/lib/ cargo build +``` diff --git a/ufos ops (move to micro-ops).md b/legacy/ufos ops (move to micro-ops).md similarity index 100% rename from ufos ops (move to micro-ops).md rename to legacy/ufos ops (move to micro-ops).md diff --git a/readme.md b/readme.md index 606b016..403f859 100644 --- a/readme.md +++ b/readme.md @@ -1,167 +1,71 @@ -microcosm: links -================ +microcosm +========= -this repo contains libraries and apps for working with cross-record references in at-protocol. +This repo contains APIs and libraries for [atproto](https://atproto.com/) services from [microcosm](https://microcosm.blue): -App: [Constellation](./constellation/) --------------------------------------------- - -A global atproto backlink index ✨ - -- Self hostable: handles the full write throughput of the global atproto firehose on a raspberry pi 4b + single SSD -- Storage efficient: less than 2GB/day disk consumption indexing all references in all lexicons and all non-atproto URLs -- Handles record deletion, account de/re-activation, and account deletion, ensuring accurate link counts and respecting users data choices -- Simple JSON API - -All social interactions in atproto tend to be represented by links (or references) between PDS records. This index can answer questions like "how many likes does a bsky post have", "who follows an account", "what are all the comments on a [frontpage](https://frontpage.fyi/) post", and more. - -- **status**: works! api is unstable and likely to change, and no known instances have a full network backfill yet. -- source: [./constellation/](./constellation/) -- public instance: [constellation.microcosm.blue](https://constellation.microcosm.blue/) - -_note: the public instance currently runs on a little raspberry pi in my house, feel free to use it! it comes with only with best-effort uptime, no commitment to not breaking the api for now, and possible rate-limiting. if you want to be nice you can put your project name and bsky username (or email) in your user-agent header for api requests._ - - -App: Spacedust --------------- - -A notification subscription service 💫 - -using the same "link source" concept as [constellation](./constellation/), offer webhook notifications for new references created to records - -- **status**: in design - - -Library: [links](./links/) +🌌 [Constellation](./constellation/) ------------------------------------ -A rust crate (not published on crates.io yet) for optimistically parsing links out of arbitrary atproto PDS records, and potentially canonicalizing them - -- **status**: unstable, might remain an internal lib for constellation (and spacedust, soon) - - - ---- - - -old notes follow, ignore ------------------------- - - -as far as i can tell, atproto lexicons today don't follow much of a convention for referencing across documents: sometimes it's a StrongRef, sometimes it's a DID, sometimes it's a bare at-uri. lexicon authors choose any old link-sounding key name for the key in their document. - -it's pretty messy so embrace the mess: atproto wants to be part of the web, so this library will also extract URLs and other URIs if you want it to. all the links. - - -why ---- +A global atproto interactions backlink index as a simple JSON API. Works with every lexicon, runs on a raspberry pi, consumes less than 2GiB of disk per day. Handles record deletion, account de/re-activation, and account deletion, ensuring accurate link counts while respecting users' data choices. -the atproto firehose that bluesky sprays at you will contain raw _contents_ from peoples' pdses. these are isolated, decontextualized updates. it's very easy to build some kinds of interesting downstream apps off of this feed. - -- bluesky posts (firesky, deletions, ) -- blueksy post stats (emojis, ) -- trending keywords () - -but bringing almost kind of _context_ into your project requires a big step up in complexity and potentially cost: you're entering "appview" territory. _how many likes does a post have? who follows this account?_ - -you own your atproto data: it's kept in your personal data repository (PDS) and noone else can write to it. when someone likes your post, they create a "like" record in their _own_ pds, and that like belongs to _them_, not to you/your post. - -in the firehose you'll see a `app.bsky.feed.post` record created, with no details about who has liked it. then you'll see separate `app.bsky.feed.like` records show up for each like that comes in on that post, with no context about the post except a random-looking reference to it. storing these in order to do so is up to you! - -**so, why** - -everything is links, and they're a mess, but they all kinda work the same, so maybe some tooling can bring down that big step in complexity from firehose raw-content apps -> apps requiring any social context. - -everything is links: - -- likes -- follows -- blocks -- reposts -- quotes - -some low-level things you could make from links: - -- notification streams (part of ucosm) -- a global reverse index (part of ucosm) - -i think that making these low-level services as easy to use as jetstream could open up pathways for building more atproto apps that operate at full scale with interesting features for reasonable effort at low cost to operate. - - -extracting links ---------------- - - -- low-level: pass a &str of a field value and get a parsed link back - -- med-level: pass a &str of record in json form and get a list of parsed links + json paths back. (todo: should also handle dag-cbor prob?) - -- high-ish level: pass the json record and maybe apply some pre-loaded rules based on known lexicons to get the best result. - -for now, a link is only considered if it matches for the entire value of the record's field -- links embedded in text content are not included. note that urls in bluesky posts _will_ still be extracted, since they are broken out into facets. - - -resolving / canonicalizing links --------------------------------- - - -### at-uris - -every at-uri has at least two equivalent forms, one with a `DID`, and one with an account handle. the at-uri spec [illustrates this by example](https://atproto.com/specs/at-uri-scheme): +- source: [./constellation/](./constellation/) +- [public instance + API docs](https://constellation.microcosm.blue/) +- status: used in production. APIs will change but backwards compatibility will be maintained as long as needed. -- `at://did:plc:44ybard66vv44zksje25o7dz/app.bsky.feed.post/3jwdwj2ctlk26` -- `at://bnewbold.bsky.team/app.bsky.feed.post/3jwdwj2ctlk26` -some applications, like a reverse link index, may wish to canonicalize at-uris to a single form. the `DID`-form is stable as an account changes its handle and probably the right choice to canonicalize to, but maybe some apps would actually perfer to canonicalise to handles? +🎇 [Spacedust](./spacedust/) +---------------------------- -hopefully atrium will make it easy to resolve at-uris. +A global atproto interactions firehose. Extracts all at-uris, DIDs, and URLs from every lexicon in the firehose, and exposes them over a websocket modelled after [jetstream](github.com/bluesky-social/jetstream). +- source: [./spacedust/](./spacedust/) +- [public instance + API docs](https://spacedust.microcosm.blue/) +- status: v0: the basics work and the APIs are in place! missing cursor replay, forward link storage, and delete event link hydration. -### urls +Demos: -canonicalizing URLs is more annoying but also a bit more established. lots of details. +- [Spacedust notifications](https://notifications.microcosm.blue/): web push notifications for _every_ atproto app +- [Zero-Bluesky real-time interaction-updating post embed](https://bsky.bad-example.com/zero-bluesky-realtime-embed/) -- do we have to deal with punycode? -- follow redirects (todo: only permanent ones, or all?) -- check for rel=canonical http header and possibly follow it -- check link rel=canonical meta tag and possibly follow it -- do we need to check site maps?? -- do we have to care at all about AMP? -- do we want anything to do with url shorteners?? -- how do multilingual sites affect this? -- do we have to care about `script type="application/ld+json"` ??? -ugh. is there a crate for this. +🛰️ [Slingshot](./slingshot) +--------------------------- +A fast, eager, production-grade edge cache for atproto records and identities. Pre-caches all records from the firehose and maintains a longer-term cache of requested records on disk. -### relative uris? +- source: [./slingshot/](./slingshot/) +- [public instance + API docs](https://slingshot.microcosm.blue/) +- status: v0: most XRPC APIs are working. cache storage is being reworked. -links might be relative, in which case they might need to be made absolute before being useful. is that a concern for this library, or up to the user? (seems like we might not have context here to determine its absolute) +🛸 [UFOs API](./ufos) +--------------------- -### canonicalizing +Timeseries stats and sample records for every [collection](https://atproto.com/guides/glossary#collection) ever seen in the atproto firehose. Unique users are counted in hyperloglog sketches enabling arbitrary cardinality aggregation across time buckets and/or NSIDs. -there should be a few async functions available to canonicalize already-parsed links. +- source: [./ufos/](./ufos/) +- [public instance + API docs](https://ufos-api.microcosm.blue/) +- status: Used in production. It has APIs and they work! Needs improvement on indexing; needs more indexes and some more APIs to the data exposed. -- what happens if a link can't be resolved? +See also: [UFOs atproto explorer](https://ufos.microcosm.blue/) built on UFOs API. ([source](github.com/at-microcosm/spacedust-utils)) ---- +💫 [Links](./links) +------------------- -- using `tinyjson` because it's nice -- maybe should switch to serde_json to share deps with atrium? +Rust library for parsing and extracting links (at-uris, DIDs, and URLs) from atproto records. -- would use atrium for parsing at-uris, but it's not in there. there's a did-only version in the non-lib commands.rs. its identifier parser is strict to did + handle, which makes sense, but for our purposes we might want to allow unknown methods too? +- source: [./links/](./links/) +- status: not yet published to crates.io; needs some rework - - rsky-syntax has an aturi - - adenosyne also - - might come back to these +🔭 Deprecated: [Who am I](./who-am-i) +------------------------------------- -------- +An identity bridge for microcosm demos, that kinda worked. Fixing its problems is about equivalent to reinventing a lot of OIDC, so it's being retired. -rocks +- source: [./who-am-i/](./who-am-i/) +- status: ready for retirement. -```bash -ROCKSDB_LIB_DIR=/nix/store/z2chn0hsik0clridr8mlprx1cngh1g3c-rocksdb-9.7.3/lib/ cargo build -``` +Still in use for the Spacedust Notifications demo, but that will hopefully be migrated to use atproto oauth directly instead. -- 2.51.2 From 1e19d78959b5d06a3e8d30733923d915a234122f Mon Sep 17 00:00:00 2001 From: phil Date: Thu, 7 Aug 2025 11:14:38 -0400 Subject: [PATCH 110/134] more readme stuff jestream lib, badges, intro with links etc --- readme.md | 68 +++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 48 insertions(+), 20 deletions(-) diff --git a/readme.md b/readme.md index 403f859..cc31afc 100644 --- a/readme.md +++ b/readme.md @@ -1,7 +1,20 @@ microcosm ========= -This repo contains APIs and libraries for [atproto](https://atproto.com/) services from [microcosm](https://microcosm.blue): +HTTP APIs and rust libraries for [atproto](https://atproto.com/) services from [microcosm](https://microcosm.blue). + +[![@microcosm.blue: bluesky](https://img.shields.io/badge/@microcosm.blue-bluesky-blue)](https://bsky.app/profile/microcosm.blue) +[![microcosm discord: join](https://img.shields.io/badge/microcosm_discord-join-purple)](https://discord.gg/tcDfe4PGVB) +[![github sponsors: support](https://img.shields.io/badge/github_sponsors-support-pink)](https://github.com/sponsors/uniphil/) +[![ko-fi: support](https://img.shields.io/badge/ko--fi-support-pink)](https://ko-fi.com/bad_example) + +Welcome! + +The documentation for microcosm services is under active development. If you like reading API docs, you'll probably hit the ground running! + +Tutorials, how-to guides, and client SDK libraries are all in the works for gentler on-ramps, but are not quite ready yet. But don't let that stop you! Hop in the [microcosm discord](https://img.shields.io/badge/microcosm_discord-join-purple), or post questions and tag [@bad-example.com](https://bsky.app/profile/bad-example.com) on Bluesky if you get stuck anywhere! + +This repository's primary home is moving to tangled: [@microcosm.blue/microcosm-rs](https://tangled.sh/@microcosm.blue/microcosm-rs). It will continue to be mirrored on [github](https://github.com/at-microcosm/microcosm-rs) for the forseeable future, and it's fine to open issues or pulls in either place! 🌌 [Constellation](./constellation/) @@ -9,9 +22,9 @@ This repo contains APIs and libraries for [atproto](https://atproto.com/) servic A global atproto interactions backlink index as a simple JSON API. Works with every lexicon, runs on a raspberry pi, consumes less than 2GiB of disk per day. Handles record deletion, account de/re-activation, and account deletion, ensuring accurate link counts while respecting users' data choices. -- source: [./constellation/](./constellation/) -- [public instance + API docs](https://constellation.microcosm.blue/) -- status: used in production. APIs will change but backwards compatibility will be maintained as long as needed. +- Source: [./constellation/](./constellation/) +- [Public instance/API docs](https://constellation.microcosm.blue/) +- Status: used in production. APIs will change but backwards compatibility will be maintained as long as needed. 🎇 [Spacedust](./spacedust/) @@ -19,11 +32,11 @@ A global atproto interactions backlink index as a simple JSON API. Works with ev A global atproto interactions firehose. Extracts all at-uris, DIDs, and URLs from every lexicon in the firehose, and exposes them over a websocket modelled after [jetstream](github.com/bluesky-social/jetstream). -- source: [./spacedust/](./spacedust/) -- [public instance + API docs](https://spacedust.microcosm.blue/) -- status: v0: the basics work and the APIs are in place! missing cursor replay, forward link storage, and delete event link hydration. +- Source: [./spacedust/](./spacedust/) +- [Public instance/API docs](https://spacedust.microcosm.blue/) +- Status: v0: the basics work and the APIs are in place! missing cursor replay, forward link storage, and delete event link hydration. -Demos: +### Demos: - [Spacedust notifications](https://notifications.microcosm.blue/): web push notifications for _every_ atproto app - [Zero-Bluesky real-time interaction-updating post embed](https://bsky.bad-example.com/zero-bluesky-realtime-embed/) @@ -34,9 +47,9 @@ Demos: A fast, eager, production-grade edge cache for atproto records and identities. Pre-caches all records from the firehose and maintains a longer-term cache of requested records on disk. -- source: [./slingshot/](./slingshot/) -- [public instance + API docs](https://slingshot.microcosm.blue/) -- status: v0: most XRPC APIs are working. cache storage is being reworked. +- Source: [./slingshot/](./slingshot/) +- [Public instance/API docs](https://slingshot.microcosm.blue/) +- Status: v0: most XRPC APIs are working. cache storage is being reworked. 🛸 [UFOs API](./ufos) @@ -44,11 +57,12 @@ A fast, eager, production-grade edge cache for atproto records and identities. P Timeseries stats and sample records for every [collection](https://atproto.com/guides/glossary#collection) ever seen in the atproto firehose. Unique users are counted in hyperloglog sketches enabling arbitrary cardinality aggregation across time buckets and/or NSIDs. -- source: [./ufos/](./ufos/) -- [public instance + API docs](https://ufos-api.microcosm.blue/) -- status: Used in production. It has APIs and they work! Needs improvement on indexing; needs more indexes and some more APIs to the data exposed. +- Source: [./ufos/](./ufos/) +- [Public instance/API docs](https://ufos-api.microcosm.blue/) +- Status: Used in production. It has APIs and they work! Needs improvement on indexing; needs more indexes and some more APIs to the data exposed. -See also: [UFOs atproto explorer](https://ufos.microcosm.blue/) built on UFOs API. ([source](github.com/at-microcosm/spacedust-utils)) +> [!info] +> See also: [UFOs atproto explorer](https://ufos.microcosm.blue/) built on UFOs API. ([source](github.com/at-microcosm/spacedust-utils)) 💫 [Links](./links) @@ -56,8 +70,21 @@ See also: [UFOs atproto explorer](https://ufos.microcosm.blue/) built on UFOs AP Rust library for parsing and extracting links (at-uris, DIDs, and URLs) from atproto records. -- source: [./links/](./links/) -- status: not yet published to crates.io; needs some rework +- Source: [./links/](./links/) +- Status: not yet published to crates.io; needs some rework + + +🛩️ [Jetstream](./jetstream) +--------------------------- + +A low-overhead jetstream client with cursor handling and automatic reconnect. + +- Source: [./links/](./links/) +- Status: used in multiple apps in production, but not yet published to crates.io; some rework planned + +> [!info] +> See also: [Rocketman](https://github.com/teal-fm/cadet/tree/main/rocketman), another excellent rust jetstream client which shares some lineage and _is_ published on crates.io. + 🔭 Deprecated: [Who am I](./who-am-i) @@ -65,7 +92,8 @@ Rust library for parsing and extracting links (at-uris, DIDs, and URLs) from atp An identity bridge for microcosm demos, that kinda worked. Fixing its problems is about equivalent to reinventing a lot of OIDC, so it's being retired. -- source: [./who-am-i/](./who-am-i/) -- status: ready for retirement. +- Source: [./who-am-i/](./who-am-i/) +- Status: ready for retirement. -Still in use for the Spacedust Notifications demo, but that will hopefully be migrated to use atproto oauth directly instead. +> [!warning] +> Still in use for the Spacedust Notifications demo, but that will hopefully be migrated to use atproto oauth directly instead. -- 2.51.2 From f8776810ea001c17fd52f9c78f4ea25202a4742f Mon Sep 17 00:00:00 2001 From: phil Date: Thu, 7 Aug 2025 11:17:50 -0400 Subject: [PATCH 111/134] almost like i should use a github md preview or sm --- readme.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/readme.md b/readme.md index cc31afc..be74e4a 100644 --- a/readme.md +++ b/readme.md @@ -1,20 +1,19 @@ microcosm ========= -HTTP APIs and rust libraries for [atproto](https://atproto.com/) services from [microcosm](https://microcosm.blue). - [![@microcosm.blue: bluesky](https://img.shields.io/badge/@microcosm.blue-bluesky-blue)](https://bsky.app/profile/microcosm.blue) [![microcosm discord: join](https://img.shields.io/badge/microcosm_discord-join-purple)](https://discord.gg/tcDfe4PGVB) [![github sponsors: support](https://img.shields.io/badge/github_sponsors-support-pink)](https://github.com/sponsors/uniphil/) [![ko-fi: support](https://img.shields.io/badge/ko--fi-support-pink)](https://ko-fi.com/bad_example) -Welcome! +Welcome! Here you'll find HTTP APIs and rust libraries for [atproto](https://atproto.com/) services from [microcosm](https://microcosm.blue). -The documentation for microcosm services is under active development. If you like reading API docs, you'll probably hit the ground running! +Documentation is under active development. If you like reading API docs, you'll probably hit the ground running! -Tutorials, how-to guides, and client SDK libraries are all in the works for gentler on-ramps, but are not quite ready yet. But don't let that stop you! Hop in the [microcosm discord](https://img.shields.io/badge/microcosm_discord-join-purple), or post questions and tag [@bad-example.com](https://bsky.app/profile/bad-example.com) on Bluesky if you get stuck anywhere! +Tutorials, how-to guides, and client SDK libraries are all in the works for gentler on-ramps, but are not quite ready yet. But don't let that stop you! Hop in the [microcosm discord](https://discord.gg/tcDfe4PGVB), or post questions and tag [@bad-example.com](https://bsky.app/profile/bad-example.com) on Bluesky if you get stuck anywhere. -This repository's primary home is moving to tangled: [@microcosm.blue/microcosm-rs](https://tangled.sh/@microcosm.blue/microcosm-rs). It will continue to be mirrored on [github](https://github.com/at-microcosm/microcosm-rs) for the forseeable future, and it's fine to open issues or pulls in either place! +> [!tip] +> This repository's primary home is moving to tangled: [@microcosm.blue/microcosm-rs](https://tangled.sh/@microcosm.blue/microcosm-rs). It will continue to be mirrored on [github](https://github.com/at-microcosm/microcosm-rs) for the forseeable future, and it's fine to open issues or pulls in either place! 🌌 [Constellation](./constellation/) @@ -61,7 +60,7 @@ Timeseries stats and sample records for every [collection](https://atproto.com/g - [Public instance/API docs](https://ufos-api.microcosm.blue/) - Status: Used in production. It has APIs and they work! Needs improvement on indexing; needs more indexes and some more APIs to the data exposed. -> [!info] +> [!tip] > See also: [UFOs atproto explorer](https://ufos.microcosm.blue/) built on UFOs API. ([source](github.com/at-microcosm/spacedust-utils)) @@ -82,7 +81,7 @@ A low-overhead jetstream client with cursor handling and automatic reconnect. - Source: [./links/](./links/) - Status: used in multiple apps in production, but not yet published to crates.io; some rework planned -> [!info] +> [!tip] > See also: [Rocketman](https://github.com/teal-fm/cadet/tree/main/rocketman), another excellent rust jetstream client which shares some lineage and _is_ published on crates.io. @@ -96,4 +95,4 @@ An identity bridge for microcosm demos, that kinda worked. Fixing its problems i - Status: ready for retirement. > [!warning] -> Still in use for the Spacedust Notifications demo, but that will hopefully be migrated to use atproto oauth directly instead. +> `who-am-i` is still in use for the Spacedust Notifications demo, but that will hopefully be migrated to use atproto oauth directly instead. -- 2.51.2 From fbb2f92dfa557bedac342934675600f889abbc94 Mon Sep 17 00:00:00 2001 From: phil Date: Thu, 7 Aug 2025 11:21:09 -0400 Subject: [PATCH 112/134] tweaks --- readme.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/readme.md b/readme.md index be74e4a..045652b 100644 --- a/readme.md +++ b/readme.md @@ -1,14 +1,11 @@ -microcosm -========= - +microcosm HTTP APIs + rust crates +================================= [![@microcosm.blue: bluesky](https://img.shields.io/badge/@microcosm.blue-bluesky-blue)](https://bsky.app/profile/microcosm.blue) [![microcosm discord: join](https://img.shields.io/badge/microcosm_discord-join-purple)](https://discord.gg/tcDfe4PGVB) [![github sponsors: support](https://img.shields.io/badge/github_sponsors-support-pink)](https://github.com/sponsors/uniphil/) [![ko-fi: support](https://img.shields.io/badge/ko--fi-support-pink)](https://ko-fi.com/bad_example) -Welcome! Here you'll find HTTP APIs and rust libraries for [atproto](https://atproto.com/) services from [microcosm](https://microcosm.blue). - -Documentation is under active development. If you like reading API docs, you'll probably hit the ground running! +Welcome! Documentation is under active development. If you like reading API docs, you'll probably hit the ground running! Tutorials, how-to guides, and client SDK libraries are all in the works for gentler on-ramps, but are not quite ready yet. But don't let that stop you! Hop in the [microcosm discord](https://discord.gg/tcDfe4PGVB), or post questions and tag [@bad-example.com](https://bsky.app/profile/bad-example.com) on Bluesky if you get stuck anywhere. -- 2.51.2 From 59028924a275e6d5c34230262dfd3686b723ce65 Mon Sep 17 00:00:00 2001 From: phil Date: Tue, 12 Aug 2025 15:36:08 -0400 Subject: [PATCH 113/134] use git-main fjall for weak delete --- Cargo.lock | 31 +++++++++++++++---------------- ufos/Cargo.toml | 2 +- ufos/src/storage_fjall.rs | 7 ++++++- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3f51793..e9cda1e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1220,7 +1220,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18e4fdb82bd54a12e42fb58a800dcae6b9e13982238ce2296dc3570b92148e1f" dependencies = [ "data-encoding", - "syn 2.0.103", + "syn 1.0.109", ] [[package]] @@ -1569,7 +1569,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -1611,9 +1611,8 @@ dependencies = [ [[package]] name = "fjall" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b2ced3483989a62b3533c9f99054d73b527c6c0045cf22b00fe87956f1a46f" +version = "2.11.2" +source = "git+https://github.com/fjall-rs/fjall.git#42d811f7c8cc9004407d520d37d2a1d8d246c03d" dependencies = [ "byteorder", "byteview", @@ -2595,7 +2594,7 @@ checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -2800,7 +2799,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", - "windows-targets 0.52.6", + "windows-targets 0.48.5", ] [[package]] @@ -2931,9 +2930,9 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "lsm-tree" -version = "2.8.0" +version = "2.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a63a5e98a38b51765274137d8aedfbd848da5f4d016867e186b673fcc06a8c" +checksum = "55b6d7475a8dd22e749186968daacf8e2a77932b061b1bd263157987bbfc0c6c" dependencies = [ "byteorder", "crossbeam-skiplist", @@ -4054,7 +4053,7 @@ dependencies = [ "once_cell", "socket2 0.5.9", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -4436,7 +4435,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -4449,7 +4448,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.9.4", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -5188,7 +5187,7 @@ dependencies = [ "getrandom 0.3.3", "once_cell", "rustix 1.0.5", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -5821,9 +5820,9 @@ checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" [[package]] name = "value-log" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd29b17c041f94e0885179637289815cd038f0c9fc19c4549d5a97017404fb7d" +checksum = "62fc7c4ce161f049607ecea654dca3f2d727da5371ae85e2e4f14ce2b98ed67c" dependencies = [ "byteorder", "byteview", @@ -6073,7 +6072,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.48.0", ] [[package]] diff --git a/ufos/Cargo.toml b/ufos/Cargo.toml index ec22d3b..bbabe69 100644 --- a/ufos/Cargo.toml +++ b/ufos/Cargo.toml @@ -13,7 +13,7 @@ chrono = { version = "0.4.41", features = ["serde"] } clap = { version = "4.5.31", features = ["derive"] } dropshot = "0.16.0" env_logger = "0.11.7" -fjall = { version = "2.8.0", features = ["lz4"] } +fjall = { git = "https://github.com/fjall-rs/fjall.git", features = ["lz4"] } getrandom = "0.3.3" http = "1.3.1" jetstream = { path = "../jetstream", features = ["metrics"] } diff --git a/ufos/src/storage_fjall.rs b/ufos/src/storage_fjall.rs index 4e88844..87edafb 100644 --- a/ufos/src/storage_fjall.rs +++ b/ufos/src/storage_fjall.rs @@ -1619,7 +1619,12 @@ impl StoreBackground for FjallBackground { histogram!("storage_trim_dirty_nsids").record(completed.len() as f64); histogram!("storage_trim_duration").record(dt.as_micros() as f64); counter!("storage_trim_removed", "dangling" => "true").increment(total_danglers as u64); - counter!("storage_trim_removed", "dangling" => "false").increment((total_deleted - total_danglers) as u64); + if total_deleted >= total_danglers { + counter!("storage_trim_removed", "dangling" => "false").increment((total_deleted - total_danglers) as u64); + } else { + // TODO: probably think through what's happening here + log::warn!("weird trim case: more danglers than deleted? metric will be missing for dangling=false. deleted={total_deleted} danglers={total_danglers}"); + } for c in completed { dirty_nsids.remove(&c); } -- 2.51.2 From 11d04045e141146374aef8d8d180e29a40c94e2d Mon Sep 17 00:00:00 2001 From: phil Date: Tue, 12 Aug 2025 16:14:11 -0400 Subject: [PATCH 114/134] use remove_weak for secondary rank indexes *hopefully* reduces write amplification, read amplification, space amplification, and maybe even compaction stalls? --- ufos/src/storage_fjall.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ufos/src/storage_fjall.rs b/ufos/src/storage_fjall.rs index 87edafb..d93795f 100644 --- a/ufos/src/storage_fjall.rs +++ b/ufos/src/storage_fjall.rs @@ -1222,7 +1222,8 @@ impl FjallWriter { AllTimeRecordsKey::new(new_creates_count.into(), &nsid).to_db_bytes()?, ), }; - batch.remove(&self.rollups, &old_k); // TODO: when fjall gets weak delete, this will hopefully work way better + // remove_weak is allowed here because the secondary ranking index only ever inserts once at a key + batch.remove_weak(&self.rollups, &old_k); batch.insert(&self.rollups, &new_k, ""); } @@ -1246,7 +1247,8 @@ impl FjallWriter { AllTimeDidsKey::new(new_dids_estimate.into(), &nsid).to_db_bytes()?, ), }; - batch.remove(&self.rollups, &old_k); // TODO: when fjall gets weak delete, this will hopefully work way better + // remove_weak is allowed here because the secondary ranking index only ever inserts once at a key + batch.remove_weak(&self.rollups, &old_k); batch.insert(&self.rollups, &new_k, ""); } -- 2.51.2 From 19f31140174b65b34770d299b48c3051edb434e4 Mon Sep 17 00:00:00 2001 From: phil Date: Wed, 13 Aug 2025 12:24:44 -0400 Subject: [PATCH 115/134] track all partitions' l0_run_count in metrics --- ufos/src/main.rs | 1 + ufos/src/storage.rs | 2 ++ ufos/src/storage_fjall.rs | 28 +++++++++++++++++++++++++++- 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/ufos/src/main.rs b/ufos/src/main.rs index e0c2936..d2088e8 100644 --- a/ufos/src/main.rs +++ b/ufos/src/main.rs @@ -162,6 +162,7 @@ async fn do_update_stuff(read_store: impl StoreReader) { interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); loop { interval.tick().await; + read_store.update_metrics(); match read_store.get_consumer_info().await { Err(e) => log::warn!("failed to get jetstream consumer info: {e:?}"), Ok(ConsumerInfo::Jetstream { diff --git a/ufos/src/storage.rs b/ufos/src/storage.rs index 222da1b..079d804 100644 --- a/ufos/src/storage.rs +++ b/ufos/src/storage.rs @@ -104,6 +104,8 @@ pub trait StoreBackground: Send + Sync { pub trait StoreReader: Send + Sync { fn name(&self) -> String; + fn update_metrics(&self) {} + async fn get_storage_stats(&self) -> StorageResult; async fn get_consumer_info(&self) -> StorageResult; diff --git a/ufos/src/storage_fjall.rs b/ufos/src/storage_fjall.rs index d93795f..395e9ac 100644 --- a/ufos/src/storage_fjall.rs +++ b/ufos/src/storage_fjall.rs @@ -23,7 +23,10 @@ use fjall::{ Batch as FjallBatch, Config, Keyspace, PartitionCreateOptions, PartitionHandle, Snapshot, }; use jetstream::events::Cursor; -use metrics::{counter, describe_counter, describe_histogram, histogram, Unit}; +use lsm_tree::AbstractTree; +use metrics::{ + counter, describe_counter, describe_gauge, describe_histogram, gauge, histogram, Unit, +}; use std::collections::{HashMap, HashSet}; use std::iter::Peekable; use std::ops::Bound; @@ -227,7 +230,9 @@ impl StorageWhatever for feeds: feeds.clone(), records: records.clone(), rollups: rollups.clone(), + queues: queues.clone(), }; + reader.describe_metrics(); let writer = FjallWriter { bg_taken: Arc::new(AtomicBool::new(false)), keyspace, @@ -250,6 +255,7 @@ pub struct FjallReader { feeds: PartitionHandle, records: PartitionHandle, rollups: PartitionHandle, + queues: PartitionHandle, } /// An iterator that knows how to skip over deleted/invalidated records @@ -381,6 +387,14 @@ fn get_lookup_iter( type CollectionSerieses = HashMap>; impl FjallReader { + fn describe_metrics(&self) { + describe_gauge!( + "storage_fjall_l0_run_count", + Unit::Count, + "number of L0 runs in a partition" + ); + } + fn get_storage_stats(&self) -> StorageResult { let rollup_cursor = get_static_neu::(&self.global)? @@ -1000,6 +1014,18 @@ impl StoreReader for FjallReader { fn name(&self) -> String { "fjall storage v2".into() } + fn update_metrics(&self) { + gauge!("storage_fjall_l0_run_count", "partition" => "global") + .set(self.global.tree.l0_run_count() as f64); + gauge!("storage_fjall_l0_run_count", "partition" => "feeds") + .set(self.feeds.tree.l0_run_count() as f64); + gauge!("storage_fjall_l0_run_count", "partition" => "records") + .set(self.records.tree.l0_run_count() as f64); + gauge!("storage_fjall_l0_run_count", "partition" => "rollups") + .set(self.rollups.tree.l0_run_count() as f64); + gauge!("storage_fjall_l0_run_count", "partition" => "queues") + .set(self.queues.tree.l0_run_count() as f64); + } async fn get_storage_stats(&self) -> StorageResult { let s = self.clone(); tokio::task::spawn_blocking(move || FjallReader::get_storage_stats(&s)).await? -- 2.51.2 From 471f35e28b4062a9663c5f998ba6e3a9ea6ef42a Mon Sep 17 00:00:00 2001 From: phil Date: Wed, 13 Aug 2025 15:08:57 -0400 Subject: [PATCH 116/134] run tasks in actually separate tokio tasks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit the more you know 🎶 --- ufos/src/main.rs | 47 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/ufos/src/main.rs b/ufos/src/main.rs index d2088e8..23beff9 100644 --- a/ufos/src/main.rs +++ b/ufos/src/main.rs @@ -4,6 +4,7 @@ use metrics::{describe_gauge, gauge, Unit}; use metrics_exporter_prometheus::PrometheusBuilder; use std::path::PathBuf; use std::time::{Duration, SystemTime}; +use tokio::task::JoinSet; use ufos::consumer; use ufos::file_consumer; use ufos::server; @@ -72,19 +73,31 @@ async fn main() -> anyhow::Result<()> { Ok(()) } -async fn go( +async fn go( args: Args, read_store: impl StoreReader + 'static + Clone, mut write_store: impl StoreWriter + 'static, cursor: Option, sketch_secret: SketchSecretPrefix, ) -> anyhow::Result<()> { + let mut tasks: JoinSet> = JoinSet::new(); + println!("starting server with storage..."); let serving = server::serve(read_store.clone()); + tasks.spawn(async move { + serving.await.map_err(|e| { + log::warn!("server ended: {e}"); + anyhow::anyhow!(e) + }) + }); if args.pause_writer { log::info!("not starting jetstream or the write loop."); - serving.await.map_err(|e| anyhow::anyhow!(e))?; + for t in tasks.join_all().await { + if let Err(e) = t { + return Err(anyhow::anyhow!(e)); + } + } return Ok(()); } @@ -102,18 +115,32 @@ async fn go( let rolling = write_store .background_tasks(args.reroll)? .run(args.backfill); - let consuming = write_store.receive_batches(batches); + tasks.spawn(async move { + rolling + .await + .inspect_err(|e| log::warn!("rollup ended: {e}"))?; + Ok(()) + }); - let stating = do_update_stuff(read_store); + tasks.spawn(async move { + write_store + .receive_batches(batches) + .await + .inspect_err(|e| log::warn!("consumer ended: {e}"))?; + Ok(()) + }); + + tasks.spawn(async move { + do_update_stuff(read_store).await; + log::warn!("status task ended"); + Ok(()) + }); install_metrics_server()?; - tokio::select! { - z = serving => log::warn!("serve task ended: {z:?}"), - z = rolling => log::warn!("rollup task ended: {z:?}"), - z = consuming => log::warn!("consuming task ended: {z:?}"), - z = stating => log::warn!("status task ended: {z:?}"), - }; + for (i, t) in tasks.join_all().await.iter().enumerate() { + log::warn!("task {i} done: {t:?}"); + } println!("bye!"); -- 2.51.2 From 1899b5864e3bc3a96f0569a2e8a328e4311a9a6c Mon Sep 17 00:00:00 2001 From: phil Date: Wed, 13 Aug 2025 15:50:55 -0400 Subject: [PATCH 117/134] moar metrics --- ufos/src/storage_fjall.rs | 95 ++++++++++++++++++++++++++++++++------- 1 file changed, 80 insertions(+), 15 deletions(-) diff --git a/ufos/src/storage_fjall.rs b/ufos/src/storage_fjall.rs index 395e9ac..6dc9b48 100644 --- a/ufos/src/storage_fjall.rs +++ b/ufos/src/storage_fjall.rs @@ -242,6 +242,7 @@ impl StorageWhatever for rollups, queues, }; + writer.describe_metrics(); Ok((reader, writer, js_cursor, sketch_secret)) } } @@ -393,6 +394,21 @@ impl FjallReader { Unit::Count, "number of L0 runs in a partition" ); + describe_gauge!( + "storage_fjall_keyspace_disk_space", + Unit::Bytes, + "total storage used according to fjall" + ); + describe_gauge!( + "storage_fjall_journal_count", + Unit::Count, + "total keyspace journals according to fjall" + ); + describe_gauge!( + "storage_fjall_keyspace_sequence", + Unit::Count, + "fjall keyspace sequence" + ); } fn get_storage_stats(&self) -> StorageResult { @@ -1025,6 +1041,9 @@ impl StoreReader for FjallReader { .set(self.rollups.tree.l0_run_count() as f64); gauge!("storage_fjall_l0_run_count", "partition" => "queues") .set(self.queues.tree.l0_run_count() as f64); + gauge!("storage_fjall_keyspace_disk_space").set(self.keyspace.disk_space() as f64); + gauge!("storage_fjall_journal_count").set(self.keyspace.journal_count() as f64); + gauge!("storage_fjall_keyspace_sequence").set(self.keyspace.instant() as f64); } async fn get_storage_stats(&self) -> StorageResult { let s = self.clone(); @@ -1117,6 +1136,58 @@ pub struct FjallWriter { } impl FjallWriter { + fn describe_metrics(&self) { + describe_histogram!( + "storage_insert_batch_db_batch_items", + Unit::Count, + "how many items are in the fjall batch for batched inserts" + ); + describe_histogram!( + "storage_insert_batch_db_batch_size", + Unit::Count, + "in-memory size of the fjall batch for batched inserts" + ); + describe_histogram!( + "storage_rollup_counts_db_batch_items", + Unit::Count, + "how many items are in the fjall batch for a timlies rollup" + ); + describe_histogram!( + "storage_rollup_counts_db_batch_size", + Unit::Count, + "in-memory size of the fjall batch for a timelies rollup" + ); + describe_counter!( + "storage_delete_account_partial_commits", + Unit::Count, + "fjall checkpoint commits for cleaning up accounts with too many records" + ); + describe_counter!( + "storage_delete_account_completions", + Unit::Count, + "total count of account deletes handled" + ); + describe_counter!( + "storage_delete_account_records_deleted", + Unit::Count, + "total records deleted when handling account deletes" + ); + describe_histogram!( + "storage_trim_dirty_nsids", + Unit::Count, + "number of NSIDs trimmed" + ); + describe_histogram!( + "storage_trim_duration", + Unit::Microseconds, + "how long it took to trim the dirty NSIDs" + ); + describe_counter!( + "storage_trim_removed", + Unit::Count, + "how many records were removed during trim" + ); + } fn rollup_delete_account( &mut self, cursor: Cursor, @@ -1284,6 +1355,9 @@ impl FjallWriter { insert_batch_static_neu::(&mut batch, &self.global, last_cursor)?; + histogram!("storage_rollup_counts_db_batch_items").record(batch.len() as f64); + histogram!("storage_rollup_counts_db_batch_size") + .record(std::mem::size_of_val(&batch) as f64); batch.commit()?; Ok((cursors_advanced, dirty_nsids)) } @@ -1294,21 +1368,6 @@ impl StoreWriter for FjallWriter { if self.bg_taken.swap(true, Ordering::SeqCst) { return Err(StorageError::BackgroundAlreadyStarted); } - describe_histogram!( - "storage_trim_dirty_nsids", - Unit::Count, - "number of NSIDs trimmed" - ); - describe_histogram!( - "storage_trim_duration", - Unit::Microseconds, - "how long it took to trim the dirty NSIDs" - ); - describe_counter!( - "storage_trim_removed", - Unit::Count, - "how many records were removed during trim" - ); if reroll { log::info!("reroll: resetting rollup cursor..."); insert_static_neu::(&self.global, Cursor::from_start())?; @@ -1403,6 +1462,9 @@ impl StoreWriter for FjallWriter { latest.to_db_bytes()?, ); + histogram!("storage_insert_batch_db_batch_items").record(batch.len() as f64); + histogram!("storage_insert_batch_db_batch_size") + .record(std::mem::size_of_val(&batch) as f64); batch.commit()?; Ok(()) } @@ -1584,10 +1646,13 @@ impl StoreWriter for FjallWriter { batch.remove(&self.records, key_bytes); records_deleted += 1; if batch.len() >= MAX_BATCHED_ACCOUNT_DELETE_RECORDS { + counter!("storage_delete_account_partial_commits").increment(1); batch.commit()?; batch = self.keyspace.batch(); } } + counter!("storage_delete_account_completions").increment(1); + counter!("storage_delete_account_records_deleted").increment(records_deleted as u64); batch.commit()?; Ok(records_deleted) } -- 2.51.2 From 820b5658bf2a722e3715f121a05e3ff7c287dd1e Mon Sep 17 00:00:00 2001 From: phil Date: Wed, 13 Aug 2025 16:23:08 -0400 Subject: [PATCH 118/134] didn't work (not surprising) --- ufos/src/storage_fjall.rs | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/ufos/src/storage_fjall.rs b/ufos/src/storage_fjall.rs index 6dc9b48..5a9ffc4 100644 --- a/ufos/src/storage_fjall.rs +++ b/ufos/src/storage_fjall.rs @@ -1142,21 +1142,11 @@ impl FjallWriter { Unit::Count, "how many items are in the fjall batch for batched inserts" ); - describe_histogram!( - "storage_insert_batch_db_batch_size", - Unit::Count, - "in-memory size of the fjall batch for batched inserts" - ); describe_histogram!( "storage_rollup_counts_db_batch_items", Unit::Count, "how many items are in the fjall batch for a timlies rollup" ); - describe_histogram!( - "storage_rollup_counts_db_batch_size", - Unit::Count, - "in-memory size of the fjall batch for a timelies rollup" - ); describe_counter!( "storage_delete_account_partial_commits", Unit::Count, @@ -1356,8 +1346,6 @@ impl FjallWriter { insert_batch_static_neu::(&mut batch, &self.global, last_cursor)?; histogram!("storage_rollup_counts_db_batch_items").record(batch.len() as f64); - histogram!("storage_rollup_counts_db_batch_size") - .record(std::mem::size_of_val(&batch) as f64); batch.commit()?; Ok((cursors_advanced, dirty_nsids)) } @@ -1463,8 +1451,6 @@ impl StoreWriter for FjallWriter { ); histogram!("storage_insert_batch_db_batch_items").record(batch.len() as f64); - histogram!("storage_insert_batch_db_batch_size") - .record(std::mem::size_of_val(&batch) as f64); batch.commit()?; Ok(()) } -- 2.51.2 From aa47761e4cffe0f52a92d2c4adb4d33616704899 Mon Sep 17 00:00:00 2001 From: phil Date: Thu, 14 Aug 2025 15:51:19 -0400 Subject: [PATCH 119/134] log a few errors that were dropped + atrium fork atrium fork for the https handle method fix --- Cargo.lock | 126 ++++++++++++++++++++++++++++++++------ jetstream/Cargo.toml | 2 +- slingshot/Cargo.toml | 8 +-- slingshot/src/identity.rs | 7 ++- slingshot/src/server.rs | 53 +++++++++------- 5 files changed, 148 insertions(+), 48 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e9cda1e..30951c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -306,8 +306,28 @@ version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46355d3245edc7b3160b2a45fe55d09a6963ebd3eee0252feb6b72fb0eb71463" dependencies = [ - "atrium-common", - "atrium-xrpc", + "atrium-common 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "atrium-xrpc 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", + "chrono", + "http", + "ipld-core", + "langtag", + "regex", + "serde", + "serde_bytes", + "serde_json", + "thiserror 1.0.69", + "tokio", + "trait-variant", +] + +[[package]] +name = "atrium-api" +version = "0.25.4" +source = "git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace#80a355991ac9b48ba3f559d12aac74f071fc638c" +dependencies = [ + "atrium-common 0.1.2 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)", + "atrium-xrpc 0.12.3 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)", "chrono", "http", "ipld-core", @@ -336,15 +356,44 @@ dependencies = [ "web-time", ] +[[package]] +name = "atrium-common" +version = "0.1.2" +source = "git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace#80a355991ac9b48ba3f559d12aac74f071fc638c" +dependencies = [ + "dashmap", + "lru", + "moka", + "thiserror 1.0.69", + "tokio", + "trait-variant", + "web-time", +] + [[package]] name = "atrium-identity" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9e2d42bb4dbea038f4f5f45e3af2a89d61a9894a75f06aa550b74a60d2be380" dependencies = [ - "atrium-api", - "atrium-common", - "atrium-xrpc", + "atrium-api 0.25.4 (registry+https://github.com/rust-lang/crates.io-index)", + "atrium-common 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "atrium-xrpc 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", + "serde", + "serde_html_form", + "serde_json", + "thiserror 1.0.69", + "trait-variant", +] + +[[package]] +name = "atrium-identity" +version = "0.1.5" +source = "git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace#80a355991ac9b48ba3f559d12aac74f071fc638c" +dependencies = [ + "atrium-api 0.25.4 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)", + "atrium-common 0.1.2 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)", + "atrium-xrpc 0.12.3 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)", "serde", "serde_html_form", "serde_json", @@ -358,10 +407,38 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca22dc4eaf77fd9bf050b21192ac58cd654a437d28e000ec114ebd93a51d36f5" dependencies = [ - "atrium-api", - "atrium-common", - "atrium-identity", - "atrium-xrpc", + "atrium-api 0.25.4 (registry+https://github.com/rust-lang/crates.io-index)", + "atrium-common 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "atrium-identity 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "atrium-xrpc 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", + "base64 0.22.1", + "chrono", + "dashmap", + "ecdsa", + "elliptic-curve", + "jose-jwa", + "jose-jwk", + "p256", + "rand 0.8.5", + "reqwest", + "serde", + "serde_html_form", + "serde_json", + "sha2", + "thiserror 1.0.69", + "tokio", + "trait-variant", +] + +[[package]] +name = "atrium-oauth" +version = "0.1.3" +source = "git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace#80a355991ac9b48ba3f559d12aac74f071fc638c" +dependencies = [ + "atrium-api 0.25.4 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)", + "atrium-common 0.1.2 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)", + "atrium-identity 0.1.5 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)", + "atrium-xrpc 0.12.3 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)", "base64 0.22.1", "chrono", "dashmap", @@ -395,6 +472,19 @@ dependencies = [ "trait-variant", ] +[[package]] +name = "atrium-xrpc" +version = "0.12.3" +source = "git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace#80a355991ac9b48ba3f559d12aac74f071fc638c" +dependencies = [ + "http", + "serde", + "serde_html_form", + "serde_json", + "thiserror 1.0.69", + "trait-variant", +] + [[package]] name = "auto_enums" version = "0.8.7" @@ -2642,7 +2732,7 @@ version = "0.1.1" dependencies = [ "anyhow", "async-trait", - "atrium-api", + "atrium-api 0.25.4 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)", "chrono", "clap", "futures-util", @@ -4916,10 +5006,10 @@ dependencies = [ name = "slingshot" version = "0.1.0" dependencies = [ - "atrium-api", - "atrium-common", - "atrium-identity", - "atrium-oauth", + "atrium-api 0.25.4 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)", + "atrium-common 0.1.2 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)", + "atrium-identity 0.1.5 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)", + "atrium-oauth 0.1.3 (git+https://github.com/uniphil/atrium.git?branch=fix%2Fresolve-handle-https-accept-whitespace)", "clap", "ctrlc", "foyer", @@ -6009,10 +6099,10 @@ dependencies = [ name = "who-am-i" version = "0.1.0" dependencies = [ - "atrium-api", - "atrium-common", - "atrium-identity", - "atrium-oauth", + "atrium-api 0.25.4 (registry+https://github.com/rust-lang/crates.io-index)", + "atrium-common 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "atrium-identity 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "atrium-oauth 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "axum", "axum-extra", "axum-template", diff --git a/jetstream/Cargo.toml b/jetstream/Cargo.toml index 96bd42d..1eadecd 100644 --- a/jetstream/Cargo.toml +++ b/jetstream/Cargo.toml @@ -10,7 +10,7 @@ readme = "README.md" [dependencies] async-trait = "0.1.83" -atrium-api = { version = "0.25.4", default-features = false, features = [ +atrium-api = { git = "https://github.com/uniphil/atrium.git", branch = "fix/resolve-handle-https-accept-whitespace", default-features = false, features = [ "namespace-appbsky", ] } tokio = { version = "1.44.2", features = ["full", "sync", "time"] } diff --git a/slingshot/Cargo.toml b/slingshot/Cargo.toml index 06a97a5..9991b65 100644 --- a/slingshot/Cargo.toml +++ b/slingshot/Cargo.toml @@ -4,10 +4,10 @@ version = "0.1.0" edition = "2024" [dependencies] -atrium-api = { version = "0.25.4", default-features = false } -atrium-common = "0.1.2" -atrium-identity = "0.1.5" -atrium-oauth = "0.1.3" +atrium-api = { git = "https://github.com/uniphil/atrium.git", branch = "fix/resolve-handle-https-accept-whitespace", default-features = false } +atrium-common = { git = "https://github.com/uniphil/atrium.git", branch = "fix/resolve-handle-https-accept-whitespace" } +atrium-identity = { git = "https://github.com/uniphil/atrium.git", branch = "fix/resolve-handle-https-accept-whitespace" } +atrium-oauth = { git = "https://github.com/uniphil/atrium.git", branch = "fix/resolve-handle-https-accept-whitespace" } clap = { version = "4.5.41", features = ["derive"] } ctrlc = "3.4.7" foyer = { version = "0.18.0", features = ["serde"] } diff --git a/slingshot/src/identity.rs b/slingshot/src/identity.rs index 1abd0d0..80e5286 100644 --- a/slingshot/src/identity.rs +++ b/slingshot/src/identity.rs @@ -240,9 +240,10 @@ impl Identity { Err(atrium_identity::Error::NotFound) => { Ok(IdentityVal(UtcDateTime::now(), IdentityData::NotFound)) } - Err(other) => Err(foyer::Error::Other(Box::new( - IdentityError::ResolutionFailed(other), - ))), + Err(other) => Err(foyer::Error::Other(Box::new({ + log::debug!("other error resolving handle: {other:?}"); + IdentityError::ResolutionFailed(other) + }))), } } }) diff --git a/slingshot/src/server.rs b/slingshot/src/server.rs index 9066da1..5abec05 100644 --- a/slingshot/src/server.rs +++ b/slingshot/src/server.rs @@ -439,17 +439,22 @@ impl Xrpc { let Ok(alleged_handle) = Handle::new(identifier) else { return invalid("identifier was not a valid DID or handle"); }; - if let Ok(res) = self.identity.handle_to_did(alleged_handle.clone()).await { - if let Some(did) = res { - // we did it joe - unverified_handle = Some(alleged_handle); - did - } else { - return invalid("Could not resolve handle identifier to a DID"); + + match self.identity.handle_to_did(alleged_handle.clone()).await { + Ok(res) => { + if let Some(did) = res { + // we did it joe + unverified_handle = Some(alleged_handle); + did + } else { + return invalid("Could not resolve handle identifier to a DID"); + } + } + Err(e) => { + log::debug!("failed to resolve handle: {e}"); + // TODO: ServerError not BadRequest + return invalid("errored while trying to resolve handle to DID"); } - } else { - // TODO: ServerError not BadRequest - return invalid("errored while trying to resolve handle to DID"); } } }; @@ -514,20 +519,24 @@ impl Xrpc { "repo was not a valid DID or handle", )); }; - if let Ok(res) = self.identity.handle_to_did(handle).await { - if let Some(did) = res { - did - } else { - return GetRecordResponse::BadRequest(xrpc_error( - "InvalidRequest", - "Could not resolve handle repo to a DID", + match self.identity.handle_to_did(handle).await { + Ok(res) => { + if let Some(did) = res { + did + } else { + return GetRecordResponse::BadRequest(xrpc_error( + "InvalidRequest", + "Could not resolve handle repo to a DID", + )); + } + } + Err(e) => { + log::debug!("handle resolution failed: {e}"); + return GetRecordResponse::ServerError(xrpc_error( + "ResolutionFailed", + "errored while trying to resolve handle to DID", )); } - } else { - return GetRecordResponse::ServerError(xrpc_error( - "ResolutionFailed", - "errored while trying to resolve handle to DID", - )); } } }; -- 2.51.2 From 55b37053ccde86876bdc94b41fa511c1b361600d Mon Sep 17 00:00:00 2001 From: phil Date: Thu, 14 Aug 2025 16:56:22 -0400 Subject: [PATCH 120/134] separate task joinsets so that we actually quit when the consumer quits --- ufos/src/main.rs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/ufos/src/main.rs b/ufos/src/main.rs index 23beff9..5b82c0f 100644 --- a/ufos/src/main.rs +++ b/ufos/src/main.rs @@ -80,11 +80,12 @@ async fn go( cursor: Option, sketch_secret: SketchSecretPrefix, ) -> anyhow::Result<()> { - let mut tasks: JoinSet> = JoinSet::new(); + let mut whatever_tasks: JoinSet> = JoinSet::new(); + let mut consumer_tasks: JoinSet> = JoinSet::new(); println!("starting server with storage..."); let serving = server::serve(read_store.clone()); - tasks.spawn(async move { + whatever_tasks.spawn(async move { serving.await.map_err(|e| { log::warn!("server ended: {e}"); anyhow::anyhow!(e) @@ -93,7 +94,7 @@ async fn go( if args.pause_writer { log::info!("not starting jetstream or the write loop."); - for t in tasks.join_all().await { + for t in whatever_tasks.join_all().await { if let Err(e) = t { return Err(anyhow::anyhow!(e)); } @@ -115,14 +116,14 @@ async fn go( let rolling = write_store .background_tasks(args.reroll)? .run(args.backfill); - tasks.spawn(async move { + consumer_tasks.spawn(async move { rolling .await .inspect_err(|e| log::warn!("rollup ended: {e}"))?; Ok(()) }); - tasks.spawn(async move { + consumer_tasks.spawn(async move { write_store .receive_batches(batches) .await @@ -130,7 +131,7 @@ async fn go( Ok(()) }); - tasks.spawn(async move { + whatever_tasks.spawn(async move { do_update_stuff(read_store).await; log::warn!("status task ended"); Ok(()) @@ -138,10 +139,13 @@ async fn go( install_metrics_server()?; - for (i, t) in tasks.join_all().await.iter().enumerate() { + for (i, t) in consumer_tasks.join_all().await.iter().enumerate() { log::warn!("task {i} done: {t:?}"); } + println!("consumer tasks all completed, killing the others"); + whatever_tasks.shutdown().await; + println!("bye!"); Ok(()) -- 2.51.2 From bcc7abce39eab28586e9fe0f746c0b368f73150b Mon Sep 17 00:00:00 2001 From: phil Date: Thu, 14 Aug 2025 18:42:00 -0400 Subject: [PATCH 121/134] apparently the rollup is whatever --- ufos/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ufos/src/main.rs b/ufos/src/main.rs index 5b82c0f..949db21 100644 --- a/ufos/src/main.rs +++ b/ufos/src/main.rs @@ -116,7 +116,7 @@ async fn go( let rolling = write_store .background_tasks(args.reroll)? .run(args.backfill); - consumer_tasks.spawn(async move { + whatever_tasks.spawn(async move { rolling .await .inspect_err(|e| log::warn!("rollup ended: {e}"))?; -- 2.51.2 From a9f6627aa8f2132721a77740a3f576692f297b4c Mon Sep 17 00:00:00 2001 From: phil Date: Fri, 15 Aug 2025 10:49:32 -0400 Subject: [PATCH 122/134] record insert time for every batch insert and without relying on indirect tokio scheduling for timing measurements --- ufos/src/storage.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ufos/src/storage.rs b/ufos/src/storage.rs index 079d804..e501e5a 100644 --- a/ufos/src/storage.rs +++ b/ufos/src/storage.rs @@ -41,6 +41,11 @@ where Unit::Microseconds, "batches that took more than 3s to insert" ); + describe_histogram!( + "storage_batch_insert_time", + Unit::Microseconds, + "total time to insert one commit batch" + ); while let Some(event_batch) = batches.recv().await { let token = CancellationToken::new(); let cancelled = token.clone(); @@ -69,7 +74,10 @@ where let mut me = self.clone(); move || { let _guard = token.drop_guard(); - me.insert_batch(event_batch) + let t0 = Instant::now(); + let r = me.insert_batch(event_batch); + histogram!("storage_batch_insert_time").record(t0.elapsed().as_micros() as f64); + r } }) .await??; -- 2.51.2 From 12491c8ea45b386d0a56eb75e9fc5b66f51b219e Mon Sep 17 00:00:00 2001 From: phil Date: Fri, 15 Aug 2025 13:22:50 -0400 Subject: [PATCH 123/134] =?UTF-8?q?remove=20records=20from=20the=20records?= =?UTF-8?q?=20collection=20=F0=9F=98=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit noooooooooooooooooooooooo --- ufos/src/storage_fjall.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ufos/src/storage_fjall.rs b/ufos/src/storage_fjall.rs index 5a9ffc4..ec46301 100644 --- a/ufos/src/storage_fjall.rs +++ b/ufos/src/storage_fjall.rs @@ -1605,7 +1605,7 @@ impl StoreWriter for FjallWriter { candidate_new_feed_lower_cursor = Some(feed_key.cursor()); } - self.feeds.remove(&location_key_bytes)?; + self.records.remove(&location_key_bytes)?; self.feeds.remove(key_bytes)?; records_deleted += 1; } -- 2.51.2 From cf9f6ca8ac8a32140fef58570cc8d1a66b425b86 Mon Sep 17 00:00:00 2001 From: phil Date: Wed, 3 Sep 2025 09:33:58 -0400 Subject: [PATCH 124/134] quasar stub --- Cargo.lock | 57 +++++++++++++++++++++++++++++++------------ Cargo.toml | 1 + quasar/Cargo.toml | 8 ++++++ quasar/readme.md | 3 +++ quasar/src/lib.rs | 1 + quasar/src/main.rs | 3 +++ quasar/src/storage.rs | 0 7 files changed, 57 insertions(+), 16 deletions(-) create mode 100644 quasar/Cargo.toml create mode 100644 quasar/readme.md create mode 100644 quasar/src/lib.rs create mode 100644 quasar/src/main.rs create mode 100644 quasar/src/storage.rs diff --git a/Cargo.lock b/Cargo.lock index 30951c3..7fc7650 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -933,9 +933,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.41" +version = "4.5.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be92d32e80243a54711e5d7ce823c35c41c9d929dc4ab58e1276f625841aadf9" +checksum = "2c5e4fcf9c21d2e544ca1ee9d8552de13019a42aa7dbf32747fa7aaf1df76e57" dependencies = [ "clap_builder", "clap_derive", @@ -943,9 +943,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.41" +version = "4.5.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707eab41e9622f9139419d573eca0900137718000c517d47da73045f54331c3d" +checksum = "fecb53a0e6fcfb055f686001bc2e2592fa527efaf38dbe81a6a9563562e57d41" dependencies = [ "anstream", "anstyle", @@ -955,9 +955,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.41" +version = "4.5.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef4f52386a59ca4c860f7393bcf8abd8dfd91ecccc0f774635ff68e92eeef491" +checksum = "14cb31bb0a7d536caef2639baa7fad459e15c3144efefa6dbd1c84562c4739f6" dependencies = [ "heck", "proc-macro2", @@ -1310,7 +1310,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18e4fdb82bd54a12e42fb58a800dcae6b9e13982238ce2296dc3570b92148e1f" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.103", ] [[package]] @@ -1659,7 +1659,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -1699,6 +1699,23 @@ dependencies = [ "subtle", ] +[[package]] +name = "fjall" +version = "2.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b25ad44cd4360a0448a9b5a0a6f1c7a621101cca4578706d43c9a821418aebc" +dependencies = [ + "byteorder", + "byteview", + "dashmap", + "log", + "lsm-tree", + "path-absolutize", + "std-semaphore", + "tempfile", + "xxhash-rust", +] + [[package]] name = "fjall" version = "2.11.2" @@ -2684,7 +2701,7 @@ checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2889,7 +2906,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] @@ -4071,6 +4088,14 @@ dependencies = [ "winapi", ] +[[package]] +name = "quasar" +version = "0.1.0" +dependencies = [ + "clap", + "fjall 2.11.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "quick-xml" version = "0.36.2" @@ -4143,7 +4168,7 @@ dependencies = [ "once_cell", "socket2 0.5.9", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4525,7 +4550,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4538,7 +4563,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.9.4", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5277,7 +5302,7 @@ dependencies = [ "getrandom 0.3.3", "once_cell", "rustix 1.0.5", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5773,7 +5798,7 @@ dependencies = [ "clap", "dropshot", "env_logger", - "fjall", + "fjall 2.11.2 (git+https://github.com/fjall-rs/fjall.git)", "getrandom 0.3.3", "http", "jetstream", @@ -6162,7 +6187,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 6604a84..e87ea96 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,4 +9,5 @@ members = [ "spacedust", "who-am-i", "slingshot", + "quasar", ] diff --git a/quasar/Cargo.toml b/quasar/Cargo.toml new file mode 100644 index 0000000..0206b37 --- /dev/null +++ b/quasar/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "quasar" +version = "0.1.0" +edition = "2024" + +[dependencies] +clap = { version = "4.5.46", features = ["derive"] } +fjall = "2.11.2" diff --git a/quasar/readme.md b/quasar/readme.md new file mode 100644 index 0000000..79a18e6 --- /dev/null +++ b/quasar/readme.md @@ -0,0 +1,3 @@ +# quasar + +indexed replay and fan-out for event stream services (wip) diff --git a/quasar/src/lib.rs b/quasar/src/lib.rs new file mode 100644 index 0000000..80aaac2 --- /dev/null +++ b/quasar/src/lib.rs @@ -0,0 +1 @@ +mod storage; diff --git a/quasar/src/main.rs b/quasar/src/main.rs new file mode 100644 index 0000000..e7a11a9 --- /dev/null +++ b/quasar/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} diff --git a/quasar/src/storage.rs b/quasar/src/storage.rs new file mode 100644 index 0000000..e69de29 -- 2.51.2 From 11279d97ac2f5945a282c6d2bbff6343e76875d5 Mon Sep 17 00:00:00 2001 From: phil Date: Wed, 3 Sep 2025 15:00:57 -0400 Subject: [PATCH 125/134] basic token verification works --- Cargo.lock | 276 +++++++++++++++++++++++++++++++++++++----- Cargo.toml | 1 + pocket/Cargo.toml | 19 +++ pocket/src/lib.rs | 5 + pocket/src/main.rs | 9 ++ pocket/src/server.rs | 208 +++++++++++++++++++++++++++++++ pocket/src/token.rs | 72 +++++++++++ quasar/src/storage.rs | 4 + 8 files changed, 561 insertions(+), 33 deletions(-) create mode 100644 pocket/Cargo.toml create mode 100644 pocket/src/lib.rs create mode 100644 pocket/src/main.rs create mode 100644 pocket/src/server.rs create mode 100644 pocket/src/token.rs diff --git a/Cargo.lock b/Cargo.lock index 7fc7650..eaf4675 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -192,7 +192,7 @@ dependencies = [ "nom", "num-traits", "rusticata-macros", - "thiserror 2.0.12", + "thiserror 2.0.16", "time", ] @@ -370,6 +370,19 @@ dependencies = [ "web-time", ] +[[package]] +name = "atrium-crypto" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73a3da430c71dd9006d61072c20771f264e5c498420a49c32305ceab8bd71955" +dependencies = [ + "ecdsa", + "k256", + "multibase", + "p256", + "thiserror 1.0.69", +] + [[package]] name = "atrium-identity" version = "0.1.5" @@ -628,7 +641,7 @@ dependencies = [ "axum", "handlebars", "serde", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -773,6 +786,22 @@ dependencies = [ "syn 2.0.103", ] +[[package]] +name = "bitcoin-io" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b47c4ab7a93edb0c7198c5535ed9b52b63095f4e9b45279c6736cec4b856baf" + +[[package]] +name = "bitcoin_hashes" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb18c03d0db0247e147a21a6faafd5a7eb851c743db062de72018b6b7e8e4d16" +dependencies = [ + "bitcoin-io", + "hex-conservative", +] + [[package]] name = "bitflags" version = "2.9.0" @@ -906,6 +935,33 @@ dependencies = [ "windows-link", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "cid" version = "0.11.1" @@ -1171,6 +1227,12 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-bigint" version = "0.5.5" @@ -1514,7 +1576,7 @@ dependencies = [ "slog-bunyan", "slog-json", "slog-term", - "thiserror 2.0.12", + "thiserror 2.0.16", "tokio", "tokio-rustls 0.25.0", "toml", @@ -1683,6 +1745,18 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fastrand" version = "2.3.0" @@ -1814,7 +1888,7 @@ dependencies = [ "mixtrics", "pin-project", "serde", - "thiserror 2.0.12", + "thiserror 2.0.16", "tokio", "tracing", ] @@ -1834,7 +1908,7 @@ dependencies = [ "parking_lot", "pin-project", "serde", - "thiserror 2.0.12", + "thiserror 2.0.16", "tokio", "twox-hash", ] @@ -1867,7 +1941,7 @@ dependencies = [ "parking_lot", "pin-project", "serde", - "thiserror 2.0.12", + "thiserror 2.0.16", "tokio", "tracing", ] @@ -1899,7 +1973,7 @@ dependencies = [ "pin-project", "rand 0.9.1", "serde", - "thiserror 2.0.12", + "thiserror 2.0.16", "tokio", "tracing", "twox-hash", @@ -2120,6 +2194,16 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" +dependencies = [ + "cfg-if", + "crunchy", +] + [[package]] name = "handlebars" version = "6.3.2" @@ -2133,7 +2217,7 @@ dependencies = [ "pest_derive", "serde", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.16", "walkdir", ] @@ -2169,6 +2253,15 @@ dependencies = [ "foldhash", ] +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.2", +] + [[package]] name = "headers" version = "0.4.0" @@ -2222,6 +2315,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hex-conservative" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5313b072ce3c597065a808dbf612c4c8e8590bdbf8b579508bf7a762c5eae6cd" +dependencies = [ + "arrayvec", +] + [[package]] name = "hickory-proto" version = "0.25.2" @@ -2240,7 +2342,7 @@ dependencies = [ "once_cell", "rand 0.9.1", "ring", - "thiserror 2.0.12", + "thiserror 2.0.16", "tinyvec", "tokio", "tracing", @@ -2263,7 +2365,7 @@ dependencies = [ "rand 0.9.1", "resolv-conf", "smallvec", - "thiserror 2.0.12", + "thiserror 2.0.16", "tokio", "tracing", ] @@ -2757,7 +2859,7 @@ dependencies = [ "metrics", "serde", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.16", "tokio", "tokio-tungstenite 0.26.2", "url", @@ -2859,6 +2961,39 @@ dependencies = [ "simple_asn1", ] +[[package]] +name = "jwt-compact" +version = "0.9.0-beta.1" +source = "git+https://github.com/fatfingers23/jwt-compact.git#aed088b8ff5ad44ef2785c453f6a4b7916728b1c" +dependencies = [ + "anyhow", + "base64ct", + "chrono", + "ciborium", + "hmac", + "lazy_static", + "rand_core 0.6.4", + "secp256k1", + "serde", + "serde_json", + "sha2", + "smallvec", + "subtle", + "zeroize", +] + +[[package]] +name = "k256" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" +dependencies = [ + "cfg-if", + "ecdsa", + "elliptic-curve", + "sha2", +] + [[package]] name = "langtag" version = "0.3.4" @@ -2951,6 +3086,16 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "libsqlite3-sys" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "133c182a6a2c87864fe97778797e46c7e999672690dc9fa3ee8e241aa4a9c13f" +dependencies = [ + "pkg-config", + "vcpkg", +] + [[package]] name = "libz-sys" version = "1.1.22" @@ -2969,7 +3114,7 @@ dependencies = [ "anyhow", "fluent-uri", "nom", - "thiserror 2.0.12", + "thiserror 2.0.16", "tinyjson", ] @@ -3235,7 +3380,7 @@ dependencies = [ "metrics", "metrics-util 0.20.0", "quanta", - "thiserror 2.0.12", + "thiserror 2.0.16", "tokio", "tracing", ] @@ -3784,7 +3929,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1db05f56d34358a8b1066f67cbb203ee3e7ed2ba674a6263a1d5ec6db2204323" dependencies = [ "memchr", - "thiserror 2.0.12", + "thiserror 2.0.16", "ucd-trie", ] @@ -3880,6 +4025,25 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "pocket" +version = "0.1.0" +dependencies = [ + "atrium-crypto", + "clap", + "jwt-compact", + "log", + "poem", + "poem-openapi", + "reqwest", + "rusqlite", + "serde", + "serde_json", + "thiserror 2.0.16", + "tokio", + "tracing-subscriber", +] + [[package]] name = "poem" version = "3.1.12" @@ -3918,7 +4082,7 @@ dependencies = [ "smallvec", "sync_wrapper", "tempfile", - "thiserror 2.0.12", + "thiserror 2.0.16", "tokio", "tokio-rustls 0.26.2", "tokio-stream", @@ -3962,7 +4126,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "serde_yaml", - "thiserror 2.0.12", + "thiserror 2.0.16", "tokio", ] @@ -3981,7 +4145,7 @@ dependencies = [ "quote", "regex", "syn 2.0.103", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -4130,7 +4294,7 @@ dependencies = [ "rustc-hash 2.1.1", "rustls 0.23.31", "socket2 0.5.9", - "thiserror 2.0.12", + "thiserror 2.0.16", "tokio", "tracing", "web-time", @@ -4151,7 +4315,7 @@ dependencies = [ "rustls 0.23.31", "rustls-pki-types", "slab", - "thiserror 2.0.12", + "thiserror 2.0.16", "tinyvec", "tracing", "web-time", @@ -4504,6 +4668,20 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rusqlite" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "165ca6e57b20e1351573e3729b958bc62f0e48025386970b6e4d29e7a7e71f3f" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + [[package]] name = "rustc-demangle" version = "0.1.24" @@ -4732,6 +4910,26 @@ dependencies = [ "zeroize", ] +[[package]] +name = "secp256k1" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b50c5943d326858130af85e049f2661ba3c78b26589b8ab98e65e80ae44a1252" +dependencies = [ + "bitcoin_hashes", + "rand 0.8.5", + "secp256k1-sys", +] + +[[package]] +name = "secp256k1-sys" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4387882333d3aa8cb20530a17c69a3752e97837832f34f6dccc760e715001d9" +dependencies = [ + "cc", +] + [[package]] name = "security-framework" version = "2.11.1" @@ -4865,7 +5063,7 @@ dependencies = [ "percent-encoding", "ryu", "serde", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -5008,7 +5206,7 @@ checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb" dependencies = [ "num-bigint", "num-traits", - "thiserror 2.0.12", + "thiserror 2.0.16", "time", ] @@ -5050,7 +5248,7 @@ dependencies = [ "rustls 0.23.31", "serde", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.16", "time", "tokio", "tokio-util", @@ -5161,7 +5359,7 @@ dependencies = [ "serde", "serde_json", "serde_qs", - "thiserror 2.0.12", + "thiserror 2.0.16", "tinyjson", "tokio", "tokio-tungstenite 0.27.0", @@ -5327,11 +5525,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.12" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" dependencies = [ - "thiserror-impl 2.0.12", + "thiserror-impl 2.0.16", ] [[package]] @@ -5347,9 +5545,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.12" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" dependencies = [ "proc-macro2", "quote", @@ -5742,7 +5940,7 @@ dependencies = [ "native-tls", "rand 0.9.1", "sha1", - "thiserror 2.0.12", + "thiserror 2.0.16", "url", "utf-8", ] @@ -5760,7 +5958,7 @@ dependencies = [ "log", "rand 0.9.1", "sha1", - "thiserror 2.0.12", + "thiserror 2.0.16", "utf-8", ] @@ -5813,7 +6011,7 @@ dependencies = [ "serde_qs", "sha2", "tempfile", - "thiserror 2.0.12", + "thiserror 2.0.16", "tikv-jemallocator", "tokio", "tokio-util", @@ -6147,7 +6345,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.16", "tokio", "tokio-util", "url", @@ -6540,7 +6738,7 @@ dependencies = [ "nom", "oid-registry", "rusticata-macros", - "thiserror 2.0.12", + "thiserror 2.0.16", "time", ] @@ -6651,6 +6849,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" dependencies = [ "serde", + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.103", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index e87ea96..25cbcc0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,4 +10,5 @@ members = [ "who-am-i", "slingshot", "quasar", + "pocket", ] diff --git a/pocket/Cargo.toml b/pocket/Cargo.toml new file mode 100644 index 0000000..3b785ff --- /dev/null +++ b/pocket/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "pocket" +version = "0.1.0" +edition = "2024" + +[dependencies] +atrium-crypto = "0.1.2" +clap = { version = "4.5.41", features = ["derive"] } +jwt-compact = { git = "https://github.com/fatfingers23/jwt-compact.git", features = ["es256k"] } +log = "0.4.27" +poem = { version = "3.1.12", features = ["acme", "static-files"] } +poem-openapi = { version = "5.1.16", features = ["scalar"] } +reqwest = { version = "0.12.22", features = ["json"] } +rusqlite = "0.37.0" +serde = { version = "1.0.219", features = ["derive"] } +serde_json = { version = "1.0.141" } +thiserror = "2.0.16" +tokio = { version = "1.47.0", features = ["full"] } +tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } diff --git a/pocket/src/lib.rs b/pocket/src/lib.rs new file mode 100644 index 0000000..1a7a0be --- /dev/null +++ b/pocket/src/lib.rs @@ -0,0 +1,5 @@ +mod server; +mod token; + +pub use server::serve; +pub use token::verify; diff --git a/pocket/src/main.rs b/pocket/src/main.rs new file mode 100644 index 0000000..a68f1b8 --- /dev/null +++ b/pocket/src/main.rs @@ -0,0 +1,9 @@ +use pocket::serve; + +#[tokio::main] +async fn main() { + tracing_subscriber::fmt::init(); + println!("Hello, world!"); + serve("mac.cinnebar-tet.ts.net").await +} + diff --git a/pocket/src/server.rs b/pocket/src/server.rs new file mode 100644 index 0000000..940630d --- /dev/null +++ b/pocket/src/server.rs @@ -0,0 +1,208 @@ +use poem::{ + endpoint::make_sync, + Endpoint, + Route, + Server, + EndpointExt, + http::{Method, HeaderMap}, + middleware::{CatchPanic, Cors, Tracing}, + listener::TcpListener, +}; +use poem_openapi::{ + ContactObject, + ExternalDocumentObject, + OpenApi, + OpenApiService, + Tags, + Object, + ApiResponse, + types::Example, + auth::Bearer, + payload::Json, + SecurityScheme, +}; +use crate::verify; +use serde::Serialize; +use serde_json::{Value, json}; + + +#[derive(Debug, SecurityScheme)] +#[oai(ty = "bearer")] +struct BlahAuth(Bearer); + + +#[derive(Tags)] +enum ApiTags { + /// Bluesky-compatible APIs. + #[oai(rename = "app.bsky.* queries")] + AppBsky, +} + +#[derive(Object)] +#[oai(example = true)] +struct XrpcErrorResponseObject { + /// Should correspond an error `name` in the lexicon errors array + error: String, + /// Human-readable description and possibly additonal context + message: String, +} +impl Example for XrpcErrorResponseObject { + fn example() -> Self { + Self { + error: "PreferencesNotFound".to_string(), + message: "No preferences were found for this user".to_string(), + } + } +} +type XrpcError = Json; +fn xrpc_error(error: impl AsRef, message: impl AsRef) -> XrpcError { + Json(XrpcErrorResponseObject { + error: error.as_ref().to_string(), + message: message.as_ref().to_string(), + }) +} + +#[derive(Object)] +#[oai(example = true)] +struct GetBskyPrefsResponseObject { + /// at-uri for this record + preferences: Value, +} +impl Example for GetBskyPrefsResponseObject { + fn example() -> Self { + Self { + preferences: json!({ + "hello": "world", + }), + } + } +} + +#[derive(ApiResponse)] +enum GetBskyPrefsResponse { + /// Record found + #[oai(status = 200)] + Ok(Json), + /// Bad request or no preferences to return + #[oai(status = 400)] + BadRequest(XrpcError), + // /// Server errors + // #[oai(status = 500)] + // ServerError(XrpcError), +} + +struct Xrpc { + domain: String, +} + +#[OpenApi] +impl Xrpc { + /// app.bsky.actor.getPreferences + /// + /// get stored bluesky prefs + #[oai( + path = "/app.bsky.actor.getPreferences", + method = "get", + tag = "ApiTags::AppBsky" + )] + async fn app_bsky_get_prefs( + &self, + BlahAuth(auth): BlahAuth, + m: &HeaderMap, + ) -> GetBskyPrefsResponse { + log::warn!("hm: {m:?}"); + match verify( + &format!("did:web:{}#bsky_appview", self.domain), + "app.bsky.actor.getPreferences", + &auth.token, + ).await { + Ok(did) => log::info!("wooo! {did}"), + Err(err) => return GetBskyPrefsResponse::BadRequest(xrpc_error("booo", err)), + }; + log::warn!("got bearer: {:?}", auth.token); + GetBskyPrefsResponse::Ok(Json(GetBskyPrefsResponseObject::example())) + } + + /// app.bsky.actor.putPreferences + /// + /// store bluesky prefs + #[oai( + path = "/app.bsky.actor.putPreferences", + method = "post", + tag = "ApiTags::AppBsky" + )] + async fn app_bsky_put_prefs( + &self, + Json(prefs): Json, + ) -> () { + log::warn!("received prefs: {prefs:?}"); + () + } +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct AppViewService { + id: String, + r#type: String, + service_endpoint: String, +} +#[derive(Debug, Clone, Serialize)] +struct AppViewDoc { + id: String, + service: [AppViewService; 1], +} +/// Serve a did document for did:web for this to be an xrpc appview +fn get_did_doc(domain: &str) -> impl Endpoint + use<> { + let doc = poem::web::Json(AppViewDoc { + id: format!("did:web:{domain}"), + service: [AppViewService { + id: "#bsky_appview".to_string(), + r#type: "PocketBlueskyPreferences".to_string(), + service_endpoint: format!("https://{domain}"), + }], + }); + make_sync(move |_| doc.clone()) +} + +pub async fn serve( + domain: &str, +) -> () { + let api_service = OpenApiService::new( + Xrpc { domain: domain.to_string() }, + "Pocket", + env!("CARGO_PKG_VERSION"), + ) + .server(domain) + .url_prefix("/xrpc") + .contact( + ContactObject::new() + .name("@microcosm.blue") + .url("https://bsky.app/profile/microcosm.blue"), + ) + // .description(include_str!("../api-description.md")) + .external_document(ExternalDocumentObject::new( + "https://microcosm.blue/pocket", + )); + + let app = Route::new() + .at("/.well-known/did.json", get_did_doc(&domain)) + .nest("/xrpc/", api_service) + // .at("/", StaticFileEndpoint::new("./static/index.html")) + // .nest("/openapi", api_service.spec_endpoint()) + .with( + Cors::new() + .allow_method(Method::GET) + .allow_method(Method::POST) + ) + .with(CatchPanic::new()) + .with(Tracing); + + let listener = TcpListener::bind("127.0.0.1:3000"); + Server::new(listener) + .name("pocket") + .run(app) + .await + .unwrap(); + +} diff --git a/pocket/src/token.rs b/pocket/src/token.rs new file mode 100644 index 0000000..23e513f --- /dev/null +++ b/pocket/src/token.rs @@ -0,0 +1,72 @@ +use jwt_compact::{Claims, UntrustedToken}; +use atrium_crypto::did::parse_multikey; +use atrium_crypto::verify::Verifier; +use std::collections::HashMap; +use serde::Deserialize; + +#[derive(Debug, Deserialize)] +struct MiniDoc { + signing_key: String, +} + +pub async fn verify( + expected_aud: &str, + expected_lxm: &str, + token: &str, +) -> Result { + let untrusted = UntrustedToken::new(token).unwrap(); + + let claims: Claims> = untrusted.deserialize_claims_unchecked().unwrap(); + + let Some(did) = claims.custom.get("iss") else { + return Err("jwt must include the user's did in `iss`"); + }; + + if !did.starts_with("did:") { + return Err("iss should be a did"); + } + if did.contains("#") { + return Err("iss should be a user did without a service identifier"); + } + + println!("Claims: {claims:#?}"); + println!("did: {did:#?}"); + + let endpoint = "https://slingshot.microcosm.blue/xrpc/com.bad-example.identity.resolveMiniDoc"; + let doc: MiniDoc = reqwest::get(format!("{endpoint}?identifier={did}")) + .await + .unwrap() + .error_for_status() + .unwrap() + .json() + .await + .unwrap(); + + log::info!("got minidoc response: {doc:?}"); + + let (alg, public_key) = parse_multikey(&doc.signing_key).unwrap(); + log::info!("parsed key: {public_key:?}"); + + Verifier::default().verify( + alg, + &public_key, + &untrusted.signed_data, + untrusted.signature_bytes(), + ).unwrap(); + // if this passes, then our claims were trustworthy after all(??) + + let Some(aud) = claims.custom.get("aud") else { + return Err("missing aud"); + }; + if aud != expected_aud { + return Err("wrong aud"); + } + let Some(lxm) = claims.custom.get("lxm") else { + return Err("missing lxm"); + }; + if lxm != expected_lxm { + return Err("wrong lxm"); + } + + Ok(did.to_string()) +} diff --git a/quasar/src/storage.rs b/quasar/src/storage.rs index e69de29..4a26f98 100644 --- a/quasar/src/storage.rs +++ b/quasar/src/storage.rs @@ -0,0 +1,4 @@ + +trait Storage { + +} -- 2.51.2 From 2b3a4515b60e94f49fafc52836b570280099b5e6 Mon Sep 17 00:00:00 2001 From: phil Date: Wed, 3 Sep 2025 20:00:55 -0400 Subject: [PATCH 126/134] little cleanup basic api stuff --- .github/workflows/checks.yml | 2 +- Makefile | 2 +- pocket/api-description.md | 17 ++++ pocket/src/lib.rs | 2 +- pocket/src/main.rs | 1 - pocket/src/server.rs | 150 +++++++++++++++---------------- pocket/src/token.rs | 165 ++++++++++++++++++++++++----------- pocket/static/index.html | 67 ++++++++++++++ quasar/src/lib.rs | 2 + quasar/src/storage.rs | 2 +- 10 files changed, 275 insertions(+), 135 deletions(-) create mode 100644 pocket/api-description.md create mode 100644 pocket/static/index.html diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index f26327e..2a1b2cb 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -28,7 +28,7 @@ jobs: - name: get nightly toolchain for jetstream fmt run: rustup toolchain install nightly --allow-downgrade -c rustfmt - name: fmt - run: cargo fmt --package links --package constellation --package ufos --package spacedust --package who-am-i --package slingshot -- --check + run: cargo fmt --package links --package constellation --package ufos --package spacedust --package who-am-i --package slingshot --package pocket -- --check - name: fmt jetstream (nightly) run: cargo +nightly fmt --package jetstream -- --check - name: clippy diff --git a/Makefile b/Makefile index eb29af2..7e52ceb 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ test: cargo test --all-features fmt: - cargo fmt --package links --package constellation --package ufos --package spacedust --package who-am-i --package slingshot + cargo fmt --package links --package constellation --package ufos --package spacedust --package who-am-i --package slingshot --package pocket cargo +nightly fmt --package jetstream clippy: diff --git a/pocket/api-description.md b/pocket/api-description.md new file mode 100644 index 0000000..3d2a5be --- /dev/null +++ b/pocket/api-description.md @@ -0,0 +1,17 @@ +_A pocket dimension to stash a bit of non-public user data._ + + +# Pocket: user preference storage + +This API leverages atproto service proxying to offer a bit of per-user per-app non-public data storage. +Perfect for things like application preferences that might be better left out of the public PDS data. + +The intent is to use oauth scopes to isolate storage on a per-application basis, and to allow easy data migration from a community hosted instance to your own if you end up needing that. + + +### Current status + +> [!important] +> Pocket is currently in a **v0, pre-release state**. There is one production instance and you can use it! Expect short downtimes for restarts as development progresses and occaisional data loss until it's stable. + +ATProto might end up adding a similar feature to [PDSs](https://atproto.com/guides/glossary#pds-personal-data-server). If/when that happens, you should use it instead of this! diff --git a/pocket/src/lib.rs b/pocket/src/lib.rs index 1a7a0be..c6a7fd8 100644 --- a/pocket/src/lib.rs +++ b/pocket/src/lib.rs @@ -2,4 +2,4 @@ mod server; mod token; pub use server::serve; -pub use token::verify; +pub use token::TokenVerifier; diff --git a/pocket/src/main.rs b/pocket/src/main.rs index a68f1b8..46a7ad9 100644 --- a/pocket/src/main.rs +++ b/pocket/src/main.rs @@ -6,4 +6,3 @@ async fn main() { println!("Hello, world!"); serve("mac.cinnebar-tet.ts.net").await } - diff --git a/pocket/src/server.rs b/pocket/src/server.rs index 940630d..9c031e0 100644 --- a/pocket/src/server.rs +++ b/pocket/src/server.rs @@ -1,41 +1,30 @@ +use crate::TokenVerifier; use poem::{ - endpoint::make_sync, - Endpoint, - Route, - Server, - EndpointExt, - http::{Method, HeaderMap}, - middleware::{CatchPanic, Cors, Tracing}, + Endpoint, EndpointExt, Route, Server, + endpoint::{StaticFileEndpoint, make_sync}, + http::Method, listener::TcpListener, + middleware::{CatchPanic, Cors, Tracing}, }; use poem_openapi::{ - ContactObject, - ExternalDocumentObject, - OpenApi, - OpenApiService, - Tags, - Object, - ApiResponse, - types::Example, + ApiResponse, ContactObject, ExternalDocumentObject, Object, OpenApi, OpenApiService, + SecurityScheme, Tags, auth::Bearer, - payload::Json, - SecurityScheme, + payload::{Json, PlainText}, + types::Example, }; -use crate::verify; use serde::Serialize; use serde_json::{Value, json}; - #[derive(Debug, SecurityScheme)] #[oai(ty = "bearer")] -struct BlahAuth(Bearer); - +struct XrpcAuth(Bearer); #[derive(Tags)] enum ApiTags { - /// Bluesky-compatible APIs. - #[oai(rename = "app.bsky.* queries")] - AppBsky, + /// Custom pocket APIs + #[oai(rename = "Pocket APIs")] + Pocket, } #[derive(Object)] @@ -86,57 +75,74 @@ enum GetBskyPrefsResponse { /// Bad request or no preferences to return #[oai(status = 400)] BadRequest(XrpcError), +} + +#[derive(ApiResponse)] +enum PutBskyPrefsResponse { + /// Record found + #[oai(status = 200)] + Ok(PlainText), + /// Bad request or no preferences to return + #[oai(status = 400)] + BadRequest(XrpcError), // /// Server errors // #[oai(status = 500)] // ServerError(XrpcError), } struct Xrpc { - domain: String, + verifier: TokenVerifier, } #[OpenApi] impl Xrpc { - /// app.bsky.actor.getPreferences + /// com.bad-example.pocket.getPreferences /// /// get stored bluesky prefs #[oai( - path = "/app.bsky.actor.getPreferences", + path = "/com.bad-example.pocket.getPreferences", method = "get", - tag = "ApiTags::AppBsky" + tag = "ApiTags::Pocket" )] - async fn app_bsky_get_prefs( - &self, - BlahAuth(auth): BlahAuth, - m: &HeaderMap, - ) -> GetBskyPrefsResponse { - log::warn!("hm: {m:?}"); - match verify( - &format!("did:web:{}#bsky_appview", self.domain), - "app.bsky.actor.getPreferences", - &auth.token, - ).await { - Ok(did) => log::info!("wooo! {did}"), - Err(err) => return GetBskyPrefsResponse::BadRequest(xrpc_error("booo", err)), + async fn app_bsky_get_prefs(&self, XrpcAuth(auth): XrpcAuth) -> GetBskyPrefsResponse { + let did = match self + .verifier + .verify("app.bsky.actor.getPreferences", &auth.token) + .await + { + Ok(d) => d, + Err(e) => return GetBskyPrefsResponse::BadRequest(xrpc_error("boooo", e.to_string())), }; - log::warn!("got bearer: {:?}", auth.token); + log::info!("verified did: {did}"); + // TODO: fetch from storage GetBskyPrefsResponse::Ok(Json(GetBskyPrefsResponseObject::example())) } - /// app.bsky.actor.putPreferences + /// com.bad-example.pocket.putPreferences /// /// store bluesky prefs #[oai( - path = "/app.bsky.actor.putPreferences", + path = "/com.bad-example.pocket.putPreferences", method = "post", - tag = "ApiTags::AppBsky" + tag = "ApiTags::Pocket" )] async fn app_bsky_put_prefs( &self, + XrpcAuth(auth): XrpcAuth, Json(prefs): Json, - ) -> () { + ) -> PutBskyPrefsResponse { + let did = match self + .verifier + .verify("app.bsky.actor.getPreferences", &auth.token) + .await + { + Ok(d) => d, + Err(e) => return PutBskyPrefsResponse::BadRequest(xrpc_error("boooo", e.to_string())), + }; + log::info!("verified did: {did}"); log::warn!("received prefs: {prefs:?}"); - () + // TODO: put prefs into storage + PutBskyPrefsResponse::Ok(PlainText("hiiiiii".to_string())) } } @@ -157,52 +163,40 @@ fn get_did_doc(domain: &str) -> impl Endpoint + use<> { let doc = poem::web::Json(AppViewDoc { id: format!("did:web:{domain}"), service: [AppViewService { - id: "#bsky_appview".to_string(), - r#type: "PocketBlueskyPreferences".to_string(), + id: "#pocket_prefs".to_string(), + r#type: "PocketPreferences".to_string(), service_endpoint: format!("https://{domain}"), }], }); make_sync(move |_| doc.clone()) } -pub async fn serve( - domain: &str, -) -> () { - let api_service = OpenApiService::new( - Xrpc { domain: domain.to_string() }, - "Pocket", - env!("CARGO_PKG_VERSION"), - ) - .server(domain) - .url_prefix("/xrpc") - .contact( - ContactObject::new() - .name("@microcosm.blue") - .url("https://bsky.app/profile/microcosm.blue"), - ) - // .description(include_str!("../api-description.md")) - .external_document(ExternalDocumentObject::new( - "https://microcosm.blue/pocket", - )); +pub async fn serve(domain: &str) -> () { + let verifier = TokenVerifier::new(domain); + let api_service = OpenApiService::new(Xrpc { verifier }, "Pocket", env!("CARGO_PKG_VERSION")) + .server(domain) + .url_prefix("/xrpc") + .contact( + ContactObject::new() + .name("@microcosm.blue") + .url("https://bsky.app/profile/microcosm.blue"), + ) + .description(include_str!("../api-description.md")) + .external_document(ExternalDocumentObject::new("https://microcosm.blue/pocket")); let app = Route::new() - .at("/.well-known/did.json", get_did_doc(&domain)) + .nest("/openapi", api_service.spec_endpoint()) .nest("/xrpc/", api_service) - // .at("/", StaticFileEndpoint::new("./static/index.html")) - // .nest("/openapi", api_service.spec_endpoint()) + .at("/.well-known/did.json", get_did_doc(domain)) + .at("/", StaticFileEndpoint::new("./static/index.html")) .with( Cors::new() .allow_method(Method::GET) - .allow_method(Method::POST) + .allow_method(Method::POST), ) .with(CatchPanic::new()) .with(Tracing); let listener = TcpListener::bind("127.0.0.1:3000"); - Server::new(listener) - .name("pocket") - .run(app) - .await - .unwrap(); - + Server::new(listener).name("pocket").run(app).await.unwrap(); } diff --git a/pocket/src/token.rs b/pocket/src/token.rs index 23e513f..33a7394 100644 --- a/pocket/src/token.rs +++ b/pocket/src/token.rs @@ -1,72 +1,133 @@ -use jwt_compact::{Claims, UntrustedToken}; use atrium_crypto::did::parse_multikey; use atrium_crypto::verify::Verifier; -use std::collections::HashMap; +use jwt_compact::UntrustedToken; use serde::Deserialize; +use std::collections::HashMap; +use std::time::Duration; +use thiserror::Error; #[derive(Debug, Deserialize)] struct MiniDoc { signing_key: String, + did: String, } -pub async fn verify( - expected_aud: &str, - expected_lxm: &str, - token: &str, -) -> Result { - let untrusted = UntrustedToken::new(token).unwrap(); - - let claims: Claims> = untrusted.deserialize_claims_unchecked().unwrap(); +#[derive(Error, Debug)] +pub enum VerifyError { + #[error("The cross-service authorization token failed verification: {0}")] + VerificationFailed(&'static str), + #[error("Error trying to resolve the DID to a signing key, retry in a moment: {0}")] + ResolutionFailed(&'static str), +} - let Some(did) = claims.custom.get("iss") else { - return Err("jwt must include the user's did in `iss`"); - }; +pub struct TokenVerifier { + domain: String, + client: reqwest::Client, +} - if !did.starts_with("did:") { - return Err("iss should be a did"); - } - if did.contains("#") { - return Err("iss should be a user did without a service identifier"); +impl TokenVerifier { + pub fn new(domain: &str) -> Self { + let client = reqwest::Client::builder() + .user_agent(format!( + "microcosm pocket v{} (dev: @bad-example.com)", + env!("CARGO_PKG_VERSION") + )) + .no_proxy() + .timeout(Duration::from_secs(12)) // slingshot timeout is 10s + .build() + .unwrap(); + Self { + client, + domain: domain.to_string(), + } } - println!("Claims: {claims:#?}"); - println!("did: {did:#?}"); + pub async fn verify(&self, expected_lxm: &str, token: &str) -> Result { + let untrusted = UntrustedToken::new(token).unwrap(); - let endpoint = "https://slingshot.microcosm.blue/xrpc/com.bad-example.identity.resolveMiniDoc"; - let doc: MiniDoc = reqwest::get(format!("{endpoint}?identifier={did}")) - .await - .unwrap() - .error_for_status() - .unwrap() - .json() - .await - .unwrap(); + // danger! unfortunately we need to decode the DID from the jwt body before we have a public key to verify the jwt with + let Ok(untrusted_claims) = + untrusted.deserialize_claims_unchecked::>() + else { + return Err(VerifyError::VerificationFailed( + "could not deserialize jtw claims", + )); + }; - log::info!("got minidoc response: {doc:?}"); + // get the (untrusted!) claimed DID + let Some(untrusted_did) = untrusted_claims.custom.get("iss") else { + return Err(VerifyError::VerificationFailed( + "jwt must include the user's did in `iss`", + )); + }; - let (alg, public_key) = parse_multikey(&doc.signing_key).unwrap(); - log::info!("parsed key: {public_key:?}"); + // bail if it's not even a user-ish did + if !untrusted_did.starts_with("did:") { + return Err(VerifyError::VerificationFailed("iss should be a did")); + } + if untrusted_did.contains("#") { + return Err(VerifyError::VerificationFailed( + "iss should be a user did without a service identifier", + )); + } - Verifier::default().verify( - alg, - &public_key, - &untrusted.signed_data, - untrusted.signature_bytes(), - ).unwrap(); - // if this passes, then our claims were trustworthy after all(??) + let endpoint = + "https://slingshot.microcosm.blue/xrpc/com.bad-example.identity.resolveMiniDoc"; + let doc: MiniDoc = self + .client + .get(format!("{endpoint}?identifier={untrusted_did}")) + .send() + .await + .map_err(|_| VerifyError::ResolutionFailed("failed to fetch minidoc"))? + .error_for_status() + .map_err(|_| VerifyError::ResolutionFailed("non-ok response for minidoc"))? + .json() + .await + .map_err(|_| VerifyError::ResolutionFailed("failed to parse json to minidoc"))?; - let Some(aud) = claims.custom.get("aud") else { - return Err("missing aud"); - }; - if aud != expected_aud { - return Err("wrong aud"); - } - let Some(lxm) = claims.custom.get("lxm") else { - return Err("missing lxm"); - }; - if lxm != expected_lxm { - return Err("wrong lxm"); - } + // sanity check before we go ahead with this signing key + if doc.did != *untrusted_did { + return Err(VerifyError::VerificationFailed( + "wtf, resolveMiniDoc returned a doc for a different DID, slingshot bug", + )); + } + + let Ok((alg, public_key)) = parse_multikey(&doc.signing_key) else { + return Err(VerifyError::VerificationFailed( + "could not parse signing key form minidoc", + )); + }; + + // i _guess_ we've successfully bootstrapped the verification of the jwt unless this fails + if let Err(e) = Verifier::default().verify( + alg, + &public_key, + &untrusted.signed_data, + untrusted.signature_bytes(), + ) { + log::warn!("jwt verification failed: {e}"); + return Err(VerifyError::VerificationFailed( + "jwt signature verification failed", + )); + } - Ok(did.to_string()) + // past this point we're should have established trust. crossing ts and dotting is. + let did = &untrusted_did; + let claims = &untrusted_claims; + + let Some(aud) = claims.custom.get("aud") else { + return Err(VerifyError::VerificationFailed("missing aud")); + }; + if *aud != format!("did:web:{}#bsky_appview", self.domain) { + return Err(VerifyError::VerificationFailed("wrong aud")); + } + let Some(lxm) = claims.custom.get("lxm") else { + return Err(VerifyError::VerificationFailed("missing lxm")); + }; + if lxm != expected_lxm { + return Err(VerifyError::VerificationFailed("wrong lxm")); + } + + Ok(did.to_string()) + } } diff --git a/pocket/static/index.html b/pocket/static/index.html new file mode 100644 index 0000000..05f8bd3 --- /dev/null +++ b/pocket/static/index.html @@ -0,0 +1,67 @@ + + + + + Pocket: atproto user preference storage + + + + + +
+

+ TODO: thing +

+ +
+ + + + + + + + diff --git a/quasar/src/lib.rs b/quasar/src/lib.rs index 80aaac2..851d8a6 100644 --- a/quasar/src/lib.rs +++ b/quasar/src/lib.rs @@ -1 +1,3 @@ mod storage; + +pub use storage::Storage; diff --git a/quasar/src/storage.rs b/quasar/src/storage.rs index 4a26f98..e243f5f 100644 --- a/quasar/src/storage.rs +++ b/quasar/src/storage.rs @@ -1,4 +1,4 @@ -trait Storage { +pub trait Storage { } -- 2.51.2 From b515a72c279f7295b4c3376a64b85b67a389d8d9 Mon Sep 17 00:00:00 2001 From: phil Date: Thu, 4 Sep 2025 16:05:52 -0400 Subject: [PATCH 127/134] reflector: an any-domain did:web->service mapper --- Cargo.lock | 78 ++++++++++++++++++---------------------- Cargo.toml | 1 + Makefile | 9 ++++- reflector/Cargo.toml | 12 +++++++ reflector/readme.md | 9 +++++ reflector/src/main.rs | 83 +++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 147 insertions(+), 45 deletions(-) create mode 100644 reflector/Cargo.toml create mode 100644 reflector/readme.md create mode 100644 reflector/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index eaf4675..15af227 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -989,9 +989,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.46" +version = "4.5.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c5e4fcf9c21d2e544ca1ee9d8552de13019a42aa7dbf32747fa7aaf1df76e57" +checksum = "7eac00902d9d136acd712710d71823fb8ac8004ca445a89e73a41d45aa712931" dependencies = [ "clap_builder", "clap_derive", @@ -999,9 +999,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.46" +version = "4.5.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fecb53a0e6fcfb055f686001bc2e2592fa527efaf38dbe81a6a9563562e57d41" +checksum = "2ad9bbf750e73b5884fb8a211a9424a1906c1e156724260fdae972f31d70e1d6" dependencies = [ "anstream", "anstyle", @@ -1011,9 +1011,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.45" +version = "4.5.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14cb31bb0a7d536caef2639baa7fad459e15c3144efefa6dbd1c84562c4739f6" +checksum = "bbfd7eae0b0f1a6e63d4b13c9c478de77c2eb546fba158ad50b4203dc24b9f9c" dependencies = [ "heck", "proc-macro2", @@ -3148,9 +3148,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.27" +version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" [[package]] name = "loom" @@ -3300,11 +3300,11 @@ checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" [[package]] name = "matchers" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" dependencies = [ - "regex-automata 0.1.10", + "regex-automata", ] [[package]] @@ -3603,12 +3603,11 @@ dependencies = [ [[package]] name = "nu-ansi-term" -version = "0.46.0" +version = "0.50.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399" dependencies = [ - "overload", - "winapi", + "windows-sys 0.52.0", ] [[package]] @@ -3810,12 +3809,6 @@ dependencies = [ "hashbrown 0.13.2", ] -[[package]] -name = "overload" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" - [[package]] name = "p256" version = "0.13.2" @@ -4499,6 +4492,18 @@ dependencies = [ "syn 2.0.103", ] +[[package]] +name = "reflector" +version = "0.1.0" +dependencies = [ + "clap", + "log", + "poem", + "serde", + "tokio", + "tracing-subscriber", +] + [[package]] name = "regex" version = "1.11.1" @@ -4507,17 +4512,8 @@ checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.9", - "regex-syntax 0.8.5", -] - -[[package]] -name = "regex-automata" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" -dependencies = [ - "regex-syntax 0.6.29", + "regex-automata", + "regex-syntax", ] [[package]] @@ -4528,15 +4524,9 @@ checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.5", + "regex-syntax", ] -[[package]] -name = "regex-syntax" -version = "0.6.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" - [[package]] name = "regex-syntax" version = "0.8.5" @@ -5650,9 +5640,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.47.0" +version = "1.47.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43864ed400b6043a4757a25c7a64a8efde741aed79a056a2fb348a406701bb35" +checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" dependencies = [ "backtrace", "bytes", @@ -5893,14 +5883,14 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.19" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" +checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5" dependencies = [ "matchers", "nu-ansi-term", "once_cell", - "regex", + "regex-automata", "sharded-slab", "smallvec", "thread_local", diff --git a/Cargo.toml b/Cargo.toml index 25cbcc0..a119dd2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,4 +11,5 @@ members = [ "slingshot", "quasar", "pocket", + "reflector", ] diff --git a/Makefile b/Makefile index 7e52ceb..a495ad9 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,14 @@ test: cargo test --all-features fmt: - cargo fmt --package links --package constellation --package ufos --package spacedust --package who-am-i --package slingshot --package pocket + cargo fmt --package links \ + --package constellation \ + --package ufos \ + --package spacedust \ + --package who-am-i \ + --package slingshot \ + --package pocket \ + --package reflector cargo +nightly fmt --package jetstream clippy: diff --git a/reflector/Cargo.toml b/reflector/Cargo.toml new file mode 100644 index 0000000..9911736 --- /dev/null +++ b/reflector/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "reflector" +version = "0.1.0" +edition = "2024" + +[dependencies] +clap = { version = "4.5.47", features = ["derive"] } +log = "0.4.28" +poem = "3.1.12" +serde = { version = "1.0.219", features = ["derive"] } +tokio = "1.47.1" +tracing-subscriber = { version = "0.3.20", features = ["env-filter"] } diff --git a/reflector/readme.md b/reflector/readme.md new file mode 100644 index 0000000..18ea187 --- /dev/null +++ b/reflector/readme.md @@ -0,0 +1,9 @@ +# reflector + +a tiny did:web service server that maps subdomains to a single service endpoint + +receiving requests from multiple subdomains is left as a problem for the reverse proxy to solve, since acme wildcard certificates (ie. letsencrypt) require the most complicated and involved challenge type (DNS). + +caddy [has good support for](https://caddyserver.com/docs/caddyfile/patterns#wildcard-certificates) configuring the wildcard DNS challenge with various DNS providers, and also supports [on-demand](https://caddyserver.com/docs/automatic-https#using-on-demand-tls) provisioning via the simpler methods. + +if you only need a small fixed number of subdomains, you can also use certbot or otherwise individually configure them in your reverse proxy. diff --git a/reflector/src/main.rs b/reflector/src/main.rs new file mode 100644 index 0000000..6edadac --- /dev/null +++ b/reflector/src/main.rs @@ -0,0 +1,83 @@ +use clap::Parser; +use poem::{ + EndpointExt, Route, Server, get, handler, + listener::TcpListener, + middleware::{AddData, Tracing}, + web::{Data, Json, TypedHeader, headers::Host}, +}; +use serde::Serialize; + +#[handler] +fn hello() -> String { + "ɹoʇɔǝʅⅎǝɹ".to_string() +} + +#[derive(Debug, Serialize)] +struct DidDoc { + id: String, + service: [DidService; 1], +} + +#[derive(Debug, Clone, Serialize)] +struct DidService { + id: String, + r#type: String, + service_endpoint: String, +} + +#[handler] +fn did_doc(TypedHeader(host): TypedHeader, service: Data<&DidService>) -> Json { + Json(DidDoc { + id: format!("did:web:{}", host.hostname()), + service: [service.clone()], + }) +} + +/// Slingshot record edge cache +#[derive(Parser, Debug, Clone)] +#[command(version, about, long_about = None)] +struct Args { + /// The DID document service ID to serve + /// + /// must start with a '#', like `#bsky_appview' + #[arg(long)] + id: String, + /// Service type + /// + /// Not sure exactly what its requirements are. 'BlueskyAppview' for example + #[arg(long)] + r#type: String, + /// The HTTPS endpoint for the service + #[arg(long)] + service_endpoint: String, +} + +impl From for DidService { + fn from(a: Args) -> Self { + Self { + id: a.id, + r#type: a.r#type, + service_endpoint: a.service_endpoint, + } + } +} + +#[tokio::main(flavor = "current_thread")] +async fn main() { + tracing_subscriber::fmt::init(); + log::info!("ɹoʇɔǝʅⅎǝɹ"); + + let args = Args::parse(); + let service: DidService = args.into(); + + Server::new(TcpListener::bind("0.0.0.0:3001")) + .run( + Route::new() + .at("/", get(hello)) + .at("/.well-known/did.json", get(did_doc)) + .with(AddData::new(service)) + .with(Tracing), + ) + .await + .unwrap() +} -- 2.51.2 From ad0011afd6331381b3c05a78c6b226f66ffccbda Mon Sep 17 00:00:00 2001 From: phil Date: Thu, 4 Sep 2025 17:11:30 -0400 Subject: [PATCH 128/134] maybe try building on gh surely it will work the first time --- .github/workflows/build.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..bf0d3ec --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,19 @@ +name: Build + +on: + push: + tags: + - "reflector-v*.*.*" + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: build reflector + run: cargo build --bin reflector --release && mv target/release/reflector target/release/reflector_amd64 + - name: release + uses: softprops/action-gh-release@v2 + with: + files: target/release/reflector_amd64 -- 2.51.2 From 3839338b6b73dbee5183540958141344466cb7c6 Mon Sep 17 00:00:00 2001 From: phil Date: Thu, 4 Sep 2025 17:28:28 -0400 Subject: [PATCH 129/134] permission? --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bf0d3ec..876ad54 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,6 +8,8 @@ on: jobs: build: runs-on: ubuntu-latest + permissions: + contents: write steps: - uses: actions/checkout@v4 -- 2.51.2 From d42f70cf91464c75f9fbcd4501e27ddf7b02ac5d Mon Sep 17 00:00:00 2001 From: phil Date: Fri, 5 Sep 2025 12:35:24 -0400 Subject: [PATCH 130/134] ask endpoint for caddy should probably put this behind a config flag... --- reflector/src/main.rs | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/reflector/src/main.rs b/reflector/src/main.rs index 6edadac..69bf087 100644 --- a/reflector/src/main.rs +++ b/reflector/src/main.rs @@ -1,11 +1,12 @@ use clap::Parser; use poem::{ - EndpointExt, Route, Server, get, handler, + EndpointExt, Response, Route, Server, get, handler, + http::StatusCode, listener::TcpListener, middleware::{AddData, Tracing}, - web::{Data, Json, TypedHeader, headers::Host}, + web::{Data, Json, Query, TypedHeader, headers::Host}, }; -use serde::Serialize; +use serde::{Deserialize, Serialize}; #[handler] fn hello() -> String { @@ -33,6 +34,28 @@ fn did_doc(TypedHeader(host): TypedHeader, service: Data<&DidService>) -> }) } +#[derive(Deserialize)] +struct AskQuery { + domain: String, +} +#[handler] +fn ask_caddy( + Data(parent): Data<&Option>, + Query(AskQuery { domain }): Query, +) -> Response { + if let Some(parent) = parent { + if let Some(prefix) = domain.strip_suffix(&format!(".{parent}")) { + if !prefix.contains('.') { + // no sub-sub-domains allowed + return Response::builder().body("ok"); + } + } + }; + Response::builder() + .status(StatusCode::FORBIDDEN) + .body("nope") +} + /// Slingshot record edge cache #[derive(Parser, Debug, Clone)] #[command(version, about, long_about = None)] @@ -50,6 +73,9 @@ struct Args { /// The HTTPS endpoint for the service #[arg(long)] service_endpoint: String, + /// The parent domain; requests should come from subdomains of this + #[arg(long)] + domain: Option, } impl From for DidService { @@ -68,6 +94,7 @@ async fn main() { log::info!("ɹoʇɔǝʅⅎǝɹ"); let args = Args::parse(); + let domain = args.domain.clone(); let service: DidService = args.into(); Server::new(TcpListener::bind("0.0.0.0:3001")) @@ -75,7 +102,9 @@ async fn main() { Route::new() .at("/", get(hello)) .at("/.well-known/did.json", get(did_doc)) + .at("/ask", get(ask_caddy)) .with(AddData::new(service)) + .with(AddData::new(domain)) .with(Tracing), ) .await -- 2.51.2 From 4e0bca4847421d5fab0c7e41d828160f5da99e40 Mon Sep 17 00:00:00 2001 From: phil Date: Fri, 5 Sep 2025 13:44:50 -0400 Subject: [PATCH 131/134] debugging serviceEndpoint casing lksdfjlkasdj --- pocket/src/server.rs | 26 +++++++++++++++++++------- reflector/src/main.rs | 1 + 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/pocket/src/server.rs b/pocket/src/server.rs index 9c031e0..0dc7879 100644 --- a/pocket/src/server.rs +++ b/pocket/src/server.rs @@ -94,13 +94,16 @@ struct Xrpc { verifier: TokenVerifier, } +// app.bsky.actor.getPreferences +// com.bad-example.pocket.getPreferences + #[OpenApi] impl Xrpc { /// com.bad-example.pocket.getPreferences /// /// get stored bluesky prefs #[oai( - path = "/com.bad-example.pocket.getPreferences", + path = "/app.bsky.actor.getPreferences", method = "get", tag = "ApiTags::Pocket" )] @@ -156,17 +159,26 @@ struct AppViewService { #[derive(Debug, Clone, Serialize)] struct AppViewDoc { id: String, - service: [AppViewService; 1], + service: [AppViewService; 2], } /// Serve a did document for did:web for this to be an xrpc appview fn get_did_doc(domain: &str) -> impl Endpoint + use<> { let doc = poem::web::Json(AppViewDoc { id: format!("did:web:{domain}"), - service: [AppViewService { - id: "#pocket_prefs".to_string(), - r#type: "PocketPreferences".to_string(), - service_endpoint: format!("https://{domain}"), - }], + service: [ + AppViewService { + id: "#pocket_prefs".to_string(), + // id: "#bsky_appview".to_string(), + r#type: "PocketPreferences".to_string(), + service_endpoint: format!("https://{domain}"), + }, + AppViewService { + id: "#bsky_appview".to_string(), + // id: "#bsky_appview".to_string(), + r#type: "BlueskyAppview".to_string(), + service_endpoint: format!("https://{domain}"), + }, + ], }); make_sync(move |_| doc.clone()) } diff --git a/reflector/src/main.rs b/reflector/src/main.rs index 69bf087..777dc30 100644 --- a/reflector/src/main.rs +++ b/reflector/src/main.rs @@ -20,6 +20,7 @@ struct DidDoc { } #[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] struct DidService { id: String, r#type: String, -- 2.51.2 From fb5ed31c22f0dd2013ee099845ff1423196b4451 Mon Sep 17 00:00:00 2001 From: phil Date: Fri, 5 Sep 2025 14:02:35 -0400 Subject: [PATCH 132/134] working with bsky_appview... --- pocket/src/server.rs | 10 +++++----- pocket/src/token.rs | 31 ++++++++++++++++++++----------- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/pocket/src/server.rs b/pocket/src/server.rs index 0dc7879..5c49a09 100644 --- a/pocket/src/server.rs +++ b/pocket/src/server.rs @@ -108,7 +108,7 @@ impl Xrpc { tag = "ApiTags::Pocket" )] async fn app_bsky_get_prefs(&self, XrpcAuth(auth): XrpcAuth) -> GetBskyPrefsResponse { - let did = match self + let (did, aud) = match self .verifier .verify("app.bsky.actor.getPreferences", &auth.token) .await @@ -116,7 +116,7 @@ impl Xrpc { Ok(d) => d, Err(e) => return GetBskyPrefsResponse::BadRequest(xrpc_error("boooo", e.to_string())), }; - log::info!("verified did: {did}"); + log::info!("verified did: {did}/{aud}"); // TODO: fetch from storage GetBskyPrefsResponse::Ok(Json(GetBskyPrefsResponseObject::example())) } @@ -134,7 +134,7 @@ impl Xrpc { XrpcAuth(auth): XrpcAuth, Json(prefs): Json, ) -> PutBskyPrefsResponse { - let did = match self + let (did, aud) = match self .verifier .verify("app.bsky.actor.getPreferences", &auth.token) .await @@ -142,7 +142,7 @@ impl Xrpc { Ok(d) => d, Err(e) => return PutBskyPrefsResponse::BadRequest(xrpc_error("boooo", e.to_string())), }; - log::info!("verified did: {did}"); + log::info!("verified did: {did}/{aud}"); log::warn!("received prefs: {prefs:?}"); // TODO: put prefs into storage PutBskyPrefsResponse::Ok(PlainText("hiiiiii".to_string())) @@ -184,7 +184,7 @@ fn get_did_doc(domain: &str) -> impl Endpoint + use<> { } pub async fn serve(domain: &str) -> () { - let verifier = TokenVerifier::new(domain); + let verifier = TokenVerifier::default(); let api_service = OpenApiService::new(Xrpc { verifier }, "Pocket", env!("CARGO_PKG_VERSION")) .server(domain) .url_prefix("/xrpc") diff --git a/pocket/src/token.rs b/pocket/src/token.rs index 33a7394..1133fd6 100644 --- a/pocket/src/token.rs +++ b/pocket/src/token.rs @@ -21,12 +21,11 @@ pub enum VerifyError { } pub struct TokenVerifier { - domain: String, client: reqwest::Client, } impl TokenVerifier { - pub fn new(domain: &str) -> Self { + pub fn new() -> Self { let client = reqwest::Client::builder() .user_agent(format!( "microcosm pocket v{} (dev: @bad-example.com)", @@ -36,13 +35,14 @@ impl TokenVerifier { .timeout(Duration::from_secs(12)) // slingshot timeout is 10s .build() .unwrap(); - Self { - client, - domain: domain.to_string(), - } + Self { client } } - pub async fn verify(&self, expected_lxm: &str, token: &str) -> Result { + pub async fn verify( + &self, + expected_lxm: &str, + token: &str, + ) -> Result<(String, String), VerifyError> { let untrusted = UntrustedToken::new(token).unwrap(); // danger! unfortunately we need to decode the DID from the jwt body before we have a public key to verify the jwt with @@ -118,9 +118,12 @@ impl TokenVerifier { let Some(aud) = claims.custom.get("aud") else { return Err(VerifyError::VerificationFailed("missing aud")); }; - if *aud != format!("did:web:{}#bsky_appview", self.domain) { - return Err(VerifyError::VerificationFailed("wrong aud")); - } + let Some(aud) = aud.strip_prefix("did:web:") else { + return Err(VerifyError::VerificationFailed("expected a did:web aud")); + }; + let Some((aud, _)) = aud.split_once("#") else { + return Err(VerifyError::VerificationFailed("aud missing #fragment")); + }; let Some(lxm) = claims.custom.get("lxm") else { return Err(VerifyError::VerificationFailed("missing lxm")); }; @@ -128,6 +131,12 @@ impl TokenVerifier { return Err(VerifyError::VerificationFailed("wrong lxm")); } - Ok(did.to_string()) + Ok((did.to_string(), aud.to_string())) + } +} + +impl Default for TokenVerifier { + fn default() -> Self { + Self::new() } } -- 2.51.2 From 69d717a0be1e71e9ffcd63f1754125380999f482 Mon Sep 17 00:00:00 2001 From: phil Date: Fri, 5 Sep 2025 15:50:48 -0400 Subject: [PATCH 133/134] clean up and add size limit to requests --- pocket/src/server.rs | 20 ++++++++------------ pocket/src/token.rs | 9 +++++---- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/pocket/src/server.rs b/pocket/src/server.rs index 5c49a09..309c107 100644 --- a/pocket/src/server.rs +++ b/pocket/src/server.rs @@ -4,7 +4,7 @@ use poem::{ endpoint::{StaticFileEndpoint, make_sync}, http::Method, listener::TcpListener, - middleware::{CatchPanic, Cors, Tracing}, + middleware::{CatchPanic, Cors, SizeLimit, Tracing}, }; use poem_openapi::{ ApiResponse, ContactObject, ExternalDocumentObject, Object, OpenApi, OpenApiService, @@ -94,23 +94,20 @@ struct Xrpc { verifier: TokenVerifier, } -// app.bsky.actor.getPreferences -// com.bad-example.pocket.getPreferences - #[OpenApi] impl Xrpc { /// com.bad-example.pocket.getPreferences /// - /// get stored bluesky prefs + /// get stored preferencess #[oai( - path = "/app.bsky.actor.getPreferences", + path = "/com.bad-example.pocket.getPreferences", method = "get", tag = "ApiTags::Pocket" )] - async fn app_bsky_get_prefs(&self, XrpcAuth(auth): XrpcAuth) -> GetBskyPrefsResponse { + async fn pocket_get_prefs(&self, XrpcAuth(auth): XrpcAuth) -> GetBskyPrefsResponse { let (did, aud) = match self .verifier - .verify("app.bsky.actor.getPreferences", &auth.token) + .verify("com.bad-example.pocket.getPreferences", &auth.token) .await { Ok(d) => d, @@ -129,14 +126,14 @@ impl Xrpc { method = "post", tag = "ApiTags::Pocket" )] - async fn app_bsky_put_prefs( + async fn pocket_put_prefs( &self, XrpcAuth(auth): XrpcAuth, Json(prefs): Json, ) -> PutBskyPrefsResponse { let (did, aud) = match self .verifier - .verify("app.bsky.actor.getPreferences", &auth.token) + .verify("com.bad-example.pocket.putPreferences", &auth.token) .await { Ok(d) => d, @@ -168,13 +165,11 @@ fn get_did_doc(domain: &str) -> impl Endpoint + use<> { service: [ AppViewService { id: "#pocket_prefs".to_string(), - // id: "#bsky_appview".to_string(), r#type: "PocketPreferences".to_string(), service_endpoint: format!("https://{domain}"), }, AppViewService { id: "#bsky_appview".to_string(), - // id: "#bsky_appview".to_string(), r#type: "BlueskyAppview".to_string(), service_endpoint: format!("https://{domain}"), }, @@ -201,6 +196,7 @@ pub async fn serve(domain: &str) -> () { .nest("/xrpc/", api_service) .at("/.well-known/did.json", get_did_doc(domain)) .at("/", StaticFileEndpoint::new("./static/index.html")) + .with(SizeLimit::new(100 * 2_usize.pow(10))) .with( Cors::new() .allow_method(Method::GET) diff --git a/pocket/src/token.rs b/pocket/src/token.rs index 1133fd6..128dbc8 100644 --- a/pocket/src/token.rs +++ b/pocket/src/token.rs @@ -118,12 +118,13 @@ impl TokenVerifier { let Some(aud) = claims.custom.get("aud") else { return Err(VerifyError::VerificationFailed("missing aud")); }; - let Some(aud) = aud.strip_prefix("did:web:") else { + let Some(mut aud) = aud.strip_prefix("did:web:") else { return Err(VerifyError::VerificationFailed("expected a did:web aud")); }; - let Some((aud, _)) = aud.split_once("#") else { - return Err(VerifyError::VerificationFailed("aud missing #fragment")); - }; + if let Some((aud_without_hash, _)) = aud.split_once("#") { + log::warn!("aud claim is missing service id fragment: {aud:?}"); + aud = aud_without_hash; + } let Some(lxm) = claims.custom.get("lxm") else { return Err(VerifyError::VerificationFailed("missing lxm")); }; -- 2.51.2 From b0a66a102261d0b4e8a90d34cec3421073a7b728 Mon Sep 17 00:00:00 2001 From: phil Date: Fri, 5 Sep 2025 17:30:27 -0400 Subject: [PATCH 134/134] sqlite get/put pretty much works mutex-wrapping the storage accidentally serializes all access which oops but also whatever --- pocket/.gitignore | 1 + pocket/src/lib.rs | 2 + pocket/src/main.rs | 32 +++++++++++-- pocket/src/server.rs | 101 ++++++++++++++++++++++++++++++++---------- pocket/src/storage.rs | 50 +++++++++++++++++++++ 5 files changed, 160 insertions(+), 26 deletions(-) create mode 100644 pocket/.gitignore create mode 100644 pocket/src/storage.rs diff --git a/pocket/.gitignore b/pocket/.gitignore new file mode 100644 index 0000000..540ddfe --- /dev/null +++ b/pocket/.gitignore @@ -0,0 +1 @@ +prefs.sqlite3* diff --git a/pocket/src/lib.rs b/pocket/src/lib.rs index c6a7fd8..67b3a79 100644 --- a/pocket/src/lib.rs +++ b/pocket/src/lib.rs @@ -1,5 +1,7 @@ mod server; +mod storage; mod token; pub use server::serve; +pub use storage::Storage; pub use token::TokenVerifier; diff --git a/pocket/src/main.rs b/pocket/src/main.rs index 46a7ad9..986bef6 100644 --- a/pocket/src/main.rs +++ b/pocket/src/main.rs @@ -1,8 +1,34 @@ -use pocket::serve; +use clap::Parser; +use pocket::{Storage, serve}; +use std::path::PathBuf; + +/// Slingshot record edge cache +#[derive(Parser, Debug, Clone)] +#[command(version, about, long_about = None)] +struct Args { + /// path to the sqlite db file + #[arg(long)] + db: Option, + /// just initialize the db and exit + #[arg(long, action)] + init_db: bool, + /// the domain for serving a did doc (unused if running behind reflector) + #[arg(long)] + domain: Option, +} #[tokio::main] async fn main() { tracing_subscriber::fmt::init(); - println!("Hello, world!"); - serve("mac.cinnebar-tet.ts.net").await + log::info!("👖 hi"); + let args = Args::parse(); + let domain = args.domain.unwrap_or("bad-example.com".into()); + let db_path = args.db.unwrap_or("prefs.sqlite3".into()); + if args.init_db { + Storage::init(&db_path).unwrap(); + log::info!("👖 initialized db at {db_path:?}. bye") + } else { + let storage = Storage::connect(db_path).unwrap(); + serve(&domain, storage).await + } } diff --git a/pocket/src/server.rs b/pocket/src/server.rs index 309c107..f507db1 100644 --- a/pocket/src/server.rs +++ b/pocket/src/server.rs @@ -1,10 +1,10 @@ -use crate::TokenVerifier; +use crate::{Storage, TokenVerifier}; use poem::{ Endpoint, EndpointExt, Route, Server, endpoint::{StaticFileEndpoint, make_sync}, http::Method, listener::TcpListener, - middleware::{CatchPanic, Cors, SizeLimit, Tracing}, + middleware::{CatchPanic, Cors, Tracing}, }; use poem_openapi::{ ApiResponse, ContactObject, ExternalDocumentObject, Object, OpenApi, OpenApiService, @@ -15,6 +15,7 @@ use poem_openapi::{ }; use serde::Serialize; use serde_json::{Value, json}; +use std::sync::{Arc, Mutex}; #[derive(Debug, SecurityScheme)] #[oai(ty = "bearer")] @@ -51,13 +52,13 @@ fn xrpc_error(error: impl AsRef, message: impl AsRef) -> XrpcError { }) } -#[derive(Object)] +#[derive(Debug, Object)] #[oai(example = true)] -struct GetBskyPrefsResponseObject { +struct BskyPrefsObject { /// at-uri for this record preferences: Value, } -impl Example for GetBskyPrefsResponseObject { +impl Example for BskyPrefsObject { fn example() -> Self { Self { preferences: json!({ @@ -71,7 +72,7 @@ impl Example for GetBskyPrefsResponseObject { enum GetBskyPrefsResponse { /// Record found #[oai(status = 200)] - Ok(Json), + Ok(Json), /// Bad request or no preferences to return #[oai(status = 400)] BadRequest(XrpcError), @@ -92,6 +93,7 @@ enum PutBskyPrefsResponse { struct Xrpc { verifier: TokenVerifier, + storage: Arc>, } #[OpenApi] @@ -114,8 +116,40 @@ impl Xrpc { Err(e) => return GetBskyPrefsResponse::BadRequest(xrpc_error("boooo", e.to_string())), }; log::info!("verified did: {did}/{aud}"); - // TODO: fetch from storage - GetBskyPrefsResponse::Ok(Json(GetBskyPrefsResponseObject::example())) + + let storage = self.storage.clone(); + + let Ok(Ok(res)) = tokio::task::spawn_blocking(move || { + storage + .lock() + .unwrap() + .get(&did, &aud) + .inspect_err(|e| log::error!("failed to get prefs: {e}")) + }) + .await + else { + return GetBskyPrefsResponse::BadRequest(xrpc_error("boooo", "failed to get from db")); + }; + + let Some(serialized) = res else { + return GetBskyPrefsResponse::BadRequest(xrpc_error( + "NotFound", + "could not find prefs for u", + )); + }; + + let preferences = match serde_json::from_str(&serialized) { + Ok(v) => v, + Err(e) => { + log::error!("failed to deserialize prefs: {e}"); + return GetBskyPrefsResponse::BadRequest(xrpc_error( + "boooo", + "failed to deserialize prefs", + )); + } + }; + + GetBskyPrefsResponse::Ok(Json(BskyPrefsObject { preferences })) } /// com.bad-example.pocket.putPreferences @@ -129,7 +163,7 @@ impl Xrpc { async fn pocket_put_prefs( &self, XrpcAuth(auth): XrpcAuth, - Json(prefs): Json, + Json(prefs): Json, ) -> PutBskyPrefsResponse { let (did, aud) = match self .verifier @@ -141,8 +175,23 @@ impl Xrpc { }; log::info!("verified did: {did}/{aud}"); log::warn!("received prefs: {prefs:?}"); - // TODO: put prefs into storage - PutBskyPrefsResponse::Ok(PlainText("hiiiiii".to_string())) + + let storage = self.storage.clone(); + let serialized = prefs.preferences.to_string(); + + let Ok(Ok(())) = tokio::task::spawn_blocking(move || { + storage + .lock() + .unwrap() + .put(&did, &aud, &serialized) + .inspect_err(|e| log::error!("failed to insert prefs: {e}")) + }) + .await + else { + return PutBskyPrefsResponse::BadRequest(xrpc_error("boooo", "failed to put to db")); + }; + + PutBskyPrefsResponse::Ok(PlainText("saved.".to_string())) } } @@ -178,25 +227,31 @@ fn get_did_doc(domain: &str) -> impl Endpoint + use<> { make_sync(move |_| doc.clone()) } -pub async fn serve(domain: &str) -> () { +pub async fn serve(domain: &str, storage: Storage) -> () { let verifier = TokenVerifier::default(); - let api_service = OpenApiService::new(Xrpc { verifier }, "Pocket", env!("CARGO_PKG_VERSION")) - .server(domain) - .url_prefix("/xrpc") - .contact( - ContactObject::new() - .name("@microcosm.blue") - .url("https://bsky.app/profile/microcosm.blue"), - ) - .description(include_str!("../api-description.md")) - .external_document(ExternalDocumentObject::new("https://microcosm.blue/pocket")); + let api_service = OpenApiService::new( + Xrpc { + verifier, + storage: Arc::new(Mutex::new(storage)), + }, + "Pocket", + env!("CARGO_PKG_VERSION"), + ) + .server(domain) + .url_prefix("/xrpc") + .contact( + ContactObject::new() + .name("@microcosm.blue") + .url("https://bsky.app/profile/microcosm.blue"), + ) + .description(include_str!("../api-description.md")) + .external_document(ExternalDocumentObject::new("https://microcosm.blue/pocket")); let app = Route::new() .nest("/openapi", api_service.spec_endpoint()) .nest("/xrpc/", api_service) .at("/.well-known/did.json", get_did_doc(domain)) .at("/", StaticFileEndpoint::new("./static/index.html")) - .with(SizeLimit::new(100 * 2_usize.pow(10))) .with( Cors::new() .allow_method(Method::GET) diff --git a/pocket/src/storage.rs b/pocket/src/storage.rs new file mode 100644 index 0000000..1b6ff96 --- /dev/null +++ b/pocket/src/storage.rs @@ -0,0 +1,50 @@ +use rusqlite::{Connection, OptionalExtension, Result}; +use std::path::Path; + +pub struct Storage { + con: Connection, +} + +impl Storage { + pub fn connect(path: impl AsRef) -> Result { + let con = Connection::open(path)?; + con.pragma_update(None, "journal_mode", "WAL")?; + con.pragma_update(None, "synchronous", "NORMAL")?; + con.pragma_update(None, "busy_timeout", "100")?; + con.pragma_update(None, "foreign_keys", "ON")?; + Ok(Self { con }) + } + pub fn init(path: impl AsRef) -> Result { + let me = Self::connect(path)?; + me.con.execute( + r#" + create table prefs ( + actor text not null, + aud text not null, + pref text not null, + primary key (actor, aud) + ) strict"#, + (), + )?; + Ok(me) + } + pub fn put(&self, actor: &str, aud: &str, pref: &str) -> Result<()> { + self.con.execute( + r#"insert into prefs (actor, aud, pref) + values (?1, ?2, ?3) + on conflict do update set pref = excluded.pref"#, + [actor, aud, pref], + )?; + Ok(()) + } + pub fn get(&self, actor: &str, aud: &str) -> Result> { + self.con + .query_one( + r#"select pref from prefs + where actor = ?1 and aud = ?2"#, + [actor, aud], + |row| row.get(0), + ) + .optional() + } +}