From f04ded2e92cacc4829bcf8a9cc523fe108ebea1f Mon Sep 17 00:00:00 2001 From: "@permadeath.com" Date: Thu, 27 Aug 2026 22:57:59 -0400 Subject: [PATCH] test(local-dev): the whole path, from a hook payload to a stored record A real listener, the real hook handler and the real scrobble host: a session opens, a payload becomes a stamped call, the call becomes a record, and the session ends taking its accounts with it. It needs a socket where the other suites do not, because the hook and the host are HTTP clients and substituting the transport would skip exactly the code that has to agree. Change-Id: I85ca52e9d2fb015afa1ed92b2302d30d507d0ce4 --- Cargo.lock | 2 + crates/vibescrobble/Cargo.toml | 5 + crates/vibescrobble/tests/hook_to_record.rs | 344 ++++++++++++++++++++ plan/local-dev.md | 17 +- 4 files changed, 364 insertions(+), 4 deletions(-) create mode 100644 crates/vibescrobble/tests/hook_to_record.rs diff --git a/Cargo.lock b/Cargo.lock index be7e35d0..1b983bea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2773,6 +2773,7 @@ dependencies = [ "vibescrobble-data", "vibescrobble-dns", "vibescrobble-hook", + "vibescrobble-hookd", "vibescrobble-identity", "vibescrobble-index", "vibescrobble-key", @@ -3015,6 +3016,7 @@ dependencies = [ "toml", "vibescrobble-hook", "vibescrobble-hookd", + "vibescrobble-mcp", "vibescrobble-stack", ] diff --git a/crates/vibescrobble/Cargo.toml b/crates/vibescrobble/Cargo.toml index 61dabdee..f837e33a 100644 --- a/crates/vibescrobble/Cargo.toml +++ b/crates/vibescrobble/Cargo.toml @@ -40,6 +40,11 @@ vibescrobble-mcp.workspace = true # filter the development scripts export, and the server is what that filter has # to mean the same thing to. vibescrobble-stack.workspace = true +# And the hook handler, so that `tests/hook_to_record.rs` can drive the whole +# path from a harness payload to a stored record. It is the only crate that can +# see the hook, the scrobble host and the server at once, which is the same +# reason the conformance suites live here. +vibescrobble-hookd.workspace = true # `reqwest` is here so one test can read this server's sync endpoints the way # anything else would: over HTTP, from outside the process. reqwest.workspace = true diff --git a/crates/vibescrobble/tests/hook_to_record.rs b/crates/vibescrobble/tests/hook_to_record.rs new file mode 100644 index 00000000..dcff3794 --- /dev/null +++ b/crates/vibescrobble/tests/hook_to_record.rs @@ -0,0 +1,344 @@ +//! From a hook payload to a stored record, through every real component. +//! +//! [`end_to_end`](../end_to_end.rs) proves the server agrees with itself. +//! This proves the three processes agree with *each other*: the hook that +//! stamps a tool call, the scrobble host that turns one into a record, and the +//! personal data server that stores it. Each is exercised by its own tests +//! against its own idea of the other two, and the gap between those ideas is +//! what this closes. +//! +//! It was exercised by hand until now — five terminals, a session, and reading +//! a log — which is the slowest possible way to find out that a stamp key was +//! renamed on one side only. +//! +//! # Why this one needs a socket +//! +//! The other suites hand requests to the router directly, which is faster and +//! cannot be flaky. They can, because what they exercise is a function of the +//! request. This cannot: the hook and the scrobble host are HTTP *clients*, +//! with their own URL building, their own headers and their own error mapping, +//! and substituting the transport would skip exactly the code that has to +//! agree. So one listener, on `127.0.0.1:0`, for the length of one test. + +use std::sync::Arc; + +use serde_json::{json, Value}; +use vibescrobble::attest::SharedSecretBackend; +use vibescrobble::dns::LoopbackDns; +use vibescrobble::identity::Zone; +use vibescrobble::pds::{MemoryAccountStore, Provisioner, Registry}; +use vibescrobble_hook::{STAMP_KEY, TURN_KEY}; + +/// The defaults `vibescrobble-hookd` uses when nothing overrides them, so that +/// a hook configured with no environment at all reaches this server. +const SECRET: &[u8] = b"quernstone-placeholder-secret"; +const NODE: &str = "dev-node"; +const ZONE: &str = "agents.localhost"; + +/// A session identifier shaped like the harness's. +const SESSION: &str = "3f8a1c22-91d4-4a7e-b6f0-2c5e9d7a1b03"; + +/// The harness's own pairing identifier for one tool call. +const TURN: &str = "toolu_01HarnessTurn"; + +/// A server on a real port, and the URL to reach it at. +struct Server { + url: String, + /// Dropped with the server, which is what stops it: nothing here needs to + /// shut it down politely, and a test that outlives its listener is a test + /// leaking a port. + _task: tokio::task::JoinHandle<()>, +} + +impl Server { + /// Starts one, on whatever port the operating system hands out. + async fn start() -> Self { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("a loopback listener"); + let port = listener.local_addr().expect("a bound address").port(); + let zone = Zone::new(ZONE) + .expect("the zone parses") + .with_port(port) + .expect("a port is allowed on localhost"); + let registry: Arc = Arc::new(Provisioner::new( + zone, + format!("http://127.0.0.1:{port}"), + SharedSecretBackend::new(SECRET.to_vec(), [NODE]), + LoopbackDns::new(), + MemoryAccountStore::new(), + )); + let app = vibescrobble::serve::app(registry); + let task = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + Self { + url: format!("http://127.0.0.1:{port}"), + _task: task, + } + } +} + +/// A hook configured to reach `server`, writing its state under `state`. +fn hook_config(server: &Server, state: &std::path::Path) -> vibescrobble_hookd::Config { + vibescrobble_hookd::Config { + pds_url: server.url.clone(), + // Pinned, so this test reaches its own server whatever stack + // configuration the machine running it happens to have. + pinned_pds_url: true, + shared_secret: String::from_utf8(SECRET.to_vec()).expect("the secret is text"), + node_id: NODE.to_owned(), + state_path: state.to_owned(), + debug: false, + } +} + +/// A directory nothing else writes into. +fn scratch(name: &str) -> std::path::PathBuf { + let dir = std::env::temp_dir().join(format!( + "vibescrobble-hook-to-record-{name}-{}-{}", + std::process::id(), + time::OffsetDateTime::now_utc().unix_timestamp_nanos() + )); + std::fs::create_dir_all(&dir).expect("the scratch directory is creatable"); + dir.join("hook-agents.json") +} + +/// A payload of the shape the harness writes to the hook's stdin. +fn payload(event: &str, extra: Value) -> String { + let mut value = json!({ + "session_id": SESSION, + "cwd": "/nowhere/in/particular", + "hook_event_name": event, + }); + for (key, item) in extra.as_object().expect("an object").clone() { + value[key] = item; + } + value.to_string() +} + +/// The `updatedInput` a hook response carries, if any. +fn stamped(output: &vibescrobble_hook::HookOutput) -> Option { + serde_json::to_value(output) + .expect("the response serializes") + .get("hookSpecificOutput") + .and_then(|specific| specific.get("updatedInput")) + .cloned() +} + +/// Reads one record back over HTTP, as anything outside this process would. +async fn read_record(server: &Server, did: &str, rkey: &str) -> (u16, Value) { + let response = reqwest::Client::new() + .get(format!("{}/xrpc/com.atproto.repo.getRecord", server.url)) + .query(&[ + ("repo", did), + ("collection", "zone.quernstone.scrobble"), + ("rkey", rkey), + ]) + .send() + .await + .expect("the server answers"); + let status = response.status().as_u16(); + (status, response.json().await.unwrap_or(Value::Null)) +} + +/// The whole path: a session begins, says something, and ends. +#[tokio::test] +async fn a_hook_payload_becomes_a_record_and_the_account_goes_away() { + let server = Server::start().await; + let state = scratch("whole-path"); + let config = hook_config(&server, &state); + + // 1. The harness opens a session. The hook mints an account for it. + let start = vibescrobble_hookd::handle(&config, &payload("SessionStart", json!({}))).await; + assert!( + serde_json::to_value(&start) + .expect("serializes") + .pointer("/hookSpecificOutput/additionalContext") + .is_some(), + "the agent is told the scrobble tool exists" + ); + + let did = vibescrobble_hookd::StateStore::new(&state) + .load() + .agents + .get(SESSION) + .cloned() + .expect("SessionStart provisioned an account"); + + // 2. The model calls the scrobble tool. The hook rewrites the call to + // carry an identity the model did not choose, and the turn it is in. + let call = vibescrobble_hookd::handle( + &config, + &payload( + "PreToolUse", + json!({ + "tool_name": "mcp__vibescrobble__scrobble", + "tool_use_id": TURN, + "tool_input": {"text": "closing the loop by hand no longer", "emoji": "\u{1f9ea}"}, + }), + ), + ) + .await; + let arguments = stamped(&call).expect("the call is rewritten"); + assert_eq!(arguments[STAMP_KEY], json!(did)); + assert_eq!(arguments[TURN_KEY], json!(TURN)); + + // 3. The scrobble host receives exactly those arguments and writes. + let host = vibescrobble_mcp::ScrobbleServer::new(vibescrobble_mcp::PdsClient::new(&server.url)); + let result = host.scrobble(arguments.as_object()).await; + assert_ne!( + result.is_error, + Some(true), + "the host refused what the hook produced: {result:?}" + ); + + // 4. The record is there, over HTTP, with what the model said and the + // identity it did not choose. + let listed = reqwest::Client::new() + .get(format!("{}/xrpc/com.atproto.repo.listRecords", server.url)) + .query(&[ + ("repo", did.as_str()), + ("collection", "zone.quernstone.scrobble"), + ]) + .send() + .await + .expect("the server answers") + .json::() + .await + .expect("a JSON listing"); + let uri = listed["records"][0]["uri"] + .as_str() + .expect("one record was written"); + let rkey = uri.rsplit('/').next().expect("a record key"); + + let (status, body) = read_record(&server, &did, rkey).await; + assert_eq!(status, 200, "{body}"); + assert_eq!( + body["value"]["text"], + json!("closing the loop by hand no longer") + ); + assert_eq!(body["value"]["emoji"], json!("\u{1f9ea}")); + assert!( + !body["value"].to_string().contains(TURN), + "the turn identifier was stored: {}", + body["value"] + ); + + // 5. The session ends. The account and everything in it go with it. + vibescrobble_hookd::handle(&config, &payload("SessionEnd", json!({"reason": "clear"}))).await; + let (status, _) = read_record(&server, &did, rkey).await; + assert_eq!( + status, 404, + "the account outlived the session that owned it" + ); + assert!( + vibescrobble_hookd::StateStore::new(&state) + .load() + .agents + .is_empty(), + "the hook still remembers an account it deleted" + ); +} + +/// A subagent is its own account, and says who spawned it. +#[tokio::test] +async fn a_subagent_gets_its_own_account_and_is_torn_down_with_the_session() { + let server = Server::start().await; + let state = scratch("subagent"); + let config = hook_config(&server, &state); + + vibescrobble_hookd::handle(&config, &payload("SessionStart", json!({}))).await; + vibescrobble_hookd::handle( + &config, + &payload( + "SubagentStart", + json!({"agent_id": "agent-7b2f", "agent_type": "Explore"}), + ), + ) + .await; + + let remembered = vibescrobble_hookd::StateStore::new(&state).load(); + let session = remembered.agents.get(SESSION).expect("the session has one"); + let subagent = remembered + .agents + .get(&format!("{SESSION}.agent-7b2f")) + .expect("the subagent has one"); + assert_ne!(session, subagent, "two agents, two accounts"); + + // A subagent killed rather than stopped never fires SubagentStop, so + // SessionEnd is what has to take it. + vibescrobble_hookd::handle(&config, &payload("SessionEnd", json!({"reason": "exit"}))).await; + for did in [session, subagent] { + let response = reqwest::Client::new() + .get(format!("{}/xrpc/com.atproto.repo.describeRepo", server.url)) + .query(&[("repo", did.as_str())]) + .send() + .await + .expect("the server answers"); + assert_eq!( + response.status().as_u16(), + 404, + "{did} outlived its session" + ); + } +} + +/// The server being down is never the session's problem, and never permanent. +#[tokio::test] +async fn a_session_that_opened_with_no_server_scrobbles_once_there_is_one() { + let state = scratch("late-server"); + + // Nothing is listening on this port yet: the address is bound, read, and + // released, so the hook's request is refused rather than hanging. + let dead = { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("a loopback listener"); + let port = listener.local_addr().expect("a bound address").port(); + drop(listener); + format!("http://127.0.0.1:{port}") + }; + let absent = vibescrobble_hookd::Config { + pds_url: dead, + pinned_pds_url: true, + shared_secret: String::from_utf8(SECRET.to_vec()).expect("the secret is text"), + node_id: NODE.to_owned(), + state_path: state.clone(), + debug: false, + }; + + // The session opens anyway. This is the whole point of the hook never + // failing: a developer with the stack switched off still gets a session. + vibescrobble_hookd::handle(&absent, &payload("SessionStart", json!({}))).await; + assert!( + vibescrobble_hookd::StateStore::new(&state) + .load() + .agents + .is_empty(), + "an account was minted against a server that is not there" + ); + + // The developer starts the stack. SessionStart has been and gone, and the + // harness will not read hook configuration again until the next session — + // so the first scrobble is the second chance. + let server = Server::start().await; + let config = hook_config(&server, &state); + let call = vibescrobble_hookd::handle( + &config, + &payload( + "PreToolUse", + json!({ + "tool_name": "mcp__vibescrobble__scrobble", + "tool_use_id": TURN, + "tool_input": {"text": "the server came back", "emoji": "\u{1f331}"}, + }), + ), + ) + .await; + + let arguments = stamped(&call).expect("the call is stamped after all"); + let host = vibescrobble_mcp::ScrobbleServer::new(vibescrobble_mcp::PdsClient::new(&server.url)); + let result = host.scrobble(arguments.as_object()).await; + assert_ne!(result.is_error, Some(true), "{result:?}"); +} diff --git a/plan/local-dev.md b/plan/local-dev.md index a74bf69f..b7f8c8a6 100644 --- a/plan/local-dev.md +++ b/plan/local-dev.md @@ -18,10 +18,8 @@ workable without a deployment. - [ ] **A runner for [scripts/ci.sh](../scripts/ci.sh).** Tangled's CI is spindle, and a spindle is self-hosted: this repo has none attached, so the script is run by a person until one exists. -- [ ] **An integration test harness: a server and a fake agent, in process.** - The path from a hook payload to a stored record is exercised by hand. -- [ ] **Simulate failure**, not only the happy path: a server that goes away - mid-stream, a torn log, a session that dies without a `SessionEnd`. +- [ ] **Simulate the failures still exercised by hand:** a server that goes + away mid-stream, and a torn log. - [ ] **Keep [docs/running-locally.md](../docs/running-locally.md) true.** - [ ] **Say where to run it, and what each posture costs.** Server and agents on one laptop is the development posture and the weakest: one compromise takes @@ -36,6 +34,17 @@ workable without a deployment. ## Done +- [x] **An integration test harness: a server and a fake agent, in process.** + `crates/vibescrobble/tests/hook_to_record.rs` drives a real listener, the + real hook handler and the real scrobble host: a session opens, a payload + becomes a stamped call, the call becomes a record, the record reads back + over HTTP, and the session ends taking its accounts with it. It needs a + socket where the other suites do not, because the hook and the scrobble + host are HTTP *clients* and substituting the transport would skip exactly + the code that has to agree. +- [x] **A session that never reached a live server** is covered by the same + harness: one opens with nothing listening, gets no account, and scrobbles + anyway once a server exists. - [x] `dev-pds.sh` and `dev-mcp.sh` stop a previous run by pidfile rather than by command-line pattern. `pkill -f "vibescrobble-dev --port ${PORT}"` matched anything whose arguments contained that string, including another -- 2.51.2