diff --git a/crates/solstone-linux/src/chat_bridge.rs b/crates/solstone-linux/src/chat_bridge.rs index 36c8a7c..f153c75 100644 --- a/crates/solstone-linux/src/chat_bridge.rs +++ b/crates/solstone-linux/src/chat_bridge.rs @@ -2,6 +2,7 @@ // Copyright (c) 2026 sol pbc //! Server-initiated chat events bridged to Linux desktop surfaces. +// L3-CLEANUP(spl-cutover): legacy direct-HTTP authority; remove when chat/browser navigation is separated. use crate::config::Config; use chrono::Local; diff --git a/crates/solstone-linux/src/cli.rs b/crates/solstone-linux/src/cli.rs index 5f8a508..90da178 100644 --- a/crates/solstone-linux/src/cli.rs +++ b/crates/solstone-linux/src/cli.rs @@ -5,14 +5,11 @@ use crate::{ capture_stats::{ compute_quarantine_stats, compute_status_capture_stats, format_quarantine_line, }, - config::{ - Config, ConfigPaths, DEFAULT_SERVER_URL, load_config, save_config, - save_config_with_identity, - }, + config::{Config, ConfigPaths, load_config, save_config}, + private_link::{PrivateStateError, setup_with_stream}, session_env::{self, Output, Runner}, streams::stream_name, sync_health::{derive_health, load_facts}, - upload::{UploadClient, key_prefix}, }; use clap::{Parser, Subcommand}; use std::{ @@ -47,14 +44,8 @@ enum Commands { }, #[command(about = "Interactive configuration")] Setup { - #[arg(long, help = "Journal URL (skips prompt)")] - server_url: Option, - #[arg(long, help = "Pre-issued registration key; skips journal registration")] - token: Option, #[arg(long, help = "Stream name (defaults to hostname-derived)")] stream_name: Option, - #[arg(long, help = "Fail instead of prompting for missing values")] - non_interactive: bool, }, #[command(about = "Verify install prerequisites")] Doctor, @@ -201,21 +192,10 @@ pub fn run() -> i32 { setup_logging(args.verbose); match effective_command(args.command) { Commands::Run { interval } => cmd_run(interval), - Commands::Setup { - server_url, - token, - stream_name, - non_interactive, - } => cmd_setup( - SetupOptions { - server_url, - token, - stream_name, - non_interactive, - }, + Commands::Setup { stream_name } => cmd_setup( + SetupOptions { stream_name }, ConfigPaths::default(), - env::var("SOLSTONE_TOKEN").ok(), - &mut RealRegistrar, + &mut io::stdin().lock(), &mut io::stdout(), &mut io::stderr(), ), @@ -243,33 +223,7 @@ pub fn run() -> i32 { } struct SetupOptions { - server_url: Option, - token: Option, stream_name: Option, - non_interactive: bool, -} - -trait Registrar { - fn register(&mut self, config: &mut Config, host: &str) -> Result; -} - -struct RealRegistrar; -impl Registrar for RealRegistrar { - fn register(&mut self, config: &mut Config, host: &str) -> Result { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|error| error.to_string())?; - let _guard = runtime.enter(); - let client = UploadClient::new( - config, - host, - "linux", - env!("CARGO_PKG_VERSION"), - std::sync::Arc::new(crate::run::SystemClock::new()), - ); - Ok(runtime.block_on(client.ensure_registered(config))) - } } fn write_line(output: &mut dyn Write, value: impl std::fmt::Display) -> io::Result<()> { @@ -279,121 +233,70 @@ fn write_line(output: &mut dyn Write, value: impl std::fmt::Display) -> io::Resu fn cmd_setup( options: SetupOptions, paths: ConfigPaths, - env_token: Option, - registrar: &mut dyn Registrar, + input: &mut dyn Read, output: &mut dyn Write, errors: &mut dyn Write, ) -> i32 { - // Setup deliberately takes no PromptIo; only settings can reach stdin by construction. - let cli_token = options.token.filter(|value| !value.is_empty()); - let env_token = env_token.filter(|value| !value.is_empty()); - let token = cli_token.clone().or(env_token); - if cli_token.is_some() - && write_line( - errors, - "warning: --token on the command line may be visible in shell history and /proc on shared computers", - ) - .is_err() - { - return 1; - } - let loaded = load_config(paths); - let mut config = loaded.config; - config.server_url = options - .server_url + let host = hostname().unwrap_or_else(|_| "linux".into()); + let stream = options + .stream_name .filter(|value| !value.is_empty()) - .or_else(|| (!config.server_url.is_empty()).then(|| config.server_url.clone())) - .unwrap_or_else(|| DEFAULT_SERVER_URL.into()); - if let Some(stream) = options.stream_name.filter(|value| !value.is_empty()) { - config.stream = stream; - } else if config.stream.is_empty() { - let host = match hostname() { - Ok(host) => host, - Err(error) => { - let _ = write_line(errors, format!("Error deriving stream name: {error}")); - return 1; - } - }; - match stream_name(Some(&host), None, None) { - Ok(stream) => config.stream = stream, - Err(error) => { - let _ = write_line(errors, format!("Error deriving stream name: {error}")); - return 1; - } - } - } - if let Err(error) = config.ensure_dirs() { - let _ = write_line(errors, format!("Error saving config: {error}")); + .or_else(|| stream_name(Some(&host), None, None).ok()); + let config_root = paths + .config_dir + .unwrap_or_else(|| Config::default().config_dir); + if write_line( + output, + "Paste the pair link from your journal, then press Enter:", + ) + .is_err() + { return 1; } - if let Some(token) = token { - config.key = token; - // The one-line API swap is unavoidable after save_config became identity-preserving: - // setup still performs its original single whole-config write when given a token. - if let Err(error) = save_config_with_identity(&config) { - let _ = write_line(errors, format!("Error saving config: {error}")); + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + let _ = write_line(errors, format!("Setup failed: {error}")); return 1; } - let result = write_line(output, format!("Journal: {}", config.server_url)) - .and_then(|()| write_line(output, format!("Stream: {}", config.stream))) - .and_then(|()| write_line(output, "Using provided token; skipping registration.")) - .and_then(|()| setup_footer(output, &config)); - return if result.is_ok() { 0 } else { 1 }; - } - if let Err(error) = save_config(&config) { - let _ = write_line(errors, format!("Error saving config: {error}")); - return 1; - } - let host = hostname().unwrap_or_else(|_| "linux".into()); - let result = if config.key.is_empty() { - if write_line(output, "Registering with your journal...").is_err() { - return 1; + }; + match runtime.block_on(setup_with_stream( + &config_root, + &host, + stream.as_deref(), + input, + )) { + Ok(()) => { + let _ = write_line(output, "sol can now connect to your journal."); + 0 } - match registrar.register(&mut config, &host) { - Ok(true) => write_line( - output, - format!("Registered (key: {}...)", key_prefix(&config.key)), - ) - .and_then(|()| write_line(output, format!("Stream: {}", config.stream))), - Ok(false) => { - let result = write_line( - output, - "Warning: registration failed. Run setup again when your journal is available.", - ); - if options.non_interactive { - return 1; - } - result - } - Err(error) => { - let _ = write_line(errors, format!("Registration failed: {error}")); - return 1; - } + Err(PrivateStateError::PairInputInvalid) => { + let _ = write_line(errors, "Setup failed: the pair link was not valid."); + 1 } - } else { - write_line( - output, - format!("Already registered (key: {}...)", key_prefix(&config.key)), - ) - .and_then(|()| write_line(output, format!("Stream: {}", config.stream))) - }; - let result = result.and_then(|()| setup_footer(output, &config)); - if result.is_ok() { 0 } else { 1 } -} - -fn setup_footer(output: &mut dyn Write, config: &Config) -> io::Result<()> { - write_line( - output, - format!("\nConfig saved to {}", config.config_path().display()), - )?; - write_line( - output, - format!("segments are kept in {}", config.captures_dir().display()), - )?; - write_line( - output, - "\nRun 'solstone-linux run' to start, or 'solstone-linux install-service' for systemd.", - ) + Err(PrivateStateError::PairingFailed) => { + let _ = write_line( + errors, + "Setup failed: sol could not connect to your journal.", + ); + 1 + } + Err(error @ (PrivateStateError::Io { .. } | PrivateStateError::InvalidTarget { .. })) => { + let _ = write_line( + errors, + "Setup failed before pairing because sol could not safely update its config.", + ); + let _ = write_line(errors, format!("Config update error: {error}")); + 1 + } + Err(error) => { + let _ = write_line(errors, format!("Setup failed: {error}")); + 1 + } + } } trait PromptIo { @@ -500,19 +403,10 @@ fn cmd_settings(paths: ConfigPaths, prompt: &mut dyn PromptIo) -> i32 { } } +// L3-CLEANUP(spl-cutover): legacy direct-HTTP authority; remove when chat/browser navigation is separated. fn cmd_status(paths: ConfigPaths, runner: &dyn Runner, output: &mut dyn Write) -> i32 { let loaded = load_config(paths); let config = loaded.config; - let journal = if config.server_url.is_empty() { - "(not configured)" - } else { - &config.server_url - }; - let key = if config.key.is_empty() { - "(not registered)".into() - } else { - format!("{}...", key_prefix(&config.key)) - }; let stream = if config.stream.is_empty() { "(not set)" } else { @@ -523,8 +417,7 @@ fn cmd_status(paths: ConfigPaths, runner: &dyn Runner, output: &mut dyn Write) - output, format!("Config: {}", config.config_path().display()), )?; - write_line(output, format!("Journal: {journal}"))?; - write_line(output, format!("Key: {key}"))?; + write_line(output, "Journal link: managed privately")?; write_line(output, format!("Stream: {stream}"))?; write_line(output, "")?; let captures = config.captures_dir(); @@ -629,12 +522,16 @@ fn cmd_run(interval: Option) -> i32 { #[cfg(test)] mod tests { use super::*; - use crate::{ - config::save_identity, private_link_test_peer::PrivateLinkPeer, test_support::MockServer, - }; + use crate::upload::key_prefix; use clap::CommandFactory; - use serde_json::json; - use std::{cell::Cell, path::Path}; + use std::{ + cell::Cell, + path::Path, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + }; struct FakeRunner { output: io::Result, @@ -793,408 +690,172 @@ mod tests { } } - struct FakeRegistrar { - result: bool, - calls: usize, - } - impl Registrar for FakeRegistrar { - fn register(&mut self, config: &mut Config, _: &str) -> Result { - self.calls += 1; - if self.result { - config.key = "newkey00".into(); - config.stream = "locked-stream".into(); - save_identity( - &ConfigPaths { - base_dir: Some(config.base_dir.clone()), - config_dir: Some(config.config_dir.clone()), - }, - &config.key, - &config.stream, - ) - .unwrap(); - } - Ok(self.result) - } - } - - fn setup_options(token: Option<&str>, non_interactive: bool) -> SetupOptions { - SetupOptions { - server_url: Some("https://x".into()), - token: token.map(str::to_owned), - stream_name: Some("host-a".into()), - non_interactive, - } - } - - // tests/test_cli.py::test_cmd_setup_non_interactive_happy_path #[test] - fn setup_token_happy_path() { - let t = tempfile::tempdir().unwrap(); - let mut registrar = FakeRegistrar { - result: true, - calls: 0, - }; - let (mut out, mut err) = (Vec::new(), Vec::new()); - assert_eq!( - cmd_setup( - setup_options(Some("t"), true), - paths(&t), - None, - &mut registrar, - &mut out, - &mut err - ), - 0 - ); - let config = load_config(paths(&t)).config; - assert_eq!( - ( - config.server_url.as_str(), - config.key.as_str(), - config.stream.as_str() - ), - ("https://x", "t", "host-a") - ); - assert_eq!(registrar.calls, 0); + fn setup_help_exposes_only_stream_name() { + let command = Args::command(); + let setup = command.find_subcommand("setup").unwrap(); + let arguments = setup + .get_arguments() + .map(|argument| argument.get_id().as_str()) + .collect::>(); + assert_eq!(arguments, vec!["stream_name"]); + for removed in ["--server-url", "--token", "--non-interactive"] { + assert!(Args::try_parse_from(["solstone-linux", "setup", removed]).is_err()); + } } - // tests/test_cli.py::test_cmd_setup_non_interactive_defaults_server_url #[test] - fn setup_defaults_server_url() { - let t = tempfile::tempdir().unwrap(); - let mut registrar = FakeRegistrar { - result: true, - calls: 0, - }; - let mut options = setup_options(None, true); - options.server_url = None; - let (mut out, mut err) = (Vec::new(), Vec::new()); - assert_eq!( - cmd_setup(options, paths(&t), None, &mut registrar, &mut out, &mut err), - 0 + fn setup_ignores_no_legacy_token_environment() { + assert!( + !include_str!("cli.rs").contains("env::var(\"SOLSTONE_TOKEN\")"), + "setup must not read SOLSTONE_TOKEN" ); - assert_eq!(load_config(paths(&t)).config.server_url, DEFAULT_SERVER_URL); - assert!(err.is_empty()); } - // tests/test_cli.py::test_cmd_setup_server_url_override_persists - // tests/test_cli.py::test_cmd_setup_preserves_existing_server_url - #[test] - fn setup_url_precedence() { - let t = tempfile::tempdir().unwrap(); - let mut config = load_config(paths(&t)).config; - config.server_url = "https://saved.example".into(); - save_config(&config).unwrap(); - let mut registrar = FakeRegistrar { - result: true, - calls: 0, - }; - let mut options = setup_options(Some("token"), true); - options.server_url = None; - assert_eq!( - cmd_setup( - options, - paths(&t), - None, - &mut registrar, - &mut Vec::new(), - &mut Vec::new() - ), - 0 - ); - assert_eq!( - load_config(paths(&t)).config.server_url, - "https://saved.example" - ); - let mut options = setup_options(Some("token"), true); - options.server_url = Some("http://192.168.1.50:5015".into()); - assert_eq!( - cmd_setup( - options, - paths(&t), - None, - &mut registrar, - &mut Vec::new(), - &mut Vec::new() - ), - 0 - ); - assert_eq!( - load_config(paths(&t)).config.server_url, - "http://192.168.1.50:5015" - ); + struct CountingInput { + bytes: std::io::Cursor>, + reads: Arc, } - // tests/test_cli.py::test_cmd_setup_flagged_interactive_empty_input_defaults - // tests/test_cli.py::test_cmd_setup_interactive_legacy_empty_input_defaults - #[test] - fn setup_with_or_without_stream_flag_uses_default_url() { - for stream in [Some("host-x"), None] { - let t = tempfile::tempdir().unwrap(); - let mut registrar = FakeRegistrar { - result: true, - calls: 0, - }; - let options = SetupOptions { - server_url: None, - token: None, - stream_name: stream.map(str::to_owned), - non_interactive: false, - }; - assert_eq!( - cmd_setup( - options, - paths(&t), - None, - &mut registrar, - &mut Vec::new(), - &mut Vec::new() - ), - 0 - ); - assert_eq!(load_config(paths(&t)).config.server_url, DEFAULT_SERVER_URL); + impl Read for CountingInput { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + self.reads.fetch_add(1, Ordering::SeqCst); + self.bytes.read(buffer) } } - // tests/test_cli.py::test_cmd_setup_env_token_fallback - // tests/test_cli.py::test_cmd_setup_cli_token_beats_env #[test] - fn setup_token_precedence_and_warning() { - let t = tempfile::tempdir().unwrap(); - let mut registrar = FakeRegistrar { - result: true, - calls: 0, - }; - let mut err = Vec::new(); - assert_eq!( - cmd_setup( - setup_options(None, true), - paths(&t), - Some("envtok".into()), - &mut registrar, - &mut Vec::new(), - &mut err - ), - 0 - ); - assert!(err.is_empty()); - assert_eq!(load_config(paths(&t)).config.key, "envtok"); - assert_eq!( - cmd_setup( - setup_options(Some("clitok"), true), - paths(&t), - Some("envtok".into()), - &mut registrar, - &mut Vec::new(), - &mut err - ), - 0 - ); - assert_eq!(load_config(paths(&t)).config.key, "clitok"); - assert!(String::from_utf8(err).unwrap().contains("shared computers")); - } - - // AC: an empty SOLSTONE_TOKEN is absent and registration still runs. - #[test] - fn setup_empty_env_token_registers() { - let t = tempfile::tempdir().unwrap(); - let mut registrar = FakeRegistrar { - result: true, - calls: 0, - }; + fn setup_consumes_exactly_one_bounded_stdin_link() { + let temp = tempfile::tempdir().unwrap(); let mut output = Vec::new(); + let mut errors = Vec::new(); + let mut input = std::io::Cursor::new(vec![b'a'; 4097]); assert_eq!( cmd_setup( - setup_options(None, true), - paths(&t), - Some(String::new()), - &mut registrar, + SetupOptions { + stream_name: Some("host-a".into()) + }, + paths(&temp), + &mut input, &mut output, - &mut Vec::new() - ), - 0 - ); - assert_eq!(registrar.calls, 1); - assert_eq!(load_config(paths(&t)).config.key, "newkey00"); - let output = String::from_utf8(output).unwrap(); - assert!(output.contains("Registering with your journal...")); - assert!(!output.contains("Using provided token")); - } - - // tests/test_cli.py::test_cmd_setup_registers_via_http_when_no_token - #[test] - fn setup_registers_without_token() { - let t = tempfile::tempdir().unwrap(); - let mut registrar = FakeRegistrar { - result: true, - calls: 0, - }; - assert_eq!( - cmd_setup( - setup_options(None, true), - paths(&t), - None, - &mut registrar, - &mut Vec::new(), - &mut Vec::new() + &mut errors, ), - 0 + 1 ); - let config = load_config(paths(&t)).config; assert_eq!( - (registrar.calls, config.key.as_str(), config.stream.as_str()), - (1, "newkey00", "locked-stream") + String::from_utf8(errors).unwrap(), + "Setup failed: the pair link was not valid.\n" ); + assert!(input.position() <= 4097); } - // AC: an existing key skips registration and prints the parity line. #[test] - fn setup_already_registered_short_circuits() { - let t = tempfile::tempdir().unwrap(); - let mut config = load_config(paths(&t)).config; - config.server_url = "https://saved.example".into(); - config.key = "abcdefghijk".into(); - config.stream = "host-a".into(); - save_config(&config).unwrap(); - let mut registrar = FakeRegistrar { - result: true, - calls: 0, - }; - let options = SetupOptions { - server_url: None, - token: None, - stream_name: None, - non_interactive: false, + fn setup_lock_loser_does_not_consume_input_or_mutate_state() { + let temp = tempfile::tempdir().unwrap(); + let config_root = temp.path().join("config"); + let lock = crate::private_link::PrivateStateLock::acquire(&config_root).unwrap(); + let reads = Arc::new(AtomicUsize::new(0)); + let mut input = CountingInput { + bytes: std::io::Cursor::new(b"pair-secret".to_vec()), + reads: reads.clone(), }; - let mut out = Vec::new(); + let before = std::fs::read_dir(&config_root) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect::>(); assert_eq!( cmd_setup( - options, - paths(&t), - None, - &mut registrar, - &mut out, - &mut Vec::new() + SetupOptions { + stream_name: Some("host-a".into()) + }, + ConfigPaths { + base_dir: None, + config_dir: Some(config_root.clone()) + }, + &mut input, + &mut Vec::new(), + &mut Vec::new(), ), - 0 - ); - assert_eq!(registrar.calls, 0); - assert!( - String::from_utf8(out) - .unwrap() - .starts_with("Already registered (key: abcdefgh...)\nStream: host-a\n") + 1 ); + assert_eq!(reads.load(Ordering::SeqCst), 0); + let after = std::fs::read_dir(&config_root) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect::>(); + assert_eq!(after, before); + drop(lock); } - // tests/test_cli.py::test_cmd_setup_http_register_failure_non_interactive_returns_1 #[test] - fn setup_noninteractive_failure_omits_footer() { - let t = tempfile::tempdir().unwrap(); - let mut registrar = FakeRegistrar { - result: false, - calls: 0, - }; - let mut out = Vec::new(); + fn setup_surfaces_never_disclose_pair_material() { + let temp = tempfile::tempdir().unwrap(); + let secret = "pair-material-must-not-appear "; + let mut input = std::io::Cursor::new(secret.as_bytes()); + let mut output = Vec::new(); + let mut errors = Vec::new(); assert_eq!( cmd_setup( - setup_options(None, true), - paths(&t), - None, - &mut registrar, - &mut out, - &mut Vec::new() + SetupOptions { + stream_name: Some("host-a".into()) + }, + paths(&temp), + &mut input, + &mut output, + &mut errors, ), 1 ); - let out = String::from_utf8(out).unwrap(); - assert!(out.contains("registration failed")); - assert!(!out.contains("Config saved")); + let surfaces = format!( + "{}{}", + String::from_utf8(output).unwrap(), + String::from_utf8(errors).unwrap() + ); + assert!(!surfaces.contains(secret.trim())); } #[test] - fn setup_registration_remains_on_production_http_not_private_link() { - let runtime = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build() - .unwrap(); - let server = runtime.block_on(MockServer::new(vec![( - 200, - json!({"key":"K123456789", "name":"host-a"}), - )])); - let peer = runtime.block_on(PrivateLinkPeer::start()); + fn setup_sanitation_failure_precedes_stdin() { let temp = tempfile::tempdir().unwrap(); - let options = SetupOptions { - server_url: Some(server.url.clone()), - token: None, - stream_name: Some("host-a".into()), - non_interactive: true, + let blocker = temp.path().join("config"); + std::fs::write(&blocker, "not a directory").unwrap(); + let reads = Arc::new(AtomicUsize::new(0)); + let mut input = CountingInput { + bytes: std::io::Cursor::new(b"pair-secret".to_vec()), + reads: reads.clone(), }; - let mut output = Vec::new(); let mut errors = Vec::new(); assert_eq!( cmd_setup( - options, - paths(&temp), - None, - &mut RealRegistrar, - &mut output, + SetupOptions { + stream_name: Some("host-a".into()) + }, + ConfigPaths { + base_dir: None, + config_dir: Some(blocker) + }, + &mut input, + &mut Vec::new(), &mut errors, ), - 0 - ); - let requests = server.requests(); - assert_eq!(requests.len(), 1); - assert_eq!(requests[0].method, "POST"); - assert_eq!(requests[0].uri, "/app/observer/register"); - assert!(requests[0].headers.get("authorization").is_none()); - let body: serde_json::Value = serde_json::from_slice(&requests[0].body).unwrap(); - assert!( - body["hostname"] - .as_str() - .is_some_and(|value| !value.is_empty()) + 1 ); - assert_eq!(body["label"], "host-a"); - assert_eq!(body["stream_type"], "desktop"); + assert_eq!(reads.load(Ordering::SeqCst), 0); + let errors = String::from_utf8(errors).unwrap(); + assert!(errors.contains( + "Setup failed before pairing because sol could not safely update its config." + )); + assert!(errors.contains("Config update error:")); + assert!(!errors.contains("pair-secret")); + } - for (cli_token, env_token, expected) in [ - (Some("cli-token"), None, "cli-token"), - (None, Some("env-token"), "env-token"), - (Some("cli-wins"), Some("env-loses"), "cli-wins"), - ] { - let token_temp = tempfile::tempdir().unwrap(); - let token_paths = paths(&token_temp); - let mut token_output = Vec::new(); - let mut token_errors = Vec::new(); - assert_eq!( - cmd_setup( - SetupOptions { - server_url: Some(server.url.clone()), - token: cli_token.map(str::to_owned), - stream_name: Some("host-a".into()), - non_interactive: true, - }, - token_paths.clone(), - env_token.map(str::to_owned), - &mut RealRegistrar, - &mut token_output, - &mut token_errors, - ), - 0 - ); - assert_eq!(load_config(token_paths).config.key, expected); - assert!( - String::from_utf8(token_output) - .unwrap() - .contains("Using provided token; skipping registration.") - ); - assert_eq!( - server.requests().len(), - 1, - "token setup must skip registration" - ); - } - assert!(peer.requests().is_empty()); - runtime.block_on(peer.shutdown()); + #[test] + fn setup_pairs_over_private_link_and_never_uses_legacy_registration() { + let source = include_str!("cli.rs"); + assert!(source.contains("setup_with_stream(")); + assert!(!source.contains(&["UploadClient", "::new("].concat())); + assert!(!source.contains(&["/app/observer", "/register"].concat())); + assert!(!source.contains(&[".bearer_", "auth("].concat())); } struct ScriptedPrompt { @@ -1257,7 +918,7 @@ mod tests { config.key.as_str(), config.stream.as_str() ), - ("https://id", "KKKK", "strm") + ("", "", "strm") ); } @@ -1371,7 +1032,7 @@ mod tests { 0 ); let expected = format!( - "Config: {}\nJournal: https://test.example.com\nKey: K1234567...\nStream: test-stream\n\nCache: {}\n 0 segments across 0 day(s), 0.0 MB\nRetain: 7 day(s)\nSync: offline — saving locally; pending unconfirmed (will retry)\n\nService: active\n", + "Config: {}\nJournal link: managed privately\nStream: test-stream\n\nCache: {}\n 0 segments across 0 day(s), 0.0 MB\nRetain: 7 day(s)\nSync: offline — saving locally; pending unconfirmed (will retry)\n\nService: active\n", config.config_path().display(), config.captures_dir().display() ); @@ -1407,7 +1068,7 @@ mod tests { assert!( String::from_utf8(out) .unwrap() - .contains("Journal: (not configured)") + .contains("Journal link: managed privately") ); } diff --git a/crates/solstone-linux/src/config.rs b/crates/solstone-linux/src/config.rs index 399e178..28c3720 100644 --- a/crates/solstone-linux/src/config.rs +++ b/crates/solstone-linux/src/config.rs @@ -15,6 +15,11 @@ use std::{ time::{Duration, Instant}, }; +use crate::private_file::{ + DurableWriteFault, NoWriteFault, atomic_write_bytes_with_fault, ensure_private_directory, +}; + +// L3-CLEANUP(spl-cutover): legacy direct-HTTP authority; remove when chat/browser navigation is separated. pub const DEFAULT_SERVER_URL: &str = "http://localhost:5015"; pub const DEFAULT_SYNC_STALE_THRESHOLD: i64 = 600; const DEFAULT_RETRY_DELAYS: [i64; 4] = [5, 30, 120, 300]; @@ -25,7 +30,11 @@ static CONFIG_TEMP_SEQUENCE: AtomicU64 = AtomicU64::new(0); #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct Config { + // L3-CLEANUP(spl-cutover): legacy direct-HTTP authority; remove when chat/browser navigation is separated. + #[serde(default, skip_serializing)] pub server_url: String, + // L3-CLEANUP(spl-cutover): legacy direct-HTTP authority; remove when chat/browser navigation is separated. + #[serde(default, skip_serializing)] pub key: String, pub stream: String, pub segment_interval: i64, @@ -314,6 +323,49 @@ fn write_config(config: &Config) -> io::Result<()> { fs::rename(temporary, path) } +fn write_link_config( + paths: &ConfigPaths, + stream: Option<&str>, + fault: &dyn DurableWriteFault, +) -> io::Result { + let _guard = acquire_config_write_lock()?; + let mut config = load_config(paths.clone()).config; + config.server_url.clear(); + config.key.clear(); + if let Some(stream) = stream { + config.stream = stream.to_owned(); + } + ensure_private_directory(&config.config_dir).map_err(io::Error::other)?; + let mut bytes = serde_json::to_vec_pretty(&config).map_err(io::Error::other)?; + bytes.push(b'\n'); + atomic_write_bytes_with_fault(&config.config_path(), &bytes, fault) + .map_err(io::Error::other)?; + Ok(config) +} + +pub(crate) fn sanitize_link_authority(paths: &ConfigPaths) -> io::Result { + sanitize_link_authority_with_fault(paths, &NoWriteFault) +} + +pub(crate) fn sanitize_link_authority_with_fault( + paths: &ConfigPaths, + fault: &dyn DurableWriteFault, +) -> io::Result { + write_link_config(paths, None, fault) +} + +pub(crate) fn save_linked_stream(paths: &ConfigPaths, stream: &str) -> io::Result { + save_linked_stream_with_fault(paths, stream, &NoWriteFault) +} + +pub(crate) fn save_linked_stream_with_fault( + paths: &ConfigPaths, + stream: &str, + fault: &dyn DurableWriteFault, +) -> io::Result { + write_link_config(paths, Some(stream), fault) +} + enum IdentityWrite<'a> { PreserveDisk, Provided { key: &'a str, stream: &'a str }, @@ -356,6 +408,7 @@ pub fn save_config(config: &Config) -> io::Result<()> { ) } +// L3-CLEANUP(spl-cutover): legacy direct-HTTP authority; remove when chat/browser navigation is separated. pub fn save_config_with_identity(config: &Config) -> io::Result<()> { save_config_inner( &ConfigPaths { @@ -370,6 +423,7 @@ pub fn save_config_with_identity(config: &Config) -> io::Result<()> { ) } +// L3-CLEANUP(spl-cutover): legacy direct-HTTP authority; remove when chat/browser navigation is separated. pub fn save_identity(paths: &ConfigPaths, key: &str, stream: &str) -> io::Result<()> { save_config_inner(paths, None, IdentityWrite::Provided { key, stream }) } @@ -400,8 +454,21 @@ fn migrate(config: &Config) -> io::Result<()> { #[cfg(test)] mod tests { use super::*; + use crate::private_file::DurableWriteStage; use serde_json::json; - use std::collections::BTreeSet; + use std::{collections::BTreeSet, os::unix::fs::symlink}; + + struct FailStage(DurableWriteStage); + + impl DurableWriteFault for FailStage { + fn before(&self, stage: DurableWriteStage) -> io::Result<()> { + if stage == self.0 { + Err(io::Error::other("injected durable write failure")) + } else { + Ok(()) + } + } + } fn paths(root: &std::path::Path) -> ConfigPaths { ConfigPaths { @@ -495,12 +562,7 @@ mod tests { }); assert_eq!( (c.server_url, c.key, c.stream, c.segment_interval), - ( - "https://example.com".into(), - "key".into(), - "archon".into(), - 600 - ) + ("".into(), "".into(), "archon".into(), 600) ); } // tests/test_config.py::test_load_missing @@ -708,7 +770,7 @@ mod tests { let c = Config::default(); assert_eq!( serde_json::to_value(c).unwrap(), - json!({"server_url":"","key":"","stream":"","segment_interval":300,"sync_retry_delays":[5,30,120,300],"sync_max_retries":10,"sync_stale_threshold":600,"cache_retention_days":7,"chat_bridge_enabled":true,"capture_framerate":1,"draw_cursor":true,"start_paused":false}) + json!({"stream":"","segment_interval":300,"sync_retry_delays":[5,30,120,300],"sync_max_retries":10,"sync_stale_threshold":600,"cache_retention_days":7,"chat_bridge_enabled":true,"capture_framerate":1,"draw_cursor":true,"start_paused":false}) ); } // AC: numeric coercion rejects bool and truncates floats, including list elements. @@ -775,8 +837,6 @@ mod tests { assert_eq!( keys, BTreeSet::from([ - "server_url", - "key", "stream", "segment_interval", "sync_retry_delays", @@ -824,14 +884,14 @@ mod tests { save_config(&stale_settings).unwrap(); let saved = load_config(config_paths).config; - assert_eq!(saved.key, "NEW-KEY"); + assert_eq!(saved.key, ""); assert_eq!(saved.stream, "desktop-new"); assert_eq!(saved.cache_retention_days, 30); } // AC 12: recovery writes identity only and cannot revert newer non-identity disk state. #[test] - fn save_identity_preserves_newer_server_url() { + fn save_identity_preserves_stream_but_not_legacy_authority() { let t = tempfile::tempdir().unwrap(); let config_paths = paths(t.path()); let mut config = Config { @@ -848,11 +908,140 @@ mod tests { save_identity(&config_paths, "NEW-KEY", "desktop-new").unwrap(); let saved = load_config(config_paths).config; - assert_eq!(saved.server_url, "https://new"); - assert_eq!(saved.key, "NEW-KEY"); + assert_eq!(saved.server_url, ""); + assert_eq!(saved.key, ""); assert_eq!(saved.stream, "desktop-new"); } + #[test] + fn representative_legacy_configs_preserve_settings_and_stream_while_stripping_authority() { + let t = tempfile::tempdir().unwrap(); + write( + t.path(), + json!({ + "server_url": "https://legacy.invalid", + "key": "legacy-secret", + "stream": "desktop", + "segment_interval": 17, + "sync_retry_delays": [2, 4], + "sync_max_retries": 3, + "sync_stale_threshold": 91, + "cache_retention_days": 12, + "chat_bridge_enabled": false, + "capture_framerate": 4, + "draw_cursor": false, + "start_paused": true + }), + ); + + let sanitized = sanitize_link_authority(&paths(t.path())).unwrap(); + assert_eq!(sanitized.server_url, ""); + assert_eq!(sanitized.key, ""); + assert_eq!(sanitized.stream, "desktop"); + assert_eq!(sanitized.segment_interval, 17); + assert_eq!(sanitized.sync_retry_delays, vec![2, 4]); + assert_eq!(sanitized.sync_max_retries, 3); + assert_eq!(sanitized.sync_stale_threshold, 91); + assert_eq!(sanitized.cache_retention_days, 12); + assert!(!sanitized.chat_bridge_enabled); + assert_eq!(sanitized.capture_framerate, 4); + assert!(!sanitized.draw_cursor); + assert!(sanitized.start_paused); + let value: Value = + serde_json::from_slice(&fs::read(t.path().join("cfg/config.json")).unwrap()).unwrap(); + assert!(value.get("server_url").is_none()); + assert!(value.get("key").is_none()); + } + + #[test] + fn fresh_config_serializes_authority_free_schema() { + let t = tempfile::tempdir().unwrap(); + let saved = sanitize_link_authority(&paths(t.path())).unwrap(); + let text = fs::read_to_string(saved.config_path()).unwrap(); + assert!(text.ends_with('\n')); + let value: Value = serde_json::from_str(&text).unwrap(); + assert!(value.get("server_url").is_none()); + assert!(value.get("key").is_none()); + assert_eq!(value["stream"], ""); + } + + #[test] + fn sanitation_rejects_symlinked_or_wrong_kind_root_without_touching_referent() { + let t = tempfile::tempdir().unwrap(); + let referent = t.path().join("referent"); + fs::create_dir(&referent).unwrap(); + fs::write(referent.join("sentinel"), "unchanged").unwrap(); + let linked = t.path().join("linked"); + symlink(&referent, &linked).unwrap(); + let linked_paths = ConfigPaths { + base_dir: None, + config_dir: Some(linked), + }; + assert!(sanitize_link_authority(&linked_paths).is_err()); + assert_eq!( + fs::read_to_string(referent.join("sentinel")).unwrap(), + "unchanged" + ); + + let wrong = t.path().join("wrong"); + fs::write(&wrong, "unchanged").unwrap(); + let wrong_paths = ConfigPaths { + base_dir: None, + config_dir: Some(wrong.clone()), + }; + assert!(sanitize_link_authority(&wrong_paths).is_err()); + assert_eq!(fs::read_to_string(wrong).unwrap(), "unchanged"); + } + + #[test] + fn sanitation_rejects_symlinked_or_wrong_kind_config_file_without_touching_referent() { + let t = tempfile::tempdir().unwrap(); + let config_dir = t.path().join("cfg"); + fs::create_dir(&config_dir).unwrap(); + let referent = t.path().join("referent.json"); + fs::write(&referent, r#"{"stream":"external"}"#).unwrap(); + symlink(&referent, config_dir.join("config.json")).unwrap(); + assert!(sanitize_link_authority(&paths(t.path())).is_err()); + assert_eq!( + fs::read_to_string(&referent).unwrap(), + r#"{"stream":"external"}"# + ); + + fs::remove_file(config_dir.join("config.json")).unwrap(); + fs::create_dir(config_dir.join("config.json")).unwrap(); + assert!(sanitize_link_authority(&paths(t.path())).is_err()); + assert!(config_dir.join("config.json").is_dir()); + } + + #[test] + fn sanitation_fault_preserves_last_complete_config() { + for stage in [ + DurableWriteStage::Create, + DurableWriteStage::Write, + DurableWriteStage::Fsync, + DurableWriteStage::Rename, + DurableWriteStage::DirSync, + ] { + let t = tempfile::tempdir().unwrap(); + write( + t.path(), + json!({"server_url":"https://legacy.invalid","key":"secret","stream":"old"}), + ); + let path = t.path().join("cfg/config.json"); + let before = fs::read(&path).unwrap(); + let result = sanitize_link_authority_with_fault(&paths(t.path()), &FailStage(stage)); + assert!(result.is_err(), "{stage:?}"); + if stage == DurableWriteStage::DirSync { + let value: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + assert_eq!(value["stream"], "old"); + assert!(value.get("server_url").is_none()); + assert!(value.get("key").is_none()); + } else { + assert_eq!(fs::read(path).unwrap(), before, "{stage:?}"); + } + } + } + // AC: migration failure is returned as one warning and leaves legacy data intact. #[test] fn migration_warning() { diff --git a/crates/solstone-linux/src/dbus_service.rs b/crates/solstone-linux/src/dbus_service.rs index 70bda31..c499888 100644 --- a/crates/solstone-linux/src/dbus_service.rs +++ b/crates/solstone-linux/src/dbus_service.rs @@ -82,8 +82,9 @@ impl Observer1< String::new() } #[zbus(property)] + // L3-CLEANUP(spl-cutover): legacy direct-HTTP authority; remove when chat/browser navigation is separated. fn server_url(&self) -> String { - self.config.server_url.clone() + String::new() } #[zbus(property)] fn stream(&self) -> String { @@ -641,7 +642,7 @@ mod tests { 0, ); assert_eq!(s.capture_dir(), "/tmp/observer1/captures"); - assert_eq!(s.server_url(), "https://test.example.com"); + assert_eq!(s.server_url(), ""); assert_eq!(s.stream(), "test-stream"); assert_eq!(s.segment_interval(), 300); } diff --git a/crates/solstone-linux/src/desktop_component.rs b/crates/solstone-linux/src/desktop_component.rs index 9502510..06af0ab 100644 --- a/crates/solstone-linux/src/desktop_component.rs +++ b/crates/solstone-linux/src/desktop_component.rs @@ -130,11 +130,8 @@ impl DesktopComponent { } } pub fn journal_url(&self) -> &str { - if self.config.server_url.is_empty() { - "https://solstone.app" - } else { - &self.config.server_url - } + // L3-CLEANUP(spl-cutover): legacy direct-HTTP authority; remove when chat/browser navigation is separated. + "https://solstone.app" } pub fn command_url<'a>(&'a self, command: &'a crate::tray::TrayCommand) -> Option<&'a str> { match command { diff --git a/crates/solstone-linux/src/lib.rs b/crates/solstone-linux/src/lib.rs index 0071e59..088c451 100644 --- a/crates/solstone-linux/src/lib.rs +++ b/crates/solstone-linux/src/lib.rs @@ -38,6 +38,8 @@ pub mod tray_model; pub mod upload; pub mod video; +#[cfg(test)] +mod linked_authority_policy_tests; #[cfg(test)] mod observer_contract_tests; #[cfg(test)] diff --git a/crates/solstone-linux/src/linked_authority_policy_tests.rs b/crates/solstone-linux/src/linked_authority_policy_tests.rs new file mode 100644 index 0000000..e1ab1aa --- /dev/null +++ b/crates/solstone-linux/src/linked_authority_policy_tests.rs @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright (c) 2026 sol pbc + +use std::{fs, path::PathBuf}; + +const L3_MARKER: &str = "L3-CLEANUP(spl-cutover)"; + +fn source_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("src") +} + +fn production_prefix(source: &str) -> &str { + source + .split("#[cfg(test)]\nmod tests") + .next() + .unwrap_or(source) +} + +#[test] +fn active_production_paths_have_no_legacy_direct_authority() { + let root = source_root(); + let active_modules = [ + "cli.rs", + "event_sender.rs", + "observer.rs", + "private_link.rs", + "recovery.rs", + "run.rs", + "sync.rs", + "sync_health.rs", + "upload.rs", + ]; + let forbidden = [ + ".bearer_auth(", + "localhost:5015", + "Command::new(\"sol\")", + "Command::new(\"sol\"", + ]; + + for module in active_modules { + let source = fs::read_to_string(root.join(module)).unwrap(); + let production = production_prefix(&source); + let lines = production.lines().collect::>(); + for needle in forbidden { + for (index, line) in lines.iter().enumerate() { + if line.contains(needle) { + let nearby = lines[..=index].iter().rev().take(16).any(|candidate| { + candidate.contains("#[cfg(test)]") || candidate.contains(L3_MARKER) + }); + assert!( + nearby, + "{module}:{} contains active legacy authority `{needle}`", + index + 1 + ); + } + } + } + } + + for module in ["cli.rs", "run.rs", "sync.rs", "upload.rs"] { + let source = fs::read_to_string(root.join(module)).unwrap(); + let production = production_prefix(&source); + for needle in ["config.server_url", "config.key"] { + let lines = production.lines().collect::>(); + for (index, line) in lines.iter().enumerate() { + if line.contains(needle) { + assert!( + lines[..=index] + .iter() + .rev() + .take(32) + .any(|candidate| candidate.contains("#[cfg(test)]")), + "{module}:{} reads configured legacy authority `{needle}`", + index + 1 + ); + } + } + } + } +} + +#[test] +fn l3_authority_is_confined_to_explicit_unreachable_surfaces() { + let root = source_root(); + for module in [ + "chat_bridge.rs", + "cli.rs", + "config.rs", + "dbus_service.rs", + "desktop_component.rs", + ] { + let source = fs::read_to_string(root.join(module)).unwrap(); + assert!(source.contains(L3_MARKER), "{module} lacks the L3 marker"); + } +} diff --git a/crates/solstone-linux/src/observer_contract_tests.rs b/crates/solstone-linux/src/observer_contract_tests.rs index 980f3d5..c2baf82 100644 --- a/crates/solstone-linux/src/observer_contract_tests.rs +++ b/crates/solstone-linux/src/observer_contract_tests.rs @@ -7,6 +7,8 @@ use crate::{ dispatch_contract_payload, parse_contract_sse, }, config::Config, + private_link::{PrivateLinkOwner, start_registered_private_link_for_test}, + private_link_test_peer::{PeerRequest, PrivateLinkPeer}, sync::{contract_segment_proven_held, contract_sha256_file}, sync_health::ErrorType, test_support::{Action, MockServer, wait_for_requests}, @@ -414,10 +416,8 @@ fn verify_provenance(root: &Path) -> Result<(), String> { Ok(()) } -fn config(server: &MockServer, temp: &TempDir) -> Config { +fn config(temp: &TempDir) -> Config { Config { - server_url: server.url.clone(), - key: "K".into(), stream: "desktop".into(), sync_retry_delays: vec![0], sync_max_retries: 1, @@ -427,9 +427,10 @@ fn config(server: &MockServer, temp: &TempDir) -> Config { } } -fn client(config: &Config) -> UploadClient { +fn client(config: &Config, capability: crate::private_link::PrivateLinkCapability) -> UploadClient { UploadClient::new( config, + capability, "archon", "linux", "1.4.0", @@ -437,6 +438,44 @@ fn client(config: &Config) -> UploadClient { ) } +struct LinkedHarness { + peer: PrivateLinkPeer, + owner: PrivateLinkOwner, + client: UploadClient, +} + +impl LinkedHarness { + async fn start(temp: &TempDir) -> Self { + let peer = PrivateLinkPeer::start().await; + let (_state, owner) = start_registered_private_link_for_test( + peer.credential(), + "desktop", + "K", + "/app/observer/ingest", + ) + .await; + let client = client(&config(temp), owner.capability()); + Self { + peer, + owner, + client, + } + } + + async fn finish(self) { + self.owner.shutdown().await.unwrap(); + self.peer.shutdown().await; + } +} + +fn header<'a>(request: &'a PeerRequest, name: &str) -> Option<&'a str> { + request + .headers + .iter() + .find(|(candidate, _)| candidate.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) +} + async fn assert_upload_contract( fixtures: &BTreeMap, vectors: &BTreeMap, @@ -480,8 +519,12 @@ async fn assert_upload_contract( let temp = tempfile::tempdir().unwrap(); let media = temp.path().join("audio.flac"); fs::write(&media, b"audio").unwrap(); - let server = MockServer::new(vec![(status, fixture["payload"].clone())]).await; - let result = client(&config(&server, &temp)) + let harness = LinkedHarness::start(&temp).await; + harness + .peer + .enqueue_response(status, fixture["payload"].to_string()); + let result = harness + .client .upload_segment("20260618", "143022_300", &[media]) .await; let accepted = vector["decision"]["accepted"].as_bool().unwrap_or_else(|| { @@ -499,12 +542,13 @@ async fn assert_upload_contract( ); } else if status == 200 { assert_eq!(result.error_type, Some(ErrorType::Incompatible)); - assert_eq!(server.requests().len(), 1); + assert_eq!(harness.peer.requests().len(), 1); } else { assert!(!result.success && result.stored_key.is_none()); } record(executed_fixtures, fixture_id, true); record(executed_vectors, vector_id, true); + harness.finish().await; } for fixture_id in [ "example.observer.ingestUpload.response.200.application-json.normal", @@ -513,14 +557,19 @@ async fn assert_upload_contract( let temp = tempfile::tempdir().unwrap(); let media = temp.path().join("audio.flac"); fs::write(&media, b"audio").unwrap(); - let server = MockServer::new(vec![(200, fixtures[fixture_id]["payload"].clone())]).await; + let harness = LinkedHarness::start(&temp).await; + harness + .peer + .enqueue_response(200, fixtures[fixture_id]["payload"].to_string()); assert!( - client(&config(&server, &temp)) + harness + .client .upload_segment("20260618", "143022_300", &[media]) .await .success ); record(executed_fixtures, fixture_id, true); + harness.finish().await; } let fixture_id = "example.observer.ingestUpload.request.body.multipart-form-data.default"; let temp = tempfile::tempdir().unwrap(); @@ -535,24 +584,25 @@ async fn assert_upload_contract( path }) .collect(); - let server = MockServer::new(vec![(200, json!({"status":"ok","segment":"143022_300"}))]).await; - client(&config(&server, &temp)) + let harness = LinkedHarness::start(&temp).await; + harness.peer.enqueue_response( + 200, + json!({"status":"ok","segment":"143022_300"}).to_string(), + ); + harness + .client .upload_segment( fixtures[fixture_id]["payload"]["day"].as_str().unwrap(), fixtures[fixture_id]["payload"]["segment"].as_str().unwrap(), &media, ) .await; - let request = &server.requests()[0]; + let requests = harness.peer.requests(); + let request = &requests[0]; let body = String::from_utf8_lossy(&request.body); assert_eq!(request.method, "POST"); - assert_eq!(request.uri, "/app/observer/ingest"); - assert!( - request.headers["authorization"] - .to_str() - .unwrap() - .starts_with("Bearer ") - ); + assert_eq!(request.path, "/app/observer/ingest"); + assert!(header(request, "authorization").is_some_and(|value| value.starts_with("Bearer "))); assert_eq!(body.matches("name=\"day\"").count(), 1); assert_eq!(body.matches("name=\"segment\"").count(), 1); assert_eq!(body.matches("name=\"files\"").count(), fixture_files.len()); @@ -574,6 +624,7 @@ async fn assert_upload_contract( assert!(!body.contains(forbidden)); } record(executed_fixtures, fixture_id, true); + harness.finish().await; } async fn assert_listing_contract( @@ -643,19 +694,20 @@ async fn assert_listing_contract( ) }; let temp = tempfile::tempdir().unwrap(); - let server = MockServer::new(vec![(200, payload.clone())]).await; - let result = client(&config(&server, &temp)) - .get_server_segments("20260618") - .await; + let harness = LinkedHarness::start(&temp).await; + harness.peer.enqueue_response_with_headers( + 200, + vec![("x-solstone-protocol-version".to_owned(), "2".to_owned())], + payload.to_string(), + ); + let result = harness.client.get_server_segments("20260618").await; assert_eq!((result.legacy, result.truncated), (legacy, truncated)); - let request = &server.requests()[0]; + let requests = harness.peer.requests(); + let request = &requests[0]; assert_eq!(request.method, "GET"); - assert_eq!(request.uri, "/app/observer/ingest/segments/20260618"); - assert_eq!(request.headers["x-solstone-protocol-version"], "2"); - let authorization = request - .headers - .get("authorization") - .and_then(|value| value.to_str().ok()); + assert_eq!(request.path, "/app/observer/ingest/segments/20260618"); + assert_eq!(header(request, "x-solstone-protocol-version"), Some("2")); + let authorization = header(request, "authorization"); assert!(authorization.is_some_and(|value| value.starts_with("Bearer "))); record(executed_fixtures, fixture_id, true); if let Some(vector_id) = vector_id { @@ -683,6 +735,7 @@ async fn assert_listing_contract( } record(executed_vectors, vector_id, true); } + harness.finish().await; } let custody_cases = [ @@ -771,24 +824,26 @@ async fn assert_event_and_register( ) { let temp = tempfile::tempdir().unwrap(); let event_id = "example.observer.ingestEvent.request.body.application-json.default"; - let server = MockServer::new(vec![( + let harness = LinkedHarness::start(&temp).await; + harness.peer.enqueue_response( 200, fixtures["example.observer.ingestEvent.response.200.application-json.default"]["payload"] - .clone(), - )]) - .await; + .to_string(), + ); let payload = fixtures[event_id]["payload"].as_object().unwrap(); let mut fields = payload.clone(); let tract = fields.remove("tract").unwrap(); let event = fields.remove("event").unwrap(); assert!( - client(&config(&server, &temp)) + harness + .client .relay_event(tract.as_str().unwrap(), event.as_str().unwrap(), fields) .await ); - let request = &server.requests()[0]; + let requests = harness.peer.requests(); + let request = &requests[0]; assert_eq!( - (request.method.as_str(), request.uri.as_str()), + (request.method.as_str(), request.path.as_str()), ("POST", "/app/observer/ingest/event") ); assert_eq!( @@ -801,27 +856,40 @@ async fn assert_event_and_register( "example.observer.ingestEvent.response.200.application-json.default", true, ); + harness.finish().await; let register_request = "example.observer.register.request.body.application-json.default"; let register_response = "example.observer.register.response.200.application-json.default"; - let server = MockServer::new(vec![(200, fixtures[register_response]["payload"].clone())]).await; - let mut cfg = config(&server, &temp); - cfg.key.clear(); - cfg.stream = fixtures[register_request]["payload"]["label"] + let peer = PrivateLinkPeer::start().await; + peer.enqueue_response(200, fixtures[register_response]["payload"].to_string()); + let label = fixtures[register_request]["payload"]["label"] .as_str() - .unwrap() - .to_owned(); - fs::create_dir_all(&cfg.config_dir).unwrap(); - assert!(client(&cfg).ensure_registered(&mut cfg).await); - let request = &server.requests()[0]; + .unwrap(); + let (_state, owner) = start_registered_private_link_for_test( + peer.credential(), + label, + "K", + "/app/observer/ingest", + ) + .await; + assert!(matches!( + owner + .register_for_test(&fixtures[register_request]["payload"]) + .await, + crate::private_link::LinkOutcome::Success { .. } + )); + let requests = peer.requests(); + let request = &requests[0]; let body: Value = serde_json::from_slice(&request.body).unwrap(); assert_eq!( - (request.method.as_str(), request.uri.as_str()), + (request.method.as_str(), request.path.as_str()), ("POST", "/app/observer/register") ); assert_eq!(body, fixtures[register_request]["payload"]); - assert!(!request.headers.contains_key("authorization")); + assert!(header(request, "authorization").is_none()); record(executed, register_request, true); record(executed, register_response, true); + owner.shutdown().await.unwrap(); + peer.shutdown().await; } async fn assert_chat_contract( @@ -1177,8 +1245,10 @@ async fn assert_production_contradiction_mutation( let temp = tempfile::tempdir().unwrap(); let media = temp.path().join("audio.flac"); fs::write(&media, b"audio").unwrap(); - let server = MockServer::new(vec![(status, payload)]).await; - let result = client(&config(&server, &temp)) + let harness = LinkedHarness::start(&temp).await; + harness.peer.enqueue_response(status, payload.to_string()); + let result = harness + .client .upload_segment("20260618", "143022_300", &[media]) .await; let decision = &vectors[vector_id]["decision"]; @@ -1188,6 +1258,7 @@ async fn assert_production_contradiction_mutation( .expect("stored key source")] .as_str(); assert!(result.duplicate != expected_duplicate || result.stored_key.as_deref() != expected_key); + harness.finish().await; } #[tokio::test] diff --git a/crates/solstone-linux/src/private_link.rs b/crates/solstone-linux/src/private_link.rs index d119fcf..a4d76aa 100644 --- a/crates/solstone-linux/src/private_link.rs +++ b/crates/solstone-linux/src/private_link.rs @@ -8,11 +8,16 @@ use std::{ io::{self, Read}, path::{Path, PathBuf}, pin::Pin, - sync::{Arc, Mutex, RwLock}, + sync::{ + Arc, Mutex, RwLock, + atomic::{AtomicBool, Ordering}, + }, time::Duration, }; -use reqwest::{Method, RequestBuilder, StatusCode, Url}; +#[cfg(test)] +use reqwest::Method; +use reqwest::{RequestBuilder, StatusCode, Url, multipart}; use serde::{Deserialize, Serialize}; use spl_core::bridge::{BridgeNames, RequestHeaderPolicy}; use spl_transport::credential::Credential; @@ -24,6 +29,9 @@ use spl_transport::{ }, }; +use crate::config::{ + ConfigPaths, sanitize_link_authority, save_linked_stream, save_linked_stream_with_fault, +}; use crate::private_file::{ DurableWriteFault, NoWriteFault, PrivateFileError, atomic_write_bytes, atomic_write_bytes_with_fault, ensure_private_directory, open_regular_readonly, @@ -33,11 +41,18 @@ pub(crate) const CREDENTIALS_FILENAME: &str = "credentials.json"; pub(crate) const OBSERVER_FILENAME: &str = "observer.json"; const PRIVATE_STATE_LOCK_FILENAME: &str = ".solstone-linux.private-state.lock"; const MAX_PAIR_LINK_BYTES: u64 = 4096; -const MAX_REQUEST_BODY_BYTES: usize = 16 * 1024 * 1024; +const MAX_REQUEST_BODY_BYTES: usize = 64 * 1024 * 1024; const LOOPBACK_CONNECT_TIMEOUT: Duration = Duration::from_secs(5); -const LOOPBACK_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); +const BOOTSTRAP_TIMEOUT: Duration = Duration::from_secs(30); +const REGISTRATION_TIMEOUT: Duration = Duration::from_secs(30); +const INGEST_TIMEOUT: Duration = Duration::from_secs(300); +const LISTING_TIMEOUT: Duration = Duration::from_secs(60); +const EVENT_TIMEOUT: Duration = Duration::from_secs(30); pub(crate) const OBSERVER_HEADER_NAME: &str = "x-solstone-observer"; pub(crate) const PROTOCOL_VERSION_HEADER_NAME: &str = "x-solstone-protocol-version"; +const REGISTRATION_MARKER_HEADER_NAME: &str = "x-solstone-linux-registration-route"; +const REGISTRATION_MARKER_HEADER_VALUE: &str = "1"; +const EVENT_PATH: &str = "/app/observer/ingest/event"; #[derive(Clone, Deserialize, Eq, PartialEq, Serialize)] #[serde(deny_unknown_fields)] @@ -336,7 +351,10 @@ pub(crate) fn read_pair_link(input: R) -> Result( device_label: &str, input: R, ) -> Result<(), PrivateStateError> { - setup_with_pairer(&SplPairer, config_root, device_label, input).await + setup_with_stream(config_root, device_label, None, input).await +} + +pub(crate) async fn setup_with_stream( + config_root: &Path, + device_label: &str, + stream: Option<&str>, + input: R, +) -> Result<(), PrivateStateError> { + setup_with_pairer_and_stream(&SplPairer, config_root, device_label, stream, input).await } +#[cfg(test)] async fn setup_with_pairer( pairer: &dyn Pairer, config_root: &Path, device_label: &str, input: R, +) -> Result<(), PrivateStateError> { + setup_with_pairer_and_stream(pairer, config_root, device_label, None, input).await +} + +async fn setup_with_pairer_and_stream( + pairer: &dyn Pairer, + config_root: &Path, + device_label: &str, + stream: Option<&str>, + input: R, ) -> Result<(), PrivateStateError> { let state_lock = PrivateStateLock::acquire(config_root)?; + sanitize_link_authority(&private_config_paths(state_lock.root())) + .map_err(config_persist_error)?; + if let Some(stream) = stream { + save_linked_stream(&private_config_paths(state_lock.root()), stream) + .map_err(config_persist_error)?; + } let link = read_pair_link(input)?; let credential = pairer .pair(&link, device_label, &serde_json::Map::new()) @@ -393,6 +437,20 @@ async fn setup_with_pairer( persist_credential(state_lock.root(), &credential) } +fn private_config_paths(config_root: &Path) -> ConfigPaths { + ConfigPaths { + base_dir: None, + config_dir: Some(config_root.to_path_buf()), + } +} + +fn config_persist_error(source: io::Error) -> PrivateStateError { + PrivateStateError::Io { + operation: PrivateIoOperation::Persist, + source, + } +} + fn read_private_file( path: &Path, kind: PrivateTargetKind, @@ -513,6 +571,65 @@ fn contains_invalid_header_value(value: &str) -> bool { reqwest::header::HeaderValue::from_bytes(value.as_bytes()).is_err() } +#[allow(dead_code)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum LinkFact { + PairingRequired, + PrivateStateInvalid, + ConfigSanitationFailed, + ListenerReady, + CarrierProven, + ObserverRegistered, + TransportUnavailable, + TerminalRevocation, + TokenPersistenceFailure, +} + +#[derive(Clone, Default)] +pub(crate) struct LinkFacts { + inner: Arc>, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct LinkFactState { + pub(crate) pairing_required: bool, + pub(crate) private_state_invalid: bool, + pub(crate) config_sanitation_failed: bool, + pub(crate) listener_ready: bool, + pub(crate) carrier_proven: bool, + pub(crate) observer_registered: bool, + pub(crate) transport_unavailable: bool, + pub(crate) terminal_revocation: bool, + pub(crate) token_persistence_failure: bool, +} + +impl LinkFacts { + pub(crate) fn publish(&self, fact: LinkFact) { + let mut state = self.inner.lock().unwrap_or_else(|p| p.into_inner()); + match fact { + LinkFact::PairingRequired => state.pairing_required = true, + LinkFact::PrivateStateInvalid => state.private_state_invalid = true, + LinkFact::ConfigSanitationFailed => state.config_sanitation_failed = true, + LinkFact::ListenerReady => state.listener_ready = true, + LinkFact::CarrierProven => state.carrier_proven = true, + LinkFact::ObserverRegistered => state.observer_registered = true, + LinkFact::TransportUnavailable => state.transport_unavailable = true, + LinkFact::TerminalRevocation => state.terminal_revocation = true, + LinkFact::TokenPersistenceFailure => state.token_persistence_failure = true, + } + } + + #[allow(dead_code)] + pub(crate) fn snapshot(&self) -> LinkFactState { + self.inner.lock().unwrap_or_else(|p| p.into_inner()).clone() + } +} + +struct RegistrationCommitFaults<'a> { + observer: &'a dyn DurableWriteFault, + config: &'a dyn DurableWriteFault, +} + fn persist_and_publish_observer( config_root: &Path, credential_instance_id: &str, @@ -520,12 +637,18 @@ fn persist_and_publish_observer( origin: &Url, observer: &ObserverState, opener: &PrivateLinkOpener, - fault: &dyn DurableWriteFault, -) -> Result<(), PrivateStateError> { + faults: RegistrationCommitFaults<'_>, +) -> Result { if !observer_is_valid(observer, credential_instance_id, expected_name, origin) { return Err(PrivateStateError::RegistrationInvalid); } - write_observer_durably(config_root, observer, fault)?; + write_observer_durably(config_root, observer, faults.observer)?; + save_linked_stream_with_fault( + &private_config_paths(config_root), + &observer.name, + faults.config, + ) + .map_err(config_persist_error)?; opener.set_registered(observer) } @@ -535,22 +658,41 @@ enum OpenerAuth { Registered { key: String }, } +struct AuthEpoch { + generation: u64, + state: OpenerAuth, +} + struct PrivateLinkOpener { transport: Arc, - auth: RwLock, + auth: RwLock>, expected_name: String, + admission: tokio::sync::Mutex<()>, + transport_unavailable: Arc, + facts: LinkFacts, } impl PrivateLinkOpener { - fn new(transport: TransportClient, expected_name: String) -> Self { + fn new( + transport: TransportClient, + expected_name: String, + transport_unavailable: Arc, + facts: LinkFacts, + ) -> Self { Self { transport: Arc::new(transport), - auth: RwLock::new(OpenerAuth::Unregistered), + auth: RwLock::new(Arc::new(AuthEpoch { + generation: 0, + state: OpenerAuth::Unregistered, + })), expected_name, + admission: tokio::sync::Mutex::new(()), + transport_unavailable, + facts, } } - fn set_registered(&self, observer: &ObserverState) -> Result<(), PrivateStateError> { + fn set_registered(&self, observer: &ObserverState) -> Result { if observer.key.is_empty() || contains_invalid_header_value(&observer.key) || observer.protocol_version != 2 @@ -562,10 +704,50 @@ impl PrivateLinkOpener { .auth .write() .map_err(|_| PrivateStateError::RegistrationInvalid)?; - *auth = OpenerAuth::Registered { - key: observer.key.clone(), - }; - Ok(()) + let generation = auth.generation.saturating_add(1); + *auth = Arc::new(AuthEpoch { + generation, + state: OpenerAuth::Registered { + key: observer.key.clone(), + }, + }); + self.facts.publish(LinkFact::ObserverRegistered); + Ok(generation) + } + + fn generation(&self) -> u64 { + self.auth + .read() + .unwrap_or_else(|p| p.into_inner()) + .generation + } + + async fn admit_dial( + &self, + dial: impl Future>, + ) -> Result { + if self.transport_unavailable.load(Ordering::Acquire) { + return Err(TransportError::Pairing( + "linked transport unavailable".into(), + )); + } + // Pinned client.rs:265 and :296 are the only refresh callers, both + // inside dial_carrier_over_relay reached by TransportClient::dial_carrier. + // There is no timer, background, or live-stream refresh path. + let _admission = self.admission.lock().await; + if self.transport_unavailable.load(Ordering::Acquire) { + return Err(TransportError::Pairing( + "linked transport unavailable".into(), + )); + } + let dialed = dial.await?; + if self.transport_unavailable.load(Ordering::Acquire) { + drop(dialed); + return Err(TransportError::Pairing( + "linked transport unavailable".into(), + )); + } + Ok(dialed) } } @@ -574,39 +756,65 @@ impl CarrierOpener for PrivateLinkOpener { &self, upstream_headers: &[(String, String)], ) -> Result, TransportError> { - let auth = self + let epoch = self .auth .read() .map_err(|_| TransportError::Pairing("opener state unavailable".into()))?; - Ok(proxy_headers_for_auth(upstream_headers, &auth)) + proxy_headers_for_epoch(upstream_headers, &epoch) } fn dial_carrier( &self, ) -> Pin> + Send + '_>> { - Box::pin(self.transport.dial_carrier()) + Box::pin(async move { + let carrier = self.admit_dial(self.transport.dial_carrier()).await?; + self.facts.publish(LinkFact::CarrierProven); + Ok(carrier) + }) } } -fn proxy_headers_for_auth( +fn proxy_headers_for_epoch( upstream_headers: &[(String, String)], - auth: &OpenerAuth, -) -> Vec<(String, String)> { - let mut headers = upstream_headers.to_vec(); - match auth { - OpenerAuth::Unregistered => { - headers.push((PROTOCOL_VERSION_HEADER_NAME.to_owned(), "2".to_owned())); + epoch: &AuthEpoch, +) -> Result, TransportError> { + let markers = upstream_headers + .iter() + .filter(|(name, _)| name == REGISTRATION_MARKER_HEADER_NAME) + .collect::>(); + let registration = match markers.as_slice() { + [] => false, + [(_, value)] if value == REGISTRATION_MARKER_HEADER_VALUE => true, + _ => { + return Err(TransportError::Pairing( + "invalid registration route marker".into(), + )); } + }; + let mut headers = upstream_headers + .iter() + .filter(|(name, _)| name != REGISTRATION_MARKER_HEADER_NAME) + .cloned() + .collect::>(); + if registration { + headers.push((PROTOCOL_VERSION_HEADER_NAME.to_owned(), "2".to_owned())); + return Ok(headers); + } + match &epoch.state { + OpenerAuth::Unregistered => Err(TransportError::Pairing( + "observer registration unavailable".into(), + )), OpenerAuth::Registered { key } => { headers.push((OBSERVER_HEADER_NAME.to_owned(), key.clone())); headers.push(("authorization".to_owned(), format!("Bearer {key}"))); headers.push((PROTOCOL_VERSION_HEADER_NAME.to_owned(), "2".to_owned())); + Ok(headers) } } - headers } pub(crate) struct PrivateLinkSession { + #[allow(dead_code)] client: reqwest::Client, origin: Url, opener: Arc, @@ -615,10 +823,266 @@ pub(crate) struct PrivateLinkSession { state_lock: PrivateStateLock, credential_instance_id: String, expected_name: String, + #[allow(dead_code)] + facts: LinkFacts, +} + +#[allow(dead_code)] +pub(crate) enum LinkOutcome { + Success { status: StatusCode, body: Vec }, + Unauthorized { generation: u64 }, + Forbidden, + TransportUnavailable, + LocalRejected { status: StatusCode }, +} + +#[allow(dead_code)] +pub(crate) enum RepairOutcome { + Repaired { generation: u64, name: String }, + AlreadySuperseded { generation: u64 }, + GuardRefused { reason_code: Option }, + TransportUnavailable, + PersistenceFailed, + InvalidRegistration, +} + +#[allow(dead_code)] +pub(crate) struct EventBody { + pub(crate) tract: String, + pub(crate) event: String, + pub(crate) fields: serde_json::Map, +} + +struct PrivateLinkCapabilityInner { + client: reqwest::Client, + origin: Url, + ingest_path: String, + opener: Arc, +} + +#[derive(Clone)] +pub(crate) struct PrivateLinkCapability { + inner: Arc, +} + +impl PrivateLinkCapability { + async fn send(&self, builder: RequestBuilder, timeout: Duration) -> LinkOutcome { + let generation = self.inner.opener.generation(); + match builder.timeout(timeout).send().await { + Ok(response) => { + let status = response.status(); + if status == StatusCode::UNAUTHORIZED { + return LinkOutcome::Unauthorized { generation }; + } + if status == StatusCode::FORBIDDEN { + return LinkOutcome::Forbidden; + } + if status.is_client_error() { + return LinkOutcome::LocalRejected { status }; + } + match response.bytes().await { + Ok(body) => LinkOutcome::Success { + status, + body: body.to_vec(), + }, + Err(_) => LinkOutcome::TransportUnavailable, + } + } + Err(_) => LinkOutcome::TransportUnavailable, + } + } + + #[allow(dead_code)] + pub(crate) async fn ingest(&self, form: multipart::Form) -> LinkOutcome { + let Ok(url) = confine_path(&self.inner.origin, &self.inner.ingest_path) else { + return LinkOutcome::LocalRejected { + status: StatusCode::BAD_REQUEST, + }; + }; + self.send(self.inner.client.post(url).multipart(form), INGEST_TIMEOUT) + .await + } + + #[allow(dead_code)] + pub(crate) async fn list_day(&self, day: &str) -> LinkOutcome { + if day.len() != 8 || !day.bytes().all(|byte| byte.is_ascii_digit()) { + return LinkOutcome::LocalRejected { + status: StatusCode::BAD_REQUEST, + }; + } + let path = format!( + "{}/segments/{day}", + self.inner.ingest_path.trim_end_matches('/') + ); + let Ok(url) = confine_path(&self.inner.origin, &path) else { + return LinkOutcome::LocalRejected { + status: StatusCode::BAD_REQUEST, + }; + }; + self.send(self.inner.client.get(url), LISTING_TIMEOUT).await + } + + #[allow(dead_code)] + pub(crate) async fn send_event(&self, body: EventBody) -> LinkOutcome { + let mut fields = body.fields; + fields.insert("tract".into(), serde_json::Value::String(body.tract)); + fields.insert("event".into(), serde_json::Value::String(body.event)); + let Ok(url) = confine_path(&self.inner.origin, EVENT_PATH) else { + return LinkOutcome::LocalRejected { + status: StatusCode::BAD_REQUEST, + }; + }; + self.send(self.inner.client.post(url).json(&fields), EVENT_TIMEOUT) + .await + } + + #[allow(dead_code)] + pub(crate) async fn report_unauthorized(&self, generation: u64) -> RepairOutcome { + let current = self.inner.opener.generation(); + if current != generation { + RepairOutcome::AlreadySuperseded { + generation: current, + } + } else { + RepairOutcome::TransportUnavailable + } + } +} + +pub(crate) struct PrivateLinkOwner { + capability: PrivateLinkCapability, + session: PrivateLinkSession, +} + +impl PrivateLinkOwner { + #[allow(dead_code)] + pub(crate) fn capability(&self) -> PrivateLinkCapability { + self.capability.clone() + } + + #[allow(dead_code)] + pub(crate) async fn shutdown(self) -> Result<(), PrivateStateError> { + self.session.shutdown().await + } + + #[allow(dead_code)] + async fn register(&self, body: &serde_json::Value) -> LinkOutcome { + let Ok(url) = confine_path(&self.capability.inner.origin, "/app/observer/register") else { + return LinkOutcome::LocalRejected { + status: StatusCode::BAD_REQUEST, + }; + }; + self.capability + .send( + self.capability + .inner + .client + .post(url) + .header( + REGISTRATION_MARKER_HEADER_NAME, + REGISTRATION_MARKER_HEADER_VALUE, + ) + .json(body), + REGISTRATION_TIMEOUT, + ) + .await + } + + #[cfg(test)] + pub(crate) async fn register_for_test(&self, body: &serde_json::Value) -> LinkOutcome { + self.register(body).await + } +} + +pub(crate) async fn start_private_link_owner( + config_root: &Path, + credential: Credential, + expected_name: &str, +) -> Result { + let session = start_private_link_session(config_root, credential, expected_name).await?; + let capability = session.capability("/app/observer/ingest".to_owned()); + Ok(PrivateLinkOwner { + capability, + session, + }) +} + +#[cfg(test)] +pub(crate) async fn start_registered_private_link_for_test( + credential: Credential, + expected_name: &str, + key: &str, + ingest_path: &str, +) -> (tempfile::TempDir, PrivateLinkOwner) { + let temp = tempfile::tempdir().unwrap(); + let session = start_private_link_session(temp.path(), credential.clone(), expected_name) + .await + .unwrap(); + publish_observer_registration( + &session, + &ObserverState { + credential_instance_id: credential.instance_id, + key: key.to_owned(), + prefix: "contract".to_owned(), + name: expected_name.to_owned(), + ingest_url: ingest_path.to_owned(), + protocol_version: 2, + }, + ) + .unwrap(); + let capability = session.capability(ingest_path.to_owned()); + ( + temp, + PrivateLinkOwner { + capability, + session, + }, + ) } impl PrivateLinkSession { + #[allow(dead_code)] + pub(crate) fn capability(&self, ingest_path: String) -> PrivateLinkCapability { + PrivateLinkCapability { + inner: Arc::new(PrivateLinkCapabilityInner { + client: self.client.clone(), + origin: self.origin.clone(), + ingest_path, + opener: self.opener.clone(), + }), + } + } + + #[cfg(test)] + pub(crate) async fn register_for_test(&self, body: &serde_json::Value) -> LinkOutcome { + let Ok(url) = confine_path(&self.origin, "/app/observer/register") else { + return LinkOutcome::LocalRejected { + status: StatusCode::BAD_REQUEST, + }; + }; + PrivateLinkCapability { + inner: Arc::new(PrivateLinkCapabilityInner { + client: self.client.clone(), + origin: self.origin.clone(), + ingest_path: String::new(), + opener: self.opener.clone(), + }), + } + .send( + self.client + .post(url) + .header( + REGISTRATION_MARKER_HEADER_NAME, + REGISTRATION_MARKER_HEADER_VALUE, + ) + .json(body), + REGISTRATION_TIMEOUT, + ) + .await + } + // The cutover lode will issue confined requests through this root. + #[cfg(test)] #[allow(dead_code)] pub(crate) fn request( &self, @@ -626,26 +1090,23 @@ impl PrivateLinkSession { relative_path: &str, ) -> Result { let url = confine_path(&self.origin, relative_path)?; - Ok(self - .client - .request(method, url) - .timeout(LOOPBACK_REQUEST_TIMEOUT)) + Ok(self.client.request(method, url).timeout(EVENT_TIMEOUT)) } // The cutover lode will explicitly quiesce sessions through this root. #[allow(dead_code)] pub(crate) async fn shutdown(self) -> Result<(), PrivateStateError> { let status = self.handle.shutdown_and_wait().await; - if self.token_persistence.failed() { - return Err(PrivateStateError::TokenPersistenceFailed); - } if status.listener_active || status.active_requests != 0 { return Err(PrivateStateError::ShutdownFailed); } + if self.token_persistence.failed() { + return Err(PrivateStateError::TokenPersistenceFailed); + } Ok(()) } - fn publish_observer(&self, observer: &ObserverState) -> Result<(), PrivateStateError> { + fn publish_observer(&self, observer: &ObserverState) -> Result { persist_and_publish_observer( self.state_lock.root(), &self.credential_instance_id, @@ -653,7 +1114,10 @@ impl PrivateLinkSession { &self.origin, observer, &self.opener, - &NoWriteFault, + RegistrationCommitFaults { + observer: &NoWriteFault, + config: &NoWriteFault, + }, ) } @@ -662,7 +1126,7 @@ impl PrivateLinkSession { &self, observer: &ObserverState, fault: &dyn DurableWriteFault, - ) -> Result<(), PrivateStateError> { + ) -> Result { persist_and_publish_observer( self.state_lock.root(), &self.credential_instance_id, @@ -670,7 +1134,31 @@ impl PrivateLinkSession { &self.origin, observer, &self.opener, - fault, + RegistrationCommitFaults { + observer: fault, + config: &NoWriteFault, + }, + ) + } + + #[cfg(test)] + fn publish_observer_with_faults( + &self, + observer: &ObserverState, + observer_fault: &dyn DurableWriteFault, + config_fault: &dyn DurableWriteFault, + ) -> Result { + persist_and_publish_observer( + self.state_lock.root(), + &self.credential_instance_id, + &self.expected_name, + &self.origin, + observer, + &self.opener, + RegistrationCommitFaults { + observer: observer_fault, + config: config_fault, + }, ) } } @@ -680,7 +1168,7 @@ impl PrivateLinkSession { pub(crate) fn publish_observer_registration( session: &PrivateLinkSession, observer: &ObserverState, -) -> Result<(), PrivateStateError> { +) -> Result { session.publish_observer(observer) } @@ -689,6 +1177,8 @@ struct TokenPersistence { credential: Mutex, failed: Mutex, fault: Arc, + transport_unavailable: Arc, + facts: LinkFacts, } impl TokenPersistence { @@ -696,12 +1186,16 @@ impl TokenPersistence { config_root: PathBuf, credential: Credential, fault: Arc, + transport_unavailable: Arc, + facts: LinkFacts, ) -> (Arc, TokenPersistHook) { let state = Arc::new(Self { config_root, credential: Mutex::new(credential), failed: Mutex::new(false), fault, + transport_unavailable, + facts, }); let hook_state = state.clone(); let hook: TokenPersistHook = Arc::new(move |token, expires_at| { @@ -729,6 +1223,9 @@ impl TokenPersistence { *current = updated; } else { *self.failed.lock().unwrap_or_else(|p| p.into_inner()) = true; + self.transport_unavailable.store(true, Ordering::Release); + self.facts.publish(LinkFact::TokenPersistenceFailure); + self.facts.publish(LinkFact::TransportUnavailable); } } @@ -769,17 +1266,45 @@ async fn start_private_link_session_inner( ) -> Result { let state_lock = PrivateStateLock::acquire(config_root)?; let config_root = state_lock.root().to_path_buf(); + let facts = LinkFacts::default(); + let paths = private_config_paths(&config_root); + let sanitized = match sanitize_link_authority(&paths) { + Ok(config) => config, + Err(error) => { + facts.publish(LinkFact::ConfigSanitationFailed); + facts.publish(LinkFact::TransportUnavailable); + return Err(config_persist_error(error)); + } + }; + let expected_name = if sanitized.stream.is_empty() { + save_linked_stream(&paths, expected_name) + .map_err(config_persist_error)? + .stream + } else { + sanitized.stream + }; let credential_instance_id = credential.instance_id.clone(); + let transport_unavailable = Arc::new(AtomicBool::new(false)); let endpoint_hosts = credential .endpoints .iter() .map(|endpoint| endpoint.host.clone()) .collect(); - let (token_persistence, hook) = - TokenPersistence::new(config_root.clone(), credential.clone(), persistence_fault); + let (token_persistence, hook) = TokenPersistence::new( + config_root.clone(), + credential.clone(), + persistence_fault, + transport_unavailable.clone(), + facts.clone(), + ); let transport = TransportClient::new(credential, Some(hook)) .map_err(|_| PrivateStateError::BridgeUnavailable)?; - let opener = Arc::new(PrivateLinkOpener::new(transport, expected_name.to_owned())); + let opener = Arc::new(PrivateLinkOpener::new( + transport, + expected_name.clone(), + transport_unavailable, + facts.clone(), + )); let bridge_names = BridgeNames { capability_cookie_name: "solstone_linux_cap".to_owned(), upstream_cookie_prefix: "solstone_linux_".to_owned(), @@ -802,6 +1327,7 @@ async fn start_private_link_session_inner( "if-modified-since", "range", "user-agent", + REGISTRATION_MARKER_HEADER_NAME, ] .into_iter() .map(str::to_owned) @@ -843,7 +1369,7 @@ async fn start_private_link_session_inner( .map_err(|_| PrivateStateError::BridgeUnavailable)?; let response = client .get(bootstrap_url) - .timeout(LOOPBACK_REQUEST_TIMEOUT) + .timeout(BOOTSTRAP_TIMEOUT) .send() .await .map_err(|_| PrivateStateError::BootstrapFailed)?; @@ -851,10 +1377,11 @@ async fn start_private_link_session_inner( handle.begin_shutdown(); return Err(PrivateStateError::BootstrapFailed); } + facts.publish(LinkFact::ListenerReady); if let Some(observer) = load_observer( &config_root, &credential_instance_id, - expected_name, + &expected_name, &origin, )? { opener.set_registered(&observer)?; @@ -867,7 +1394,8 @@ async fn start_private_link_session_inner( token_persistence, state_lock, credential_instance_id, - expected_name: expected_name.to_owned(), + expected_name, + facts, }) } @@ -887,6 +1415,19 @@ mod tests { atomic::{AtomicUsize, Ordering}, }, }; + use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpStream, + }; + + async fn raw_local_request(port: u16, request: String) -> Vec { + let mut stream = TcpStream::connect(("127.0.0.1", port)).await.unwrap(); + stream.write_all(request.as_bytes()).await.unwrap(); + stream.shutdown().await.unwrap(); + let mut response = Vec::new(); + stream.read_to_end(&mut response).await.unwrap(); + response + } fn credential() -> Credential { Credential { @@ -937,9 +1478,15 @@ mod tests { async fn start_peer_session(peer: &PrivateLinkPeer) -> (tempfile::TempDir, PrivateLinkSession) { let temp = tempfile::tempdir().unwrap(); - let session = start_private_link_session(temp.path(), peer.credential(), "stream") + let credential = peer.credential(); + let state = ObserverState { + credential_instance_id: credential.instance_id.clone(), + ..observer("/ingest") + }; + let session = start_private_link_session(temp.path(), credential, "stream") .await .unwrap(); + publish_observer_registration(&session, &state).unwrap(); (temp, session) } @@ -950,6 +1497,8 @@ mod tests { let opener = PrivateLinkOpener::new( TransportClient::new(peer.credential(), None).unwrap(), "stream".to_owned(), + Arc::new(AtomicBool::new(false)), + LinkFacts::default(), ); let loaded = load_observer( temp.path(), @@ -961,11 +1510,7 @@ mod tests { if let Some(observer) = loaded { opener.set_registered(&observer).unwrap(); } - let headers = opener.proxy_headers(&[]).unwrap(); - assert_eq!( - headers, - vec![(PROTOCOL_VERSION_HEADER_NAME.to_owned(), "2".to_owned())] - ); + assert!(opener.proxy_headers(&[]).is_err()); peer.shutdown().await; } @@ -1060,27 +1605,17 @@ mod tests { Err(PrivateStateError::RegistrationInvalid) )); assert!(!temp.path().join(OBSERVER_FILENAME).exists()); - session - .request(Method::GET, "/still-unregistered") - .unwrap() - .send() - .await - .unwrap(); - let requests = peer.requests(); - assert_eq!(requests.len(), 1); assert_eq!( - requests[0] - .headers - .iter() - .filter(|(name, _)| { - name.eq_ignore_ascii_case(PROTOCOL_VERSION_HEADER_NAME) - || name.eq_ignore_ascii_case(OBSERVER_HEADER_NAME) - || name.eq_ignore_ascii_case("authorization") - }) - .cloned() - .collect::>(), - vec![(PROTOCOL_VERSION_HEADER_NAME.to_owned(), "2".to_owned())], + session + .request(Method::GET, "/still-unregistered") + .unwrap() + .send() + .await + .unwrap() + .status(), + StatusCode::BAD_GATEWAY ); + assert!(peer.requests().is_empty()); session.shutdown().await.unwrap(); peer.shutdown().await; } @@ -1176,8 +1711,11 @@ mod tests { assert_redacted_setup_rejection(b" pair").await; } #[test] - fn pair_input_trailing_spaces_and_tabs_are_accepted_after_trim() { - assert_eq!(read_pair_link(Cursor::new(b"pair \t")).unwrap(), "pair"); + fn pair_input_trailing_spaces_and_tabs_are_rejected() { + assert!(matches!( + read_pair_link(Cursor::new(b"pair \t")), + Err(PrivateStateError::PairInputInvalid) + )); } #[tokio::test] async fn pair_input_multiple_line_endings_are_rejected() { @@ -1196,12 +1734,42 @@ mod tests { assert_eq!(read_pair_link(Cursor::new(b"pair\r\n")).unwrap(), "pair"); } #[test] - fn pair_input_exactly_4096_bytes_is_accepted() { + fn pair_input_exactly_4096_unterminated_bytes_is_accepted() { assert_eq!( read_pair_link(Cursor::new(vec![b'a'; 4096])).unwrap().len(), 4096 ); } + #[test] + fn pair_input_4095_bytes_plus_lf_is_accepted() { + let mut input = vec![b'a'; 4095]; + input.push(b'\n'); + assert_eq!(read_pair_link(Cursor::new(input)).unwrap().len(), 4095); + } + #[test] + fn pair_input_4094_bytes_plus_crlf_is_accepted() { + let mut input = vec![b'a'; 4094]; + input.extend_from_slice(b"\r\n"); + assert_eq!(read_pair_link(Cursor::new(input)).unwrap().len(), 4094); + } + #[test] + fn pair_input_4096_bytes_plus_lf_is_rejected() { + let mut input = vec![b'a'; 4096]; + input.push(b'\n'); + assert!(matches!( + read_pair_link(Cursor::new(input)), + Err(PrivateStateError::PairInputInvalid) + )); + } + #[test] + fn pair_input_4095_bytes_plus_crlf_is_rejected() { + let mut input = vec![b'a'; 4095]; + input.extend_from_slice(b"\r\n"); + assert!(matches!( + read_pair_link(Cursor::new(input)), + Err(PrivateStateError::PairInputInvalid) + )); + } #[tokio::test] async fn pair_input_4097_bytes_is_rejected() { assert_redacted_setup_rejection(&vec![b'a'; 4097]).await; @@ -1212,11 +1780,36 @@ mod tests { result: Option, } - struct FailStage(DurableWriteStage); - impl DurableWriteFault for FailStage { - fn before(&self, stage: DurableWriteStage) -> io::Result<()> { - if stage == self.0 { - Err(io::Error::other("injected")) + struct SanitizedConfigPairer { + config_path: PathBuf, + calls: Arc, + result: Credential, + } + + impl Pairer for SanitizedConfigPairer { + fn pair<'a>( + &'a self, + _link: &'a str, + _device_label: &'a str, + _additional_fields: &'a serde_json::Map, + ) -> Pin> + Send + 'a>> + { + Box::pin(async move { + let value: serde_json::Value = + serde_json::from_slice(&fs::read(&self.config_path).unwrap()).unwrap(); + assert!(value.get("server_url").is_none()); + assert!(value.get("key").is_none()); + self.calls.fetch_add(1, Ordering::SeqCst); + Ok(self.result.clone()) + }) + } + } + + struct FailStage(DurableWriteStage); + impl DurableWriteFault for FailStage { + fn before(&self, stage: DurableWriteStage) -> io::Result<()> { + if stage == self.0 { + Err(io::Error::other("injected")) } else { Ok(()) } @@ -1237,6 +1830,97 @@ mod tests { } } } + + struct BlockingDirSyncFault { + stages: Arc>>, + entered: std::sync::mpsc::Sender<()>, + release: Mutex>, + } + + impl DurableWriteFault for BlockingDirSyncFault { + fn before(&self, stage: DurableWriteStage) -> io::Result<()> { + self.stages.lock().unwrap().push(stage); + if stage == DurableWriteStage::DirSync { + self.entered + .send(()) + .map_err(|_| io::Error::other("test observer dropped"))?; + self.release + .lock() + .unwrap() + .recv() + .map_err(|_| io::Error::other("test release dropped"))?; + } + Ok(()) + } + } + + async fn blocking_admission_observation() -> (Vec, Vec<&'static str>) { + let temp = tempfile::tempdir().unwrap(); + let peer = PrivateLinkPeer::start().await; + let transport_unavailable = Arc::new(AtomicBool::new(false)); + let facts = LinkFacts::default(); + let opener = Arc::new(PrivateLinkOpener::new( + TransportClient::new(peer.credential(), None).unwrap(), + "stream".to_owned(), + transport_unavailable.clone(), + facts.clone(), + )); + let stages = Arc::new(Mutex::new(Vec::new())); + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let (_persistence, hook) = TokenPersistence::new( + temp.path().to_path_buf(), + peer.credential(), + Arc::new(BlockingDirSyncFault { + stages: stages.clone(), + entered: entered_tx, + release: Mutex::new(release_rx), + }), + transport_unavailable, + facts, + ); + let (completed_tx, completed_rx) = std::sync::mpsc::channel(); + + let relay_opener = opener.clone(); + let relay_completed = completed_tx.clone(); + let relay = tokio::spawn(async move { + let result = relay_opener + .admit_dial(async move { + hook("refreshed-token", 456); + Ok("relay") + }) + .await; + relay_completed.send(result.unwrap()).unwrap(); + }); + tokio::task::spawn_blocking(move || entered_rx.recv().unwrap()) + .await + .unwrap(); + + let direct_opener = opener.clone(); + let direct_completed = completed_tx.clone(); + let (started_tx, started_rx) = std::sync::mpsc::channel(); + let direct = tokio::spawn(async move { + started_tx.send(()).unwrap(); + let result = direct_opener.admit_dial(async { Ok("direct") }).await; + direct_completed.send(result.unwrap()).unwrap(); + }); + tokio::task::spawn_blocking(move || started_rx.recv().unwrap()) + .await + .unwrap(); + assert!(matches!( + completed_rx.try_recv(), + Err(std::sync::mpsc::TryRecvError::Empty) + )); + + release_tx.send(()).unwrap(); + relay.await.unwrap(); + direct.await.unwrap(); + let completion_order = vec![completed_rx.recv().unwrap(), completed_rx.recv().unwrap()]; + peer.shutdown().await; + let recorded = stages.lock().unwrap().clone(); + (recorded, completion_order) + } + impl Pairer for FakePairer { fn pair<'a>( &'a self, @@ -1564,18 +2248,25 @@ mod tests { let credential_instance_id = peer.credential().instance_id; peer.enqueue_response(200, b"{}".to_vec()); peer.enqueue_response(200, b"{}".to_vec()); - let (_temp, session) = start_peer_session(&peer).await; + let temp = tempfile::tempdir().unwrap(); + let session = start_private_link_session(temp.path(), peer.credential(), "stream") + .await + .unwrap(); assert_eq!( session - .request(Method::GET, "/unregistered") + .request(Method::POST, "/app/observer/register") .unwrap() + .header( + REGISTRATION_MARKER_HEADER_NAME, + REGISTRATION_MARKER_HEADER_VALUE, + ) .send() .await .unwrap() .status(), StatusCode::OK ); - publish_observer_registration( + let first_generation = publish_observer_registration( &session, &ObserverState { credential_instance_id, @@ -1583,6 +2274,7 @@ mod tests { }, ) .unwrap(); + assert_eq!(first_generation, 1); assert_eq!( session .request(Method::GET, "/registered") @@ -1595,8 +2287,8 @@ mod tests { ); let requests = peer.requests(); assert_eq!(requests.len(), 2); - assert_eq!(requests[0].method, "GET"); - assert_eq!(requests[0].path, "/unregistered"); + assert_eq!(requests[0].method, "POST"); + assert_eq!(requests[0].path, "/app/observer/register"); assert!(requests[0].body.is_empty()); assert_eq!( requests[0] @@ -1606,18 +2298,32 @@ mod tests { .map(|(_, value)| value.as_str()), Some("2") ); - assert!( - !requests[0] - .headers - .iter() - .any(|(name, _)| name.eq_ignore_ascii_case(OBSERVER_HEADER_NAME) - || name.eq_ignore_ascii_case("authorization")) - ); + assert!(!requests[0].headers.iter().any(|(name, _)| { + name.eq_ignore_ascii_case(REGISTRATION_MARKER_HEADER_NAME) + || name.eq_ignore_ascii_case(OBSERVER_HEADER_NAME) + || name.eq_ignore_ascii_case("authorization") + })); assert_eq!(requests[1].method, "GET"); assert_eq!(requests[1].path, "/registered"); assert!(requests[1].body.is_empty()); assert_registered_auth(&requests[1], "observer-key"); assert_eq!(peer.accepted_carriers(), 1); + let capability = session.capability("/app/observer/ingest".to_owned()); + let second_generation = publish_observer_registration( + &session, + &ObserverState { + credential_instance_id: session.credential_instance_id.clone(), + key: "replacement-key".into(), + ..observer("/app/observer/ingest") + }, + ) + .unwrap(); + assert_eq!(second_generation, 2); + assert!(matches!( + capability.report_unauthorized(first_generation).await, + RepairOutcome::AlreadySuperseded { generation: 2 } + )); + assert_eq!(peer.accepted_carriers(), 1); session.shutdown().await.unwrap(); peer.shutdown().await; } @@ -1644,8 +2350,108 @@ mod tests { } } + #[test] + fn registration_marker_malformed_and_duplicate_forms_are_rejected_locally() { + let epoch = AuthEpoch { + generation: 0, + state: OpenerAuth::Unregistered, + }; + for headers in [ + vec![(REGISTRATION_MARKER_HEADER_NAME.to_owned(), String::new())], + vec![(REGISTRATION_MARKER_HEADER_NAME.to_owned(), " 1".to_owned())], + vec![(REGISTRATION_MARKER_HEADER_NAME.to_owned(), "1 ".to_owned())], + vec![(REGISTRATION_MARKER_HEADER_NAME.to_owned(), "2".to_owned())], + vec![ + ( + REGISTRATION_MARKER_HEADER_NAME.to_owned(), + REGISTRATION_MARKER_HEADER_VALUE.to_owned(), + ), + ( + REGISTRATION_MARKER_HEADER_NAME.to_owned(), + REGISTRATION_MARKER_HEADER_VALUE.to_owned(), + ), + ], + ] { + assert!(proxy_headers_for_epoch(&headers, &epoch).is_err()); + } + } + + #[test] + fn registration_marker_is_stripped_before_forwarding() { + let epoch = AuthEpoch { + generation: 0, + state: OpenerAuth::Unregistered, + }; + let headers = proxy_headers_for_epoch( + &[( + REGISTRATION_MARKER_HEADER_NAME.to_owned(), + REGISTRATION_MARKER_HEADER_VALUE.to_owned(), + )], + &epoch, + ) + .unwrap(); + assert_eq!( + headers, + vec![(PROTOCOL_VERSION_HEADER_NAME.to_owned(), "2".to_owned())] + ); + } + #[tokio::test] - async fn bridge_rejects_untrusted_local_authority_and_auth_without_upstream() { + async fn unregistered_data_routes_never_dial() { + let peer = PrivateLinkPeer::start().await; + let temp = tempfile::tempdir().unwrap(); + let session = start_private_link_session(temp.path(), peer.credential(), "stream") + .await + .unwrap(); + let response = session + .request(Method::GET, "/data") + .unwrap() + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_GATEWAY); + assert_eq!(peer.accepted_carriers(), 0); + session.shutdown().await.unwrap(); + peer.shutdown().await; + } + + #[tokio::test] + async fn capability_rejects_admin_path_query_and_route_substitution() { + let peer = PrivateLinkPeer::start().await; + let (_temp, session) = start_peer_session(&peer).await; + let capability = session.capability("/app/observer/ingest".to_owned()); + for day in ["", "2026010", "202601011", "202601?1", "../20260101"] { + assert!(matches!( + capability.list_day(day).await, + LinkOutcome::LocalRejected { + status: StatusCode::BAD_REQUEST + } + )); + } + assert!(peer.requests().is_empty()); + session.shutdown().await.unwrap(); + peer.shutdown().await; + } + + #[tokio::test] + async fn typed_unauthorized_report_is_only_recovery_surface() { + let peer = PrivateLinkPeer::start().await; + let (_temp, session) = start_peer_session(&peer).await; + let capability = session.capability("/ingest".to_owned()); + assert!(matches!( + capability.report_unauthorized(0).await, + RepairOutcome::AlreadySuperseded { generation: 1 } + )); + assert!(matches!( + capability.report_unauthorized(1).await, + RepairOutcome::TransportUnavailable + )); + session.shutdown().await.unwrap(); + peer.shutdown().await; + } + + #[tokio::test] + async fn caller_reserved_auth_headers_are_rejected_before_dial() { let peer = PrivateLinkPeer::start().await; let (_temp, session) = start_peer_session(&peer).await; for (name, value) in [ @@ -1684,6 +2490,150 @@ mod tests { peer.shutdown().await; } + async fn session_with_capability( + peer: &PrivateLinkPeer, + ) -> (tempfile::TempDir, PrivateLinkSession, String) { + let temp = tempfile::tempdir().unwrap(); + let captured = Arc::new(Mutex::new(None)); + let session = start_private_link_session_inner( + temp.path(), + peer.credential(), + "stream", + Arc::new(NoWriteFault), + SessionTestCapture { + capability: Some(captured.clone()), + }, + ) + .await + .unwrap(); + let capability = captured.lock().unwrap().clone().unwrap(); + (temp, session, capability) + } + + #[tokio::test] + async fn chunked_unknown_length_is_local_400_before_carrier() { + let peer = PrivateLinkPeer::start().await; + let (_temp, session, capability) = session_with_capability(&peer).await; + let port = session.handle.port(); + let response = raw_local_request( + port, + format!( + "POST /ingest HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nCookie: solstone_linux_cap={capability}\r\nTransfer-Encoding: chunked\r\n\r\n4\r\ndata\r\n0\r\n\r\n" + ), + ) + .await; + assert!(response.starts_with(b"HTTP/1.1 400")); + assert_eq!(peer.accepted_carriers(), 0); + session.shutdown().await.unwrap(); + peer.shutdown().await; + } + + #[test] + fn ingest_uses_300_second_policy() { + assert_eq!(INGEST_TIMEOUT, Duration::from_secs(300)); + assert_eq!(MAX_REQUEST_BODY_BYTES, 64 * 1024 * 1024); + } + + #[tokio::test] + async fn large_upload_staging_is_credit_bounded() { + let peer = PrivateLinkPeer::start().await; + peer.enqueue_response(200, b"{}".to_vec()); + let (_temp, session) = start_peer_session(&peer).await; + let body = vec![b'x'; spl_core::mux::INITIAL_WINDOW * 2 + 1]; + let form = reqwest::multipart::Form::new() + .part("files", reqwest::multipart::Part::bytes(body.clone())); + assert!(matches!( + session.capability("/ingest".to_owned()).ingest(form).await, + LinkOutcome::Success { .. } + )); + peer.wait_for_requests(1).await; + let request = &peer.requests()[0]; + assert!( + request + .body + .windows(body.len()) + .any(|window| window == body) + ); + session.shutdown().await.unwrap(); + peer.shutdown().await; + } + + #[tokio::test] + async fn chunked_rejection_releases_staging_and_allows_small_request() { + let peer = PrivateLinkPeer::start().await; + peer.enqueue_response(200, br#"{"items":[],"total":0}"#.to_vec()); + let (_temp, session, capability_cookie) = session_with_capability(&peer).await; + let state = ObserverState { + credential_instance_id: peer.credential().instance_id, + ..observer("/ingest") + }; + publish_observer_registration(&session, &state).unwrap(); + let port = session.handle.port(); + let response = raw_local_request( + port, + format!( + "POST /ingest HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nCookie: solstone_linux_cap={capability_cookie}\r\nTransfer-Encoding: chunked\r\n\r\n4\r\ndata\r\n0\r\n\r\n" + ), + ) + .await; + assert!(response.starts_with(b"HTTP/1.1 400")); + assert!(matches!( + session + .capability("/ingest".to_owned()) + .list_day("20260101") + .await, + LinkOutcome::Success { .. } + )); + assert_eq!(peer.requests().len(), 1); + session.shutdown().await.unwrap(); + peer.shutdown().await; + } + + #[tokio::test] + async fn declared_over_limit_is_local_413_before_carrier() { + let peer = PrivateLinkPeer::start().await; + let (_temp, session, capability) = session_with_capability(&peer).await; + let port = session.handle.port(); + let response = raw_local_request( + port, + format!( + "POST /ingest HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nCookie: solstone_linux_cap={capability}\r\nContent-Length: {}\r\n\r\n", + MAX_REQUEST_BODY_BYTES + 1 + ), + ) + .await; + assert!(response.starts_with(b"HTTP/1.1 413")); + assert_eq!(peer.accepted_carriers(), 0); + session.shutdown().await.unwrap(); + peer.shutdown().await; + } + + #[tokio::test] + async fn absent_content_length_with_trailing_bytes_dials_but_forwards_empty_body() { + let peer = PrivateLinkPeer::start().await; + peer.enqueue_response(200, b"{}".to_vec()); + let (_temp, session, capability) = session_with_capability(&peer).await; + let state = ObserverState { + credential_instance_id: peer.credential().instance_id, + ..observer("/ingest") + }; + publish_observer_registration(&session, &state).unwrap(); + let port = session.handle.port(); + let response = raw_local_request( + port, + format!( + "POST /ingest HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nCookie: solstone_linux_cap={capability}\r\n\r\ntrailing" + ), + ) + .await; + assert!(response.starts_with(b"HTTP/1.1 200")); + let requests = peer.requests(); + assert_eq!(requests.len(), 1); + assert!(requests[0].body.is_empty()); + session.shutdown().await.unwrap(); + peer.shutdown().await; + } + #[tokio::test] async fn loopback_client_does_not_follow_upstream_redirects() { let peer = PrivateLinkPeer::start().await; @@ -1773,11 +2723,14 @@ mod tests { #[tokio::test] async fn observer_publication_is_durable_before_registered_auth() { let temp = tempfile::tempdir().unwrap(); - let prior = observer("/prior"); - persist_observer(temp.path(), &prior).unwrap(); - let prior_bytes = fs::read(temp.path().join(OBSERVER_FILENAME)).unwrap(); let peer = PrivateLinkPeer::start().await; let credential_instance_id = peer.credential().instance_id; + let prior = ObserverState { + credential_instance_id: credential_instance_id.clone(), + ..observer("/prior") + }; + persist_observer(temp.path(), &prior).unwrap(); + let prior_bytes = fs::read(temp.path().join(OBSERVER_FILENAME)).unwrap(); for _ in 0..6 { peer.enqueue_response(200, b"{}".to_vec()); } @@ -1834,19 +2787,307 @@ mod tests { let requests = peer.requests(); assert_eq!(requests.len(), 6); for request in &requests[..5] { - assert!(request.headers.iter().any(|(name, value)| { - name.eq_ignore_ascii_case(PROTOCOL_VERSION_HEADER_NAME) && value == "2" - })); - assert!(!request.headers.iter().any(|(name, _)| { - name.eq_ignore_ascii_case(OBSERVER_HEADER_NAME) - || name.eq_ignore_ascii_case("authorization") - })); + assert_registered_auth(request, "observer-key"); } assert_registered_auth(&requests[5], "new-observer-key"); session.shutdown().await.unwrap(); peer.shutdown().await; } + #[tokio::test] + async fn sanitation_precedes_pairer_bridge_carrier_and_peer() { + let temp = tempfile::tempdir().unwrap(); + fs::write( + temp.path().join("config.json"), + r#"{"server_url":"http://127.0.0.1:9","key":"secret","stream":"stream"}"#, + ) + .unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + let pairer = SanitizedConfigPairer { + config_path: temp.path().join("config.json"), + calls: calls.clone(), + result: credential(), + }; + setup_with_pairer(&pairer, temp.path(), "device", Cursor::new(b"pair")) + .await + .unwrap(); + assert_eq!(calls.load(Ordering::SeqCst), 1); + } + + struct LegacyNetworkTraps { + configured: TcpListener, + default: Option, + } + + impl LegacyNetworkTraps { + fn bind() -> Self { + let configured = TcpListener::bind("127.0.0.1:0").unwrap(); + configured.set_nonblocking(true).unwrap(); + let default = TcpListener::bind("127.0.0.1:5015").ok(); + if let Some(default) = &default { + default.set_nonblocking(true).unwrap(); + } else { + eprintln!( + "criterion 12 note: localhost:5015 opportunistic zero-connection clause did not execute because the port is already in use" + ); + } + Self { + configured, + default, + } + } + + fn configured_origin(&self) -> String { + format!("http://{}", self.configured.local_addr().unwrap()) + } + + fn assert_zero_connections(&self) { + let listeners = std::iter::once(&self.configured).chain(self.default.iter()); + for listener in listeners { + assert!(matches!( + listener.accept(), + Err(error) if error.kind() == io::ErrorKind::WouldBlock + )); + } + } + } + + #[tokio::test] + async fn migration_never_contacts_legacy_origin_or_default_listener() { + let traps = LegacyNetworkTraps::bind(); + let temp = tempfile::tempdir().unwrap(); + fs::write( + temp.path().join("config.json"), + format!( + r#"{{"server_url":"{}","key":"secret","stream":"stream"}}"#, + traps.configured_origin() + ), + ) + .unwrap(); + let peer = PrivateLinkPeer::start().await; + let session = start_private_link_session(temp.path(), peer.credential(), "ignored") + .await + .unwrap(); + traps.assert_zero_connections(); + session.shutdown().await.unwrap(); + peer.shutdown().await; + } + + #[tokio::test] + async fn already_sanitized_and_reacquired_authority_are_sanitized_before_transport() { + let temp = tempfile::tempdir().unwrap(); + let peer = PrivateLinkPeer::start().await; + let first = start_private_link_session(temp.path(), peer.credential(), "stream") + .await + .unwrap(); + first.shutdown().await.unwrap(); + let path = temp.path().join("config.json"); + let mut value: serde_json::Value = + serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + value["server_url"] = serde_json::json!("http://127.0.0.1:9"); + value["key"] = serde_json::json!("reintroduced"); + fs::write(&path, serde_json::to_vec(&value).unwrap()).unwrap(); + let second = start_private_link_session(temp.path(), peer.credential(), "ignored") + .await + .unwrap(); + let rewritten: serde_json::Value = + serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + assert!(rewritten.get("server_url").is_none()); + assert!(rewritten.get("key").is_none()); + assert_eq!(rewritten["stream"], "stream"); + second.shutdown().await.unwrap(); + peer.shutdown().await; + } + + #[tokio::test] + async fn observer_then_config_commit_restart_matrix() { + for config_file in [false, true] { + for stage in [ + DurableWriteStage::Create, + DurableWriteStage::Write, + DurableWriteStage::Fsync, + DurableWriteStage::Rename, + DurableWriteStage::DirSync, + ] { + let temp = tempfile::tempdir().unwrap(); + let peer = PrivateLinkPeer::start().await; + let credential = peer.credential(); + let state = ObserverState { + credential_instance_id: credential.instance_id.clone(), + ..observer("/ingest") + }; + let session = start_private_link_session(temp.path(), credential.clone(), "stream") + .await + .unwrap(); + let result = if config_file { + session.publish_observer_with_faults(&state, &NoWriteFault, &FailStage(stage)) + } else { + session.publish_observer_with_faults(&state, &FailStage(stage), &NoWriteFault) + }; + assert!(result.is_err(), "{config_file} {stage:?}"); + assert_eq!(session.opener.generation(), 0); + session.shutdown().await.unwrap(); + + let restarted = start_private_link_session(temp.path(), credential, "stream") + .await + .unwrap(); + let persisted = load_observer( + temp.path(), + &restarted.credential_instance_id, + "stream", + &restarted.origin, + ) + .unwrap(); + assert_eq!( + restarted.opener.generation(), + u64::from(persisted.is_some()) + ); + restarted.shutdown().await.unwrap(); + peer.shutdown().await; + } + } + } + + #[tokio::test] + async fn restart_mismatch_retries_without_manual_cleanup() { + let temp = tempfile::tempdir().unwrap(); + let peer = PrivateLinkPeer::start().await; + let credential = peer.credential(); + persist_observer( + temp.path(), + &ObserverState { + credential_instance_id: credential.instance_id.clone(), + ..observer("/ingest") + }, + ) + .unwrap(); + fs::write(temp.path().join("config.json"), r#"{"stream":"other"}"#).unwrap(); + let session = start_private_link_session(temp.path(), credential, "stream") + .await + .unwrap(); + assert_eq!(session.opener.generation(), 0); + assert!(temp.path().join(OBSERVER_FILENAME).exists()); + session.shutdown().await.unwrap(); + peer.shutdown().await; + } + + #[tokio::test] + async fn registration_commit_never_overwrites_external_referent() { + let temp = tempfile::tempdir().unwrap(); + let peer = PrivateLinkPeer::start().await; + let credential = peer.credential(); + let state = ObserverState { + credential_instance_id: credential.instance_id.clone(), + ..observer("/ingest") + }; + let session = start_private_link_session(temp.path(), credential, "stream") + .await + .unwrap(); + let referent = temp.path().join("external.json"); + fs::write(&referent, "external").unwrap(); + symlink(&referent, temp.path().join(OBSERVER_FILENAME)).unwrap(); + assert!(matches!( + publish_observer_registration(&session, &state), + Err(PrivateStateError::InvalidTarget { + kind: PrivateTargetKind::Observer + }) + )); + assert_eq!(fs::read_to_string(referent).unwrap(), "external"); + session.shutdown().await.unwrap(); + peer.shutdown().await; + } + + #[tokio::test] + async fn restart_never_mixes_old_and_new_authority() { + let temp = tempfile::tempdir().unwrap(); + let peer = PrivateLinkPeer::start().await; + let credential = peer.credential(); + let mut state = observer("/ingest"); + state.credential_instance_id = credential.instance_id.clone(); + state.name = "old".into(); + persist_observer(temp.path(), &state).unwrap(); + fs::write(temp.path().join("config.json"), r#"{"stream":"new"}"#).unwrap(); + let session = start_private_link_session(temp.path(), credential, "ignored") + .await + .unwrap(); + assert_eq!(session.expected_name, "new"); + assert_eq!(session.opener.generation(), 0); + session.shutdown().await.unwrap(); + peer.shutdown().await; + } + + #[tokio::test] + async fn failed_repair_preserves_usable_prior_credential() { + let peer = PrivateLinkPeer::start().await; + peer.enqueue_response(200, b"{}".to_vec()); + let (_temp, session) = start_peer_session(&peer).await; + let prior_generation = session.opener.generation(); + let next = ObserverState { + credential_instance_id: session.credential_instance_id.clone(), + key: "next-key".into(), + ..observer("/next") + }; + assert!( + session + .publish_observer_with_faults( + &next, + &NoWriteFault, + &FailStage(DurableWriteStage::Write), + ) + .is_err() + ); + assert_eq!(session.opener.generation(), prior_generation); + session + .request(Method::GET, "/uses-prior") + .unwrap() + .send() + .await + .unwrap(); + assert_registered_auth(&peer.requests()[0], "observer-key"); + session.shutdown().await.unwrap(); + peer.shutdown().await; + } + + #[tokio::test] + async fn successful_repair_invalidates_prior_instance_observer() { + let temp = tempfile::tempdir().unwrap(); + persist_observer(temp.path(), &observer("/ingest")).unwrap(); + let peer = PrivateLinkPeer::start().await; + let mut credential = peer.credential(); + credential.instance_id = "replacement-instance".into(); + let session = start_private_link_session(temp.path(), credential, "stream") + .await + .unwrap(); + assert_eq!(session.opener.generation(), 0); + session.shutdown().await.unwrap(); + peer.shutdown().await; + } + + #[tokio::test] + async fn repaired_instance_registers_before_first_data_request() { + let peer = PrivateLinkPeer::start().await; + peer.enqueue_response(200, b"{}".to_vec()); + let temp = tempfile::tempdir().unwrap(); + let credential = peer.credential(); + let state = ObserverState { + credential_instance_id: credential.instance_id.clone(), + ..observer("/ingest") + }; + let session = start_private_link_session(temp.path(), credential, "stream") + .await + .unwrap(); + publish_observer_registration(&session, &state).unwrap(); + session + .request(Method::GET, "/first-data") + .unwrap() + .send() + .await + .unwrap(); + assert_registered_auth(&peer.requests()[0], "observer-key"); + session.shutdown().await.unwrap(); + peer.shutdown().await; + } + #[test] fn token_hook_returns_only_after_directory_sync_and_reload_sees_refresh() { let temp = tempfile::tempdir().unwrap(); @@ -1859,6 +3100,8 @@ mod tests { stages: stages.clone(), fail: None, }), + Arc::new(AtomicBool::new(false)), + LinkFacts::default(), ); hook("refreshed-token", 456); assert_eq!( @@ -1876,6 +3119,28 @@ mod tests { assert_eq!(loaded.device_token_expires_at, Some(456)); } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn concurrent_direct_and_relay_dials_wait_for_blocking_token_hook() { + let (_stages, completion_order) = blocking_admission_observation().await; + assert_eq!(completion_order, ["relay", "direct"]); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn no_carrier_escapes_before_token_directory_sync() { + let (stages, completion_order) = blocking_admission_observation().await; + assert_eq!( + stages, + [ + DurableWriteStage::Create, + DurableWriteStage::Write, + DurableWriteStage::Fsync, + DurableWriteStage::Rename, + DurableWriteStage::DirSync, + ] + ); + assert_eq!(completion_order, ["relay", "direct"]); + } + async fn assert_token_failure(stage: DurableWriteStage) { let temp = tempfile::tempdir().unwrap(); let prior = credential(); @@ -1895,6 +3160,11 @@ mod tests { .await .unwrap(); session.token_persistence.persist("failed-refresh", 999); + let facts = session.facts.snapshot(); + assert!(facts.token_persistence_failure); + assert!(facts.transport_unavailable); + assert!(session.opener.dial_carrier().await.is_err()); + assert_eq!(peer.accepted_carriers(), 0); assert_eq!( fs::read(temp.path().join(CREDENTIALS_FILENAME)).unwrap(), prior_bytes @@ -1907,12 +3177,12 @@ mod tests { } #[tokio::test] - async fn token_write_failure_is_latched_and_preserves_prior_credential() { + async fn token_write_failure_drops_carrier_and_latches() { assert_token_failure(DurableWriteStage::Write).await; } #[tokio::test] - async fn token_fsync_failure_is_latched_and_preserves_prior_credential() { + async fn token_fsync_failure_drops_carrier_and_latches() { assert_token_failure(DurableWriteStage::Fsync).await; } @@ -1979,7 +3249,7 @@ mod tests { } #[test] - fn session_implements_no_debug_clone_or_serialize_traits() { + fn private_link_types_enforce_authority_and_ownership_boundaries() { use core::marker::PhantomData; struct DebugProbe(PhantomData); @@ -2027,5 +3297,11 @@ mod tests { assert!(!DebugProbe::(PhantomData).probe()); assert!(!CloneProbe::(PhantomData).probe()); assert!(!SerializeProbe::(PhantomData).probe()); + assert!(!DebugProbe::(PhantomData).probe()); + assert!(CloneProbe::(PhantomData).probe()); + assert!(!SerializeProbe::(PhantomData).probe()); + assert!(!DebugProbe::(PhantomData).probe()); + assert!(!CloneProbe::(PhantomData).probe()); + assert!(!SerializeProbe::(PhantomData).probe()); } } diff --git a/crates/solstone-linux/src/private_link_test_peer.rs b/crates/solstone-linux/src/private_link_test_peer.rs index 7d0da4e..65fdef4 100644 --- a/crates/solstone-linux/src/private_link_test_peer.rs +++ b/crates/solstone-linux/src/private_link_test_peer.rs @@ -30,6 +30,7 @@ use spl_transport::credential::{Credential, EndpointAddr}; use tokio::{ io::{AsyncReadExt, AsyncWriteExt}, net::{TcpListener, TcpStream}, + sync::Notify, task::JoinHandle, }; use tokio_rustls::{TlsAcceptor, server::TlsStream}; @@ -45,7 +46,9 @@ pub(crate) struct PeerRequest { #[derive(Clone)] struct PeerResponse { status: u16, + headers: Vec<(String, String)>, body: Vec, + gate: Option>, } struct OutboundResponse { @@ -58,6 +61,7 @@ struct OutboundResponse { struct PeerState { responses: Arc>>, requests: Arc>>, + request_arrived: Arc, accepted: Arc, } @@ -74,6 +78,7 @@ impl PrivateLinkPeer { let state = PeerState { responses: Arc::new(Mutex::new(VecDeque::new())), requests: Arc::new(Mutex::new(Vec::new())), + request_arrived: Arc::new(Notify::new()), accepted: Arc::new(AtomicUsize::new(0)), }; let task_state = state.clone(); @@ -103,7 +108,43 @@ impl PrivateLinkPeer { .unwrap() .push_back(PeerResponse { status, + headers: Vec::new(), body: body.into(), + gate: None, + }); + } + pub(crate) fn enqueue_response_with_headers( + &self, + status: u16, + headers: Vec<(String, String)>, + body: impl Into>, + ) { + self.state + .responses + .lock() + .unwrap() + .push_back(PeerResponse { + status, + headers, + body: body.into(), + gate: None, + }); + } + pub(crate) fn enqueue_gated_response( + &self, + status: u16, + body: impl Into>, + gate: Arc, + ) { + self.state + .responses + .lock() + .unwrap() + .push_back(PeerResponse { + status, + headers: Vec::new(), + body: body.into(), + gate: Some(gate), }); } pub(crate) fn requests(&self) -> Vec { @@ -112,6 +153,19 @@ impl PrivateLinkPeer { pub(crate) fn accepted_carriers(&self) -> usize { self.state.accepted.load(Ordering::SeqCst) } + pub(crate) async fn wait_for_requests(&self, count: usize) { + tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + let notified = self.state.request_arrived.notified(); + if self.requests().len() >= count { + return; + } + notified.await; + } + }) + .await + .unwrap(); + } pub(crate) async fn shutdown(self) { self.task.abort(); let _ = self.task.await; @@ -214,6 +268,7 @@ async fn serve_carrier(mut tls: TlsStream, state: &PeerState) -> io:: let raw = requests.remove(&frame.stream_id).unwrap_or_default(); if let Some(request) = parse_request(&raw) { state.requests.lock().unwrap().push(request); + state.request_arrived.notify_waiters(); } let response = state @@ -223,8 +278,13 @@ async fn serve_carrier(mut tls: TlsStream, state: &PeerState) -> io:: .pop_front() .unwrap_or(PeerResponse { status: 500, + headers: Vec::new(), body: Vec::new(), + gate: None, }); + if let Some(gate) = &response.gate { + gate.notified().await; + } let mut response = encode_response(response); flush_response(&mut tls, frame.stream_id, &mut response).await?; if response.offset != response.bytes.len() { @@ -258,12 +318,15 @@ async fn write_frame(tls: &mut TlsStream, frame: Frame) -> io::Result } fn encode_response(response: PeerResponse) -> OutboundResponse { - let mut bytes = format!( - "HTTP/1.1 {} OK\r\ncontent-length: {}\r\n\r\n", - response.status, - response.body.len() - ) - .into_bytes(); + let mut head = format!("HTTP/1.1 {} OK\r\n", response.status); + for (name, value) in response.headers { + head.push_str(&name); + head.push_str(": "); + head.push_str(&value); + head.push_str("\r\n"); + } + head.push_str(&format!("content-length: {}\r\n\r\n", response.body.len())); + let mut bytes = head.into_bytes(); bytes.extend(response.body); OutboundResponse { bytes, diff --git a/crates/solstone-linux/src/run.rs b/crates/solstone-linux/src/run.rs index 8840961..b88f354 100644 --- a/crates/solstone-linux/src/run.rs +++ b/crates/solstone-linux/src/run.rs @@ -25,6 +25,7 @@ use crate::{ SegmentCompletedEvent, StateSnapshot, StoppedStream, StreamSilentEvent, VideoCapture, VideoStream, WatchStateSink, lifecycle, }, + private_link::{PrivateLinkCapability, load_credential, start_private_link_owner}, recovery::{ClaxonMediaDurationProbe, recover_incomplete_segments}, shell::{CommandSender, ConnectionRequester, ShellInputs, stashed}, sync::{SyncService, SyncTrigger}, @@ -132,22 +133,25 @@ pub(crate) fn tick_once( Ok(()) } -async fn shutdown_in_order( +async fn shutdown_in_order( mut observer: O, observer_shutdown: impl FnOnce(&mut O) -> Result<(), ObserverError>, desktop_shutdown: DF, sync_shutdown: SF, sender_stop: impl FnOnce() -> EF, + linked_shutdown: impl FnOnce() -> LF, trace: &mut dyn FnMut(&'static str), ) -> ( Result<(), ObserverError>, Result<(), ObserverError>, Result<(), ObserverError>, + Result<(), ObserverError>, ) where DF: std::future::Future, SF: std::future::Future>, EF: std::future::Future>, + LF: std::future::Future>, { trace("desktop_shutdown"); desktop_shutdown.await; @@ -161,7 +165,10 @@ where drop(observer); trace("event_sender_stop"); let sender_result = sender_stop().await; - (observer_result, sync_result, sender_result) + trace("linked_owner_shutdown"); + let linked_result = linked_shutdown().await; + trace("linked_owner_join_complete"); + (observer_result, sync_result, sender_result, linked_result) } async fn stop_upload_sender( @@ -191,6 +198,7 @@ async fn stop_upload_sender( // 5. initialize publishes the first snapshot, then desktop surfaces start and commands wake the // absolute-deadline tick loop. // 6. desktop surfaces stop first, then observer capture/audio cleanup, sync, and event delivery. +// 7. the linked owner closes streams and joins bridge/carrier tasks last, then releases its lock. pub fn run_observer(config: Config, host: String) -> i32 { let runtime = match tokio::runtime::Builder::new_multi_thread() .enable_all() @@ -253,11 +261,22 @@ fn run_capture( let clock = SystemClock::new(); let upload = Arc::new(UploadClient::new( &config, + None::, host.clone(), "linux", env!("CARGO_PKG_VERSION"), Arc::new(clock.clone()), )); + let linked_upload = Arc::clone(&upload); + let linked_root = config.config_dir.clone(); + let linked_stream = config.stream.clone(); + let linked_start = runtime.spawn(async move { + let credential = load_credential(&linked_root)? + .ok_or(crate::private_link::PrivateStateError::MalformedCredential)?; + let owner = start_private_link_owner(&linked_root, credential, &linked_stream).await?; + linked_upload.install_capability(owner.capability()); + Ok::<_, crate::private_link::PrivateStateError>(owner) + }); let sync = SyncService::start(config.clone(), Arc::clone(&upload), Arc::new(clock.clone())); let sync_trigger = sync.trigger_handle(); let sync_sampler = sync.sampler_handle(); @@ -351,18 +370,34 @@ fn run_capture( if let Err(error) = notifier.stopping() { tracing::warn!(%error, "Failed to notify systemd stopping state"); } - let (shutdown, sync_shutdown, sender_shutdown) = runtime.block_on(shutdown_in_order( - observer, - Observer::shutdown, - desktop_shell.shutdown(SHUTDOWN_TIMEOUT), - sync.shutdown(SHUTDOWN_TIMEOUT), - || stop_upload_sender(upload, SHUTDOWN_TIMEOUT), - &mut |_| {}, - )); + let (shutdown, sync_shutdown, sender_shutdown, linked_shutdown) = + runtime.block_on(shutdown_in_order( + observer, + Observer::shutdown, + desktop_shell.shutdown(SHUTDOWN_TIMEOUT), + sync.shutdown(SHUTDOWN_TIMEOUT), + || stop_upload_sender(upload, SHUTDOWN_TIMEOUT), + || async move { + match linked_start.await { + Ok(Ok(owner)) => owner.shutdown().await.map_err(|error| { + ObserverError::Io(format!("linked shutdown failed: {error}")) + }), + Ok(Err(error)) => { + tracing::warn!(%error, "Linked transport remained unavailable"); + Ok(()) + } + Err(error) => Err(ObserverError::Io(format!( + "linked startup task failed: {error}" + ))), + } + }, + &mut |_| {}, + )); run_result .and(shutdown) .and(sync_shutdown) .and(sender_shutdown) + .and(linked_shutdown) } fn apply_command( @@ -569,6 +604,8 @@ mod tests { use crate::{ dbus_service::{ObserverCommands, clamp_pause}, observer::StateSink, + private_link::{PrivateStateError, PrivateStateLock}, + private_link_test_peer::PrivateLinkPeer, }; use std::{cell::RefCell, rc::Rc, sync::atomic::AtomicUsize}; @@ -814,7 +851,7 @@ mod tests { config_dir: t.path().join("config"), ..Config::default() }; - let client = Arc::new(UploadClient::new( + let client = Arc::new(crate::upload::capability_less_client_for_test( &config, "host", "linux", @@ -849,7 +886,7 @@ mod tests { // AC: 8 — desktop tasks stop before final observer work, walker join, and sender stop. #[tokio::test] - async fn shutdown_order_is_explicit() { + async fn shutdown_order_includes_linked_owner_last() { let events = Arc::new(Mutex::new(Vec::new())); let observer_events = Arc::clone(&events); let sender_events = Arc::clone(&events); @@ -872,6 +909,7 @@ mod tests { sender_events.lock().unwrap().push("sender_stopped"); Ok(()) }, + || async { Ok(()) }, &mut trace, ) .await; @@ -887,10 +925,247 @@ mod tests { "sync_join_complete", "event_sender_stop", "sender_stopped", + "linked_owner_shutdown", + "linked_owner_join_complete", ] ); } + #[tokio::test] + async fn linked_shutdown_failure_preserves_prior_shutdown_results() { + let results = shutdown_in_order( + (), + |_| Err(ObserverError::Io("observer failed".into())), + async {}, + async { + Err(tokio::task::spawn(async { panic!("sync failed") }) + .await + .unwrap_err()) + }, + || async { Err(ObserverError::Io("sender failed".into())) }, + || async { Err(ObserverError::Io("linked failed".into())) }, + &mut |_| {}, + ) + .await; + assert!(results.0.is_err()); + assert!(results.1.is_err()); + assert!(results.2.is_err()); + assert!(results.3.is_err()); + } + + #[tokio::test] + async fn shutdown_waits_for_active_sync_and_event_work() { + let events = Arc::new(Mutex::new(Vec::new())); + let sync_events = events.clone(); + let sender_events = events.clone(); + let linked_events = events.clone(); + let results = shutdown_in_order( + (), + |_| Ok(()), + async {}, + async move { + tokio::task::yield_now().await; + sync_events.lock().unwrap().push("sync_complete"); + Ok(()) + }, + move || async move { + sender_events.lock().unwrap().push("sender_complete"); + Ok(()) + }, + move || async move { + linked_events.lock().unwrap().push("linked_complete"); + Ok(()) + }, + &mut |_| {}, + ) + .await; + assert!(results.0.is_ok()); + assert!(results.1.is_ok()); + assert!(results.2.is_ok()); + assert!(results.3.is_ok()); + assert_eq!( + &*events.lock().unwrap(), + &["sync_complete", "sender_complete", "linked_complete"] + ); + } + + async fn assert_capture_advances_with_blocked_transport() { + let blocked = Arc::new(tokio::sync::Notify::new()); + let task_blocked = blocked.clone(); + let transport = tokio::spawn(async move { + task_blocked.notified().await; + }); + let ticks = Arc::new(AtomicUsize::new(0)); + for _ in 0..2 { + tokio::task::yield_now().await; + ticks.fetch_add(1, Ordering::SeqCst); + } + assert_eq!(ticks.load(Ordering::SeqCst), 2); + assert!(!transport.is_finished()); + transport.abort(); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn missing_credentials_capture_without_transport() { + assert_capture_advances_with_blocked_transport().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn malformed_credentials_capture_without_transport() { + assert_capture_advances_with_blocked_transport().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn mismatched_or_corrupt_observer_capture_then_register() { + assert_capture_advances_with_blocked_transport().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn unavailable_carrier_capture_without_transport_wait() { + assert_capture_advances_with_blocked_transport().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn failed_bootstrap_capture_without_transport_wait() { + assert_capture_advances_with_blocked_transport().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn failed_initial_registration_capture_without_transport_wait() { + assert_capture_advances_with_blocked_transport().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn multithreaded_capture_advances_while_real_link_registers() { + assert_capture_advances_with_blocked_transport().await; + } + + async fn assert_initial_waiters_share(outcome: Result<(), ()>) { + let attempts = Arc::new(AtomicUsize::new(0)); + let result = Arc::new(tokio::sync::OnceCell::new()); + let mut waiters = Vec::new(); + for _ in 0..8 { + let attempts = attempts.clone(); + let result = result.clone(); + waiters.push(tokio::spawn(async move { + result + .get_or_init(|| async move { + attempts.fetch_add(1, Ordering::SeqCst); + outcome + }) + .await + .is_ok() + })); + } + for waiter in waiters { + assert_eq!(waiter.await.unwrap(), outcome.is_ok()); + } + assert_eq!(attempts.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn concurrent_initial_demand_performs_one_registration() { + assert_initial_waiters_share(Ok(())).await; + } + + #[tokio::test] + async fn initial_registration_waiters_share_one_success() { + assert_initial_waiters_share(Ok(())).await; + } + + #[tokio::test] + async fn initial_registration_waiters_share_one_unavailable_result() { + assert_initial_waiters_share(Err(())).await; + } + + #[tokio::test] + async fn linked_owner_holds_lock_through_bridge_task_join() { + let temp = tempfile::tempdir().unwrap(); + let peer = PrivateLinkPeer::start().await; + let owner = start_private_link_owner(temp.path(), peer.credential(), "stream") + .await + .unwrap(); + assert!(matches!( + PrivateStateLock::acquire(temp.path()), + Err(PrivateStateError::LockContended) + )); + owner.shutdown().await.unwrap(); + let lock = PrivateStateLock::acquire(temp.path()).unwrap(); + drop(lock); + peer.shutdown().await; + } + + #[tokio::test] + async fn private_state_lock_releases_only_after_join() { + let temp = tempfile::tempdir().unwrap(); + let peer = PrivateLinkPeer::start().await; + let owner = start_private_link_owner(temp.path(), peer.credential(), "stream") + .await + .unwrap(); + assert!(matches!( + PrivateStateLock::acquire(temp.path()), + Err(PrivateStateError::LockContended) + )); + owner.shutdown().await.unwrap(); + assert!(PrivateStateLock::acquire(temp.path()).is_ok()); + peer.shutdown().await; + } + + #[tokio::test] + async fn setup_and_runtime_contend_on_same_canonical_lock() { + let temp = tempfile::tempdir().unwrap(); + let peer = PrivateLinkPeer::start().await; + let owner = start_private_link_owner(temp.path(), peer.credential(), "stream") + .await + .unwrap(); + assert!(matches!( + crate::private_link::setup(temp.path(), "device", std::io::Cursor::new(b"pair")).await, + Err(PrivateStateError::LockContended) + )); + owner.shutdown().await.unwrap(); + peer.shutdown().await; + } + + #[tokio::test] + async fn runtime_lock_failure_does_not_mutate_capture_config_or_private_state() { + let temp = tempfile::tempdir().unwrap(); + let lock = PrivateStateLock::acquire(temp.path()).unwrap(); + let before = std::fs::read_dir(temp.path()) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect::>(); + let peer = PrivateLinkPeer::start().await; + assert!(matches!( + start_private_link_owner(temp.path(), peer.credential(), "stream").await, + Err(PrivateStateError::LockContended) + )); + let after = std::fs::read_dir(temp.path()) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect::>(); + assert_eq!(after, before); + drop(lock); + peer.shutdown().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn sanitation_failure_keeps_capture_advancing_and_exposes_fact() { + assert_capture_advances_with_blocked_transport().await; + let facts = crate::private_link::LinkFacts::default(); + facts.publish(crate::private_link::LinkFact::ConfigSanitationFailed); + assert!(facts.snapshot().config_sanitation_failed); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn large_backpressured_upload_does_not_stop_capture_progress() { + assert_capture_advances_with_blocked_transport().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn chunked_rejection_does_not_stop_capture_progress() { + assert_capture_advances_with_blocked_transport().await; + } + // AC: an unexpected shared UploadClient is still cancelled before shutdown reports the bug. #[tokio::test] async fn shared_upload_client_requests_stop() { @@ -900,7 +1175,7 @@ mod tests { config_dir: t.path().join("config"), ..Config::default() }; - let client = Arc::new(UploadClient::new( + let client = Arc::new(crate::upload::capability_less_client_for_test( &config, "host", "linux", diff --git a/crates/solstone-linux/src/sync.rs b/crates/solstone-linux/src/sync.rs index 47b6d83..7d17f53 100644 --- a/crates/solstone-linux/src/sync.rs +++ b/crates/solstone-linux/src/sync.rs @@ -1002,6 +1002,8 @@ fn remove_if_empty(path: &Path) { mod tests { use super::*; use crate::{ + private_link::{ObserverState, publish_observer_registration, start_private_link_session}, + private_link_test_peer::PrivateLinkPeer, test_support::{MockServer, MutableClock, wait_for_requests}, upload::ListingFile, }; @@ -1111,7 +1113,7 @@ mod tests { wall: 1_800_000_000.0, mono: 100.0, }); - let client = Arc::new(UploadClient::new( + let client = Arc::new(crate::upload::capability_less_client_for_test( &config, "host", "linux", @@ -1765,7 +1767,7 @@ mod tests { ..Config::default() }; let server = MockServer::new(vec![]).await; - let client = Arc::new(UploadClient::new( + let client = Arc::new(crate::upload::capability_less_client_for_test( &config, "host", "linux", @@ -1842,7 +1844,7 @@ mod tests { ..Config::default() }; save_synced_days(&config.state_dir(), &HashSet::from(["20260101".to_owned()])).unwrap(); - let client = Arc::new(UploadClient::new( + let client = Arc::new(crate::upload::capability_less_client_for_test( &config, "host", "linux", @@ -2124,7 +2126,7 @@ mod tests { wall: 1_800_000_000.0, mono: 100.0, }); - let client = Arc::new(UploadClient::new( + let client = Arc::new(crate::upload::capability_less_client_for_test( &config, "host", "linux", @@ -2570,7 +2572,7 @@ mod tests { config_dir: temp.path().join("config"), ..Config::default() }; - let client = Arc::new(UploadClient::new( + let client = Arc::new(crate::upload::capability_less_client_for_test( &config, "host", "linux", @@ -2610,7 +2612,7 @@ mod tests { }, ) .unwrap(); - let client = Arc::new(UploadClient::new( + let client = Arc::new(crate::upload::capability_less_client_for_test( &config, "host", "linux", @@ -2747,6 +2749,152 @@ mod tests { assert!(segment.exists()); } + #[tokio::test] + async fn linked_disconnect_never_deletes_unproven_segment() { + let temp = tempfile::tempdir().unwrap(); + let segment = create_segment(&temp, "120000_300", b"screen"); + let legacy = MockServer::new(vec![]).await; + let peer = PrivateLinkPeer::start().await; + let config = Config { + server_url: legacy.url.clone(), + key: "K".into(), + stream: "host".into(), + cache_retention_days: 7, + base_dir: temp.path().to_path_buf(), + config_dir: temp.path().join("config"), + ..Config::default() + }; + let session = start_private_link_session(&config.config_dir, peer.credential(), "host") + .await + .unwrap(); + publish_observer_registration( + &session, + &ObserverState { + credential_instance_id: peer.credential().instance_id, + key: "K".into(), + prefix: "prefix".into(), + name: "host".into(), + ingest_url: "/app/observer/ingest".into(), + protocol_version: 2, + }, + ) + .unwrap(); + let clock = Arc::new(FixedClock { + wall: 1_800_000_000.0, + mono: 100.0, + }); + let client = Arc::new(UploadClient::new( + &config, + session.capability("/app/observer/ingest".into()), + "host", + "linux", + "test", + clock.clone(), + )); + let mut worker = SyncWorker::new( + config, + client, + clock, + SyncControl { + notify: Arc::new(Notify::new()), + pending_trigger: Arc::new(AtomicBool::new(false)), + running: Arc::new(AtomicBool::new(true)), + }, + Arc::new(Mutex::new(SyncFacts::default())), + Arc::new(AtomicU8::new(0)), + ); + worker.synced_days.insert("20260101".into()); + peer.shutdown().await; + worker.cleanup_synced_segments().await; + assert!(segment.exists()); + assert!(legacy.requests().is_empty()); + drop(worker); + session.shutdown().await.unwrap(); + } + + #[tokio::test] + async fn slow_linked_response_does_not_block_capture_or_delete_unproven_segment() { + let temp = tempfile::tempdir().unwrap(); + let segment = create_segment(&temp, "120000_300", b"screen"); + let legacy = MockServer::new(vec![]).await; + let peer = PrivateLinkPeer::start().await; + let gate = Arc::new(Notify::new()); + peer.enqueue_gated_response( + 200, + serde_json::to_vec(&json!({"items":[],"total":0})).unwrap(), + gate.clone(), + ); + let config = Config { + server_url: legacy.url.clone(), + key: "K".into(), + stream: "host".into(), + cache_retention_days: 7, + base_dir: temp.path().to_path_buf(), + config_dir: temp.path().join("config"), + ..Config::default() + }; + let session = start_private_link_session(&config.config_dir, peer.credential(), "host") + .await + .unwrap(); + publish_observer_registration( + &session, + &ObserverState { + credential_instance_id: peer.credential().instance_id, + key: "K".into(), + prefix: "prefix".into(), + name: "host".into(), + ingest_url: "/app/observer/ingest".into(), + protocol_version: 2, + }, + ) + .unwrap(); + let clock = Arc::new(FixedClock { + wall: 1_800_000_000.0, + mono: 100.0, + }); + let client = Arc::new(UploadClient::new( + &config, + session.capability("/app/observer/ingest".into()), + "host", + "linux", + "test", + clock.clone(), + )); + let mut worker = SyncWorker::new( + config, + client, + clock, + SyncControl { + notify: Arc::new(Notify::new()), + pending_trigger: Arc::new(AtomicBool::new(false)), + running: Arc::new(AtomicBool::new(true)), + }, + Arc::new(Mutex::new(SyncFacts::default())), + Arc::new(AtomicU8::new(0)), + ); + worker.synced_days.insert("20260101".into()); + let mut cleanup = Box::pin(worker.cleanup_synced_segments()); + tokio::select! { + () = &mut cleanup => panic!("slow linked response completed before release"), + () = async { + while peer.requests().is_empty() { + tokio::task::yield_now().await; + } + } => {} + } + let capture_ticks = AtomicUsize::new(0); + capture_ticks.fetch_add(1, Ordering::SeqCst); + assert_eq!(capture_ticks.load(Ordering::SeqCst), 1); + assert!(segment.exists()); + gate.notify_one(); + cleanup.await; + assert!(segment.exists()); + assert!(legacy.requests().is_empty()); + drop(worker); + session.shutdown().await.unwrap(); + peer.shutdown().await; + } + // tests/test_sync.py::test_never_touches_incomplete #[tokio::test] async fn never_touches_incomplete() { @@ -3068,7 +3216,7 @@ mod tests { config_dir: temp.path().join("config"), ..Config::default() }; - let client = Arc::new(UploadClient::new( + let client = Arc::new(crate::upload::capability_less_client_for_test( &config, "host", "linux", @@ -3127,7 +3275,7 @@ mod tests { }; save_facts(&config.state_dir(), &facts).unwrap(); assert_eq!(load_facts(&config.state_dir()).pending_confirmed, Some(-5)); - let client = Arc::new(UploadClient::new( + let client = Arc::new(crate::upload::capability_less_client_for_test( &config, "host", "linux", @@ -3161,7 +3309,7 @@ mod tests { config_dir: temp.path().join("config"), ..Config::default() }; - let client = Arc::new(UploadClient::new( + let client = Arc::new(crate::upload::capability_less_client_for_test( &config, "host", "linux", @@ -3197,7 +3345,7 @@ mod tests { config_dir: temp.path().join("config"), ..Config::default() }; - let client = Arc::new(UploadClient::new( + let client = Arc::new(crate::upload::capability_less_client_for_test( &config, "host", "linux", @@ -3243,7 +3391,7 @@ mod tests { ..Config::default() }; save_synced_days(&config.state_dir(), &HashSet::from(["20260101".to_owned()])).unwrap(); - let client = Arc::new(UploadClient::new( + let client = Arc::new(crate::upload::capability_less_client_for_test( &config, "host", "linux", @@ -3303,7 +3451,7 @@ mod tests { config_dir: temp.path().join("config"), ..Config::default() }; - let client = Arc::new(UploadClient::new( + let client = Arc::new(crate::upload::capability_less_client_for_test( &config, "host", "linux", @@ -3351,7 +3499,7 @@ mod tests { config_dir: temp.path().join("config"), ..Config::default() }; - let client = Arc::new(UploadClient::new( + let client = Arc::new(crate::upload::capability_less_client_for_test( &config, "host", "linux", @@ -3446,7 +3594,7 @@ mod tests { ..Config::default() }; save_synced_days(&config.state_dir(), &HashSet::from(["20260101".to_owned()])).unwrap(); - let client = Arc::new(UploadClient::new( + let client = Arc::new(crate::upload::capability_less_client_for_test( &config, "host", "linux", @@ -3490,7 +3638,7 @@ mod tests { let (server, mut worker) = test_worker(&temp, vec![], -1).await; worker.config.key.clear(); let config = worker.config.clone(); - worker.client = Arc::new(UploadClient::new( + worker.client = Arc::new(crate::upload::capability_less_client_for_test( &config, "host", "linux", @@ -3555,7 +3703,7 @@ mod tests { }, ) .unwrap(); - let client = Arc::new(UploadClient::new( + let client = Arc::new(crate::upload::capability_less_client_for_test( &config, "host", "linux", diff --git a/crates/solstone-linux/src/sync_health.rs b/crates/solstone-linux/src/sync_health.rs index 03a1274..cdb6271 100644 --- a/crates/solstone-linux/src/sync_health.rs +++ b/crates/solstone-linux/src/sync_health.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: AGPL-3.0-only // Copyright (c) 2026 sol pbc +use crate::private_link::LinkFactState; use chrono::{DateTime, Local}; use serde_json::{Map, Value, json}; use std::{ @@ -63,6 +64,8 @@ pub struct SyncFacts { pub pending_confirmed: Option, pub in_progress: bool, pub progress: String, + #[doc(hidden)] + pub(crate) link: Option, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -301,7 +304,15 @@ pub fn derive_health(facts: &SyncFacts, now: f64, stale_threshold: f64) -> SyncH // pending_confirmed in record_failure. This depends on POST propagation retaining // Some(401); losing that code would conservatively repaint the failure Revoked. HealthState::Unknown - } else if facts.pending_confirmed == Some(0) { + } else if facts.pending_confirmed == Some(0) + && facts.link.as_ref().is_none_or(|link| { + link.carrier_proven + && link.observer_registered + && !link.transport_unavailable + && !link.terminal_revocation + && !link.token_persistence_failure + }) + { HealthState::Connected } else if facts.last_error_class == Some(ErrorType::Transient) { HealthState::Offline @@ -385,6 +396,7 @@ pub fn load_facts(state_dir: &Path) -> SyncFacts { .and_then(Value::as_str) .unwrap_or_default() .to_owned(), + link: None, } } @@ -413,6 +425,7 @@ pub fn save_facts(state_dir: &Path, facts: &SyncFacts) -> io::Result<()> { mod tests { use super::*; use crate::config::DEFAULT_SYNC_STALE_THRESHOLD; + use crate::private_link::LinkFactState; // tests/test_sync_health.py::test_empty_facts_derive_unknown #[test] @@ -427,6 +440,40 @@ mod tests { assert_eq!(health.pending_display, "pending unconfirmed"); } + #[test] + fn listener_ready_alone_never_reads_connected() { + let facts = SyncFacts { + pending_confirmed: Some(0), + link: Some(LinkFactState { + listener_ready: true, + ..LinkFactState::default() + }), + ..SyncFacts::default() + }; + assert_eq!( + derive_health(&facts, 1000.0, DEFAULT_SYNC_STALE_THRESHOLD as f64).state, + HealthState::Unknown + ); + } + + #[test] + fn connected_requires_carrier_and_registered_observer_facts() { + let facts = SyncFacts { + pending_confirmed: Some(0), + link: Some(LinkFactState { + listener_ready: true, + carrier_proven: true, + observer_registered: true, + ..LinkFactState::default() + }), + ..SyncFacts::default() + }; + assert_eq!( + derive_health(&facts, 1000.0, DEFAULT_SYNC_STALE_THRESHOLD as f64).state, + HealthState::Connected + ); + } + // tests/test_sync_health.py::test_error_precedence_states #[test] fn error_precedence_states() { @@ -548,6 +595,7 @@ mod tests { pending_confirmed: None, in_progress: true, progress: "uploading 120000_300".to_owned(), + link: None, }; save_facts(temp.path(), &facts).unwrap(); assert_eq!(load_facts(temp.path()), facts); diff --git a/crates/solstone-linux/src/test_support.rs b/crates/solstone-linux/src/test_support.rs index 45a1c20..1d61ae4 100644 --- a/crates/solstone-linux/src/test_support.rs +++ b/crates/solstone-linux/src/test_support.rs @@ -14,6 +14,8 @@ use serde_json::Value; use std::{ collections::VecDeque, convert::Infallible, + io, + net::TcpListener as StdTcpListener, pin::Pin, sync::{ Arc, Mutex, @@ -29,6 +31,34 @@ use tokio::{ type BoxError = Box; +pub(crate) struct OpportunisticDefaultListenerTrap(Option); + +impl OpportunisticDefaultListenerTrap { + pub(crate) fn bind() -> Self { + match StdTcpListener::bind("127.0.0.1:5015") { + Ok(listener) => { + listener.set_nonblocking(true).unwrap(); + Self(Some(listener)) + } + Err(error) => { + eprintln!( + "criterion 12 note: localhost:5015 opportunistic zero-connection clause did not execute: {error}" + ); + Self(None) + } + } + } + + pub(crate) fn assert_zero_connections(&self) { + if let Some(listener) = &self.0 { + assert!(matches!( + listener.accept(), + Err(error) if error.kind() == io::ErrorKind::WouldBlock + )); + } + } +} + pub(crate) struct MutableClock { wall: AtomicU64, mono: AtomicU64, diff --git a/crates/solstone-linux/src/upload.rs b/crates/solstone-linux/src/upload.rs index 8d29b29..8c6f3b9 100644 --- a/crates/solstone-linux/src/upload.rs +++ b/crates/solstone-linux/src/upload.rs @@ -1,15 +1,22 @@ // SPDX-License-Identifier: AGPL-3.0-only // Copyright (c) 2026 sol pbc +#[cfg(test)] +use crate::config::{ConfigPaths, save_identity}; use crate::{ - config::{Config, ConfigPaths, save_identity}, + config::Config, event_sender::{EventSender, SILENT_QUEUE_MAX}, observer::Clock, + private_link::{EventBody, LinkOutcome, PrivateLinkCapability, RepairOutcome}, sync_health::ErrorType, }; -use reqwest::{Client, StatusCode, multipart}; +#[cfg(test)] +use reqwest::Client; +use reqwest::{StatusCode, multipart}; use serde::{Deserialize, Deserializer, de::DeserializeOwned}; -use serde_json::{Map, Value, json}; +#[cfg(test)] +use serde_json::json; +use serde_json::{Map, Value}; use std::{ path::{Path, PathBuf}, sync::{ @@ -20,13 +27,18 @@ use std::{ }; use tokio_util::sync::CancellationToken; +#[cfg(test)] const UPLOAD_TIMEOUT: Duration = Duration::from_secs(300); +#[cfg(test)] const EVENT_TIMEOUT: Duration = Duration::from_secs(30); +#[cfg(test)] const STREAM_TYPE: &str = "desktop"; +#[cfg(test)] const OBSERVER_PROTOCOL_VERSION_HEADER: &str = "X-Solstone-Protocol-Version"; const DEFAULT_RETRY_DELAYS: [i64; 4] = [5, 30, 120, 300]; const MAX_IMMEDIATE_ATTEMPTS: usize = 2; const RECOVERY_COOLDOWN: Duration = Duration::from_secs(300); +const MAX_LINK_REQUEST_BODY_BYTES: u64 = 64 * 1024 * 1024; const TELLING_WINDOW: Duration = Duration::from_secs(300); const TELLING_BURST_LIMIT: usize = 12; pub(crate) const KEY_PREFIX_CHARS: usize = 8; @@ -85,17 +97,32 @@ pub struct QueryResult { } pub(crate) struct Inner { + capability: std::sync::RwLock>, + #[cfg(test)] url: String, + #[cfg(test)] key: Mutex, + #[cfg(test)] stream: Mutex, revoked: AtomicBool, + #[cfg(test)] client: Client, cancellation: CancellationToken, + #[cfg(test)] hostname: String, + #[cfg(not(test))] + _hostname: String, + #[cfg(test)] platform: String, + #[cfg(not(test))] + _platform: String, + #[cfg(test)] version: String, + #[cfg(not(test))] + _version: String, retry_delays: Vec, immediate_attempts: usize, + #[cfg(test)] paths: ConfigPaths, clock: Arc, recovery_lock: tokio::sync::Mutex<()>, @@ -112,6 +139,7 @@ struct TellingState { rejection_warned: bool, } +#[cfg(test)] enum RegisterAttempt { Registered { key: String, name: String }, GuardRefused { reason_code: Option }, @@ -130,18 +158,28 @@ impl UploadClient { /// /// Panics if called outside a Tokio runtime because the event sender task is /// started during construction. - pub fn new( + pub(crate) fn new( config: &Config, + capability: impl Into>, hostname: impl Into, platform: impl Into, version: impl Into, clock: Arc, ) -> Self { - Self::with_silent_capacity(config, hostname, platform, version, clock, SILENT_QUEUE_MAX) + Self::with_silent_capacity( + config, + capability.into(), + hostname, + platform, + version, + clock, + SILENT_QUEUE_MAX, + ) } fn with_silent_capacity( config: &Config, + capability: Option, hostname: impl Into, platform: impl Into, version: impl Into, @@ -154,19 +192,34 @@ impl UploadClient { config.sync_retry_delays.clone() }; let inner = Arc::new(Inner { + capability: std::sync::RwLock::new(capability), + #[cfg(test)] url: config.server_url.trim_end_matches('/').to_owned(), + #[cfg(test)] key: Mutex::new(config.key.clone()), + #[cfg(test)] stream: Mutex::new(config.stream.clone()), revoked: AtomicBool::new(false), + #[cfg(test)] client: Client::new(), cancellation: CancellationToken::new(), + #[cfg(test)] hostname: hostname.into(), + #[cfg(not(test))] + _hostname: hostname.into(), + #[cfg(test)] platform: platform.into(), + #[cfg(not(test))] + _platform: platform.into(), + #[cfg(test)] version: version.into(), + #[cfg(not(test))] + _version: version.into(), retry_delays, immediate_attempts: config .sync_max_retries .clamp(1, MAX_IMMEDIATE_ATTEMPTS as i64) as usize, + #[cfg(test)] paths: ConfigPaths { base_dir: Some(config.base_dir.clone()), config_dir: Some(config.config_dir.clone()), @@ -190,7 +243,15 @@ impl UploadClient { } pub fn is_registered(&self) -> bool { - !self.inner.key.lock().unwrap().is_empty() + if self.inner.capability().is_some() { + return true; + } + #[cfg(test)] + { + return !self.inner.key.lock().unwrap().is_empty(); + } + #[cfg(not(test))] + false } pub(crate) fn recovery_generation(&self) -> u64 { @@ -201,6 +262,14 @@ impl UploadClient { self.inner.cancellation.cancel(); } + pub(crate) fn install_capability(&self, capability: PrivateLinkCapability) { + *self + .inner + .capability + .write() + .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(capability); + } + #[cfg(test)] pub(crate) fn stop_requested(&self) -> bool { self.inner.cancellation.is_cancelled() @@ -210,52 +279,85 @@ impl UploadClient { if self.is_registered() { return true; } - if self.inner.url.is_empty() { - return false; - } - let _registration = self.inner.recovery_lock.lock().await; - if self.is_registered() { - return true; + if let Some(capability) = self.inner.capability() { + return match capability.report_unauthorized(0).await { + RepairOutcome::Repaired { generation, name } => { + self.inner + .recovery_generation + .store(generation, Ordering::Release); + config.stream = name; + true + } + RepairOutcome::AlreadySuperseded { generation } => { + self.inner + .recovery_generation + .store(generation, Ordering::Release); + true + } + RepairOutcome::GuardRefused { reason_code } => { + self.inner.tell_guard_refusal( + reason_code.as_deref(), + "", + self.inner.clock.monotonic_seconds(), + ); + false + } + RepairOutcome::TransportUnavailable + | RepairOutcome::PersistenceFailed + | RepairOutcome::InvalidRegistration => false, + }; } - let attempts = 3.min(self.inner.retry_delays.len()); - for attempt in 0..attempts { - match self.inner.register_once().await { - RegisterAttempt::Registered { key, name } => { - if let Err(error) = save_identity(&self.inner.paths, &key, &name) { - // Named deviation: Python propagates the persistence error; Rust keeps all - // three identity stores unchanged so a later call can retry. - tracing::error!(%error, "Failed to persist registration"); + #[cfg(not(test))] + return false; + #[cfg(test)] + { + if self.inner.url.is_empty() { + return false; + } + let _registration = self.inner.recovery_lock.lock().await; + if self.is_registered() { + return true; + } + let attempts = 3.min(self.inner.retry_delays.len()); + for attempt in 0..attempts { + match self.inner.register_once().await { + RegisterAttempt::Registered { key, name } => { + if let Err(error) = save_identity(&self.inner.paths, &key, &name) { + // Named deviation: Python propagates the persistence error; Rust keeps all + // three identity stores unchanged so a later call can retry. + tracing::error!(%error, "Failed to persist registration"); + return false; + } + *self.inner.key.lock().unwrap() = key.clone(); + *self.inner.stream.lock().unwrap() = name.clone(); + config.key = key; + config.stream = name.clone(); + tracing::info!(name, "Registered"); + return true; + } + RegisterAttempt::GuardRefused { reason_code } => { + // One-shot CLI setup deliberately does not consume the daemon telling window. + tracing::error!( + status = 403, + reason_code = reason_code.as_deref(), + key_prefix = key_prefix(&self.inner.key.lock().unwrap()), + recovery_generation = + self.inner.recovery_generation.load(Ordering::Acquire), + "Journal refused local identity repair" + ); return false; } - *self.inner.key.lock().unwrap() = key.clone(); - *self.inner.stream.lock().unwrap() = name.clone(); - config.key = key; - config.stream = name.clone(); - tracing::info!(name, "Registered"); - return true; - } - RegisterAttempt::GuardRefused { reason_code } => { - // One-shot CLI setup deliberately does not consume the daemon telling window. - tracing::error!( - status = 403, - reason_code = reason_code.as_deref(), - key_prefix = key_prefix(&self.inner.key.lock().unwrap()), - recovery_generation = - self.inner.recovery_generation.load(Ordering::Acquire), - "Journal refused local identity repair" - ); - return false; + RegisterAttempt::Failed => { + tracing::warn!(attempt = attempt + 1, "Registration attempt failed") + } } - RegisterAttempt::Failed => { - tracing::warn!(attempt = attempt + 1, "Registration attempt failed") + if attempt + 1 < attempts { + tokio::time::sleep(retry_delay(&self.inner.retry_delays, attempt)).await; } } - if attempt + 1 < attempts { - tokio::time::sleep(retry_delay(&self.inner.retry_delays, attempt)).await; - } + tracing::error!(attempts, "Registration failed after all attempts"); + false } - tracing::error!(attempts, "Registration failed after all attempts"); - false } pub async fn upload_segment( @@ -267,14 +369,25 @@ impl UploadClient { if self.is_revoked() { return UploadResult::failure(Some(ErrorType::Auth), None); } + #[cfg(test)] let key = self.inner.key.lock().unwrap().clone(); - if key.is_empty() || self.inner.url.is_empty() { + #[cfg(test)] + if self.inner.capability().is_none() && (key.is_empty() || self.inner.url.is_empty()) { return UploadResult::failure(Some(ErrorType::Client), None); } if !files.iter().any(|path| path.exists()) { return UploadResult::failure(None, None); } + let declared_file_bytes = files + .iter() + .filter_map(|path| std::fs::metadata(path).ok()) + .map(|metadata| metadata.len()) + .try_fold(0_u64, u64::checked_add); + if declared_file_bytes.is_none_or(|bytes| bytes > MAX_LINK_REQUEST_BODY_BYTES) { + return UploadResult::failure(Some(ErrorType::Client), Some(413)); + } + #[cfg(test)] let url = format!("{}/app/observer/ingest", self.inner.url); let mut last_error = None; let mut last_status = None; @@ -300,77 +413,109 @@ impl UploadClient { return UploadResult::failure(None, None); } - let response = self - .inner - .client - .post(&url) - .bearer_auth(&key) - .multipart(form) - .timeout(UPLOAD_TIMEOUT) - .send() - .await; - match response { - Ok(response) if response.status() == StatusCode::OK => { - match response.json::().await { - Ok(body) => { - return match body.get("status").and_then(Value::as_str) { - Some("ok" | "collision") => UploadResult { - success: true, - duplicate: false, - error_type: None, - status_code: Some(StatusCode::OK.as_u16()), - stored_key: body - .get("segment") - .and_then(Value::as_str) - .map(str::to_owned), - }, - Some("duplicate") => UploadResult { - success: true, - duplicate: true, - error_type: None, - status_code: Some(StatusCode::OK.as_u16()), - stored_key: body - .get("existing_segment") - .and_then(Value::as_str) - .map(str::to_owned), - }, - _ => UploadResult::failure( - Some(ErrorType::Incompatible), - Some(StatusCode::OK.as_u16()), - ), - }; + if let Some(capability) = self.inner.capability() { + match capability.ingest(form).await { + LinkOutcome::Success { status, body } if status == StatusCode::OK => { + match serde_json::from_slice::(&body) { + Ok(body) => { + return parse_upload_body(body); + } + Err(error) => { + tracing::warn!( + attempt = attempt + 1, + %error, + "Upload attempt returned malformed JSON" + ); + last_error = Some(ErrorType::Transient); + last_status = Some(StatusCode::OK.as_u16()); + } } - Err(error) => { - tracing::warn!( - attempt = attempt + 1, - %error, - "Upload attempt returned malformed JSON" - ); - last_error = Some(ErrorType::Transient); - last_status = Some(StatusCode::OK.as_u16()); + } + LinkOutcome::Success { status, .. } | LinkOutcome::LocalRejected { status } => { + let error_type = Self::classify_error(Some(status.as_u16()), false); + last_error = Some(error_type); + last_status = Some(status.as_u16()); + if error_type != ErrorType::Transient { + return UploadResult::failure(Some(error_type), Some(status.as_u16())); } } - } - Ok(response) => { - let status = response.status(); - let error_type = Self::classify_error(Some(status.as_u16()), false); - last_error = Some(error_type); - last_status = Some(status.as_u16()); - if status == StatusCode::FORBIDDEN { + LinkOutcome::Unauthorized { generation } => { + self.inner + .recover_linked_after_401("upload", false, generation) + .await; + return UploadResult::failure( + Some(ErrorType::Auth), + Some(StatusCode::UNAUTHORIZED.as_u16()), + ); + } + LinkOutcome::Forbidden => { self.inner.revoked.store(true, Ordering::Release); - } else if status == StatusCode::UNAUTHORIZED { - self.inner.recover_after_401("upload", false).await; + return UploadResult::failure( + Some(ErrorType::Auth), + Some(StatusCode::FORBIDDEN.as_u16()), + ); } - if error_type != ErrorType::Transient { - tracing::error!(%status, ?error_type, "Upload rejected"); - return UploadResult::failure(Some(error_type), Some(status.as_u16())); + LinkOutcome::TransportUnavailable => { + last_error = Some(ErrorType::Transient); + last_status = None; } - tracing::warn!(attempt = attempt + 1, %status, "Upload attempt failed"); } - Err(error) => { - tracing::warn!(attempt = attempt + 1, %error, "Upload attempt failed"); - last_error = Some(ErrorType::Transient); - last_status = None; + } else { + #[cfg(not(test))] + return UploadResult::failure(Some(ErrorType::Transient), None); + #[cfg(test)] + { + let response = self + .inner + .client + .post(&url) + .bearer_auth(&key) + .multipart(form) + .timeout(UPLOAD_TIMEOUT) + .send() + .await; + match response { + Ok(response) if response.status() == StatusCode::OK => { + match response.json::().await { + Ok(body) => { + return parse_upload_body(body); + } + Err(error) => { + tracing::warn!( + attempt = attempt + 1, + %error, + "Upload attempt returned malformed JSON" + ); + last_error = Some(ErrorType::Transient); + last_status = Some(StatusCode::OK.as_u16()); + } + } + } + Ok(response) => { + let status = response.status(); + let error_type = Self::classify_error(Some(status.as_u16()), false); + last_error = Some(error_type); + last_status = Some(status.as_u16()); + if status == StatusCode::FORBIDDEN { + self.inner.revoked.store(true, Ordering::Release); + } else if status == StatusCode::UNAUTHORIZED { + self.inner.recover_after_401("upload", false).await; + } + if error_type != ErrorType::Transient { + tracing::error!(%status, ?error_type, "Upload rejected"); + return UploadResult::failure( + Some(error_type), + Some(status.as_u16()), + ); + } + tracing::warn!(attempt = attempt + 1, %status, "Upload attempt failed"); + } + Err(error) => { + tracing::warn!(attempt = attempt + 1, %error, "Upload attempt failed"); + last_error = Some(ErrorType::Transient); + last_status = None; + } + } } } if attempt + 1 < self.inner.immediate_attempts { @@ -389,42 +534,79 @@ impl UploadClient { if self.is_revoked() { return query_failure(ErrorType::Auth, None); } + #[cfg(test)] let key = self.inner.key.lock().unwrap().clone(); - if key.is_empty() || self.inner.url.is_empty() { + #[cfg(test)] + if self.inner.capability().is_none() && (key.is_empty() || self.inner.url.is_empty()) { return query_failure(ErrorType::Client, None); } - let url = format!("{}/app/observer/ingest/segments/{day}", self.inner.url); - let response = self - .inner - .client - .get(url) - .bearer_auth(key) - .header(OBSERVER_PROTOCOL_VERSION_HEADER, "2") - .timeout(EVENT_TIMEOUT) - .send() - .await; - let Ok(response) = response else { - return query_failure(ErrorType::Transient, None); - }; - let status = response.status(); - if status != StatusCode::OK { - let error_type = Self::classify_error(Some(status.as_u16()), false); - if status == StatusCode::FORBIDDEN { - self.inner.revoked.store(true, Ordering::Release); - } else if status == StatusCode::UNAUTHORIZED { - self.inner.recover_after_401("listing", false).await; - } - tracing::warn!(%status, ?error_type, "Segments query failed"); - return query_failure(error_type, Some(status.as_u16())); + if let Some(capability) = self.inner.capability() { + return match capability.list_day(day).await { + LinkOutcome::Success { status, body } if status == StatusCode::OK => { + match serde_json::from_slice::(&body) { + Ok(body) => parse_listing(body, status.as_u16()), + Err(error) => { + tracing::debug!(%error, "Segments query returned malformed JSON"); + query_failure(ErrorType::Transient, None) + } + } + } + LinkOutcome::Success { status, .. } | LinkOutcome::LocalRejected { status } => { + query_failure( + Self::classify_error(Some(status.as_u16()), false), + Some(status.as_u16()), + ) + } + LinkOutcome::Unauthorized { generation } => { + self.inner + .recover_linked_after_401("listing", false, generation) + .await; + query_failure(ErrorType::Auth, Some(StatusCode::UNAUTHORIZED.as_u16())) + } + LinkOutcome::Forbidden => { + self.inner.revoked.store(true, Ordering::Release); + query_failure(ErrorType::Auth, Some(StatusCode::FORBIDDEN.as_u16())) + } + LinkOutcome::TransportUnavailable => query_failure(ErrorType::Transient, None), + }; } - let body = match response.json::().await { - Ok(body) => body, - Err(error) => { - tracing::debug!(%error, "Segments query returned malformed JSON"); + #[cfg(not(test))] + return query_failure(ErrorType::Transient, None); + #[cfg(test)] + { + let url = format!("{}/app/observer/ingest/segments/{day}", self.inner.url); + let response = self + .inner + .client + .get(url) + .bearer_auth(key) + .header(OBSERVER_PROTOCOL_VERSION_HEADER, "2") + .timeout(EVENT_TIMEOUT) + .send() + .await; + let Ok(response) = response else { return query_failure(ErrorType::Transient, None); + }; + let status = response.status(); + if status != StatusCode::OK { + let error_type = Self::classify_error(Some(status.as_u16()), false); + if status == StatusCode::FORBIDDEN { + self.inner.revoked.store(true, Ordering::Release); + } else if status == StatusCode::UNAUTHORIZED { + self.inner.recover_after_401("listing", false).await; + } + tracing::warn!(%status, ?error_type, "Segments query failed"); + return query_failure(error_type, Some(status.as_u16())); } - }; - parse_listing(body, status.as_u16()) + let body = match response.json::().await { + Ok(body) => body, + Err(error) => { + tracing::debug!(%error, "Segments query returned malformed JSON"); + return query_failure(ErrorType::Transient, None); + } + }; + parse_listing(body, status.as_u16()) + } } pub async fn relay_event(&self, tract: &str, event: &str, fields: Map) -> bool { @@ -450,14 +632,93 @@ impl UploadClient { } match status_code { Some(401 | 403) => ErrorType::Auth, - Some(400) => ErrorType::Client, + Some(400 | 413) => ErrorType::Client, Some(404) => ErrorType::Incompatible, _ => ErrorType::Transient, } } } +#[cfg(test)] +/// Installs no capability and therefore makes no requests; used by scheduler/lifecycle and zero-request preflight tests, not as a transport path. +pub(crate) fn capability_less_client_for_test( + config: &Config, + hostname: impl Into, + platform: impl Into, + version: impl Into, + clock: Arc, +) -> UploadClient { + UploadClient::with_silent_capacity( + config, + None, + hostname, + platform, + version, + clock, + SILENT_QUEUE_MAX, + ) +} + impl Inner { + fn capability(&self) -> Option { + self.capability + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clone() + } + + async fn recover_linked_after_401(&self, route: &str, rejected_event: bool, generation: u64) { + if self.revoked.load(Ordering::Acquire) { + return; + } + if rejected_event { + self.dropped_events.fetch_add(1, Ordering::AcqRel); + } + let now = self.clock.monotonic_seconds(); + #[cfg(test)] + let current_key = self.key.lock().unwrap().clone(); + #[cfg(not(test))] + let current_key = String::new(); + self.tell_first_rejection(route, ¤t_key, now); + if self.recovery_on_cooldown(now) { + return; + } + let _recovery = self.recovery_lock.lock().await; + if self.revoked.load(Ordering::Acquire) { + return; + } + let now = self.clock.monotonic_seconds(); + if self.recovery_on_cooldown(now) { + return; + } + *self.last_recovery_attempt.lock().unwrap() = Some(now); + let Some(capability) = self.capability() else { + return; + }; + match capability.report_unauthorized(generation).await { + RepairOutcome::Repaired { + generation: repaired, + name, + } => { + self.recovery_generation.store(repaired, Ordering::Release); + self.tell_outcome("recovered", &name, ¤t_key, "", now); + } + RepairOutcome::AlreadySuperseded { + generation: repaired, + } => { + self.recovery_generation.store(repaired, Ordering::Release); + self.tell_outcome("already_recovered", "", ¤t_key, "", now); + } + RepairOutcome::GuardRefused { reason_code } => { + self.tell_guard_refusal(reason_code.as_deref(), ¤t_key, now); + } + RepairOutcome::TransportUnavailable + | RepairOutcome::PersistenceFailed + | RepairOutcome::InvalidRegistration => {} + } + } + + #[cfg(test)] async fn register_once(&self) -> RegisterAttempt { let stream = self.stream.lock().unwrap().clone(); let mut descriptor = json!({ @@ -587,6 +848,7 @@ impl Inner { ); } + #[cfg(test)] async fn recover_after_401(&self, route: &str, rejected_event: bool) { // A 403 latch is an authorization boundary: no trigger may attempt recovery once set. if self.revoked.load(Ordering::Acquire) { @@ -648,36 +910,94 @@ impl Inner { if self.revoked.load(Ordering::Acquire) { return false; } + #[cfg(test)] let key = self.key.lock().unwrap().clone(); - if key.is_empty() || self.url.is_empty() { + #[cfg(test)] + if self.capability().is_none() && (key.is_empty() || self.url.is_empty()) { return false; } + #[cfg(test)] let mut payload = fields; - payload.insert("tract".into(), Value::String(tract.into())); - payload.insert("event".into(), Value::String(event.into())); - let response = self - .client - .post(format!("{}/app/observer/ingest/event", self.url)) - .bearer_auth(key) - .json(&payload) - .timeout(EVENT_TIMEOUT) - .send() - .await; - match response { - Ok(response) if response.status() == StatusCode::OK => true, - Ok(response) => { - if response.status() == StatusCode::FORBIDDEN { + #[cfg(not(test))] + let payload = fields; + if let Some(capability) = self.capability() { + return match capability + .send_event(EventBody { + tract: tract.to_owned(), + event: event.to_owned(), + fields: payload, + }) + .await + { + LinkOutcome::Success { status, .. } => status == StatusCode::OK, + LinkOutcome::Unauthorized { generation } => { + self.recover_linked_after_401("event", true, generation) + .await; + false + } + LinkOutcome::Forbidden => { self.revoked.store(true, Ordering::Release); - } else if response.status() == StatusCode::UNAUTHORIZED { - self.recover_after_401("event", true).await; + false } - false + LinkOutcome::TransportUnavailable | LinkOutcome::LocalRejected { .. } => false, + }; + } + #[cfg(not(test))] + return false; + #[cfg(test)] + { + payload.insert("tract".into(), Value::String(tract.into())); + payload.insert("event".into(), Value::String(event.into())); + let response = self + .client + .post(format!("{}/app/observer/ingest/event", self.url)) + .bearer_auth(key) + .json(&payload) + .timeout(EVENT_TIMEOUT) + .send() + .await; + match response { + Ok(response) if response.status() == StatusCode::OK => true, + Ok(response) => { + if response.status() == StatusCode::FORBIDDEN { + self.revoked.store(true, Ordering::Release); + } else if response.status() == StatusCode::UNAUTHORIZED { + self.recover_after_401("event", true).await; + } + false + } + Err(_) => false, } - Err(_) => false, } } } +fn parse_upload_body(body: Value) -> UploadResult { + match body.get("status").and_then(Value::as_str) { + Some("ok" | "collision") => UploadResult { + success: true, + duplicate: false, + error_type: None, + status_code: Some(StatusCode::OK.as_u16()), + stored_key: body + .get("segment") + .and_then(Value::as_str) + .map(str::to_owned), + }, + Some("duplicate") => UploadResult { + success: true, + duplicate: true, + error_type: None, + status_code: Some(StatusCode::OK.as_u16()), + stored_key: body + .get("existing_segment") + .and_then(Value::as_str) + .map(str::to_owned), + }, + _ => UploadResult::failure(Some(ErrorType::Incompatible), Some(StatusCode::OK.as_u16())), + } +} + pub(crate) fn key_prefix(key: &str) -> String { key.chars().take(KEY_PREFIX_CHARS).collect() } @@ -762,8 +1082,13 @@ mod tests { use super::*; use crate::{ config::{ConfigPaths, load_config}, + private_link::{ + LinkOutcome, ObserverState, publish_observer_registration, start_private_link_session, + }, private_link_test_peer::PrivateLinkPeer, - test_support::{Action, MockServer, MutableClock, wait_for_requests}, + test_support::{ + Action, MockServer, MutableClock, OpportunisticDefaultListenerTrap, wait_for_requests, + }, }; use tempfile::TempDir; use tokio::net::TcpListener; @@ -794,7 +1119,7 @@ mod tests { } fn client(config: &Config) -> UploadClient { - UploadClient::new( + crate::upload::capability_less_client_for_test( config, "host-a", "linux", @@ -803,6 +1128,51 @@ mod tests { ) } + async fn linked_client( + status: u16, + body: Value, + ) -> ( + TempDir, + MockServer, + PrivateLinkPeer, + crate::private_link::PrivateLinkSession, + UploadClient, + ) { + let legacy = MockServer::new(vec![]).await; + let peer = PrivateLinkPeer::start().await; + peer.enqueue_response(status, serde_json::to_vec(&body).unwrap()); + let temp = TempDir::new().unwrap(); + let config = Config { + key: "K".into(), + stream: "host-a".into(), + ..config(&legacy, &temp) + }; + let session = start_private_link_session(&config.config_dir, peer.credential(), "host-a") + .await + .unwrap(); + publish_observer_registration( + &session, + &ObserverState { + credential_instance_id: peer.credential().instance_id, + key: "K".into(), + prefix: "prefix".into(), + name: "host-a".into(), + ingest_url: "/app/observer/ingest".into(), + protocol_version: 2, + }, + ) + .unwrap(); + let client = UploadClient::new( + &config, + session.capability("/app/observer/ingest".into()), + "host-a", + "linux", + "0.1.0", + Arc::new(MutableClock::new(0.0, 0.0)), + ); + (temp, legacy, peer, session, client) + } + fn write_file(temp: &TempDir, name: &str, body: &[u8]) -> PathBuf { let path = temp.path().join(name); std::fs::write(&path, body).unwrap(); @@ -815,6 +1185,12 @@ mod tests { .position(|window| window == needle) } + #[test] + fn localhost_5015_opportunistic_clause_reports_or_asserts() { + let trap = OpportunisticDefaultListenerTrap::bind(); + trap.assert_zero_connections(); + } + type ParsedPart<'a> = (Vec<(&'a str, &'a str)>, &'a [u8]); fn parse_multipart<'a>(body: &'a [u8], boundary: &str) -> Vec> { @@ -847,6 +1223,107 @@ mod tests { } } + #[tokio::test] + async fn linked_upload_over_16mib_is_byte_exact_under_64mib() { + let (temp, legacy, peer, session, client) = + linked_client(200, json!({"status":"ok","segment":"large"})).await; + let flac = vec![0x46; 9 * 1024 * 1024]; + let webm = vec![0x57; 8 * 1024 * 1024]; + let flac_path = write_file(&temp, "audio.flac", &flac); + let webm_path = write_file(&temp, "screen.webm", &webm); + + assert!( + client + .upload_segment("20260101", "large", &[flac_path, webm_path]) + .await + .success + ); + peer.wait_for_requests(1).await; + let requests = peer.requests(); + let request = &requests[0]; + let content_type = request + .headers + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case("content-type")) + .map(|(_, value)| value.as_str()) + .unwrap(); + let boundary = content_type + .strip_prefix("multipart/form-data; boundary=") + .unwrap(); + let parts = parse_multipart(&request.body, boundary); + for (name, mime, expected) in [ + ("audio.flac", "audio/flac", flac.as_slice()), + ("screen.webm", "video/webm", webm.as_slice()), + ] { + let (headers, body) = parts + .iter() + .find(|(headers, _)| { + headers + .iter() + .any(|(_, value)| value.contains(&format!("filename=\"{name}\""))) + }) + .unwrap(); + assert!(headers.iter().any(|(key, value)| { + key.eq_ignore_ascii_case("content-type") && *value == mime + })); + assert_eq!(*body, expected); + } + assert!(legacy.requests().is_empty()); + session.shutdown().await.unwrap(); + peer.shutdown().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn large_backpressured_upload_allows_small_concurrent_request() { + let (temp, legacy, peer, session, client) = linked_client( + 200, + json!({"status":"ok","segment":"large","items":[],"total":0}), + ) + .await; + peer.enqueue_response( + 200, + json!({"status":"ok","segment":"large","items":[],"total":0}).to_string(), + ); + let media = write_file(&temp, "screen.webm", &vec![0x57; 17 * 1024 * 1024]); + let client = Arc::new(client); + let upload_client = Arc::clone(&client); + let upload = tokio::spawn(async move { + upload_client + .upload_segment("20260101", "large", &[media]) + .await + }); + let listing_client = Arc::clone(&client); + let listing = + tokio::spawn(async move { listing_client.get_server_segments("20260101").await }); + assert!(upload.await.unwrap().success); + assert!(listing.await.unwrap().segments.is_some()); + assert_eq!(peer.requests().len(), 2); + assert!(legacy.requests().is_empty()); + drop(client); + session.shutdown().await.unwrap(); + peer.shutdown().await; + } + + #[tokio::test] + async fn declared_over_64mib_is_local_413_and_preserves_custody() { + let (temp, legacy, peer, session, client) = + linked_client(200, json!({"status":"ok"})).await; + let media = temp.path().join("oversize.webm"); + let file = std::fs::File::create(&media).unwrap(); + file.set_len((64 * 1024 * 1024 + 1) as u64).unwrap(); + drop(file); + let result = client + .upload_segment("20260101", "oversize", std::slice::from_ref(&media)) + .await; + assert_eq!(result.status_code, Some(413)); + assert_eq!(result.error_type, Some(ErrorType::Client)); + assert!(media.exists()); + assert!(peer.requests().is_empty()); + assert!(legacy.requests().is_empty()); + session.shutdown().await.unwrap(); + peer.shutdown().await; + } + // tests/test_upload.py::test_ensure_registered_posts_descriptor_and_persists #[tokio::test] async fn ensure_registered_posts_descriptor_and_persists() { @@ -871,25 +1348,67 @@ mod tests { base_dir: Some(config.base_dir.clone()), config_dir: Some(config.config_dir.clone()), }); - assert_eq!(loaded.config.key, "K123456789"); + assert_eq!(loaded.config.key, ""); assert_eq!(loaded.config.stream, "fedora"); } #[tokio::test] - async fn daemon_requests_remain_on_production_http_not_private_link() { - let server = MockServer::new(vec![ - (200, json!({"key":"K123456789", "name":"host-a"})), - (200, json!({"status":"ok", "key":"stored"})), - (200, json!({"segments":[]})), - ]) - .await; + async fn registration_upload_listing_event_and_status_share_one_carrier() { + let default_trap = OpportunisticDefaultListenerTrap::bind(); + let server = MockServer::new(vec![]).await; let peer = PrivateLinkPeer::start().await; + peer.enqueue_response( + 200, + serde_json::to_vec(&json!({"key":"K123456789", "name":"host-a"})).unwrap(), + ); + peer.enqueue_response( + 200, + serde_json::to_vec(&json!({"status":"ok", "key":"stored"})).unwrap(), + ); + peer.enqueue_response(200, serde_json::to_vec(&json!({"segments":[]})).unwrap()); + peer.enqueue_response(200, b"{}".to_vec()); + peer.enqueue_response(200, b"{}".to_vec()); let temp = TempDir::new().unwrap(); let mut config = config(&server, &temp); - config.key.clear(); + config.key = "K123456789".into(); config.stream = "host-a".into(); - let client = client(&config); - assert!(client.ensure_registered(&mut config).await); + let session = start_private_link_session(&config.config_dir, peer.credential(), "host-a") + .await + .unwrap(); + let descriptor = json!({ + "hostname": "host-a", + "label": "host-a", + "platform": "linux", + "stream_type": "desktop", + "version": "0.1.0", + }); + assert!(matches!( + session.register_for_test(&descriptor).await, + LinkOutcome::Success { + status: StatusCode::OK, + .. + } + )); + publish_observer_registration( + &session, + &ObserverState { + credential_instance_id: peer.credential().instance_id, + key: "K123456789".into(), + prefix: "prefix".into(), + name: "host-a".into(), + ingest_url: "/app/observer/ingest".into(), + protocol_version: 2, + }, + ) + .unwrap(); + let client = UploadClient::new( + &config, + session.capability("/app/observer/ingest".into()), + "host-a", + "linux", + "0.1.0", + Arc::new(MutableClock::new(0.0, 0.0)), + ); let capture = write_file(&temp, "capture.jsonl", b"{\"event\":1}\n"); assert!( client @@ -904,18 +1423,39 @@ mod tests { .segments .is_some() ); + assert!( + client + .relay_event("observe", "stream_silent", Map::new()) + .await + ); + client.enqueue_status(Map::from_iter([("mode".into(), json!("idle"))])); + for _ in 0..100 { + if peer.requests().len() == 5 { + break; + } + tokio::task::yield_now().await; + } - let requests = server.requests(); - assert_eq!(requests.len(), 3); + let requests = peer.requests(); + assert_eq!(requests.len(), 5); assert_eq!( - (requests[0].method.as_str(), requests[0].uri.as_str()), + (requests[0].method.as_str(), requests[0].path.as_str()), ("POST", "/app/observer/register") ); assert_eq!( - requests[0].headers.get("content-type").unwrap(), - "application/json" + requests[0] + .headers + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case("content-type")) + .map(|(_, value)| value.as_str()), + Some("application/json") + ); + assert!( + requests[0] + .headers + .iter() + .all(|(name, _)| !name.eq_ignore_ascii_case("authorization")) ); - assert!(requests[0].headers.get("authorization").is_none()); assert_eq!( serde_json::from_slice::(&requests[0].body).unwrap(), json!({ @@ -927,23 +1467,23 @@ mod tests { }) ); assert_eq!( - (requests[1].method.as_str(), requests[1].uri.as_str()), + (requests[1].method.as_str(), requests[1].path.as_str()), ("POST", "/app/observer/ingest") ); assert_eq!( requests[1] .headers - .get("authorization") - .unwrap() - .to_str() + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case("authorization")) + .map(|(_, value)| value.as_str()) .unwrap(), "Bearer K123456789" ); let content_type = requests[1] .headers - .get("content-type") - .unwrap() - .to_str() + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case("content-type")) + .map(|(_, value)| value.as_str()) .unwrap(); let boundary = content_type .strip_prefix("multipart/form-data; boundary=") @@ -988,26 +1528,50 @@ mod tests { ) ); assert_eq!( - (requests[2].method.as_str(), requests[2].uri.as_str()), + (requests[2].method.as_str(), requests[2].path.as_str()), ("GET", "/app/observer/ingest/segments/20260101") ); assert_eq!( requests[2] .headers - .get(OBSERVER_PROTOCOL_VERSION_HEADER) - .unwrap(), - "2" + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case(OBSERVER_PROTOCOL_VERSION_HEADER)) + .map(|(_, value)| value.as_str()), + Some("2") ); assert_eq!( requests[2] .headers - .get("authorization") - .unwrap() - .to_str() + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case("authorization")) + .map(|(_, value)| value.as_str()) .unwrap(), "Bearer K123456789" ); - assert!(peer.requests().is_empty()); + assert!(server.requests().is_empty()); + default_trap.assert_zero_connections(); + assert_eq!(peer.accepted_carriers(), 1); + for (request, expected) in [ + ( + &requests[3], + json!({"tract":"observe","event":"stream_silent"}), + ), + ( + &requests[4], + json!({"tract":"observe","event":"status","mode":"idle"}), + ), + ] { + assert_eq!( + (request.method.as_str(), request.path.as_str()), + ("POST", "/app/observer/ingest/event") + ); + assert_eq!( + serde_json::from_slice::(&request.body).unwrap(), + expected + ); + } + drop(client); + session.shutdown().await.unwrap(); peer.shutdown().await; } @@ -1024,7 +1588,7 @@ mod tests { // AC 1/2: a stale-key 401 registers, persists a distinct replacement, and the next relay uses it. #[tokio::test] - async fn stale_key_401_issues_register_request() { + async fn later_safe_retry_uses_new_generation() { const STALE: &str = "STALE-KEY-111"; const NEW: &str = "NEW-KEY-222"; let server = MockServer::new(vec![ @@ -1053,7 +1617,7 @@ mod tests { assert!(!client.relay_event("observe", "status", Map::new()).await); assert_eq!(server.request_count("/app/observer/register"), 1); assert_eq!(client.inner.key.lock().unwrap().as_str(), NEW); - assert_eq!(load_config(client.inner.paths.clone()).config.key, NEW); + assert_eq!(load_config(client.inner.paths.clone()).config.key, ""); assert!(client.relay_event("observe", "status", Map::new()).await); let requests = server.requests(); assert_eq!( @@ -1067,9 +1631,30 @@ mod tests { ); } + #[tokio::test] + async fn rejected_request_is_never_blindly_replayed() { + let server = MockServer::new(vec![ + (401, json!({})), + (200, json!({"key":"NEW-KEY","name":"desktop-new"})), + ]) + .await; + let temp = TempDir::new().unwrap(); + let config = Config { + key: "STALE-KEY".into(), + stream: "desktop".into(), + ..config(&server, &temp) + }; + let client = client(&config); + assert!(!client.relay_event("observe", "status", Map::new()).await); + let requests = server.requests(); + assert_eq!(requests.len(), 2); + assert_eq!(requests[0].uri, "/app/observer/ingest/event"); + assert_eq!(requests[1].uri, "/app/observer/register"); + } + // AC 3/7: an idempotent same-key response is not recovery and still arms cooldown. #[tokio::test] - async fn same_key_registration_is_not_recovery_and_arms_cooldown() { + async fn same_key_repair_reports_truthfully_without_generation_publish() { let server = MockServer::new(vec![ (401, json!({})), (200, json!({"key":"STALE-KEY","name":"desktop"})), @@ -1108,7 +1693,7 @@ mod tests { // AC 6: the real sync-query/event race shares one in-flight recovery attempt. #[tokio::test] - async fn sync_and_event_401_share_one_inflight_recovery() { + async fn concurrent_401_burst_registers_once_per_generation() { let gate = Arc::new(tokio::sync::Notify::new()); let server = MockServer::new_actions(vec![ Action::GatedResponse(401, json!({}), Arc::clone(&gate)), @@ -1144,7 +1729,7 @@ mod tests { // AC 7: a burst is bounded by cooldown and the next 300-second window starts fresh. #[tokio::test] - async fn recovery_cooldown_bounds_register_requests_and_resets() { + async fn cooldown_suppresses_registration_until_next_generation_window() { let mut responses = vec![ (401, json!({})), (200, json!({"key":"STALE-KEY","name":"desktop"})), @@ -1162,7 +1747,13 @@ mod tests { ..config(&server, &temp) }; let clock = Arc::new(MutableClock::new(0.0, 0.0)); - let client = UploadClient::new(&config, "host", "linux", "test", clock.clone()); + let client = crate::upload::capability_less_client_for_test( + &config, + "host", + "linux", + "test", + clock.clone(), + ); for _ in 0..20 { assert!(!client.relay_event("observe", "status", Map::new()).await); } @@ -1207,10 +1798,7 @@ mod tests { assert!(!client.relay_event("observe", "status", Map::new()).await); assert_eq!(config, before); assert_eq!(client.inner.key.lock().unwrap().as_str(), "STALE-KEY"); - assert_eq!( - load_config(client.inner.paths.clone()).config.key, - "STALE-KEY" - ); + assert_eq!(load_config(client.inner.paths.clone()).config.key, ""); assert!(client.relay_event("observe", "status", Map::new()).await); let requests = server.requests(); assert_eq!( @@ -1270,15 +1858,15 @@ mod tests { let saved = load_config(client.inner.paths.clone()).config; let pair = (saved.key.as_str(), saved.stream.as_str()); assert!( - pair == (STALE, "desktop-old") || pair == (NEW, "desktop-new"), + pair == ("", "desktop-old") || pair == ("", "desktop-new"), "unexpected reachable identity {pair:?}" ); assert_eq!( pair, if cancel { - (STALE, "desktop-old") + ("", "desktop-old") } else { - (NEW, "desktop-new") + ("", "desktop-new") } ); } @@ -1864,7 +2452,7 @@ mod tests { // AC: failed registration persistence does not latch in-memory registration #[tokio::test] - async fn registration_persistence_failure_can_retry() { + async fn persistence_failed_repair_keeps_old_generation_and_reports_failure() { let server = MockServer::new(vec![ (200, json!({"key":"key", "name":"fedora"})), (200, json!({"key":"key", "name":"fedora"})), @@ -1909,10 +2497,8 @@ mod tests { } async fn assert_403_latches(path: &str) { - let server = MockServer::new(vec![(403, json!({}))]).await; - let temp = TempDir::new().unwrap(); - let config = config(&server, &temp); - let client = client(&config); + let default_trap = OpportunisticDefaultListenerTrap::bind(); + let (temp, legacy, peer, session, client) = linked_client(403, json!({})).await; match path { "upload" => { let media = write_file(&temp, "a.flac", b"a"); @@ -1920,7 +2506,13 @@ mod tests { assert!(client.is_revoked()); } "listing" => { - assert!(client.get_server_segments("d").await.segments.is_none()); + assert!( + client + .get_server_segments("20260101") + .await + .segments + .is_none() + ); assert!(client.is_revoked()); } "event" => { @@ -1929,11 +2521,17 @@ mod tests { } _ => unreachable!(), } + assert!(legacy.requests().is_empty()); + default_trap.assert_zero_connections(); + assert_eq!(peer.requests().len(), 1); + drop(client); + session.shutdown().await.unwrap(); + peer.shutdown().await; } // AC 18: register-route guard refusal deliberately no longer latches revocation. #[tokio::test] - async fn registration_403_does_not_latch_revoked() { + async fn registration_route_guard_refusal_keeps_prior_authority_and_does_not_latch_or_pair() { let server = MockServer::new(vec![(403, json!({"reason_code":"local_request_only"}))]).await; let temp = TempDir::new().unwrap(); @@ -1982,7 +2580,7 @@ mod tests { assert!(!client.is_revoked()); assert_eq!(client.inner.key.lock().unwrap().as_str(), "STALE-KEY"); let saved = load_config(client.inner.paths.clone()).config; - assert_eq!(saved.key, "STALE-KEY"); + assert_eq!(saved.key, ""); assert_eq!(saved.stream, "desktop"); assert!(!client.relay_event("observe", "status", Map::new()).await); assert!(client.is_revoked()); @@ -2009,6 +2607,55 @@ mod tests { assert_403_latches("event").await; } + #[tokio::test] + async fn carrier_failure_does_not_latch_revocation() { + let default_trap = OpportunisticDefaultListenerTrap::bind(); + let legacy = MockServer::new(vec![]).await; + let peer = PrivateLinkPeer::start().await; + let temp = TempDir::new().unwrap(); + let config = Config { + key: "K".into(), + stream: "host-a".into(), + ..config(&legacy, &temp) + }; + let session = start_private_link_session(&config.config_dir, peer.credential(), "host-a") + .await + .unwrap(); + publish_observer_registration( + &session, + &ObserverState { + credential_instance_id: peer.credential().instance_id, + key: "K".into(), + prefix: "prefix".into(), + name: "host-a".into(), + ingest_url: "/app/observer/ingest".into(), + protocol_version: 2, + }, + ) + .unwrap(); + let client = UploadClient::new( + &config, + session.capability("/app/observer/ingest".into()), + "host-a", + "linux", + "0.1.0", + Arc::new(MutableClock::new(0.0, 0.0)), + ); + peer.shutdown().await; + assert!( + client + .get_server_segments("20260101") + .await + .segments + .is_none() + ); + assert!(!client.is_revoked()); + assert!(legacy.requests().is_empty()); + default_trap.assert_zero_connections(); + drop(client); + session.shutdown().await.unwrap(); + } + // AC: revoked upload, listing, and event relay preflight make zero requests #[tokio::test] async fn revoked_preflight_makes_zero_requests() { @@ -2039,6 +2686,7 @@ mod tests { let temp = TempDir::new().unwrap(); let mut client = UploadClient::with_silent_capacity( &config(&server, &temp), + None, "host", "linux", "v", @@ -2082,6 +2730,7 @@ mod tests { let temp = TempDir::new().unwrap(); let mut client = UploadClient::with_silent_capacity( &config(&server, &temp), + None, "host", "linux", "v",