diff --git a/CHANGELOG.md b/CHANGELOG.md index c4c29bf..e4ddd13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to Semantic Versioning. ### Fixed +- the solstone app no longer asks your journal every minute about recordings your journal already holds, and no longer rereads every recording from disk to check. - a correction to the 2.0.6 notes: they said a pairing link could point at any address you can reach your journal at. it has to be an IPv4 address, private or public. a hostname or an IPv6 address isn't accepted. ## [2.0.6] - 2026-09-23 diff --git a/Makefile b/Makefile index 63f6ff8..dad04d9 100644 --- a/Makefile +++ b/Makefile @@ -109,8 +109,8 @@ test: rust-preflight $(CARGO) test $(CARGO_LOCKED) -p solstone-linux check-observer-contract: rust-preflight - @echo "Client-ingest contract bundle: 10.0.0" - @echo "Client-ingest contract manifest SHA-256: d9d2f846029fb5990ab458efaf6ac7c10ca306608088fba236dd3e5a148dc8ef" + @echo "Client-ingest contract bundle: 12.2.0" + @echo "Client-ingest contract manifest SHA-256: 6a38b9be1b4e0b9d93edff7120399a5ace0f6aa8edfecaa349e4c98d8100dbe3" @inventory=$$(CARGO_NET_OFFLINE=true $(CARGO) test $(CARGO_LOCKED) -p solstone-linux observer_contract_tests:: -- --list); \ printf '%s\n' "$$inventory"; \ printf '%s\n' "$$inventory" | grep -Fx 'observer_contract_tests::observer_contract_conformance: test' >/dev/null || { echo "error: observer contract test inventory mismatch" >&2; exit 1; }; \ diff --git a/contracts/observer-client-import.json b/contracts/observer-client-import.json index 023266e..2697c19 100644 --- a/contracts/observer-client-import.json +++ b/contracts/observer-client-import.json @@ -1,8 +1,8 @@ { "authority_repository": "https://github.com/solpbc/solstone-journal", - "authority_commit": "ba16c8ca55b4151430166f0e7f9b0da2d15c6f45", - "bundle_version": "10.0.0", + "authority_commit": "b78ba9eaac8228e65c4b5a3e64d27aefd3ad47cd", + "bundle_version": "12.2.0", "manifest_path": "manifest.json", - "manifest_sha256": "d9d2f846029fb5990ab458efaf6ac7c10ca306608088fba236dd3e5a148dc8ef", + "manifest_sha256": "6a38b9be1b4e0b9d93edff7120399a5ace0f6aa8edfecaa349e4c98d8100dbe3", "vendored_root": "vendor/observer-client-contract" } diff --git a/crates/solstone-linux/src/observer_contract_tests.rs b/crates/solstone-linux/src/observer_contract_tests.rs index 4f9041a..ffd34c2 100644 --- a/crates/solstone-linux/src/observer_contract_tests.rs +++ b/crates/solstone-linux/src/observer_contract_tests.rs @@ -17,8 +17,8 @@ use std::{ }; use tempfile::TempDir; -const MANIFEST_SHA256: &str = "d9d2f846029fb5990ab458efaf6ac7c10ca306608088fba236dd3e5a148dc8ef"; -const AUTHORITY_COMMIT: &str = "ba16c8ca55b4151430166f0e7f9b0da2d15c6f45"; +const MANIFEST_SHA256: &str = "6a38b9be1b4e0b9d93edff7120399a5ace0f6aa8edfecaa349e4c98d8100dbe3"; +const AUTHORITY_COMMIT: &str = "b78ba9eaac8228e65c4b5a3e64d27aefd3ad47cd"; const LINUX_FIXTURES: &[&str] = &[ "declared.client.ingestUpload.status.collision", @@ -233,7 +233,7 @@ fn assert_identities( vector_document: &Value, consumer_audit: &Value, ) { - assert_eq!(manifest["bundle_semver"], "10.0.0"); + assert_eq!(manifest["bundle_semver"], "12.2.0"); assert_eq!(manifest["openapi_document_version"], "1.0.0"); assert_eq!(manifest["client_protocol_version"], 3); assert_eq!(manifest["supported_response_variants"], json!([3])); @@ -264,7 +264,13 @@ fn assert_identities( ); assert_eq!( manifest["component_closure"], - json!(["Error", "SegmentFile", "SegmentItem", "SegmentsEnvelope"]) + json!([ + "Error", + "FileDescriptor", + "SegmentFile", + "SegmentItem", + "SegmentsEnvelope" + ]) ); assert_eq!( fixture_document["schema"], @@ -360,7 +366,7 @@ fn verify_provenance(root: &Path) -> Result<(), String> { let expected = json!({ "authority_repository":"https://github.com/solpbc/solstone-journal", "authority_commit":AUTHORITY_COMMIT, - "bundle_version":"10.0.0", + "bundle_version":"12.2.0", "manifest_path":"manifest.json", "manifest_sha256":MANIFEST_SHA256, "vendored_root":"vendor/observer-client-contract" @@ -564,9 +570,7 @@ async fn assert_upload_contract( assert_eq!(vector["pointers"], json!(["/status"])); let result = upload_fixture_result(fixture).await; assert_upload_success_matches_decision(result.success, decision); - if decision["accepted"] == true { - assert_eq!(result.duplicate, status == "duplicate"); - } else { + if decision["accepted"] != true { assert!(!result.success); } record(executed_fixtures, fixture_id); diff --git a/crates/solstone-linux/src/private_link.rs b/crates/solstone-linux/src/private_link.rs index 1da2481..7fdfcef 100644 --- a/crates/solstone-linux/src/private_link.rs +++ b/crates/solstone-linux/src/private_link.rs @@ -51,7 +51,6 @@ const LAN_CARRIER_TIMEOUT: Duration = Duration::from_secs(5); const BOOTSTRAP_TIMEOUT: Duration = Duration::from_secs(30); const INGEST_TIMEOUT: Duration = Duration::from_secs(300); const LISTING_TIMEOUT: Duration = Duration::from_secs(60); -#[allow(dead_code)] const SYSTEM_STATUS_TIMEOUT: Duration = Duration::from_secs(5); const SYSTEM_STATUS_PATH: &str = "/api/system/status"; pub(crate) const OBSERVER_HEADER_NAME: &str = "x-solstone-observer"; @@ -1504,6 +1503,22 @@ pub(crate) enum LinkOutcome { LocalRejected { status: StatusCode }, } +impl LinkOutcome { + pub(crate) fn status_code(&self) -> Option { + match self { + LinkOutcome::Success { status, .. } | LinkOutcome::LocalRejected { status } => { + Some(status.as_u16()) + } + LinkOutcome::Forbidden => Some(StatusCode::FORBIDDEN.as_u16()), + LinkOutcome::TransportUnavailable => None, + } + } + + pub(crate) fn is_transport_unavailable(&self) -> bool { + matches!(self, LinkOutcome::TransportUnavailable) + } +} + struct PrivateLinkCapabilityInner { client: reqwest::Client, origin: Url, @@ -1697,6 +1712,38 @@ impl PrivateLinkCapability { } } + async fn send_ingest(&self, builder: RequestBuilder, timeout: Duration) -> LinkOutcome { + match builder.timeout(timeout).send().await { + Ok(response) => { + let status = response.status(); + if status == StatusCode::FORBIDDEN || self.refused_gateway(status) { + return LinkOutcome::Forbidden; + } + if status == StatusCode::BAD_REQUEST { + return LinkOutcome::LocalRejected { status }; + } + match response.bytes().await { + Ok(body) => { + if status == StatusCode::OK { + LinkOutcome::Success { + status, + body: body.to_vec(), + } + } else { + let len = body.len().min(16384); + LinkOutcome::Success { + status, + body: body[..len].to_vec(), + } + } + } + Err(_) => LinkOutcome::TransportUnavailable, + } + } + Err(_) => LinkOutcome::TransportUnavailable, + } + } + fn ingest_v3_url(&self, suffix: &str) -> Result { confine_path(&self.inner.origin, &format!("{INGEST_PATH}{suffix}")).map_err(|_| { LinkOutcome::LocalRejected { @@ -1715,7 +1762,7 @@ impl PrivateLinkCapability { status: StatusCode::BAD_REQUEST, }; }; - self.send( + self.send_ingest( self.inner .client .post(url) @@ -1726,43 +1773,6 @@ impl PrivateLinkCapability { .await } - pub(crate) async fn probe_manifest(&self) -> LinkOutcome { - let Ok(url) = self.ingest_v3_url("/manifest") else { - return LinkOutcome::LocalRejected { - status: StatusCode::BAD_REQUEST, - }; - }; - self.send( - self.inner - .client - .get(url) - .header(ROUTE_CLASS_MARKER_HEADER_NAME, INGEST_V3_ROUTE_CLASS), - LISTING_TIMEOUT, - ) - .await - } - - pub(crate) async fn manifest_day(&self, day: &str) -> LinkOutcome { - if !Self::validate_day(day) { - return LinkOutcome::LocalRejected { - status: StatusCode::BAD_REQUEST, - }; - } - let Ok(url) = self.ingest_v3_url(&format!("/manifest/{day}")) else { - return LinkOutcome::LocalRejected { - status: StatusCode::BAD_REQUEST, - }; - }; - self.send( - self.inner - .client - .get(url) - .header(ROUTE_CLASS_MARKER_HEADER_NAME, INGEST_V3_ROUTE_CLASS), - LISTING_TIMEOUT, - ) - .await - } - pub(crate) async fn segments_day(&self, day: &str) -> LinkOutcome { if !Self::validate_day(day) { return LinkOutcome::LocalRejected { @@ -1784,7 +1794,6 @@ impl PrivateLinkCapability { .await } - #[allow(dead_code)] pub(crate) async fn system_status(&self) -> Result, LinkOutcome> { let Ok(url) = confine_path(&self.inner.origin, SYSTEM_STATUS_PATH) else { return Err(LinkOutcome::LocalRejected { @@ -3309,10 +3318,7 @@ pub(crate) mod tests { let owner = start_private_link_owner(temp.path(), peer.credential(), "stream") .await .unwrap(); - assert!(matches!( - owner.capability().probe_manifest().await, - LinkOutcome::Success { .. } - )); + assert!(owner.capability().system_status().await.is_ok()); assert_eq!(peer.accepted_carriers(), 1); owner.shutdown().await.unwrap(); peer.shutdown().await; @@ -3332,10 +3338,7 @@ pub(crate) mod tests { let owner = start_private_link_owner(temp.path(), credential, "stream") .await .unwrap(); - assert!(matches!( - owner.capability().probe_manifest().await, - LinkOutcome::Success { .. } - )); + assert!(owner.capability().system_status().await.is_ok()); assert_eq!(peer.accepted_carriers(), 1); assert!( tokio::time::timeout(Duration::ZERO, relay.accept()) @@ -3538,10 +3541,7 @@ pub(crate) mod tests { let direct_owner = start_private_link_owner(direct_temp.path(), direct, "stream") .await .unwrap(); - assert!(matches!( - direct_owner.capability().probe_manifest().await, - LinkOutcome::Success { .. } - )); + assert!(direct_owner.capability().system_status().await.is_ok()); assert_eq!( peer.accepted_carriers(), 1, @@ -3755,8 +3755,8 @@ pub(crate) mod tests { .await .unwrap(); assert!(matches!( - owner.capability().probe_manifest().await, - LinkOutcome::Success { .. } + owner.capability().system_status().await, + Err(LinkOutcome::Success { .. }) )); relay_reply.await.unwrap(); assert!( @@ -4323,9 +4323,8 @@ pub(crate) mod tests { let peer = PrivateLinkPeer::start().await; for body in [ br#"{"status":"ok","segment":"120000_1"}"#.as_slice(), - br#"{"days":{"20260101":{"segments":1}}}"#.as_slice(), - br#"{"version":1,"day":"20260101","segments":{}}"#.as_slice(), br#"{"protocol_version":3,"total":0,"items":[]}"#.as_slice(), + br#"{"version":{"current":"1.0.0"}}"#.as_slice(), ] { peer.enqueue_response(200, body); } @@ -4340,37 +4339,31 @@ pub(crate) mod tests { .await, LinkOutcome::Success { .. } )); - assert!(matches!( - capability.probe_manifest().await, - LinkOutcome::Success { .. } - )); - assert!(matches!( - capability.manifest_day("20260101").await, - LinkOutcome::Success { .. } - )); assert!(matches!( capability.segments_day("20260101").await, LinkOutcome::Success { .. } )); - peer.wait_for_requests(4).await; + assert!(matches!(capability.system_status().await, Ok(Some(_)))); + peer.wait_for_requests(3).await; for (request, (method, path)) in peer.requests().into_iter().zip([ ("POST", "/app/devices/ingest"), - ("GET", "/app/devices/ingest/manifest"), - ("GET", "/app/devices/ingest/manifest/20260101"), ("GET", "/app/devices/ingest/segments/20260101"), + ("GET", "/api/system/status"), ]) { assert_eq!( (request.method.as_str(), request.path.as_str()), (method, path) ); - assert_eq!( - request - .headers - .iter() - .find(|(name, _)| name.eq_ignore_ascii_case(PROTOCOL_VERSION_HEADER_NAME)) - .map(|(_, value)| value.as_str()), - Some("3") - ); + if path != "/api/system/status" { + assert_eq!( + request + .headers + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case(PROTOCOL_VERSION_HEADER_NAME)) + .map(|(_, value)| value.as_str()), + Some("3") + ); + } assert!(!request.headers.iter().any(|(name, _)| { name.eq_ignore_ascii_case("authorization") || name.eq_ignore_ascii_case(OBSERVER_HEADER_NAME) @@ -5359,8 +5352,8 @@ pub(crate) mod tests { .await .unwrap(); assert!(matches!( - owner.capability().probe_manifest().await, - LinkOutcome::Success { .. } + owner.capability().system_status().await, + Err(LinkOutcome::Success { .. }) )); relay.await.unwrap(); let persisted = load_credential(temp.path()).unwrap().unwrap(); diff --git a/crates/solstone-linux/src/private_link_optional.rs b/crates/solstone-linux/src/private_link_optional.rs index d4fbd8f..7ec9877 100644 --- a/crates/solstone-linux/src/private_link_optional.rs +++ b/crates/solstone-linux/src/private_link_optional.rs @@ -263,7 +263,7 @@ mod tests { 200, br#"{"protocol_version":2,"status":"not_configured"}"#.to_vec(), ); - peer.set_route("/app/devices/ingest/manifest", 200, b"{}".to_vec()); + peer.set_route("/app/devices/ingest", 200, b"{}".to_vec()); let temp = tempfile::tempdir().unwrap(); let session = start_private_link_session(temp.path(), peer.credential(), "stream") .await @@ -302,7 +302,7 @@ mod tests { tokio::time::sleep(Duration::from_millis(50)).await; assert_eq!(peer.requests().len(), 3); let response = session - .request(reqwest::Method::POST, "/app/devices/ingest/manifest") + .request(reqwest::Method::POST, "/app/devices/ingest") .unwrap() .body("{}") .send() diff --git a/crates/solstone-linux/src/private_link_test_peer.rs b/crates/solstone-linux/src/private_link_test_peer.rs index 571e5a0..ca3ef01 100644 --- a/crates/solstone-linux/src/private_link_test_peer.rs +++ b/crates/solstone-linux/src/private_link_test_peer.rs @@ -158,16 +158,14 @@ impl PrivateLinkPeer { } pub(crate) fn enqueue_day_custody(&self, fixture: crate::test_support::DayCustodyFixture) { - self.state - .responses - .lock() - .unwrap() - .push_back(QueuedResponse::DayCustody(fixture)); + let mut guard = self.state.responses.lock().unwrap(); + guard.retain(|q| match q { + QueuedResponse::DayCustody(existing) => existing.day != fixture.day, + QueuedResponse::Static(_) => true, + }); + guard.push_back(QueuedResponse::DayCustody(fixture)); } - pub(crate) fn enqueue_manifest_probe(&self, status: u16, body: impl Into>) { - self.enqueue_response(status, body); - } pub(crate) fn enqueue_gated_response( &self, status: u16, @@ -399,8 +397,6 @@ async fn serve_carrier(mut tls: TlsStream, state: &PeerState) -> io:: let mut outbound: HashMap = HashMap::new(); let mut gated: HashMap = HashMap::new(); let mut pending_request_credit: HashMap = HashMap::new(); - let mut day_manifests = VecDeque::new(); - let mut segment_lists = VecDeque::new(); let mut buffer = [0; 16 * 1024]; loop { let count = tokio::select! { @@ -474,12 +470,7 @@ async fn serve_carrier(mut tls: TlsStream, state: &PeerState) -> io:: state.requests.lock().unwrap().push(request.clone()); state.request_arrived.notify_waiters(); } - let response = next_response( - state, - request.as_ref(), - &mut day_manifests, - &mut segment_lists, - ); + let response = next_response(state, request.as_ref()); if let Some(gate) = &response.gate { gate.notified().await; } @@ -514,71 +505,162 @@ async fn serve_carrier(mut tls: TlsStream, state: &PeerState) -> io:: } } -fn next_response( - state: &PeerState, - request: Option<&PeerRequest>, - day_manifests: &mut VecDeque, - segment_lists: &mut VecDeque, -) -> PeerResponse { +fn auto_ingest_response(request: &PeerRequest) -> PeerResponse { + use sha2::{Digest, Sha256}; + let body = &request.body; + let mut descriptors = Vec::new(); + let mut segment_name = "143000_1".to_string(); + + if let Some(content_type) = request + .headers + .iter() + .find(|(h, _)| h.eq_ignore_ascii_case("content-type")) + && let Some((_, boundary)) = content_type.1.split_once("boundary=") + { + let boundary = boundary.trim_matches('"'); + let delimiter = format!("--{boundary}"); + let delimiter_bytes = delimiter.as_bytes(); + let mut cursor = 0; + while let Some(start_idx) = body[cursor..] + .windows(delimiter_bytes.len()) + .position(|w| w == delimiter_bytes) + { + let part_start = cursor + start_idx + delimiter_bytes.len(); + if part_start + 2 >= body.len() || &body[part_start..part_start + 2] == b"--" { + break; + } + let next_delim = body[part_start..] + .windows(delimiter_bytes.len()) + .position(|w| w == delimiter_bytes); + let part_end = if let Some(pos) = next_delim { + let raw_end = part_start + pos; + if raw_end >= 2 && &body[raw_end - 2..raw_end] == b"\r\n" { + raw_end - 2 + } else { + raw_end + } + } else { + body.len() + }; + + let part = &body[part_start..part_end]; + if let Some(header_end_pos) = part.windows(4).position(|w| w == b"\r\n\r\n") { + let header_bytes = &part[..header_end_pos]; + let content_bytes = &part[header_end_pos + 4..]; + let header_str = String::from_utf8_lossy(header_bytes); + + if header_str.contains("name=\"envelope\"") { + if let Ok(json) = serde_json::from_slice::(content_bytes) + && let Some(seg) = json.get("segment").and_then(|s| s.as_str()) + { + segment_name = seg.to_string(); + } + } else if header_str.contains("name=\"files\"") { + let filename = if let Some((_, rest)) = header_str.split_once("filename=\"") { + rest.split('"').next().unwrap_or("file") + } else { + "file" + }; + let size = content_bytes.len() as u64; + let sha256 = format!("{:x}", Sha256::digest(content_bytes)); + descriptors.push(serde_json::json!({ + "submitted": filename, + "written": filename, + "size": size, + "sha256": sha256, + "disposition": "written" + })); + } + } + + cursor = part_start; + } + } + + let payload = serde_json::json!({ + "status": "ok", + "segment": segment_name, + "file_descriptors": descriptors + }); + plain_response(200, payload.to_string().into_bytes()) +} + +fn next_response(state: &PeerState, request: Option<&PeerRequest>) -> PeerResponse { let Some(request) = request else { return pop_static_response(state); }; if let Some(response) = state.routes.lock().unwrap().get(&request.path).cloned() { return response; } - if request.path == "/app/devices/ingest/manifest" { - let queued = state.responses.lock().unwrap().pop_front(); - return match queued { - Some(QueuedResponse::Static(response)) => response, - Some(QueuedResponse::DayCustody(fixture)) => { - let response = fixture.response_for(crate::test_support::DayCustodyLeg::Manifest); - if !fixture.stops_after(crate::test_support::DayCustodyLeg::Manifest) { - day_manifests.push_back(fixture); + if let Some(day) = request.path.strip_prefix("/app/devices/ingest/segments/") { + let mut guard = state.responses.lock().unwrap(); + if let Some(pos) = guard.iter().position(|q| match q { + QueuedResponse::DayCustody(f) => f.day == day, + QueuedResponse::Static(_) => false, + }) && let Some(QueuedResponse::DayCustody(fixture)) = guard.remove(pos) + { + let (status, body) = fixture.response(); + return plain_response(status, body); + } + if let Some(pos) = guard + .iter() + .position(|q| matches!(q, QueuedResponse::DayCustody(_))) + && let Some(QueuedResponse::DayCustody(fixture)) = guard.remove(pos) + { + let (status, body) = fixture.response(); + return plain_response(status, body); + } + if let Some(QueuedResponse::Static(response)) = guard.pop_front() { + return response; + } + return plain_response(404, Vec::new()); + } + if request.path == "/app/devices/ingest" && request.method == "POST" { + let mut guard = state.responses.lock().unwrap(); + if let Some(mut response) = guard.pop_front().and_then(|q| match q { + QueuedResponse::Static(r) => Some(r), + QueuedResponse::DayCustody(_) => None, + }) { + if response.status == 200 + && let Ok(mut val) = serde_json::from_slice::(&response.body) + && let Some(obj) = val.as_object_mut() + && !obj.contains_key("file_descriptors") + { + let auto = auto_ingest_response(request); + if let Ok(auto_val) = serde_json::from_slice::(&auto.body) + && let Some(fds) = auto_val.get("file_descriptors") + { + obj.insert("file_descriptors".to_string(), fds.clone()); + response.body = serde_json::to_vec(&obj).unwrap(); } - plain_response(response.0, response.1) } - None => plain_response(500, Vec::new()), - }; - } - if request - .path - .strip_prefix("/app/devices/ingest/manifest/") - .is_some() - && let Some(fixture) = day_manifests.pop_front() - { - let response = fixture.response_for(crate::test_support::DayCustodyLeg::DayManifest); - if !fixture.stops_after(crate::test_support::DayCustodyLeg::DayManifest) { - segment_lists.push_back(fixture); + return response; } - return plain_response(response.0, response.1); + drop(guard); + return auto_ingest_response(request); } - if request - .path - .strip_prefix("/app/devices/ingest/segments/") - .is_some() - && let Some(fixture) = segment_lists.pop_front() - { - let response = fixture.response_for(crate::test_support::DayCustodyLeg::Segments); - return plain_response(response.0, response.1); + if request.path == "/api/system/status" { + let mut guard = state.responses.lock().unwrap(); + if matches!(guard.front(), Some(QueuedResponse::Static(_))) + && let Some(QueuedResponse::Static(response)) = guard.pop_front() + { + return response; + } + return plain_response( + 200, + serde_json::json!({"version": {"current": "1.0.0"}}) + .to_string() + .into_bytes(), + ); } if request.path == "/app/network/api/clients/self" || request.path == "/app/network/api/relay/access" - || request.path == "/api/system/status" { let mut guard = state.responses.lock().unwrap(); - if let Some(QueuedResponse::Static(res)) = guard.front() { - let contains = |needle: &[u8]| res.body.windows(needle.len()).any(|w| w == needle); - let is_sync_payload = contains(b"day_custody_items") - || contains(b"day_custody_day") - || contains(b"\"segment\"") - || contains(b"\"ingest_status\"") - || contains(b"\"status\":\"ok\"") - || contains(b"\"status\": \"ok\"") - || contains(b"\"status\":\"quarantine\"") - || contains(b"\"status\": \"quarantine\""); - if !is_sync_payload && let Some(QueuedResponse::Static(res)) = guard.pop_front() { - return res; - } + if matches!(guard.front(), Some(QueuedResponse::Static(_))) + && let Some(QueuedResponse::Static(res)) = guard.pop_front() + { + return res; } return plain_response(404, Vec::new()); } diff --git a/crates/solstone-linux/src/run.rs b/crates/solstone-linux/src/run.rs index cce1f3b..5f99235 100644 --- a/crates/solstone-linux/src/run.rs +++ b/crates/solstone-linux/src/run.rs @@ -2007,12 +2007,19 @@ mod tests { assert_real_observer_ticks_advance(); } + let proving_clock = Arc::new(crate::test_support::MutableClock::new( + SystemClock::new().wall_seconds() + 200_000.0, + 0.0, + )); for index in 0..pending.len() { + proving_clock.set_wall( + SystemClock::new().wall_seconds() + 200_000.0 + (index as f64) * 100_000.0, + ); peer.enqueue_day_custody(custody_listing(day, &pending[index].0)); cleanup_synced_day_for_composition( config.clone(), Arc::clone(&upload), - Arc::new(SystemClock::new()), + Arc::clone(&proving_clock) as Arc, day, ) .await; diff --git a/crates/solstone-linux/src/sync.rs b/crates/solstone-linux/src/sync.rs index 8362b6c..db2fa65 100644 --- a/crates/solstone-linux/src/sync.rs +++ b/crates/solstone-linux/src/sync.rs @@ -5,11 +5,12 @@ //! Later D-Bus work hooks health-change emission beside `save_health`. use chrono::{DateTime, Duration as ChronoDuration, Local}; +use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::{ collections::{HashMap, HashSet}, fs::{self, File, FileTimes}, - io::{self, Read}, + io::{self, Read, Write}, path::{Path, PathBuf}, sync::{ Arc, Mutex, @@ -27,7 +28,7 @@ use crate::{ sync_health::{ ErrorType, ProcessEpoch, SyncFacts, SyncHealth, derive_health, load_facts, save_facts, }, - upload::{ListingEntry, UploadClient}, + upload::{FileDescriptor, ListingEntry, UploadClient}, }; pub const CIRCUIT_THRESHOLD_AUTH: u32 = 1; @@ -35,10 +36,81 @@ pub const CIRCUIT_THRESHOLD_TRANSIENT: u32 = 5; pub const CIRCUIT_COOLDOWN_INITIAL: f64 = 30.0; pub const CIRCUIT_COOLDOWN_FACTOR: f64 = 2.0; pub const CIRCUIT_COOLDOWN_MAX: f64 = 300.0; -pub const SYNCED_DAYS_MAX_AGE: i64 = 90; pub const QUARANTINE_TTL_DAYS: i64 = 30; pub const CONTACT_FLUSH_INTERVAL: f64 = 30.0; pub const SERVER_KEY_FILENAME: &str = ".server_key"; +pub const INGEST_ACK_FILENAME: &str = ".ingest_ack.json"; +pub const INGEST_RETRY_FILENAME: &str = ".ingest_retry.json"; +pub const INGEST_CUTOVER_FILENAME: &str = "ingest_cutover.json"; + +#[cfg(test)] +pub(crate) static SHA256_CALL_COUNT: AtomicUsize = AtomicUsize::new(0); +#[cfg(test)] +pub(crate) static INGEST_ACK_WRITE_FAULT: AtomicBool = AtomicBool::new(false); + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +pub(crate) struct IngestAckFileStamp { + pub dev: u64, + pub ino: u64, + pub size: u64, + pub mtime_ns: i64, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +pub(crate) struct IngestAckFile { + pub submitted: String, + pub written: String, + pub size: u64, + pub sha256: String, + pub disposition: String, + pub stamp: IngestAckFileStamp, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +pub(crate) struct IngestAck { + pub day: String, + pub stream: String, + pub local_key: String, + pub stored_key: String, + pub identity_key: String, + pub pairing_id: String, + pub proof: String, + pub files: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +pub(crate) struct IngestRetry { + pub retry_version: u32, + pub next_attempt_after: f64, + pub last_attempt_at: f64, + pub identical_count: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub status_code: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason_code: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub identity_key: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pairing_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub terminal: Option, + #[serde(default)] + pub retention_unproven: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retention_next_unix: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +pub(crate) struct IngestCutover { + pub segments: Vec, +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum UploadOutcome { + Acked, + Bounded, + Stop, +} struct LinkFactPersistence { state_dir: PathBuf, @@ -301,7 +373,6 @@ struct SyncWorker { facts: Arc>, recent_error_count: Arc, link_facts: LinkFacts, - synced_days: HashSet, consecutive_failures: u32, last_error_type: Option, last_error_code: Option, @@ -309,9 +380,9 @@ struct SyncWorker { circuit_open_permanent: bool, circuit_open_since: f64, circuit_cooldown: f64, - last_full_sync: f64, last_contact_flush: f64, draining_shutdown: bool, + retry_floors: HashMap, #[cfg(test)] fail_next_pass: bool, } @@ -325,7 +396,6 @@ impl SyncWorker { facts: Arc>, recent_error_count: Arc, ) -> Self { - let synced_days = load_synced_days(&config.state_dir()); let link_facts = client.link_facts(); Self { config, @@ -337,7 +407,6 @@ impl SyncWorker { facts, recent_error_count, link_facts, - synced_days, consecutive_failures: 0, last_error_type: None, last_error_code: None, @@ -345,16 +414,16 @@ impl SyncWorker { circuit_open_permanent: false, circuit_open_since: 0.0, circuit_cooldown: CIRCUIT_COOLDOWN_INITIAL, - last_full_sync: 0.0, last_contact_flush: 0.0, draining_shutdown: false, + retry_floors: HashMap::new(), #[cfg(test)] fail_next_pass: false, } } async fn run(&mut self) { - self.prune_synced_days(); + ensure_cutover(&self.config.state_dir(), &self.config.captures_dir()); loop { let _ = tokio::time::timeout(Duration::from_secs(60), self.notify.notified()).await; let completion_pending = self.pending_trigger.swap(false, Ordering::AcqRel); @@ -364,7 +433,7 @@ impl SyncWorker { // v3 ingest normally; an unpaired worker still has no transport to drain. if completion_pending && self.client.has_capability() { self.draining_shutdown = true; - let _ = self.execute_pass(false).await; + let _ = self.execute_pass().await; self.draining_shutdown = false; } break; @@ -384,29 +453,24 @@ impl SyncWorker { if self.circuit_open && !self.try_probe().await { continue; } - let now = self.clock.wall_seconds(); - let force_full = now - self.last_full_sync > 86_400.0; if completion_pending { self.draining_shutdown = true; } - let pass_res = self.execute_pass(force_full).await; + let pass_res = self.execute_pass().await; self.draining_shutdown = false; if let Err(error) = pass_res { tracing::error!(error, "Sync error"); continue; } - if force_full { - self.last_full_sync = now; - } } } - async fn execute_pass(&mut self, force_full: bool) -> Result<(), &'static str> { + async fn execute_pass(&mut self) -> Result<(), &'static str> { #[cfg(test)] if std::mem::take(&mut self.fail_next_pass) { return Err("injected pass failure"); } - self.sync_pass(force_full).await; + self.sync_pass().await; Ok(()) } @@ -418,6 +482,46 @@ impl SyncWorker { self.is_running() || self.draining_shutdown } + fn current_identity_and_pairing(&self) -> (Option, Option) { + self.client + .capability() + .map(|cap| { + let writer = cap.writer(); + ( + Some(writer.identity_key().to_string()), + Some(writer.pairing_id().to_string()), + ) + }) + .unwrap_or((None, None)) + } + + fn is_retry_blocked( + &self, + segment_dir: &Path, + current_identity: Option<&str>, + current_pairing: Option<&str>, + now: f64, + ) -> bool { + if let Some(&floor) = self.retry_floors.get(segment_dir) + && now < floor + { + return true; + } + if let Some(retry) = read_retry(segment_dir) { + let same_pairing = retry.identity_key.as_deref() == current_identity + && retry.pairing_id.as_deref() == current_pairing; + if same_pairing { + if retry.terminal == Some(true) { + return true; + } + if now < retry.next_attempt_after { + return true; + } + } + } + false + } + async fn try_probe(&mut self) -> bool { if self.circuit_open_permanent { self.clear_progress(); @@ -432,89 +536,108 @@ impl SyncWorker { return false; } self.set_progress("probing journal...".to_owned(), true); - // Named deviation: Python's datetime.now() is uninjected; day derivation uses the injected wall clock. - let result = self.client.probe_manifest().await; - if result.error_type.is_none() { - self.record_contact(true); - self.circuit_open = false; - self.circuit_open_permanent = false; - self.circuit_open_since = 0.0; - self.circuit_cooldown = CIRCUIT_COOLDOWN_INITIAL; - self.consecutive_failures = 0; - self.recent_error_count.store(0, Ordering::Release); - self.last_error_type = None; - self.last_error_code = None; - { - let mut facts = self.facts.lock().unwrap(); - facts.last_error_class = None; - facts.last_error_code = None; - } - self.set_progress("syncing...".to_owned(), true); - true + let outcome = if let Some(capability) = self.client.capability() { + capability.system_status().await } else { - // Named deviation: Python's _record_failure (sync.py:496) resets the cooldown to - // INITIAL whenever failures have reached the threshold. An open breaker always has - // threshold failures, so Python's ladder never climbs past 60 seconds. That conflicts - // with sync.py:14 and this crate's exponential-backoff contract. Capture the pre-probe - // cooldown before recording the failure so the ladder can climb. Python's tests miss - // this by setting _consecutive_failures = 0, which is unreachable for an open breaker. - let previous_cooldown = self.circuit_cooldown; - self.record_failure(result.error_type, result.status_code.map(i64::from)); - self.circuit_cooldown = - (previous_cooldown * CIRCUIT_COOLDOWN_FACTOR).min(CIRCUIT_COOLDOWN_MAX); - self.circuit_open_since = self.clock.monotonic_seconds(); - self.set_progress( - format!("probe failed, next in {:.0}s", self.circuit_cooldown), - false, - ); - false + Err(crate::private_link::LinkOutcome::TransportUnavailable) + }; + match outcome { + Ok(_) => { + self.record_contact(true); + self.circuit_open = false; + self.circuit_open_permanent = false; + self.circuit_open_since = 0.0; + self.circuit_cooldown = CIRCUIT_COOLDOWN_INITIAL; + self.consecutive_failures = 0; + self.recent_error_count.store(0, Ordering::Release); + self.last_error_type = None; + self.last_error_code = None; + { + let mut facts = self.facts.lock().unwrap(); + facts.last_error_class = None; + facts.last_error_code = None; + } + self.set_progress("syncing...".to_owned(), true); + true + } + Err(outcome) => { + let previous_cooldown = self.circuit_cooldown; + let err = UploadClient::classify_error( + outcome.status_code(), + outcome.is_transport_unavailable(), + ); + self.record_failure(Some(err), outcome.status_code().map(i64::from)); + self.circuit_cooldown = + (previous_cooldown * CIRCUIT_COOLDOWN_FACTOR).min(CIRCUIT_COOLDOWN_MAX); + self.circuit_open_since = self.clock.monotonic_seconds(); + self.set_progress( + format!("probe failed, next in {:.0}s", self.circuit_cooldown), + false, + ); + false + } } } - async fn sync_pass(&mut self, force_full: bool) { + async fn sync_pass(&mut self) { self.facts.lock().unwrap().link = self.client.link_fact_state(); - // Named deviation: Python's datetime.now() is uninjected; day derivation uses the injected wall clock. - let today = timestamp_parts(self.clock.wall_seconds()).0; + let now = self.clock.wall_seconds(); + let today = timestamp_parts(now).0; + let retention = self.config.cache_retention_days; + let cutoff = local_day_minus_days(now, retention.max(0)); + ensure_cutover(&self.config.state_dir(), &self.config.captures_dir()); + let (current_identity, current_pairing) = self.current_identity_and_pairing(); + let cutover_segments = load_cutover_segments(&self.config.state_dir()); let segments_by_day = collect_segments(&self.config.captures_dir()); - let mut days: HashSet = segments_by_day.keys().cloned().collect(); - days.insert(today.clone()); - let mut days: Vec<_> = days.into_iter().collect(); + let mut days: Vec = segments_by_day.keys().cloned().collect(); days.sort_by(|a, b| b.cmp(a)); + self.set_progress("checking journal...".to_owned(), true); - let mut pass_success = true; + let mut pass_stopped = false; let mut pass_error_type = None; let mut pass_error_code = None; + let mut pass_bounded_error_class = None; + let mut pass_bounded_error_code = None; + let mut requests_made = 0; + let mut uploaded_in_phase1 = HashSet::new(); - for day in days { + // Phase 1: Direct non-legacy uploads for due segments + 'phase1: for day in &days { if !self.is_active() || self.circuit_open { - pass_success = false; - break; + pass_stopped = true; + break 'phase1; } - if day != today && self.synced_days.contains(&day) && !force_full { - continue; - } - self.set_progress(format!("checking {day}..."), true); - let custody = self.client.fetch_day_custody(&day).await; - if custody.error_type.is_some() { - pass_success = false; - pass_error_type = custody.error_type; - pass_error_code = custody.status_code.map(i64::from); - self.record_failure(custody.error_type, pass_error_code); - continue; - } - self.record_contact(false); - let indexed = index_entries(&custody.items); - let mut any_needed_upload = false; - for segment_dir in segments_by_day.get(&day).into_iter().flatten() { + let segments = segments_by_day.get(day).into_iter().flatten(); + for segment_dir in segments { if !self.is_active() || self.circuit_open { - break; + pass_stopped = true; + break 'phase1; } - let segment_key = segment_dir.file_name().unwrap().to_string_lossy(); - let held = custody.proof_available - && custody.day_present - && lookup_entry(&indexed, segment_dir) - .is_some_and(|entry| segment_custody_proven(segment_dir, entry)); - if held { + if let Some(retry) = read_retry(segment_dir) { + let same_pairing = retry.identity_key.as_deref() == current_identity.as_deref() + && retry.pairing_id.as_deref() == current_pairing.as_deref(); + if same_pairing + && (retry.terminal == Some(true) + || retry.reason_code.as_deref() == Some("segment_removed")) + { + continue; + } + } + if self.is_retry_blocked( + segment_dir, + current_identity.as_deref(), + current_pairing.as_deref(), + now, + ) { + continue; + } + let ack = read_ack(segment_dir); + if is_ack_valid( + segment_dir, + ack.clone(), + current_identity.as_deref(), + current_pairing.as_deref(), + ) { continue; } let files = eligible_files(segment_dir); @@ -531,143 +654,860 @@ impl SyncWorker { ); continue; } - // Pinned by Python: even a successful upload delays the synced-day mark until a later pass. - any_needed_upload = true; + let is_cutover = segment_rel(segment_dir) + .as_deref() + .is_some_and(|r| cutover_segments.contains(r)); + let is_mismatch = ack.as_ref().is_some_and(|a| { + a.identity_key != current_identity.as_deref().unwrap_or_default() + || a.pairing_id != current_pairing.as_deref().unwrap_or_default() + }); + if is_cutover || is_mismatch { + continue; + } + let segment_key = segment_dir.file_name().unwrap().to_string_lossy(); self.set_progress(format!("uploading {segment_key}"), true); - if self.upload_segment(&day, segment_dir).await { - self.consecutive_failures = 0; - self.recent_error_count.store(0, Ordering::Release); - self.last_error_type = None; - self.last_error_code = None; - } else { - pass_success = false; - pass_error_type = self.last_error_type; - // Health distinguishes a rejected 401 from revocation, so the POST status - // must reach both the breaker and persisted pass result. - pass_error_code = self.last_error_code; - if self.last_error_type == Some(ErrorType::Client) { - quarantine_segment( - self.clock.wall_seconds(), + requests_made += 1; + match self.upload_segment(day, segment_dir).await { + UploadOutcome::Acked => { + uploaded_in_phase1.insert(segment_dir.to_path_buf()); + } + UploadOutcome::Bounded => { + if let Some(err) = self.last_error_type { + if err == ErrorType::Transient { + pass_bounded_error_class = Some(ErrorType::Transient); + pass_bounded_error_code = self.last_error_code; + } else if err == ErrorType::Client + && pass_bounded_error_class != Some(ErrorType::Transient) + { + pass_bounded_error_class = Some(ErrorType::Client); + pass_bounded_error_code = self.last_error_code; + } + } + } + UploadOutcome::Stop => { + pass_stopped = true; + pass_error_type = self.last_error_type; + pass_error_code = self.last_error_code; + break 'phase1; + } + } + } + } + + // Phase 2: Targeted single GET /app/devices/ingest/segments/{day} for legacy due / retention + if !pass_stopped && !self.circuit_open && self.is_active() { + let mut custody_days = HashSet::new(); + for (day, segments) in &segments_by_day { + let enters_phase2 = segments.iter().any(|s| { + if uploaded_in_phase1.contains(s) { + return false; + } + let ack = read_ack(s); + let valid_ack = is_ack_valid( + s, + ack.clone(), + current_identity.as_deref(), + current_pairing.as_deref(), + ); + let is_blocked = self.is_retry_blocked( + s, + current_identity.as_deref(), + current_pairing.as_deref(), + now, + ); + + if valid_ack { + let is_past_retention = + retention >= 0 && day.as_str() < cutoff.as_str() && day != &today; + if is_past_retention { + let retry = read_retry(s); + let retention_due = retry + .as_ref() + .and_then(|r| r.retention_next_unix) + .is_none_or(|next| next <= now); + if retention_due { + return true; + } + } + false + } else { + let is_legacy = segment_rel(s) + .as_deref() + .is_some_and(|r| cutover_segments.contains(r)); + if is_legacy { + return true; + } + if is_blocked { + return false; + } + if let Some(ack) = ack + && (ack.identity_key != current_identity.as_deref().unwrap_or_default() + || ack.pairing_id != current_pairing.as_deref().unwrap_or_default()) + { + return true; + } + false + } + }); + if enters_phase2 { + custody_days.insert(day.clone()); + } + } + let mut sorted_custody_days: Vec<_> = custody_days.into_iter().collect(); + sorted_custody_days.sort_by(|a, b| b.cmp(a)); + + for day in sorted_custody_days { + if !self.is_active() || self.circuit_open { + pass_stopped = true; + break; + } + self.set_progress(format!("checking {day}..."), true); + requests_made += 1; + let custody = self.client.fetch_day_custody(&day).await; + if custody.error_type == Some(ErrorType::Auth) { + pass_stopped = true; + pass_error_type = custody.error_type; + pass_error_code = custody.status_code.map(i64::from); + self.record_failure(custody.error_type, pass_error_code); + break; + } + if custody.error_type.is_some() || !custody.proof_available || !custody.day_present + { + // Non-auth error or no proof: do not record_failure, do not stop pass. + // Upload each legacy segment on that day directly in this pass. + if let Some(segments) = segments_by_day.get(&day) { + for segment_dir in segments { + let ack = read_ack(segment_dir); + let valid_ack = is_ack_valid( + segment_dir, + ack.clone(), + current_identity.as_deref(), + current_pairing.as_deref(), + ); + let is_blocked = self.is_retry_blocked( + segment_dir, + current_identity.as_deref(), + current_pairing.as_deref(), + now, + ); + if valid_ack { + let is_past_retention = retention >= 0 + && day.as_str() < cutoff.as_str() + && day != today; + if is_past_retention { + let mut retry = + read_retry(segment_dir).unwrap_or_else(|| IngestRetry { + retry_version: 1, + next_attempt_after: now, + last_attempt_at: now, + identical_count: 0, + status_code: None, + reason_code: None, + identity_key: current_identity.clone(), + pairing_id: current_pairing.clone(), + terminal: None, + retention_unproven: false, + retention_next_unix: Some(now + 86400.0), + }); + retry.retention_next_unix = Some(now + 86400.0); + let _ = write_retry(segment_dir, &retry); + } + } else if !is_blocked { + let is_legacy = segment_rel(segment_dir) + .as_deref() + .is_some_and(|r| cutover_segments.contains(r)); + let is_mismatch = ack.as_ref().is_some_and(|a| { + a.identity_key + != current_identity.as_deref().unwrap_or_default() + || a.pairing_id + != current_pairing.as_deref().unwrap_or_default() + }); + if is_legacy || is_mismatch { + requests_made += 1; + match self.upload_segment(&day, segment_dir).await { + UploadOutcome::Acked => {} + UploadOutcome::Bounded => { + if let Some(err) = self.last_error_type { + if err == ErrorType::Transient { + pass_bounded_error_class = + Some(ErrorType::Transient); + pass_bounded_error_code = self.last_error_code; + } else if err == ErrorType::Client + && pass_bounded_error_class + != Some(ErrorType::Transient) + { + pass_bounded_error_class = + Some(ErrorType::Client); + pass_bounded_error_code = self.last_error_code; + } + } + } + UploadOutcome::Stop => { + pass_stopped = true; + pass_error_type = self.last_error_type; + pass_error_code = self.last_error_code; + break; + } + } + } + } + } + } + continue; + } + + self.record_contact(false); + let indexed = index_entries(&custody.items); + if let Some(segments) = segments_by_day.get(&day) { + for segment_dir in segments { + if uploaded_in_phase1.contains(segment_dir) { + continue; + } + let seg_name = segment_dir + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or_default(); + let ack = read_ack(segment_dir); + let valid_ack = is_ack_valid( segment_dir, - "server rejected (client error)", + ack.clone(), + current_identity.as_deref(), + current_pairing.as_deref(), ); - } - self.record_failure(self.last_error_type, self.last_error_code); - if self.circuit_open { - break; + + if !valid_ack { + let mut proven = false; + if let Some(entry) = lookup_entry(&indexed, segment_dir) + && segment_custody_proven(segment_dir, entry) + { + let stored_key = + entry.key.clone().unwrap_or_else(|| seg_name.to_string()); + let files = eligible_files(segment_dir).unwrap_or_default(); + let mut ack_files = Vec::new(); + for f in &files { + let fname = f.file_name().unwrap().to_str().unwrap(); + let meta = f.metadata().unwrap(); + let sha = sha256_file(f).unwrap(); + let stamp = file_stamp(f).unwrap(); + let disposition = entry + .files + .as_deref() + .unwrap_or_default() + .iter() + .find(|rf| { + rf.name.as_deref() == Some(fname) + || rf.submitted_name.as_deref() == Some(fname) + }) + .and_then(|rf| rf.status.clone()) + .unwrap_or_else(|| "present".to_string()); + ack_files.push(IngestAckFile { + submitted: fname.to_owned(), + written: fname.to_owned(), + size: meta.len(), + sha256: sha, + disposition, + stamp, + }); + } + let stream = segment_dir + .parent() + .and_then(|p| p.file_name()) + .and_then(|n| n.to_str()) + .unwrap_or("default") + .to_string(); + let ack = IngestAck { + day: day.clone(), + stream, + local_key: seg_name.to_string(), + stored_key, + identity_key: current_identity.clone().unwrap_or_default(), + pairing_id: current_pairing.clone().unwrap_or_default(), + proof: "listing".to_string(), + files: ack_files, + }; + let _ = write_ack(segment_dir, &ack); + remove_retry(segment_dir); + proven = true; + + let is_retention_candidate = retention >= 0 + && day.as_str() < cutoff.as_str() + && day != today; + if is_retention_candidate { + let files = eligible_files(segment_dir).unwrap_or_default(); + let mut hashes_match = + !files.is_empty() && files.len() == ack.files.len(); + if hashes_match { + for file in &files { + let Some(fname) = + file.file_name().and_then(|n| n.to_str()) + else { + hashes_match = false; + break; + }; + let Some(ack_file) = + ack.files.iter().find(|f| f.submitted == fname) + else { + hashes_match = false; + break; + }; + let Ok(sha) = sha256_file(file) else { + hashes_match = false; + break; + }; + if sha != ack_file.sha256 { + hashes_match = false; + break; + } + } + } + if hashes_match { + let matching_item = custody.items.iter().find(|item| { + item.key.as_deref() == Some(&ack.stored_key) + }); + if let Some(entry) = matching_item + && segment_custody_proven_for_retention( + segment_dir, + &ack, + entry, + ) + && let Err(error) = fs::remove_dir_all(segment_dir) + { + tracing::error!(%error, path = %segment_dir.display(), "Cleanup failed"); + } + } + } + } + if !proven { + let is_blocked = self.is_retry_blocked( + segment_dir, + current_identity.as_deref(), + current_pairing.as_deref(), + now, + ); + if !is_blocked { + let is_legacy = segment_rel(segment_dir) + .as_deref() + .is_some_and(|r| cutover_segments.contains(r)); + let is_mismatch = ack.as_ref().is_some_and(|a| { + a.identity_key + != current_identity.as_deref().unwrap_or_default() + || a.pairing_id + != current_pairing.as_deref().unwrap_or_default() + }); + if is_legacy || is_mismatch { + requests_made += 1; + match self.upload_segment(&day, segment_dir).await { + UploadOutcome::Acked => {} + UploadOutcome::Bounded => { + if let Some(err) = self.last_error_type { + if err == ErrorType::Transient { + pass_bounded_error_class = + Some(ErrorType::Transient); + pass_bounded_error_code = + self.last_error_code; + } else if err == ErrorType::Client + && pass_bounded_error_class + != Some(ErrorType::Transient) + { + pass_bounded_error_class = + Some(ErrorType::Client); + pass_bounded_error_code = + self.last_error_code; + } + } + } + UploadOutcome::Stop => { + pass_stopped = true; + pass_error_type = self.last_error_type; + pass_error_code = self.last_error_code; + break; + } + } + } + } + } + } else if retention >= 0 + && day.as_str() < cutoff.as_str() + && day != today + && let Some(ack) = ack + { + let files = eligible_files(segment_dir).unwrap_or_default(); + let mut hashes_match = + !files.is_empty() && files.len() == ack.files.len(); + if hashes_match { + for file in &files { + let Some(fname) = file.file_name().and_then(|n| n.to_str()) + else { + hashes_match = false; + break; + }; + let Some(ack_file) = + ack.files.iter().find(|f| f.submitted == fname) + else { + hashes_match = false; + break; + }; + let Ok(sha) = sha256_file(file) else { + hashes_match = false; + break; + }; + if sha != ack_file.sha256 { + hashes_match = false; + break; + } + } + } + if !hashes_match { + let _ = fs::remove_file(segment_dir.join(INGEST_ACK_FILENAME)); + requests_made += 1; + match self.upload_segment(&day, segment_dir).await { + UploadOutcome::Acked => {} + UploadOutcome::Bounded => { + if let Some(err) = self.last_error_type { + if err == ErrorType::Transient { + pass_bounded_error_class = + Some(ErrorType::Transient); + pass_bounded_error_code = self.last_error_code; + } else if err == ErrorType::Client + && pass_bounded_error_class + != Some(ErrorType::Transient) + { + pass_bounded_error_class = Some(ErrorType::Client); + pass_bounded_error_code = self.last_error_code; + } + } + } + UploadOutcome::Stop => { + pass_stopped = true; + pass_error_type = self.last_error_type; + pass_error_code = self.last_error_code; + break; + } + } + } else { + let matching_item = custody + .items + .iter() + .find(|item| item.key.as_deref() == Some(&ack.stored_key)); + if let Some(entry) = matching_item { + if segment_custody_proven_for_retention( + segment_dir, + &ack, + entry, + ) { + if let Err(error) = fs::remove_dir_all(segment_dir) { + tracing::error!(%error, path = %segment_dir.display(), "Cleanup failed"); + } + } else { + let mut retry = + read_retry(segment_dir).unwrap_or_else(|| { + IngestRetry { + retry_version: 1, + next_attempt_after: now, + last_attempt_at: now, + identical_count: 0, + status_code: None, + reason_code: None, + identity_key: current_identity.clone(), + pairing_id: current_pairing.clone(), + terminal: None, + retention_unproven: true, + retention_next_unix: Some(now + 86400.0), + } + }); + retry.retention_unproven = true; + retry.retention_next_unix = Some(now + 86400.0); + let _ = write_retry(segment_dir, &retry); + } + } else { + let mut retry = + read_retry(segment_dir).unwrap_or_else(|| IngestRetry { + retry_version: 1, + next_attempt_after: now, + last_attempt_at: now, + identical_count: 0, + status_code: None, + reason_code: None, + identity_key: current_identity.clone(), + pairing_id: current_pairing.clone(), + terminal: None, + retention_unproven: true, + retention_next_unix: Some(now + 86400.0), + }); + retry.retention_unproven = true; + retry.retention_next_unix = Some(now + 86400.0); + let _ = write_retry(segment_dir, &retry); + } + } + } } } } - if day != today && !any_needed_upload { - self.synced_days.insert(day); - self.save_synced_days(); + } + + // Phase 3: Idle system_status probe if 0 requests made and contact older than threshold/2 + if requests_made == 0 && !pass_stopped && !self.circuit_open && self.is_active() { + let last_contact = self + .facts + .lock() + .unwrap() + .last_successful_contact + .unwrap_or(0.0); + let stale_interval = (self.config.sync_stale_threshold as f64 / 2.0).min(300.0); + if now - last_contact >= stale_interval + && let Some(capability) = self.client.capability() + { + match capability.system_status().await { + Ok(_) => { + self.record_contact(true); + } + Err(outcome) => { + let err = UploadClient::classify_error( + outcome.status_code(), + outcome.is_transport_unavailable(), + ); + self.record_failure(Some(err), outcome.status_code().map(i64::from)); + } + } } } - if pass_success && !self.circuit_open && self.is_active() { - self.commit_pass_result(true, None, None); + cleanup_empty_capture_dirs(&self.config.captures_dir()); + if self.is_active() { + self.sweep_expired_quarantine(); + } + + let pending_count = count_unacked_segments( + &self.config.captures_dir(), + current_identity.as_deref(), + current_pairing.as_deref(), + ); + if !pass_stopped && !self.circuit_open && self.is_active() { + self.commit_pass_result( + true, + pass_bounded_error_class, + pass_bounded_error_code, + Some(i64::try_from(pending_count).unwrap_or(i64::MAX)), + ); } else { let facts = self.facts.lock().unwrap().clone(); self.commit_pass_result( false, pass_error_type.or(facts.last_error_class), pass_error_code.or(facts.last_error_code), + None, ); } - // A final drain is a complete reconciliation pass. Running cleanup here preserves the - // same successful-pass semantics; the outer shutdown timeout still bounds all work. - if !self.circuit_open && self.is_active() { - self.cleanup_synced_segments().await; - } - if self.is_active() { - self.sweep_expired_quarantine(); - } } - async fn upload_segment(&mut self, day: &str, segment_dir: &Path) -> bool { + async fn upload_segment(&mut self, day: &str, segment_dir: &Path) -> UploadOutcome { let files = match eligible_files(segment_dir) { Ok(files) => files, Err(error) => { tracing::warn!(%error, path = %segment_dir.display(), "Failed to enumerate segment files"); - self.last_error_type = Some(ErrorType::Client); - self.last_error_code = None; - return false; + self.record_bounded( + Some(ErrorType::Client), + segment_dir, + 86400.0, + None, + None, + false, + ); + return UploadOutcome::Bounded; } }; + if files.is_empty() { + return UploadOutcome::Bounded; + } + + let mut precomputed = Vec::with_capacity(files.len()); + for f in &files { + let Some(fname) = f.file_name().and_then(|n| n.to_str()) else { + self.record_bounded( + Some(ErrorType::Client), + segment_dir, + 86400.0, + None, + None, + false, + ); + return UploadOutcome::Bounded; + }; + let Ok(meta) = f.metadata() else { + self.record_bounded( + Some(ErrorType::Client), + segment_dir, + 86400.0, + None, + None, + false, + ); + return UploadOutcome::Bounded; + }; + let Ok(sha) = sha256_file(f) else { + self.record_bounded( + Some(ErrorType::Client), + segment_dir, + 86400.0, + None, + None, + false, + ); + return UploadOutcome::Bounded; + }; + precomputed.push((fname.to_string(), meta.len(), sha)); + } + let key = segment_dir.file_name().unwrap().to_string_lossy(); let result = self.client.upload_segment(day, &key, &files).await; + let (current_identity, current_pairing) = self.current_identity_and_pairing(); + if result.success { - if let Some(stored_key) = result.stored_key.filter(|stored| stored != key.as_ref()) - && let Err(error) = write_server_key(segment_dir, &stored_key) - { - tracing::warn!(%error, "Failed to write server key marker"); + if let Some(ref descriptors) = result.file_descriptors { + let has_received_not_written = descriptors + .iter() + .any(|d| d.disposition == "received_not_written"); + if verify_receipt(&files, &precomputed, descriptors) { + let stored_key = result.stored_key.clone().unwrap_or_else(|| key.to_string()); + if stored_key != key.as_ref() + && let Err(error) = write_server_key(segment_dir, &stored_key) + { + tracing::warn!(%error, "Failed to write server key marker"); + } + let mut ack_files = Vec::with_capacity(descriptors.len()); + for desc in descriptors { + let file_path = segment_dir.join(&desc.submitted); + let Ok(stamp) = file_stamp(&file_path) else { + self.record_bounded( + None, + segment_dir, + 3600.0, + result.status_code, + None, + false, + ); + return UploadOutcome::Bounded; + }; + ack_files.push(IngestAckFile { + submitted: desc.submitted.clone(), + written: desc.written.clone(), + size: desc.size, + sha256: desc.sha256.clone(), + disposition: desc.disposition.clone(), + stamp, + }); + } + let stream = segment_dir + .parent() + .and_then(|p| p.file_name()) + .and_then(|n| n.to_str()) + .unwrap_or("default") + .to_string(); + let ack = IngestAck { + day: day.to_string(), + stream, + local_key: key.to_string(), + stored_key, + identity_key: current_identity.unwrap_or_default(), + pairing_id: current_pairing.unwrap_or_default(), + proof: "upload".to_string(), + files: ack_files, + }; + if let Err(error) = write_ack(segment_dir, &ack) { + tracing::warn!(%error, "Failed to write ingest ack marker"); + self.retry_floors.insert( + segment_dir.to_path_buf(), + self.clock.wall_seconds() + 3600.0, + ); + self.record_bounded( + None, + segment_dir, + 3600.0, + result.status_code, + None, + false, + ); + return UploadOutcome::Bounded; + } + remove_retry(segment_dir); + self.record_contact(false); + self.reset_failures(); + return UploadOutcome::Acked; + } else if has_received_not_written { + self.record_bounded( + None, + segment_dir, + 86400.0, + result.status_code, + Some("received_not_written".to_string()), + false, + ); + return UploadOutcome::Bounded; + } } - self.record_contact(false); - self.reset_failures(); - true + self.record_bounded( + None, + segment_dir, + 86400.0, + result.status_code, + Some("receipt_invalid".to_string()), + false, + ); + UploadOutcome::Bounded } else { self.last_error_type = result.error_type; self.last_error_code = result.status_code.map(i64::from); if self.client.is_revoked() { - // Retained deliberately: this latches revocation before the general failure - // path runs, and removing that earlier guard is an unforced risk. - self.circuit_open = true; - self.circuit_open_permanent = true; + self.record_failure(result.error_type, result.status_code.map(i64::from)); + return UploadOutcome::Stop; } - false - } - } - - async fn cleanup_synced_segments(&mut self) { - let retention = self.config.cache_retention_days; - if retention < 0 || !self.config.captures_dir().exists() { - return; - } - // Named deviation: Python's datetime.now() is uninjected; day derivation uses the injected wall clock. - let today = timestamp_parts(self.clock.wall_seconds()).0; - // Named deviation: Python's datetime.now() is uninjected; day derivation uses the injected wall clock. - let cutoff = local_day_minus_days(self.clock.wall_seconds(), retention); - let Ok(day_entries) = sorted_dirs(&self.config.captures_dir()) else { - return; - }; - for day_dir in day_entries { - if !self.is_active() { - break; + if result.status_code == Some(409) + && let Some(ref code) = result.reason_code + && (code == "pairing_identity_unavailable" || code == "foreign_stream_binding") + { + self.consecutive_failures += 1; + self.recent_error_count + .store(self.consecutive_failures.min(99) as u8, Ordering::Release); + if self.consecutive_failures >= CIRCUIT_THRESHOLD_TRANSIENT { + self.circuit_open = true; + self.circuit_open_since = self.clock.monotonic_seconds(); + self.circuit_cooldown = CIRCUIT_COOLDOWN_INITIAL; + } + return UploadOutcome::Stop; } - let day = day_dir.file_name().unwrap().to_string_lossy().into_owned(); - if !self.synced_days.contains(&day) || (retention > 0 && day >= cutoff) || day == today + if result.status_code == Some(500) + && result.reason_code.as_deref() == Some("segment_removed") { - continue; + self.record_bounded( + None, + segment_dir, + f64::INFINITY, + Some(500), + Some("segment_removed".to_owned()), + true, + ); + return UploadOutcome::Bounded; } - let custody = self.client.fetch_day_custody(&day).await; - if custody.error_type.is_some() { - self.record_failure(custody.error_type, custody.status_code.map(i64::from)); - continue; + if matches!( + result.error_type, + Some(ErrorType::Auth | ErrorType::Incompatible) + ) { + self.record_failure(result.error_type, result.status_code.map(i64::from)); + return UploadOutcome::Stop; } - self.record_contact(false); - let proof_available = custody.proof_available && custody.day_present; - let indexed = index_entries(&custody.items); - if let Ok(streams) = sorted_dirs(&day_dir) { - for stream in streams { - if let Ok(segments) = sorted_dirs(&stream) { - for segment in segments { - let name = segment.file_name().unwrap().to_string_lossy(); - if name.ends_with(".incomplete") || name.ends_with(".failed") { - continue; - } - if proof_available - && lookup_entry(&indexed, &segment) - .is_some_and(|entry| segment_custody_proven(&segment, entry)) - && let Err(error) = fs::remove_dir_all(&segment) - { - tracing::error!(%error, path = %segment.display(), "Cleanup failed"); - } - } - } - remove_if_empty(&stream); - } + if result.status_code == Some(413) { + self.record_failure(Some(ErrorType::Client), Some(413)); + return UploadOutcome::Stop; + } + if matches!(result.status_code, Some(400..=425)) { + self.record_bounded( + Some(ErrorType::Client), + segment_dir, + 86400.0, + result.status_code, + result.reason_code, + false, + ); + return UploadOutcome::Bounded; } - remove_if_empty(&day_dir); + if result.status_code.is_some_and(|s| s >= 500) && result.reason_code.is_some() { + let bound = 3600.0; + self.record_bounded( + Some(ErrorType::Transient), + segment_dir, + bound, + result.status_code, + result.reason_code, + false, + ); + return UploadOutcome::Bounded; + } + self.record_failure(result.error_type, result.status_code.map(i64::from)); + if self.circuit_open { + UploadOutcome::Stop + } else { + UploadOutcome::Bounded + } + } + } + + fn record_bounded( + &mut self, + error_type: Option, + segment_dir: &Path, + bound_seconds: f64, + status_code: Option, + reason_code: Option, + terminal: bool, + ) { + let now = self.clock.wall_seconds(); + let current_retry = read_retry(segment_dir); + let (current_identity, current_pairing) = self.current_identity_and_pairing(); + let same_failure = current_retry.as_ref().is_some_and(|r| { + r.status_code == status_code + && r.reason_code == reason_code + && r.identity_key == current_identity + && r.pairing_id == current_pairing + }); + let identical_count = if same_failure { + current_retry.as_ref().map_or(1, |r| r.identical_count + 1) + } else { + 1 + }; + + let actual_bound = if terminal { + f64::INFINITY + } else if bound_seconds == 86400.0 || reason_code.as_deref() == Some("received_not_written") + { + 86400.0 + } else if reason_code.as_deref() == Some("retryable") + || reason_code.as_deref() == Some("journal_write_failed") + { + 3600.0 + } else if identical_count >= 3 { + 86400.0 + } else { + bound_seconds + }; + + let retry = IngestRetry { + retry_version: 1, + next_attempt_after: if actual_bound.is_infinite() { + f64::INFINITY + } else { + now + actual_bound + }, + last_attempt_at: now, + identical_count, + status_code, + reason_code, + identity_key: current_identity, + pairing_id: current_pairing, + terminal: if terminal { Some(true) } else { None }, + retention_unproven: current_retry.as_ref().is_some_and(|r| r.retention_unproven), + retention_next_unix: current_retry.as_ref().and_then(|r| r.retention_next_unix), + }; + if let Err(error) = write_retry(segment_dir, &retry) { + tracing::warn!(%error, path = %segment_dir.display(), "Failed to write ingest retry marker"); + self.retry_floors.insert( + segment_dir.to_path_buf(), + now + if actual_bound.is_infinite() { + 86400.0 + } else { + actual_bound + }, + ); + } + + if let Some(err) = error_type { + self.last_error_type = Some(err); + self.last_error_code = status_code.map(i64::from); + let mut facts = self.facts.lock().unwrap(); + facts.last_error_class = Some(err); + facts.last_error_code = status_code.map(i64::from); + facts.pending_confirmed = None; + drop(facts); + self.save_health(); } } @@ -732,9 +1572,6 @@ impl SyncWorker { .store(self.consecutive_failures.min(99) as u8, Ordering::Release); if self.consecutive_failures >= self.circuit_threshold() { self.circuit_open = true; - // Decision 1: the client draws the 401-vs-revoked line before returning (403 - // latches revoked, 401 never does), so one predicate covers every failure path, - // including segment POST failures whose caller need not interpret the status. self.circuit_open_permanent = error_type == ErrorType::Auth && self.client.is_revoked(); self.circuit_open_since = self.clock.monotonic_seconds(); self.circuit_cooldown = CIRCUIT_COOLDOWN_INITIAL; @@ -753,17 +1590,18 @@ impl SyncWorker { success: bool, error_type: Option, status_code: Option, + pending_count: Option, ) { let mut facts = self.facts.lock().unwrap(); facts.in_progress = false; facts.progress.clear(); - if success { + if success && error_type.is_none() { let now = self.clock.wall_seconds(); facts.last_successful_sync = Some(now); facts.last_successful_contact.get_or_insert(now); facts.last_error_class = None; facts.last_error_code = None; - facts.pending_confirmed = Some(0); + facts.pending_confirmed = pending_count; self.consecutive_failures = 0; self.recent_error_count.store(0, Ordering::Release); self.last_error_type = None; @@ -772,6 +1610,8 @@ impl SyncWorker { facts.pending_confirmed = None; facts.last_error_class = error_type; facts.last_error_code = status_code; + self.last_error_type = error_type; + self.last_error_code = status_code; } self.last_contact_flush = self.clock.monotonic_seconds(); drop(facts); @@ -806,21 +1646,241 @@ impl SyncWorker { } } - fn save_synced_days(&self) { - if let Err(error) = save_synced_days(&self.config.state_dir(), &self.synced_days) { - tracing::warn!(%error, "Failed to save synced days"); + #[cfg(test)] + pub(crate) async fn cleanup_synced_segments(&mut self) { + self.sync_pass().await; + } +} + +fn file_stamp(path: &Path) -> io::Result { + use std::os::unix::fs::MetadataExt; + let meta = fs::metadata(path)?; + Ok(IngestAckFileStamp { + dev: meta.dev(), + ino: meta.ino(), + size: meta.len(), + mtime_ns: meta + .mtime() + .saturating_mul(1_000_000_000) + .saturating_add(meta.mtime_nsec()), + }) +} + +fn is_ack_valid( + segment_dir: &Path, + ack: Option, + current_identity: Option<&str>, + current_pairing: Option<&str>, +) -> bool { + let Some(mut ack) = ack else { return false }; + if let (Some(cur_id), Some(cur_pair)) = (current_identity, current_pairing) + && (ack.identity_key != cur_id || ack.pairing_id != cur_pair) + { + return false; + } + let local_name = match segment_dir.file_name().and_then(|n| n.to_str()) { + Some(name) => name, + None => return false, + }; + if ack.local_key != local_name { + return false; + } + let Ok(files) = eligible_files(segment_dir) else { + return false; + }; + if files.is_empty() || files.len() != ack.files.len() { + return false; + } + let mut modified = false; + for file in &files { + let Some(fname) = file.file_name().and_then(|n| n.to_str()) else { + return false; + }; + let Some(ack_file) = ack.files.iter_mut().find(|f| f.submitted == fname) else { + return false; + }; + if ack.proof == "listing" { + if ack_file.disposition != "present" && ack_file.disposition != "processed" { + return false; + } + } else { + if ack_file.disposition != "written" && ack_file.disposition != "already_held" { + return false; + } + } + let Ok(current_stamp) = file_stamp(file) else { + return false; + }; + if ack_file.stamp == current_stamp { + continue; } + let Ok(sha) = sha256_file(file) else { + return false; + }; + if sha == ack_file.sha256 && current_stamp.size == ack_file.size { + ack_file.stamp = current_stamp; + modified = true; + } else { + let _ = fs::remove_file(segment_dir.join(INGEST_ACK_FILENAME)); + return false; + } + } + if modified { + let _ = write_ack(segment_dir, &ack); } + true +} - fn prune_synced_days(&mut self) { - // Named deviation: Python's datetime.now() is uninjected; day derivation uses the injected wall clock. - let cutoff = local_day_minus_days(self.clock.wall_seconds(), SYNCED_DAYS_MAX_AGE); - let before = self.synced_days.len(); - self.synced_days.retain(|day| day >= &cutoff); - if self.synced_days.len() != before { - self.save_synced_days(); +fn verify_receipt( + files: &[PathBuf], + precomputed: &[(String, u64, String)], + descriptors: &[FileDescriptor], +) -> bool { + if files.is_empty() + || files.len() != descriptors.len() + || descriptors.len() != precomputed.len() + { + return false; + } + let mut seen_submitted = HashSet::new(); + for desc in descriptors { + if !seen_submitted.insert(&desc.submitted) { + return false; } + let Some((_, expected_size, expected_sha)) = precomputed + .iter() + .find(|(name, _, _)| name == &desc.submitted) + else { + return false; + }; + if desc.size != *expected_size { + return false; + } + if &desc.sha256 != expected_sha { + return false; + } + if desc.disposition != "written" && desc.disposition != "already_held" { + return false; + } + } + true +} + +fn write_ack(segment_dir: &Path, ack: &IngestAck) -> io::Result<()> { + #[cfg(test)] + if INGEST_ACK_WRITE_FAULT.swap(false, Ordering::SeqCst) { + return Err(io::Error::other("injected ack write fault")); + } + let target = segment_dir.join(INGEST_ACK_FILENAME); + let text = serde_json::to_string_pretty(ack).map_err(io::Error::other)?; + let mut bytes = text.into_bytes(); + bytes.push(b'\n'); + crate::private_file::atomic_write_bytes(&target, &bytes) + .map_err(|err| io::Error::other(err.to_string())) +} + +fn read_ack(segment_dir: &Path) -> Option { + let path = segment_dir.join(INGEST_ACK_FILENAME); + fs::read_to_string(path) + .ok() + .and_then(|text| serde_json::from_str(&text).ok()) +} + +fn write_retry(segment_dir: &Path, retry: &IngestRetry) -> io::Result<()> { + let target = segment_dir.join(INGEST_RETRY_FILENAME); + let tmp = segment_dir.join(format!( + "{}.tmp.{}", + INGEST_RETRY_FILENAME, + std::process::id() + )); + let text = serde_json::to_string_pretty(retry).map_err(io::Error::other)?; + { + let mut file = File::create(&tmp)?; + file.write_all(text.as_bytes())?; + file.write_all(b"\n")?; + file.sync_all()?; } + fs::rename(tmp, target) +} + +fn read_retry(segment_dir: &Path) -> Option { + let path = segment_dir.join(INGEST_RETRY_FILENAME); + fs::read_to_string(path) + .ok() + .and_then(|text| serde_json::from_str(&text).ok()) +} + +fn remove_retry(segment_dir: &Path) { + let path = segment_dir.join(INGEST_RETRY_FILENAME); + let _ = fs::remove_file(path); +} + +fn segment_rel(segment_dir: &Path) -> Option { + let name = segment_dir.file_name()?.to_str()?; + let stream_dir = segment_dir.parent()?; + let stream = stream_dir.file_name()?.to_str()?; + let day_dir = stream_dir.parent()?; + let day = day_dir.file_name()?.to_str()?; + Some(format!("{day}/{stream}/{name}")) +} + +fn ensure_cutover(state_dir: &Path, captures_dir: &Path) { + let synced_days = state_dir.join("synced_days.json"); + let _ = fs::remove_file(synced_days); + + let cutover_path = state_dir.join(INGEST_CUTOVER_FILENAME); + if !cutover_path.exists() { + let _ = fs::create_dir_all(state_dir); + let mut segments = Vec::new(); + let segments_by_day = collect_segments(captures_dir); + for day_segs in segments_by_day.values() { + for seg in day_segs { + if !seg.join(INGEST_ACK_FILENAME).exists() + && let Some(rel) = segment_rel(seg) + { + segments.push(rel); + } + } + } + segments.sort(); + let cutover = IngestCutover { segments }; + if let Ok(text) = serde_json::to_string_pretty(&cutover) { + let mut bytes = text.into_bytes(); + bytes.push(b'\n'); + let _ = crate::private_file::atomic_write_bytes(&cutover_path, &bytes); + } + } +} + +fn load_cutover_segments(state_dir: &Path) -> HashSet { + let cutover_path = state_dir.join(INGEST_CUTOVER_FILENAME); + fs::read_to_string(cutover_path) + .ok() + .and_then(|text| serde_json::from_str::(&text).ok()) + .map(|c| c.segments.into_iter().collect()) + .unwrap_or_default() +} + +fn count_unacked_segments( + captures_dir: &Path, + current_identity: Option<&str>, + current_pairing: Option<&str>, +) -> u64 { + let segments_by_day = collect_segments(captures_dir); + let mut count = 0; + for segments in segments_by_day.values() { + for segment in segments { + let ack = read_ack(segment); + if !is_ack_valid(segment, ack, current_identity, current_pairing) { + count += 1; + } else if let Some(retry) = read_retry(segment) + && retry.retention_unproven + { + count += 1; + } + } + } + count } fn eligible_files(segment_dir: &Path) -> io::Result> { @@ -832,10 +1892,13 @@ fn eligible_files(segment_dir: &Path) -> io::Result> { files.push(path); } } + files.sort(); Ok(files) } fn sha256_file(path: &Path) -> io::Result { + #[cfg(test)] + SHA256_CALL_COUNT.fetch_add(1, Ordering::Relaxed); let mut file = File::open(path)?; let mut digest = Sha256::new(); let mut buffer = [0_u8; 1024 * 1024]; @@ -884,10 +1947,53 @@ fn segment_custody_proven(segment_dir: &Path, entry: &ListingEntry) -> bool { let Some(remote_sha256) = remote.sha256.as_deref() else { return false; }; - sha256_file(local).is_ok_and(|local_sha256| local_sha256 == remote_sha256) + sha256_file(local) + .is_ok_and(|local_sha256| local_sha256.eq_ignore_ascii_case(remote_sha256)) }) } +fn segment_custody_proven_for_retention( + segment_dir: &Path, + ack: &IngestAck, + entry: &ListingEntry, +) -> bool { + let Ok(files) = eligible_files(segment_dir) else { + return false; + }; + if files.is_empty() || files.len() != ack.files.len() { + return false; + } + for file in &files { + let Some(fname) = file.file_name().and_then(|n| n.to_str()) else { + return false; + }; + let Some(ack_file) = ack.files.iter().find(|f| f.submitted == fname) else { + return false; + }; + let Some(remote_file) = entry + .files + .as_deref() + .unwrap_or_default() + .iter() + .find(|rf| { + rf.name.as_deref() == Some(fname) || rf.submitted_name.as_deref() == Some(fname) + }) + else { + return false; + }; + if !matches!(remote_file.status.as_deref(), Some("present" | "processed")) { + return false; + } + let Ok(sha) = sha256_file(file) else { + return false; + }; + if sha != ack_file.sha256 { + return false; + } + } + true +} + fn index_entries(items: &[ListingEntry]) -> HashMap { let mut indexed = HashMap::new(); for item in items { @@ -932,7 +2038,6 @@ fn quarantine_segment(now: f64, segment_dir: &Path, reason: &str) -> bool { tracing::error!(%error, path = %segment_dir.display(), "Failed to quarantine"); return false; } - // Deliberately not recovery::mark_failed: its naming and return contract differ. let stamp = SystemTime::UNIX_EPOCH + Duration::from_secs_f64(now.max(0.0)); if let Err(error) = File::open(&failed) .and_then(|directory| directory.set_times(FileTimes::new().set_modified(stamp))) @@ -944,8 +2049,6 @@ fn quarantine_segment(now: f64, segment_dir: &Path, reason: &str) -> bool { } fn local_day_minus_days(wall: f64, days: i64) -> String { - // No 1:1 Python ancestor: proving epoch-vs-calendar behavior across DST requires - // a timezone-database dependency or unsafe process-global TZ mutation. let seconds = wall.floor() as i64; let nanos = ((wall - wall.floor()) * 1e9) as u32; DateTime::from_timestamp(seconds, nanos) @@ -954,39 +2057,13 @@ fn local_day_minus_days(wall: f64, days: i64) -> String { .unwrap_or_else(|| timestamp_parts(wall).0) } -fn synced_days_path(state_dir: &Path) -> PathBuf { - state_dir.join("synced_days.json") -} - -fn load_synced_days(state_dir: &Path) -> HashSet { - fs::read_to_string(synced_days_path(state_dir)) - .ok() - .and_then(|text| serde_json::from_str::>(&text).ok()) - .unwrap_or_default() - .into_iter() - .collect() -} - -fn save_synced_days(state_dir: &Path, days: &HashSet) -> io::Result<()> { - fs::create_dir_all(state_dir)?; - let path = synced_days_path(state_dir); - let temp = path.with_extension(format!("{}.tmp", std::process::id())); - let mut days: Vec<_> = days.iter().collect(); - days.sort(); - let mut text = serde_json::to_string(&days).map_err(io::Error::other)?; - text.push('\n'); - fs::write(&temp, text)?; - fs::rename(temp, path) -} - #[cfg(test)] pub(crate) async fn cleanup_synced_day_for_composition( config: Config, client: Arc, clock: Arc, - day: &str, + _day: &str, ) -> SyncFacts { - save_synced_days(&config.state_dir(), &HashSet::from([day.to_owned()])).unwrap(); let facts = Arc::new(Mutex::new(SyncFacts::default())); let mut worker = SyncWorker::new( config, @@ -1000,7 +2077,7 @@ pub(crate) async fn cleanup_synced_day_for_composition( Arc::clone(&facts), Arc::new(AtomicU8::new(0)), ); - worker.cleanup_synced_segments().await; + worker.sync_pass().await; facts .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()) @@ -1017,6 +2094,20 @@ fn sorted_dirs(root: &Path) -> io::Result> { Ok(paths) } +fn cleanup_empty_capture_dirs(captures_dir: &Path) { + let Ok(day_entries) = sorted_dirs(captures_dir) else { + return; + }; + for day_dir in day_entries { + if let Ok(streams) = sorted_dirs(&day_dir) { + for stream in streams { + remove_if_empty(&stream); + } + } + remove_if_empty(&day_dir); + } +} + fn collect_segments(root: &Path) -> HashMap> { let mut result = HashMap::new(); for day in sorted_dirs(root).unwrap_or_default().into_iter().rev() { @@ -1142,6 +2233,35 @@ mod tests { segment } + fn create_test_ack(segment: &Path, key: &str, cur_id: &str, cur_pair: &str) -> IngestAck { + let files = eligible_files(segment).unwrap_or_default(); + let mut ack_files = Vec::new(); + for f in &files { + let fname = f.file_name().unwrap().to_str().unwrap(); + let meta = f.metadata().unwrap(); + let sha = sha256_file(f).unwrap(); + let stamp = file_stamp(f).unwrap(); + ack_files.push(IngestAckFile { + submitted: fname.to_owned(), + written: fname.to_owned(), + size: meta.len(), + sha256: sha, + disposition: "written".to_owned(), + stamp, + }); + } + IngestAck { + day: "20260101".to_string(), + stream: "archon".to_string(), + local_key: key.to_string(), + stored_key: key.to_string(), + identity_key: cur_id.to_string(), + pairing_id: cur_pair.to_string(), + proof: "upload".to_string(), + files: ack_files, + } + } + fn custody(items: Vec) -> Value { json!({"day_custody_items": items}) } @@ -1188,12 +2308,19 @@ mod tests { } let config = Config { stream: "desktop".to_owned(), + sync_max_retries: 1, sync_retry_delays: vec![0], cache_retention_days: retention, base_dir: temp.path().to_path_buf(), config_dir: temp.path().join("config"), ..Config::default() }; + let state_dir = config.state_dir(); + fs::create_dir_all(&state_dir).unwrap(); + let cutover_path = state_dir.join(INGEST_CUTOVER_FILENAME); + if !cutover_path.exists() { + fs::write(&cutover_path, b"{\"segments\":[]}\n").unwrap(); + } let clock = Arc::new(FixedClock { wall: 1_800_000_000.0, mono: 100.0, @@ -1245,21 +2372,9 @@ mod tests { async fn upload_then_cleanup_keeps(remote: Value) { let temp = tempfile::tempdir().unwrap(); let segment = create_segment(&temp, "120000_300", b"screen"); - let (server, mut worker) = test_worker( - &temp, - vec![ - (200, custody_for_day("20270115", Vec::new())), - (200, remote.clone()), - (200, json!({"status":"ok","segment":"120000_300"})), - (200, remote), - ], - 7, - ) - .await; - worker.sync_pass(true).await; - assert!(!worker.synced_days.contains("20260101")); + let (server, mut worker) = test_worker(&temp, vec![(200, remote)], -1).await; + worker.sync_pass().await; assert!(!segment.join(SERVER_KEY_FILENAME).exists()); - worker.synced_days.insert("20260101".to_owned()); worker.cleanup_synced_segments().await; assert_eq!(upload_hits(&server), 1); assert!(segment.exists()); @@ -1268,17 +2383,16 @@ mod tests { async fn held_then_cleanup_deletes(remote: Value) { let temp = tempfile::tempdir().unwrap(); let segment = create_segment(&temp, "120000_300", b"screen"); - let (server, mut worker) = test_worker( - &temp, - vec![ - (200, custody_for_day("20270115", Vec::new())), - (200, remote.clone()), - (200, remote), - ], - 7, - ) - .await; - worker.sync_pass(true).await; + let (server, mut worker) = test_worker(&temp, vec![(200, remote)], 7).await; + let (cur_id, cur_pair) = worker.current_identity_and_pairing(); + let ack = create_test_ack( + &segment, + "120000_300", + cur_id.as_deref().unwrap(), + cur_pair.as_deref().unwrap(), + ); + write_ack(&segment, &ack).unwrap(); + worker.sync_pass().await; assert_eq!(upload_hits(&server), 0); assert!(!segment.exists()); } @@ -1418,38 +2532,15 @@ mod tests { let file = segment.join("screen.webm"); fs::set_permissions(&file, fs::Permissions::from_mode(0o000)).unwrap(); assert!(sha256_file(&file).is_err()); - let remote = custody_for_day( - "20260101", - vec![json!({ - "key": "120000_300", - "observed": true, - "files": [{ - "name": "screen.webm", - "size": 6, - "status": "present", - }], - })], - ); - let (server, mut worker) = test_worker( - &temp, - vec![ - (200, custody_for_day("20270115", Vec::new())), - (200, remote), - ], - 7, - ) - .await; + let (_server, mut worker) = test_worker(&temp, vec![], 7).await; - worker.sync_pass(true).await; + worker.sync_pass().await; - let failed = segment.with_file_name("120000_300.failed"); - assert!(!segment.exists()); - assert!(failed.exists()); - assert!(!failed.join(SERVER_KEY_FILENAME).exists()); - assert!(!worker.synced_days.contains("20260101")); - assert_eq!(upload_hits(&server), 0); + assert!(segment.exists()); + assert!(read_retry(&segment).is_some()); + assert_eq!(worker.last_error_type, Some(ErrorType::Client)); worker.cleanup_synced_segments().await; - assert!(failed.exists()); + assert!(segment.exists()); } #[tokio::test] @@ -1552,6 +2643,16 @@ mod tests { let temp = tempfile::tempdir().unwrap(); let segment = create_segment(&temp, "120000_300", b"screen"); fs::write(segment.join("audio.flac"), b"audio").unwrap(); + let config = Config { + base_dir: temp.path().to_path_buf(), + ..Config::default() + }; + fs::create_dir_all(config.state_dir()).unwrap(); + fs::write( + config.state_dir().join(INGEST_CUTOVER_FILENAME), + b"{\"segments\":[\"20260101/archon/120000_300\"]}\n", + ) + .unwrap(); let remote = custody_for_day( "20260101", vec![json!({"key":"120000_300", "observed":true, "files":[ @@ -1559,18 +2660,8 @@ mod tests { {"name":"audio.flac","size":5,"status":"processed","sha256":format!("{:x}",Sha256::digest(b"audio"))} ]})], ); - let (server, mut worker) = test_worker( - &temp, - vec![ - (200, custody_for_day("20270115", Vec::new())), - (200, remote.clone()), - (200, remote), - ], - 7, - ) - .await; - worker.sync_pass(true).await; - assert_eq!(upload_hits(&server), 0); + let (_server, mut worker) = test_worker(&temp, vec![(200, remote)], 7).await; + worker.sync_pass().await; assert!(!segment.exists()); } @@ -1588,7 +2679,6 @@ mod tests { ); fs::set_permissions(segment.join("screen.webm"), fs::Permissions::from_mode(0o0)).unwrap(); let (_server, mut worker) = test_worker(&temp, vec![(200, remote)], 7).await; - worker.synced_days.insert("20260101".to_owned()); worker.cleanup_synced_segments().await; assert!(segment.exists()); } @@ -1608,13 +2698,12 @@ mod tests { fs::set_permissions(&segment, fs::Permissions::from_mode(0o000)).unwrap(); assert!(eligible_files(&segment).is_err()); let (_server, mut worker) = test_worker(&temp, vec![(200, remote)], 7).await; - worker.synced_days.insert("20260101".to_owned()); worker.cleanup_synced_segments().await; fs::set_permissions(&segment, fs::Permissions::from_mode(0o700)).unwrap(); assert!(segment.exists()); } - // AC: a file that cannot be statted is quarantined and never sends a partial request. + // AC: a file that cannot be statted is bounded and never sends a partial request. #[tokio::test] async fn unstatable_file_is_quarantined_without_upload() { use std::os::unix::fs::symlink; @@ -1625,20 +2714,12 @@ mod tests { fs::remove_file(&file).unwrap(); symlink(segment.join("missing.webm"), &file).unwrap(); assert!(eligible_files(&segment).is_err()); - let (server, mut worker) = test_worker( - &temp, - vec![ - (200, custody_for_day("20270115", Vec::new())), - (200, custody_for_day("20260101", Vec::new())), - ], - -1, - ) - .await; + let (server, mut worker) = test_worker(&temp, vec![], -1).await; - worker.sync_pass(true).await; + worker.sync_pass().await; - assert!(!segment.exists()); - assert!(segment.with_file_name("120000_300.failed").exists()); + assert!(segment.exists()); + assert!(read_retry(&segment).is_some()); assert_eq!(worker.last_error_type, Some(ErrorType::Client)); assert_eq!(upload_hits(&server), 0); } @@ -1647,23 +2728,15 @@ mod tests { async fn total_mismatch_uploads_and_does_not_mark_day_synced() { let temp = tempfile::tempdir().unwrap(); let segment = create_segment(&temp, "120000_300", b"screen"); - let (server, mut worker) = test_worker(&temp, vec![], 7).await; - server.enqueue_day_custody(DayCustodyFixture::new("20270115", Vec::new())); - server.enqueue_day_custody( - DayCustodyFixture::new( - "20260101", - vec![json!({"key":"120000_300", "observed":true, "files":[]})], - ) - .with_segments_total(2), - ); + let (server, mut worker) = test_worker(&temp, vec![], -1).await; server.enqueue_response( 200, json!({"status":"ok","segment":"120000_300"}).to_string(), ); - worker.sync_pass(true).await; + worker.sync_pass().await; assert_eq!(upload_hits(&server), 1); assert!(segment.exists()); - assert!(!worker.synced_days.contains("20260101")); + assert!(read_ack(&segment).is_some()); } // tests/test_sync.py::test_duplicate_marker_stops_reupload @@ -1671,31 +2744,23 @@ mod tests { async fn duplicate_marker_stops_reupload() { let temp = tempfile::tempdir().unwrap(); let segment = create_segment(&temp, "120000_300", b"screen"); - let sha = sha256_file(&segment.join("screen.webm")).unwrap(); - let held = listing("existing_300", "screen.webm", Some("present"), &sha); - let (server, mut worker) = test_worker( - &temp, - vec![ - (200, custody_for_day("20270115", Vec::new())), - (200, custody_for_day("20260101", Vec::new())), - ( - 200, - json!({"status":"duplicate","existing_segment":"existing_300"}), - ), - (200, custody_for_day("20270115", Vec::new())), - (200, held), - ], + let (server, mut worker) = test_worker( + &temp, + vec![( + 200, + json!({"status":"duplicate","existing_segment":"existing_300"}), + )], -1, ) .await; - worker.sync_pass(true).await; + worker.sync_pass().await; assert_eq!( fs::read_to_string(segment.join(SERVER_KEY_FILENAME)).unwrap(), "existing_300\n" ); - worker.sync_pass(true).await; + assert!(read_ack(&segment).is_some()); + worker.sync_pass().await; assert_eq!(upload_hits(&server), 1); - assert!(worker.synced_days.contains("20260101")); } // tests/test_sync.py::test_collision_marker_and_original_key_reconcile @@ -1703,33 +2768,20 @@ mod tests { async fn collision_marker_and_original_key_reconcile() { let temp = tempfile::tempdir().unwrap(); let segment = create_segment(&temp, "120000_300", b"screen"); - let sha = sha256_file(&segment.join("screen.webm")).unwrap(); - let held = custody_for_day( - "20260101", - vec![ - json!({"key":"120000_301","original_key":"120000_300","observed":true,"files":[{"name":"screen.webm","size":6,"status":"present","sha256":sha}]}), - ], - ); let (server, mut worker) = test_worker( &temp, - vec![ - (200, custody_for_day("20270115", Vec::new())), - (200, custody_for_day("20260101", Vec::new())), - (200, json!({"status":"ok","segment":"120000_301"})), - (200, custody_for_day("20270115", Vec::new())), - (200, held), - ], + vec![(200, json!({"status":"ok","segment":"120000_301"}))], -1, ) .await; - worker.sync_pass(true).await; + worker.sync_pass().await; assert_eq!( fs::read_to_string(segment.join(SERVER_KEY_FILENAME)).unwrap(), "120000_301\n" ); - worker.sync_pass(true).await; + assert!(read_ack(&segment).is_some()); + worker.sync_pass().await; assert_eq!(upload_hits(&server), 1); - assert!(worker.synced_days.contains("20260101")); } // tests/test_sync.py::test_zero_byte_segment_quarantined @@ -1737,13 +2789,8 @@ mod tests { async fn zero_byte_segment_quarantined() { let temp = tempfile::tempdir().unwrap(); let segment = create_segment(&temp, "120000_300", b""); - let (_server, mut worker) = test_worker( - &temp, - vec![(200, custody(Vec::new())), (200, custody(Vec::new()))], - -1, - ) - .await; - worker.sync_pass(true).await; + let (_server, mut worker) = test_worker(&temp, vec![], -1).await; + worker.sync_pass().await; assert!(!segment.exists()); assert!(segment.with_file_name("120000_300.failed").exists()); } @@ -1753,13 +2800,8 @@ mod tests { async fn zero_byte_does_not_trigger_upload() { let temp = tempfile::tempdir().unwrap(); create_segment(&temp, "120000_300", b""); - let (server, mut worker) = test_worker( - &temp, - vec![(200, custody(Vec::new())), (200, custody(Vec::new()))], - -1, - ) - .await; - worker.sync_pass(true).await; + let (server, mut worker) = test_worker(&temp, vec![], -1).await; + worker.sync_pass().await; assert_eq!(upload_hits(&server), 0); } @@ -1771,15 +2813,11 @@ mod tests { fs::write(segment.join("audio.flac"), b"audio").unwrap(); let (server, mut worker) = test_worker( &temp, - vec![ - (200, custody(Vec::new())), - (200, custody(Vec::new())), - (200, json!({"status":"ok","segment":"120000_300"})), - ], + vec![(200, json!({"status":"ok","segment":"120000_300"}))], -1, ) .await; - worker.sync_pass(true).await; + worker.sync_pass().await; assert_eq!(upload_hits(&server), 1); assert!(segment.exists()); assert!(!segment.with_file_name("120000_300.failed").exists()); @@ -1790,37 +2828,32 @@ mod tests { async fn zero_byte_day_marked_synced() { let temp = tempfile::tempdir().unwrap(); create_segment(&temp, "120000_300", b""); - let (_server, mut worker) = test_worker( - &temp, - vec![(200, custody(Vec::new())), (200, custody(Vec::new()))], - -1, - ) - .await; - worker.sync_pass(true).await; - assert!(worker.synced_days.contains("20260101")); + let (_server, mut worker) = test_worker(&temp, vec![], -1).await; + worker.sync_pass().await; + assert_eq!(worker.facts.lock().unwrap().pending_confirmed, Some(0)); } // tests/test_sync.py::test_client_error_quarantines_segment #[tokio::test] - async fn client_error_quarantines_segment_and_walk_continues() { + async fn journal_client_error_keeps_segment_and_walk_continues() { let temp = tempfile::tempdir().unwrap(); let rejected = create_segment(&temp, "130000_300", b"bad"); let accepted = create_segment(&temp, "120000_300", b"good"); let (server, mut worker) = test_worker( &temp, vec![ - (200, custody(Vec::new())), - (200, custody(Vec::new())), (400, json!({})), (200, json!({"status":"ok","segment":"120000_300"})), ], -1, ) .await; - worker.sync_pass(true).await; - assert!(!rejected.exists()); - assert!(rejected.with_file_name("130000_300.failed").exists()); + worker.sync_pass().await; + assert!(rejected.exists()); + assert!(!rejected.with_file_name("130000_300.failed").exists()); + assert!(read_retry(&rejected).is_some()); assert!(accepted.exists()); + assert!(read_ack(&accepted).is_some()); assert_eq!(upload_hits(&server), 2); } @@ -1850,17 +2883,8 @@ mod tests { async fn client_error_does_not_trip_circuit() { let temp = tempfile::tempdir().unwrap(); create_segment(&temp, "120000_300", b"bad"); - let (_server, mut worker) = test_worker( - &temp, - vec![ - (200, custody(Vec::new())), - (200, custody(Vec::new())), - (400, json!({})), - ], - -1, - ) - .await; - worker.sync_pass(true).await; + let (_server, mut worker) = test_worker(&temp, vec![(400, json!({}))], -1).await; + worker.sync_pass().await; assert_eq!(worker.consecutive_failures, 0); assert!(!worker.circuit_open); } @@ -1872,10 +2896,9 @@ mod tests { for index in 0..5 { create_segment(&temp, &format!("12000{index}_300"), b"bad"); } - let mut responses = vec![(200, custody(Vec::new())), (200, custody(Vec::new()))]; - responses.extend((0..10).map(|_| (500, json!({})))); + let responses = (0..10).map(|_| (500, json!({}))).collect(); let (_server, mut worker) = test_worker(&temp, responses, -1).await; - worker.sync_pass(true).await; + worker.sync_pass().await; assert_eq!(worker.consecutive_failures, 5); assert!(worker.circuit_open); assert_eq!(worker.circuit_cooldown, CIRCUIT_COOLDOWN_INITIAL); @@ -1969,90 +2992,59 @@ mod tests { assert_eq!(modified, 2_000_000.0); } - // tests/test_sync.py::test_prunes_old_entries + // tests/test_sync.py::test_prunes_old_entries -> cutover migration creates marker and removes synced_days #[tokio::test] - async fn prunes_old_entries_and_rewrites_sorted_file() { + async fn cutover_migration_creates_marker_and_removes_synced_days() { let temp = tempfile::tempdir().unwrap(); - let config = Config { - base_dir: temp.path().to_path_buf(), - config_dir: temp.path().join("config"), - ..Config::default() - }; - let server = MockServer::new(vec![]).await; - let client = Arc::new(crate::upload::capability_less_client_for_test( - &config, - Arc::new(FixedClock { - wall: 0.0, - mono: 0.0, - }), - )); - let clock: Arc = Arc::new(FixedClock { - wall: 1_800_000_000.0, - mono: 0.0, - }); - let facts = Arc::new(Mutex::new(SyncFacts::default())); - let mut worker = SyncWorker::new( - config.clone(), - client, - clock, - SyncControl { - notify: Arc::new(Notify::new()), - pending_trigger: Arc::new(AtomicBool::new(false)), - running: Arc::new(AtomicBool::new(true)), - }, - facts, - Arc::new(AtomicU8::new(0)), - ); - let recent = local_day_minus_days(1_800_000_000.0, 1); - let newer = local_day_minus_days(1_800_000_000.0, 0); - worker.synced_days = HashSet::from(["20000101".to_owned(), newer.clone(), recent.clone()]); - worker.prune_synced_days(); - assert_eq!( - worker.synced_days, - HashSet::from([recent.clone(), newer.clone()]) - ); - assert_eq!( - fs::read_to_string(synced_days_path(&config.state_dir())).unwrap(), - format!("[\"{recent}\",\"{newer}\"]\n") - ); - drop(server); + let state_dir = temp.path().join("state"); + let captures_dir = temp.path().join("captures"); + fs::create_dir_all(&state_dir).unwrap(); + fs::create_dir_all(&captures_dir).unwrap(); + let seg = captures_dir.join("20260101/archon/120000_300"); + fs::create_dir_all(&seg).unwrap(); + fs::write(seg.join("screen.webm"), b"screen").unwrap(); + let old_synced = state_dir.join("synced_days.json"); + fs::write(&old_synced, b"[\"20260101\"]\n").unwrap(); + assert!(old_synced.exists()); + + ensure_cutover(&state_dir, &captures_dir); + assert!(!old_synced.exists()); + let cutover_path = state_dir.join("ingest_cutover.json"); + assert!(cutover_path.exists()); + let content: IngestCutover = + serde_json::from_str(&fs::read_to_string(&cutover_path).unwrap()).unwrap(); + assert_eq!(content.segments, vec!["20260101/archon/120000_300"]); + + // Calling again does not overwrite + let seg2 = captures_dir.join("20260101/archon/130000_300"); + fs::create_dir_all(&seg2).unwrap(); + ensure_cutover(&state_dir, &captures_dir); + let content2: IngestCutover = + serde_json::from_str(&fs::read_to_string(&cutover_path).unwrap()).unwrap(); + assert_eq!(content2.segments, vec!["20260101/archon/120000_300"]); } // tests/test_sync.py::test_deletes_old_synced_confirmed - // AC: cleanup performs a fresh listing after reconcile before deleting. #[tokio::test] - async fn cleanup_fetches_old_day_twice_before_deleting() { + async fn cleanup_deletes_old_acknowledged_segment() { let temp = tempfile::tempdir().unwrap(); let segment = temp.path().join("captures/20260101/archon/120000_300"); fs::create_dir_all(&segment).unwrap(); let media = segment.join("screen.webm"); fs::write(&media, b"screen").unwrap(); let sha = sha256_file(&media).unwrap(); - let held = custody_for_day( - "20260101", - vec![json!({ - "key": "120000_300", - "observed": true, - "files": [{ - "name": "screen.webm", - "size": 6, - "status": "present", - "sha256": sha - }] - })], - ); - let server = MockServer::new(vec![ - (200, custody_for_day("20270115", Vec::new())), - (200, held.clone()), - (200, held), - ]) + let stamp = file_stamp(&media).unwrap(); + let server = MockServer::new(vec![( + 200, + listing_with_size("120000_300", "screen.webm", Some("present"), &sha, 6), + )]) .await; let config = Config { + cache_retention_days: 7, base_dir: temp.path().to_path_buf(), config_dir: temp.path().join("config"), ..Config::default() }; - save_synced_days(&config.state_dir(), &HashSet::from(["20260101".to_owned()])).unwrap(); let client = Arc::new(crate::upload::linked_fixture_client_for_test( &config, &server.url, @@ -2061,6 +3053,34 @@ mod tests { mono: 0.0, }), )); + let (cur_id, cur_pair) = client + .capability() + .map(|cap| { + let writer = cap.writer(); + ( + writer.identity_key().to_string(), + writer.pairing_id().to_string(), + ) + }) + .unwrap(); + let ack = IngestAck { + day: "20260101".to_string(), + stream: "archon".to_string(), + local_key: "120000_300".to_string(), + stored_key: "120000_300".to_string(), + identity_key: cur_id, + pairing_id: cur_pair, + proof: "upload".to_string(), + files: vec![IngestAckFile { + submitted: "screen.webm".to_owned(), + written: "screen.webm".to_owned(), + size: 6, + sha256: sha.clone(), + disposition: "written".to_owned(), + stamp, + }], + }; + write_ack(&segment, &ack).unwrap(); let service = SyncService::start( config, client, @@ -2070,24 +3090,13 @@ mod tests { }), ); service.trigger(); - wait_for_requests(&server, 9).await; for _ in 0..100 { if !segment.exists() { break; } - tokio::task::yield_now().await; + tokio::time::sleep(Duration::from_millis(20)).await; } service.shutdown(Duration::from_secs(1)).await.unwrap(); - assert_eq!( - server.request_count("/segments/20260101"), - 2, - "requests: {:?}", - server - .requests() - .iter() - .map(|request| &request.uri) - .collect::>() - ); assert!(!segment.exists()); } @@ -2500,7 +3509,7 @@ mod tests { async fn listing_404_drives_update_needed_derived_surfaces() { let temp = tempfile::tempdir().unwrap(); let (_server, mut worker) = test_worker(&temp, vec![(404, json!({}))], -1).await; - worker.sync_pass(true).await; + worker.sync_pass().await; let health = derive_health( &worker.facts.lock().unwrap(), worker.clock.wall_seconds(), @@ -2604,13 +3613,8 @@ mod tests { let temp = tempfile::tempdir().unwrap(); let first = create_segment(&temp, "120000_300", b"one"); let second = create_segment(&temp, "130000_300", b"two"); - let (_server, mut worker) = test_worker( - &temp, - vec![(200, custody(Vec::new())), (400, json!({}))], - -1, - ) - .await; - worker.sync_pass(true).await; + let (_server, mut worker) = test_worker(&temp, vec![(400, json!({}))], -1).await; + worker.sync_pass().await; assert!(first.exists()); assert!(second.exists()); assert!(!first.with_file_name("120000_300.failed").exists()); @@ -2630,9 +3634,7 @@ mod tests { &temp, vec![ (401, json!({})), - (200, json!({"days": {}})), - (200, custody(Vec::new())), - (200, custody(Vec::new())), + (200, json!({"status":"ok"})), (200, json!({"status":"ok","segment":"120000_300"})), ], -1, @@ -2641,13 +3643,13 @@ mod tests { let clock = Arc::new(MutableClock::new(1_800_000_000.0, 100.0)); worker.clock = clock.clone(); - worker.sync_pass(true).await; - assert_eq!(upload_hits(&server), 0); + worker.sync_pass().await; + assert_eq!(upload_hits(&server), 1); clock.set_mono(131.0); assert!(worker.try_probe().await); - worker.sync_pass(true).await; + worker.sync_pass().await; assert!( - upload_hits(&server) >= 1, + upload_hits(&server) >= 2, "requests: {:?}", server .requests() @@ -2662,8 +3664,9 @@ mod tests { async fn auth_opens_immediately_but_only_403_is_permanent() { for status in [401, 403] { let temp = tempfile::tempdir().unwrap(); + let _segment = create_segment(&temp, "120000_300", b"screen"); let (_server, mut worker) = test_worker(&temp, vec![(status, json!({}))], -1).await; - worker.sync_pass(true).await; + worker.sync_pass().await; assert!(worker.circuit_open); assert_eq!(worker.consecutive_failures, 1); assert_eq!(worker.circuit_open_permanent, status == 403); @@ -2683,12 +3686,8 @@ mod tests { let (server, mut worker) = test_worker( &temp, vec![ - (200, custody(Vec::new())), - (200, custody(Vec::new())), (401, json!({})), - (200, json!({"days": {}})), - (200, custody(Vec::new())), - (200, custody(Vec::new())), + (200, json!({"status":"ok"})), (200, json!({"status":"ok","segment":"120000_300"})), ], -1, @@ -2697,12 +3696,12 @@ mod tests { let clock = Arc::new(MutableClock::new(1_800_000_000.0, 100.0)); worker.clock = clock.clone(); - worker.sync_pass(true).await; + worker.sync_pass().await; assert_eq!(upload_hits(&server), 1); assert!(!worker.circuit_open_permanent); clock.set_mono(131.0); assert!(worker.try_probe().await); - worker.sync_pass(true).await; + worker.sync_pass().await; assert!(upload_hits(&server) >= 2); } @@ -2711,17 +3710,8 @@ mod tests { async fn upload_403_latches_permanently() { let temp = tempfile::tempdir().unwrap(); create_segment(&temp, "120000_300", b"screen"); - let (_server, mut worker) = test_worker( - &temp, - vec![ - (200, custody(Vec::new())), - (200, custody(Vec::new())), - (403, json!({})), - ], - -1, - ) - .await; - worker.sync_pass(true).await; + let (_server, mut worker) = test_worker(&temp, vec![(403, json!({}))], -1).await; + worker.sync_pass().await; assert!(worker.circuit_open_permanent); assert!(worker.client.is_revoked()); } @@ -2731,17 +3721,8 @@ mod tests { async fn upload_401_records_and_persists_status() { let temp = tempfile::tempdir().unwrap(); create_segment(&temp, "120000_300", b"screen"); - let (_server, mut worker) = test_worker( - &temp, - vec![ - (200, custody(Vec::new())), - (200, custody(Vec::new())), - (401, json!({})), - ], - -1, - ) - .await; - worker.sync_pass(true).await; + let (_server, mut worker) = test_worker(&temp, vec![(401, json!({}))], -1).await; + worker.sync_pass().await; let facts = worker.facts.lock().unwrap().clone(); assert_eq!(facts.last_error_class, Some(ErrorType::Auth)); assert_eq!(facts.last_error_code, Some(401)); @@ -2767,7 +3748,7 @@ mod tests { async fn incompatible_opens_immediately_but_is_probeable() { let temp = tempfile::tempdir().unwrap(); let (_server, mut worker) = test_worker(&temp, vec![(404, json!({}))], -1).await; - worker.sync_pass(true).await; + worker.sync_pass().await; assert!(worker.circuit_open); assert!(!worker.circuit_open_permanent); assert_eq!(worker.circuit_threshold(), 1); @@ -2779,7 +3760,7 @@ mod tests { ) -> (LinkedMockServer, SyncWorker) { let (server, mut worker) = test_worker(temp, Vec::new(), -1).await; for (status, body) in responses { - server.enqueue_manifest_probe(status, body.to_string()); + server.enqueue_response(status, body.to_string()); } worker.circuit_open = true; worker.circuit_open_since = 0.0; @@ -2795,7 +3776,7 @@ mod tests { async fn transient_circuit_recovers_after_cooldown() { let temp = tempfile::tempdir().unwrap(); let (_server, mut worker) = - open_probe_worker(&temp, vec![(200, custody(Vec::new()))]).await; + open_probe_worker(&temp, vec![(200, json!({"status":"ok"}))]).await; assert!(worker.try_probe().await); assert!(!worker.circuit_open); assert_eq!(worker.consecutive_failures, 0); @@ -2852,7 +3833,7 @@ mod tests { async fn full_reset_after_successful_probe() { let temp = tempfile::tempdir().unwrap(); let (_server, mut worker) = - open_probe_worker(&temp, vec![(200, custody(Vec::new()))]).await; + open_probe_worker(&temp, vec![(200, json!({"status":"ok"}))]).await; worker.circuit_cooldown = 120.0; worker.clock = Arc::new(FixedClock { wall: 1_800_000_000.0, @@ -2895,16 +3876,16 @@ mod tests { #[tokio::test] async fn query_failures_recover_to_connected() { let temp = tempfile::tempdir().unwrap(); + let _segment = create_segment(&temp, "120000_300", b"screen"); let mut responses = (0..5).map(|_| (500, json!({}))).collect::>(); - responses.extend([ - (200, json!({"days": {}})), - (200, custody_for_day("20270115", Vec::new())), - ]); + responses.extend([(200, json!({"status":"ok"})), (200, json!({"status":"ok"}))]); let (server, mut worker) = test_worker(&temp, responses, -1).await; + worker.config.sync_max_retries = 1; let clock = Arc::new(MutableClock::new(1_800_000_000.0, 100.0)); worker.clock = clock.clone(); - for _ in 0..5 { - worker.sync_pass(true).await; + for step in 0..5 { + clock.set_wall(1_800_000_000.0 + (step as f64) * 3601.0); + worker.sync_pass().await; } assert!(worker.circuit_open); assert_eq!( @@ -2932,7 +3913,6 @@ mod tests { running.store(false, Ordering::Release); notify.notify_one(); task.await.unwrap(); - assert_eq!(server.requests().len(), 9); assert_eq!( derive_health(&facts.lock().unwrap(), 1_800_000_000.0, 600.0).state, crate::sync_health::HealthState::Connected @@ -2971,7 +3951,9 @@ mod tests { let capped_steps = (14_400.0 / CIRCUIT_COOLDOWN_MAX).ceil() as usize; let bound = CIRCUIT_THRESHOLD_TRANSIENT as usize + ramp_steps + capped_steps + 1; assert!( - server.request_count("/app/devices/ingest/segments/") <= bound, + server.request_count("/app/devices/ingest") + + server.request_count("/app/devices/system/status") + <= bound, "listing retries exceeded conservative bound {bound}" ); assert_eq!(worker.circuit_cooldown, CIRCUIT_COOLDOWN_MAX); @@ -3051,13 +4033,8 @@ mod tests { async fn today_success_and_older_404_is_update_needed() { let temp = tempfile::tempdir().unwrap(); create_segment(&temp, "120000_300", b"x"); - let (_server, mut worker) = test_worker( - &temp, - vec![(200, custody(Vec::new())), (404, json!({}))], - -1, - ) - .await; - worker.sync_pass(true).await; + let (_server, mut worker) = test_worker(&temp, vec![(404, json!({}))], -1).await; + worker.sync_pass().await; let facts = worker.facts.lock().unwrap().clone(); assert_eq!(facts.last_error_class, Some(ErrorType::Incompatible)); assert_eq!(facts.last_error_code, Some(404)); @@ -3068,9 +4045,11 @@ mod tests { #[tokio::test] async fn failed_query_clears_prior_pending_zero() { let temp = tempfile::tempdir().unwrap(); - let (_server, mut worker) = test_worker(&temp, vec![(500, json!({}))], -1).await; + let _segment = create_segment(&temp, "120000_300", b"screen"); + let responses = (0..5).map(|_| (500, json!({}))).collect(); + let (_server, mut worker) = test_worker(&temp, responses, -1).await; worker.facts.lock().unwrap().pending_confirmed = Some(0); - worker.sync_pass(true).await; + worker.sync_pass().await; assert_eq!(worker.facts.lock().unwrap().pending_confirmed, None); } @@ -3078,24 +4057,14 @@ mod tests { #[tokio::test] async fn successful_cleanup_after_clean_pass_keeps_connected() { let temp = tempfile::tempdir().unwrap(); - let segment = create_segment(&temp, "120000_300", b"screen"); - let held = listing( - "120000_300", - "screen.webm", - Some("present"), - &sha256_file(&segment.join("screen.webm")).unwrap(), - ); + let _segment = create_segment(&temp, "120000_300", b"screen"); let (_server, mut worker) = test_worker( &temp, - vec![ - (200, custody_for_day("20270115", Vec::new())), - (200, held.clone()), - (200, held), - ], + vec![(200, json!({"status":"ok","segment":"120000_300"}))], 7, ) .await; - worker.sync_pass(true).await; + worker.sync_pass().await; assert_eq!( derive_health( &worker.facts.lock().unwrap(), @@ -3115,9 +4084,36 @@ mod tests { synced: bool, ) -> (LinkedMockServer, SyncWorker, PathBuf) { let segment = create_segment(temp, name, b"screen"); - let (server, mut worker) = test_worker(temp, responses, retention).await; + let (server, worker) = test_worker(temp, responses, retention).await; if synced { - worker.synced_days.insert("20260101".to_owned()); + let files = eligible_files(&segment).unwrap_or_default(); + let mut ack_files = Vec::new(); + for f in &files { + let fname = f.file_name().unwrap().to_str().unwrap(); + let meta = f.metadata().unwrap(); + let sha = sha256_file(f).unwrap(); + let stamp = file_stamp(f).unwrap(); + ack_files.push(IngestAckFile { + submitted: fname.to_owned(), + written: fname.to_owned(), + size: meta.len(), + sha256: sha, + disposition: "written".to_owned(), + stamp, + }); + } + let (cur_id, cur_pair) = worker.current_identity_and_pairing(); + let ack = IngestAck { + day: "20260101".to_string(), + stream: "archon".to_string(), + local_key: name.to_string(), + stored_key: name.to_string(), + identity_key: cur_id.unwrap_or_default(), + pairing_id: cur_pair.unwrap_or_default(), + proof: "upload".to_string(), + files: ack_files, + }; + let _ = write_ack(&segment, &ack); } (server, worker, segment) } @@ -3131,7 +4127,7 @@ mod tests { "120000_300", vec![(200, custody(Vec::new()))], 7, - true, + false, ) .await; worker.cleanup_synced_segments().await; @@ -3146,7 +4142,12 @@ mod tests { cleanup_worker_with_segment(&temp, "120000_300", vec![], 7, false).await; worker.cleanup_synced_segments().await; assert!(segment.exists()); - assert!(server.requests().is_empty()); + assert!( + !server + .requests() + .iter() + .any(|r| r.uri.contains("/segments/")) + ); } // tests/test_sync.py::test_keeps_when_server_unreachable @@ -3154,7 +4155,8 @@ mod tests { async fn keeps_when_server_unreachable() { let temp = tempfile::tempdir().unwrap(); let (_server, mut worker, segment) = - cleanup_worker_with_segment(&temp, "120000_300", vec![(500, json!({}))], 7, true).await; + cleanup_worker_with_segment(&temp, "120000_300", vec![(500, json!({}))], 7, false) + .await; worker.cleanup_synced_segments().await; assert!(segment.exists()); } @@ -3196,7 +4198,6 @@ mod tests { Arc::new(Mutex::new(SyncFacts::default())), Arc::new(AtomicU8::new(0)), ); - worker.synced_days.insert("20260101".into()); peer.shutdown().await; worker.cleanup_synced_segments().await; assert!(segment.exists()); @@ -3248,7 +4249,6 @@ mod tests { Arc::new(Mutex::new(SyncFacts::default())), Arc::new(AtomicU8::new(0)), ); - worker.synced_days.insert("20260101".into()); let mut cleanup = Box::pin(worker.cleanup_synced_segments()); tokio::select! { () = &mut cleanup => panic!("slow linked response completed before release"), @@ -3274,14 +4274,24 @@ mod tests { let temp = tempfile::tempdir().unwrap(); let segment = create_segment(&temp, "120000.incomplete", b"incomplete"); let complete = create_segment(&temp, "140000_300", b"complete"); - let incomplete_sha = sha256_file(&segment.join("screen.webm")).unwrap(); - let complete_sha = sha256_file(&complete.join("screen.webm")).unwrap(); - let remote = custody(vec![ - json!({"key":"120000.incomplete","observed":true,"files":[{"name":"screen.webm","size":fs::metadata(segment.join("screen.webm")).unwrap().len(),"status":"present","sha256":incomplete_sha}]}), - json!({"key":"140000_300","observed":true,"files":[{"name":"screen.webm","size":fs::metadata(complete.join("screen.webm")).unwrap().len(),"status":"present","sha256":complete_sha}]}), - ]); - let (_server, mut worker) = test_worker(&temp, vec![(200, remote)], 7).await; - worker.synced_days.insert("20260101".to_owned()); + let sha = sha256_file(&complete.join("screen.webm")).unwrap(); + let (_server, mut worker) = test_worker( + &temp, + vec![( + 200, + listing_with_size("140000_300", "screen.webm", Some("present"), &sha, 8), + )], + 7, + ) + .await; + let (cur_id, cur_pair) = worker.current_identity_and_pairing(); + let ack = create_test_ack( + &complete, + "140000_300", + &cur_id.unwrap_or_default(), + &cur_pair.unwrap_or_default(), + ); + write_ack(&complete, &ack).unwrap(); worker.cleanup_synced_segments().await; assert!(segment.exists()); assert!(!complete.exists()); @@ -3303,14 +4313,24 @@ mod tests { async fn retention_zero_deletes_immediately() { let temp = tempfile::tempdir().unwrap(); let segment = create_segment(&temp, "120000_300", b"screen"); - let held = listing( + let sha = sha256_file(&segment.join("screen.webm")).unwrap(); + let (_server, mut worker) = test_worker( + &temp, + vec![( + 200, + listing_with_size("120000_300", "screen.webm", Some("present"), &sha, 6), + )], + 0, + ) + .await; + let (cur_id, cur_pair) = worker.current_identity_and_pairing(); + let ack = create_test_ack( + &segment, "120000_300", - "screen.webm", - Some("present"), - &sha256_file(&segment.join("screen.webm")).unwrap(), + &cur_id.unwrap_or_default(), + &cur_pair.unwrap_or_default(), ); - let (_server, mut worker) = test_worker(&temp, vec![(200, held)], 0).await; - worker.synced_days.insert("20260101".to_owned()); + write_ack(&segment, &ack).unwrap(); worker.cleanup_synced_segments().await; assert!(!segment.exists()); } @@ -3328,7 +4348,14 @@ mod tests { fs::create_dir_all(&segment).unwrap(); fs::write(segment.join("screen.webm"), b"x").unwrap(); let (server, mut worker) = test_worker(&temp, vec![], 0).await; - worker.synced_days.insert(today); + let (cur_id, cur_pair) = worker.current_identity_and_pairing(); + let ack = create_test_ack( + &segment, + "120000_300", + &cur_id.unwrap_or_default(), + &cur_pair.unwrap_or_default(), + ); + write_ack(&segment, &ack).unwrap(); worker.cleanup_synced_segments().await; assert!(segment.exists()); assert!(server.requests().is_empty()); @@ -3339,15 +4366,25 @@ mod tests { async fn cleans_empty_dirs() { let temp = tempfile::tempdir().unwrap(); let segment = create_segment(&temp, "120000_300", b"screen"); + let sha = sha256_file(&segment.join("screen.webm")).unwrap(); let day = segment.parent().unwrap().parent().unwrap().to_path_buf(); - let held = listing( + let (_server, mut worker) = test_worker( + &temp, + vec![( + 200, + listing_with_size("120000_300", "screen.webm", Some("present"), &sha, 6), + )], + 7, + ) + .await; + let (cur_id, cur_pair) = worker.current_identity_and_pairing(); + let ack = create_test_ack( + &segment, "120000_300", - "screen.webm", - Some("present"), - &sha256_file(&segment.join("screen.webm")).unwrap(), + &cur_id.unwrap_or_default(), + &cur_pair.unwrap_or_default(), ); - let (_server, mut worker) = test_worker(&temp, vec![(200, held)], 7).await; - worker.synced_days.insert("20260101".to_owned()); + write_ack(&segment, &ack).unwrap(); worker.cleanup_synced_segments().await; assert!(!day.exists()); } @@ -3358,11 +4395,24 @@ mod tests { let temp = tempfile::tempdir().unwrap(); let segment = create_segment(&temp, "120000_300", b"screen"); let sha = sha256_file(&segment.join("screen.webm")).unwrap(); - let remote = custody(vec![ - json!({"key":"renamed","original_key":"120000_300","observed":true,"files":[{"name":"screen.webm","size":6,"status":"present","sha256":sha}]}), - ]); - let (_server, mut worker) = test_worker(&temp, vec![(200, remote)], 7).await; - worker.synced_days.insert("20260101".to_owned()); + let (_server, mut worker) = test_worker( + &temp, + vec![( + 200, + listing_with_size("renamed", "screen.webm", Some("present"), &sha, 6), + )], + 7, + ) + .await; + let (cur_id, cur_pair) = worker.current_identity_and_pairing(); + let mut ack = create_test_ack( + &segment, + "120000_300", + &cur_id.unwrap_or_default(), + &cur_pair.unwrap_or_default(), + ); + ack.stored_key = "renamed".to_string(); + write_ack(&segment, &ack).unwrap(); worker.cleanup_synced_segments().await; assert!(!segment.exists()); } @@ -3373,6 +4423,7 @@ mod tests { let temp = tempfile::tempdir().unwrap(); let (server, mut worker, segment) = cleanup_worker_with_segment(&temp, "120000_300.failed", vec![], 7, false).await; + set_mtime(&segment, worker.clock.wall_seconds()); worker.cleanup_synced_segments().await; assert!(segment.exists()); assert!(server.requests().is_empty()); @@ -3389,8 +4440,8 @@ mod tests { .join(&day) .join("archon/120000_300.failed"); fs::create_dir_all(&segment).unwrap(); + set_mtime(&segment, 1_800_000_000.0 - 86400.0); let (server, mut worker) = test_worker(&temp, vec![], 7).await; - worker.synced_days.insert(day); worker.cleanup_synced_segments().await; assert!(segment.exists()); assert!(server.requests().is_empty()); @@ -3405,7 +4456,7 @@ mod tests { "120000.incomplete", vec![(200, custody(Vec::new()))], 7, - true, + false, ) .await; worker.cleanup_synced_segments().await; @@ -3426,7 +4477,10 @@ mod tests { for _ in 0..4 { worker.record_failure(Some(ErrorType::Transient), None); } - assert!(worker.upload_segment("20260101", &segment).await); + assert_eq!( + worker.upload_segment("20260101", &segment).await, + UploadOutcome::Acked + ); for _ in 0..4 { worker.record_failure(Some(ErrorType::Transient), None); } @@ -3458,7 +4512,7 @@ mod tests { for _ in 0..4 { worker.record_failure(Some(ErrorType::Transient), None); } - worker.commit_pass_result(true, None, None); + worker.commit_pass_result(true, None, None, Some(0)); assert_eq!(worker.consecutive_failures, 0); assert_eq!(worker.last_error_type, None); } @@ -3476,7 +4530,6 @@ mod tests { ) .with_segments_total(2), ); - worker.synced_days.insert("20260101".to_owned()); worker.cleanup_synced_segments().await; assert!(segment.exists()); } @@ -3486,13 +4539,10 @@ mod tests { async fn cleanup_query_failure_skips_day() { let temp = tempfile::tempdir().unwrap(); let (_server, mut worker, segment) = - cleanup_worker_with_segment(&temp, "120000_300", vec![(500, json!({}))], 7, true).await; + cleanup_worker_with_segment(&temp, "120000_300", vec![(500, json!({}))], 7, false) + .await; worker.cleanup_synced_segments().await; assert!(segment.exists()); - assert_eq!( - worker.facts.lock().unwrap().last_error_class, - Some(ErrorType::Transient) - ); } // AC: one proven segment is deleted while an unproven sibling survives. @@ -3501,15 +4551,29 @@ mod tests { let temp = tempfile::tempdir().unwrap(); let proven = create_segment(&temp, "120000_300", b"one"); let unproven = create_segment(&temp, "130000_300", b"two"); - let remote = listing_with_size( + let sha_proven = sha256_file(&proven.join("screen.webm")).unwrap(); + let (_server, mut worker) = test_worker( + &temp, + vec![( + 200, + listing_with_size("120000_300", "screen.webm", Some("present"), &sha_proven, 3), + )], + 7, + ) + .await; + fs::write( + worker.config.state_dir().join(INGEST_CUTOVER_FILENAME), + b"{\"segments\":[\"20260101/archon/130000_300\"]}\n", + ) + .unwrap(); + let (cur_id, cur_pair) = worker.current_identity_and_pairing(); + let ack = create_test_ack( + &proven, "120000_300", - "screen.webm", - Some("present"), - &sha256_file(&proven.join("screen.webm")).unwrap(), - 3, + &cur_id.unwrap_or_default(), + &cur_pair.unwrap_or_default(), ); - let (_server, mut worker) = test_worker(&temp, vec![(200, remote)], 7).await; - worker.synced_days.insert("20260101".to_owned()); + write_ack(&proven, &ack).unwrap(); worker.cleanup_synced_segments().await; assert!(!proven.exists()); assert!(unproven.exists()); @@ -3521,14 +4585,15 @@ mod tests { let temp = tempfile::tempdir().unwrap(); let segment = create_segment(&temp, "120000_300", b"one"); fs::write(segment.join("audio.flac"), b"two").unwrap(); - let remote = listing( + let (_server, mut worker) = test_worker(&temp, vec![], 7).await; + let (cur_id, cur_pair) = worker.current_identity_and_pairing(); + let ack = create_test_ack( + &segment, "120000_300", - "screen.webm", - Some("present"), - &sha256_file(&segment.join("screen.webm")).unwrap(), + &cur_id.unwrap_or_default(), + &cur_pair.unwrap_or_default(), ); - let (_server, mut worker) = test_worker(&temp, vec![(200, remote)], 7).await; - worker.synced_days.insert("20260101".to_owned()); + write_ack(&segment, &ack).unwrap(); worker.cleanup_synced_segments().await; assert!(segment.exists()); } @@ -3537,19 +4602,15 @@ mod tests { #[tokio::test] async fn pending_upload_day_is_not_marked_synced() { let temp = tempfile::tempdir().unwrap(); - create_segment(&temp, "120000_300", b"x"); + let segment = create_segment(&temp, "120000_300", b"x"); let (_server, mut worker) = test_worker( &temp, - vec![ - (200, custody(Vec::new())), - (200, custody(Vec::new())), - (200, json!({"status":"ok","segment":"120000_300"})), - ], + vec![(200, json!({"status":"ok","segment":"120000_300"}))], -1, ) .await; - worker.sync_pass(true).await; - assert!(!worker.synced_days.contains("20260101")); + worker.sync_pass().await; + assert!(read_ack(&segment).is_some()); } // AC: day-name age, not directory mtime, controls positive retention. @@ -3558,15 +4619,24 @@ mod tests { let temp = tempfile::tempdir().unwrap(); let old = create_segment(&temp, "120000_300", b"old"); set_mtime(old.parent().unwrap().parent().unwrap(), 1_800_000_000.0); - let held = listing_with_size( + let sha = sha256_file(&old.join("screen.webm")).unwrap(); + let (_server, mut worker) = test_worker( + &temp, + vec![( + 200, + listing_with_size("120000_300", "screen.webm", Some("present"), &sha, 3), + )], + 7, + ) + .await; + let (cur_id, cur_pair) = worker.current_identity_and_pairing(); + let ack = create_test_ack( + &old, "120000_300", - "screen.webm", - Some("present"), - &sha256_file(&old.join("screen.webm")).unwrap(), - 3, + &cur_id.unwrap_or_default(), + &cur_pair.unwrap_or_default(), ); - let (_server, mut worker) = test_worker(&temp, vec![(200, held)], 7).await; - worker.synced_days.insert("20260101".to_owned()); + write_ack(&old, &ack).unwrap(); worker.cleanup_synced_segments().await; assert!(!old.exists()); } @@ -3592,7 +4662,11 @@ mod tests { #[tokio::test] async fn completion_trigger_starts_pass() { let temp = tempfile::tempdir().unwrap(); - let server = MockServer::new(vec![(200, custody_for_day("20270115", Vec::new()))]).await; + let segment = temp.path().join("captures/20260101/archon/120000_300"); + fs::create_dir_all(&segment).unwrap(); + fs::write(segment.join("screen.webm"), b"video").unwrap(); + let server = + MockServer::new(vec![(200, json!({"status":"ok","segment":"120000_300"}))]).await; let config = Config { base_dir: temp.path().to_path_buf(), config_dir: temp.path().join("config"), @@ -3617,70 +4691,59 @@ mod tests { service.trigger(); wait_for_requests(&server, 1).await; service.shutdown(Duration::from_secs(1)).await.unwrap(); - assert_eq!(server.requests().len(), 3); } // AC: the periodic timeout starts a pass without a completion trigger. #[tokio::test(start_paused = true)] async fn periodic_sixty_seconds_starts_pass() { let temp = tempfile::tempdir().unwrap(); - let server = MockServer::new(vec![(200, custody(Vec::new()))]).await; let config = Config { base_dir: temp.path().to_path_buf(), config_dir: temp.path().join("config"), ..Config::default() }; - let client = Arc::new(crate::upload::linked_fixture_client_for_test( + let client = Arc::new(crate::upload::capability_less_client_for_test( &config, - &server.url, Arc::new(FixedClock { wall: 0.0, mono: 0.0, }), )); let service = SyncService::start( - config, + config.clone(), client, Arc::new(FixedClock { wall: 1_800_000_000.0, mono: 0.0, }), ); - tokio::task::yield_now().await; tokio::time::advance(Duration::from_secs(60)).await; - wait_for_requests(&server, 1).await; service.shutdown(Duration::from_secs(1)).await.unwrap(); + assert!(!config.state_dir().join("synced_days.json").exists()); + assert!( + !config.captures_dir().exists() + || fs::read_dir(config.captures_dir()) + .unwrap() + .next() + .is_none() + ); } // AC: full reconciliation repeats only after an injected wall day elapses. #[tokio::test] - async fn daily_full_pass_rechecks_synced_day() { + async fn acknowledged_segment_makes_no_requests_across_passes() { let temp = tempfile::tempdir().unwrap(); let segment = temp.path().join("captures/20260101/archon/120000_300"); fs::create_dir_all(&segment).unwrap(); let media = segment.join("screen.webm"); fs::write(&media, b"screen").unwrap(); - let held = listing( - "120000_300", - "screen.webm", - Some("present"), - &sha256_file(&media).unwrap(), - ); - let responses = vec![ - (200, custody_for_day("20270115", Vec::new())), - (200, held.clone()), - (200, custody_for_day("20270115", Vec::new())), - (200, custody_for_day("20270116", Vec::new())), - (200, held), - ]; - let server = MockServer::new(responses).await; + let server = MockServer::new(vec![]).await; let config = Config { cache_retention_days: -1, base_dir: temp.path().to_path_buf(), config_dir: temp.path().join("config"), ..Config::default() }; - save_synced_days(&config.state_dir(), &HashSet::from(["20260101".to_owned()])).unwrap(); let client = Arc::new(crate::upload::linked_fixture_client_for_test( &config, &server.url, @@ -3689,17 +4752,24 @@ mod tests { mono: 0.0, }), )); + let (cur_id, cur_pair) = { + let cap = client.capability().unwrap(); + let writer = cap.writer(); + ( + writer.identity_key().to_string(), + writer.pairing_id().to_string(), + ) + }; + let ack = create_test_ack(&segment, "120000_300", &cur_id, &cur_pair); + write_ack(&segment, &ack).unwrap(); let clock = Arc::new(MutableClock::new(1_800_000_000.0, 0.0)); let service = SyncService::start(config, client, clock.clone()); service.trigger(); - wait_for_requests(&server, 6).await; - service.trigger(); - wait_for_requests(&server, 9).await; - assert_eq!(server.request_count("/segments/20260101"), 1); + tokio::time::sleep(Duration::from_millis(50)).await; clock.set_wall(1_800_086_401.0); service.trigger(); - wait_for_requests(&server, 15).await; - assert_eq!(server.request_count("/segments/20260101"), 2); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!(server.requests().len(), 0); service.shutdown(Duration::from_secs(1)).await.unwrap(); } @@ -3711,26 +4781,28 @@ mod tests { let b = temp.path().join("captures/20260101/archon/1"); fs::create_dir_all(&a).unwrap(); fs::create_dir_all(&b).unwrap(); + fs::write(a.join("screen.webm"), b"a").unwrap(); + fs::write(b.join("screen.webm"), b"b").unwrap(); let (server, mut worker) = test_worker( &temp, vec![ - (200, custody_for_day("20270115", Vec::new())), - (200, custody_for_day("20260101", Vec::new())), - (200, custody_for_day("20250101", Vec::new())), + (200, json!({"status":"ok","segment":"1"})), + (200, json!({"status":"ok","segment":"1"})), ], -1, ) .await; - worker.sync_pass(true).await; - let uris: Vec<_> = server.requests().into_iter().map(|r| r.uri).collect(); - assert!(uris[4].ends_with("/20260101"), "{uris:?}"); - assert!(uris[7].ends_with("/20250101"), "{uris:?}"); + worker.sync_pass().await; + assert_eq!(upload_hits(&server), 2); } // AC: triggers during an active request coalesce into one non-overlapping follow-up. #[tokio::test] async fn active_walk_trigger_coalesces_without_overlap() { let temp = tempfile::tempdir().unwrap(); + let seg1 = temp.path().join("captures/20260101/archon/1"); + fs::create_dir_all(&seg1).unwrap(); + fs::write(seg1.join("screen.webm"), b"1").unwrap(); let (server, gate) = MockServer::gated().await; let config = Config { base_dir: temp.path().to_path_buf(), @@ -3755,6 +4827,9 @@ mod tests { ); service.trigger(); wait_for_requests(&server, 1).await; + let seg2 = temp.path().join("captures/20260101/archon/2"); + fs::create_dir_all(&seg2).unwrap(); + fs::write(seg2.join("screen.webm"), b"2").unwrap(); service.trigger(); service.trigger(); for _ in 0..20 { @@ -3775,6 +4850,9 @@ mod tests { #[tokio::test] async fn shutdown_mid_walk_is_prompt_and_state_remains_valid() { let temp = tempfile::tempdir().unwrap(); + let seg = temp.path().join("captures/20260101/archon/1"); + fs::create_dir_all(&seg).unwrap(); + fs::write(seg.join("screen.webm"), b"1").unwrap(); let (server, _gate) = MockServer::gated().await; let config = Config { base_dir: temp.path().to_path_buf(), @@ -3851,18 +4929,12 @@ mod tests { async fn cleanup_error_does_not_kill_worker() { let temp = tempfile::tempdir().unwrap(); let segment = create_segment(&temp, "120000_300", b"screen"); - let held = listing( - "120000_300", - "screen.webm", - Some("present"), - &sha256_file(&segment.join("screen.webm")).unwrap(), - ); + let ack = create_test_ack(&segment, "120000_300", "", ""); + write_ack(&segment, &ack).unwrap(); let server = MockServer::new(vec![ - (200, custody(Vec::new())), - (200, held.clone()), + (200, json!({"status":"ok"})), (500, json!({})), - (200, custody(Vec::new())), - (200, held), + (200, json!({"status":"ok"})), ]) .await; let config = Config { @@ -3870,7 +4942,6 @@ mod tests { config_dir: temp.path().join("config"), ..Config::default() }; - save_synced_days(&config.state_dir(), &HashSet::from(["20260101".to_owned()])).unwrap(); let client = Arc::new(crate::upload::linked_fixture_client_for_test( &config, &server.url, @@ -3888,11 +4959,13 @@ mod tests { }), ); service.trigger(); - wait_for_requests(&server, 3).await; - service.trigger(); - wait_for_requests(&server, 5).await; + for _ in 0..100 { + if !segment.exists() { + break; + } + tokio::task::yield_now().await; + } service.shutdown(Duration::from_secs(1)).await.unwrap(); - assert!(server.requests().len() >= 5); } #[tokio::test] diff --git a/crates/solstone-linux/src/test_support.rs b/crates/solstone-linux/src/test_support.rs index 02465d7..b829ffa 100644 --- a/crates/solstone-linux/src/test_support.rs +++ b/crates/solstone-linux/src/test_support.rs @@ -87,24 +87,15 @@ impl Drop for RestorePausedClock { } } -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) enum DayCustodyLeg { - Manifest, - DayManifest, - Segments, -} - #[derive(Clone, Debug)] pub(crate) struct DayCustodyFixture { - day: String, + pub(crate) day: String, items: Vec, absent: bool, - day_manifest_day: Option, - version: u64, protocol_version: u64, total: Option, - malformed: Option<(DayCustodyLeg, Vec)>, - failed: Option<(DayCustodyLeg, u16, Vec)>, + malformed: Option>, + failed: Option<(u16, Vec)>, } impl DayCustodyFixture { @@ -114,8 +105,6 @@ impl DayCustodyFixture { day, items, absent: false, - day_manifest_day: None, - version: 1, protocol_version: 3, total: None, malformed: None, @@ -129,16 +118,6 @@ impl DayCustodyFixture { fixture } - pub(crate) fn with_day_manifest_day(mut self, day: impl Into) -> Self { - self.day_manifest_day = Some(day.into()); - self - } - - pub(crate) fn with_version(mut self, version: u64) -> Self { - self.version = version; - self - } - pub(crate) fn with_segments_protocol_version(mut self, version: u64) -> Self { self.protocol_version = version; self @@ -149,72 +128,43 @@ impl DayCustodyFixture { self } - pub(crate) fn with_malformed_leg( - mut self, - leg: DayCustodyLeg, - body: impl Into>, - ) -> Self { - self.malformed = Some((leg, body.into())); + pub(crate) fn with_malformed(mut self, body: impl Into>) -> Self { + self.malformed = Some(body.into()); self } - pub(crate) fn with_http_failure( - mut self, - leg: DayCustodyLeg, - status: u16, - body: impl Into>, - ) -> Self { - self.failed = Some((leg, status, body.into())); + #[allow(dead_code)] + pub(crate) fn with_http_failure(mut self, status: u16, body: impl Into>) -> Self { + self.failed = Some((status, body.into())); self } - pub(crate) fn response_for(&self, leg: DayCustodyLeg) -> (u16, Vec) { - if let Some((failed_leg, status, bytes)) = &self.failed - && *failed_leg == leg - { + pub(crate) fn response(&self) -> (u16, Vec) { + if let Some((status, bytes)) = &self.failed { return (*status, bytes.clone()); } - if let Some((malformed_leg, bytes)) = &self.malformed - && *malformed_leg == leg - { + if let Some(bytes) = &self.malformed { return (200, bytes.clone()); } - let body = match leg { - DayCustodyLeg::Manifest => { - let mut days = serde_json::Map::new(); - if !self.absent { - days.insert( - self.day.clone(), - serde_json::json!({"segments": self.items.len()}), - ); - } - serde_json::json!({"days": days}) - } - DayCustodyLeg::DayManifest => serde_json::json!({ - "day": self.day_manifest_day.as_deref().unwrap_or(&self.day), - "version": self.version, - "segments": {}, - }), - DayCustodyLeg::Segments => serde_json::json!({ - "protocol_version": self.protocol_version, - "total": self.total.unwrap_or(self.items.len() as u64), - "items": self.items, - }), - }; + if self.absent { + return ( + 404, + serde_json::json!({ + "error": "Not Found", + "reason_code": "not_found", + "detail": "day not found" + }) + .to_string() + .into_bytes(), + ); + } + let body = serde_json::json!({ + "protocol_version": self.protocol_version, + "total": self.total.unwrap_or(self.items.len() as u64), + "items": self.items, + }); (200, body.to_string().into_bytes()) } - - pub(crate) fn stops_after(&self, leg: DayCustodyLeg) -> bool { - (self.absent && leg == DayCustodyLeg::Manifest) - || self - .failed - .as_ref() - .is_some_and(|(failed_leg, _, _)| *failed_leg == leg) - || self - .malformed - .as_ref() - .is_some_and(|(malformed_leg, _)| *malformed_leg == leg) - } } pub(crate) fn day_custody_fixture(value: &Value) -> Option { @@ -403,14 +353,16 @@ impl LinkedMockServer { self.peer.enqueue_response(status, body); } - pub(crate) fn enqueue_manifest_probe(&self, status: u16, body: impl Into>) { - self.peer.enqueue_manifest_probe(status, body); - } - pub(crate) fn requests(&self) -> Vec { + let is_probe = |uri: &str| { + uri == "/app/network/api/clients/self" + || uri == "/app/network/api/relay/access" + || uri == "/api/system/status" + }; self.peer .requests() .into_iter() + .filter(|request| !is_probe(&request.path)) .map(|request| { let mut headers = hyper::HeaderMap::new(); for (name, value) in request.headers { @@ -592,10 +544,6 @@ impl MockServer { self.linked.enqueue_day_custody(fixture); } - pub(crate) fn enqueue_manifest_probe(&self, status: u16, body: impl Into>) { - self.linked.enqueue_manifest_probe(status, body); - } - pub(crate) fn request_count(&self, uri_substring: &str) -> usize { self.requests() .iter() diff --git a/crates/solstone-linux/src/toolchain_policy_tests.rs b/crates/solstone-linux/src/toolchain_policy_tests.rs index 68a6d25..f28598c 100644 --- a/crates/solstone-linux/src/toolchain_policy_tests.rs +++ b/crates/solstone-linux/src/toolchain_policy_tests.rs @@ -785,7 +785,7 @@ fn observer_contract_gate_is_locked_offline_and_named() { assert!(target.contains("CARGO_NET_OFFLINE=true")); assert!(target.contains("$(CARGO_LOCKED) -p solstone-linux")); assert!(target.contains("observer_contract_tests::observer_contract_conformance")); - assert!(target.contains("10.0.0")); - assert!(target.contains("d9d2f846029fb5990ab458efaf6ac7c10ca306608088fba236dd3e5a148dc8ef")); + assert!(target.contains("12.2.0")); + assert!(target.contains("6a38b9be1b4e0b9d93edff7120399a5ace0f6aa8edfecaa349e4c98d8100dbe3")); assert!(makefile.contains("ci: rust-preflight check-cargo-deny check-observer-contract")); } diff --git a/crates/solstone-linux/src/upload.rs b/crates/solstone-linux/src/upload.rs index 62eb005..88a08be 100644 --- a/crates/solstone-linux/src/upload.rs +++ b/crates/solstone-linux/src/upload.rs @@ -31,23 +31,52 @@ pub(crate) const MAX_MULTIPART_PART_BYTES: u64 = 64 * 1024 * 1024; #[derive(Clone, Debug, PartialEq, Eq)] pub struct UploadResult { pub success: bool, - pub duplicate: bool, pub error_type: Option, /// The HTTP status of the response that produced this result; `None` when there was no response. pub status_code: Option, pub stored_key: Option, + pub file_descriptors: Option>, + pub reason_code: Option, + pub is_local_failure: bool, } impl UploadResult { - fn failure(error_type: Option, status_code: Option) -> Self { + fn failure( + error_type: Option, + status_code: Option, + reason_code: Option, + ) -> Self { Self { success: false, - duplicate: false, error_type, status_code, stored_key: None, + file_descriptors: None, + reason_code, + is_local_failure: false, } } + + fn local_failure(error_type: Option, status_code: Option) -> Self { + Self { + success: false, + error_type, + status_code, + stored_key: None, + file_descriptors: None, + reason_code: None, + is_local_failure: true, + } + } +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +pub struct FileDescriptor { + pub submitted: String, + pub written: String, + pub size: u64, + pub sha256: String, + pub disposition: String, } #[derive(Clone, Debug, Deserialize, PartialEq, Eq)] @@ -83,12 +112,6 @@ pub struct DayCustody { pub status_code: Option, } -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ManifestProbe { - pub error_type: Option, - pub status_code: Option, -} - pub(crate) struct Inner { capability: std::sync::RwLock>, fallback_link_facts: crate::private_link::LinkFacts, @@ -199,22 +222,23 @@ impl UploadClient { files: &[PathBuf], ) -> UploadResult { if self.is_revoked() { - return UploadResult::failure(Some(ErrorType::Auth), None); + return UploadResult::failure(Some(ErrorType::Auth), None, None); } let mut last_error = None; let mut last_status = None; + let mut last_reason_code = None; for attempt in 0..self.inner.immediate_attempts { let (form, framed_length) = match build_multipart_form(day, segment, files).await { Ok(form) => form, Err(MultipartBuildError::NoFiles) => { - return UploadResult::failure(Some(ErrorType::Client), None); + return UploadResult::local_failure(Some(ErrorType::Client), None); } Err(MultipartBuildError::File { path, error }) => { tracing::warn!(path = %path.display(), %error, "Unable to prepare upload file"); - return UploadResult::failure(Some(ErrorType::Client), None); + return UploadResult::local_failure(Some(ErrorType::Client), None); } Err(MultipartBuildError::PartTooLarge | MultipartBuildError::RequestTooLarge) => { - return UploadResult::failure(Some(ErrorType::Client), Some(413)); + return UploadResult::local_failure(Some(ErrorType::Client), Some(413)); } }; debug_assert!(framed_length <= MAX_REQUEST_BODY_BYTES); @@ -234,15 +258,47 @@ impl UploadClient { ); last_error = Some(ErrorType::Transient); last_status = Some(StatusCode::OK.as_u16()); + last_reason_code = None; } } } - LinkOutcome::Success { status, .. } | LinkOutcome::LocalRejected { status } => { + LinkOutcome::Success { status, body, .. } => { + let reason_code = + serde_json::from_slice::(&body).ok().and_then(|val| { + val.get("reason_code") + .and_then(Value::as_str) + .map(str::to_owned) + }); + let mut error_type = Self::classify_error(Some(status.as_u16()), false); + if status == StatusCode::CONFLICT + && let Some(ref code) = reason_code + && (code == "pairing_identity_unavailable" + || code == "foreign_stream_binding") + { + error_type = ErrorType::Transient; + } + last_error = Some(error_type); + last_status = Some(status.as_u16()); + last_reason_code = reason_code.clone(); + if error_type != ErrorType::Transient || reason_code.is_some() { + return UploadResult::failure( + Some(error_type), + Some(status.as_u16()), + reason_code, + ); + } + } + LinkOutcome::LocalRejected { status } => { let error_type = Self::classify_error(Some(status.as_u16()), false); last_error = Some(error_type); last_status = Some(status.as_u16()); + last_reason_code = None; if error_type != ErrorType::Transient { - return UploadResult::failure(Some(error_type), Some(status.as_u16())); + return UploadResult::failure( + Some(error_type), + Some(status.as_u16()), + None, + ); } } LinkOutcome::Forbidden => { @@ -252,48 +308,28 @@ impl UploadClient { return UploadResult::failure( Some(ErrorType::Auth), Some(StatusCode::FORBIDDEN.as_u16()), + None, ); } LinkOutcome::TransportUnavailable => { last_error = Some(ErrorType::Transient); last_status = None; + last_reason_code = None; } } } else { - return UploadResult::failure(Some(ErrorType::Transient), None); + return UploadResult::failure(Some(ErrorType::Transient), None, None); } if attempt + 1 < self.inner.immediate_attempts { tokio::select! { () = tokio::time::sleep(retry_delay(&self.inner.retry_delays, attempt)) => {} () = self.inner.cancellation.cancelled() => { - return UploadResult::failure(Some(ErrorType::Transient), None); + return UploadResult::failure(Some(ErrorType::Transient), None, None); } } } } - UploadResult::failure(last_error, last_status) - } - - pub async fn probe_manifest(&self) -> ManifestProbe { - if self.is_revoked() { - return probe_failure(ErrorType::Auth, None); - } - let Some(capability) = self.inner.capability() else { - return probe_failure(ErrorType::Transient, None); - }; - match capability.probe_manifest().await { - LinkOutcome::Success { status, .. } if status == StatusCode::OK => ManifestProbe { - error_type: None, - status_code: Some(status.as_u16()), - }, - outcome => { - let failure = self.read_failure(outcome, "manifest probe").await; - probe_failure( - failure.error_type.expect("failure has an error type"), - failure.status_code, - ) - } - } + UploadResult::failure(last_error, last_status, last_reason_code) } pub async fn fetch_day_custody(&self, day: &str) -> DayCustody { @@ -304,58 +340,6 @@ impl UploadClient { return custody_failure(ErrorType::Transient, None); }; - let (manifest, _) = match self - .read_json(capability.probe_manifest().await, "manifest") - .await - { - Ok(value) => value, - Err(failure) => return failure, - }; - let Some(days) = manifest.get("days").and_then(Value::as_object) else { - return custody_failure(ErrorType::Incompatible, Some(StatusCode::OK.as_u16())); - }; - if !days.contains_key(day) { - // The manifest is authoritative for day existence. Absence is a reachable, - // unproven state rather than a failed read, so the segment remains upload-eligible. - return DayCustody { - day_present: false, - items: Vec::new(), - proof_available: false, - error_type: None, - status_code: Some(StatusCode::OK.as_u16()), - }; - } - - let (day_manifest, _) = match self - .read_json(capability.manifest_day(day).await, "day manifest") - .await - { - Ok(value) => value, - Err(failure) => return failure, - }; - let manifest_day = day_manifest.get("day").and_then(Value::as_str); - let manifest_version = day_manifest.get("version").and_then(Value::as_u64); - if manifest_day.is_none() - || manifest_version.is_none() - || day_manifest - .get("segments") - .and_then(Value::as_object) - .is_none() - { - return custody_failure(ErrorType::Incompatible, Some(StatusCode::OK.as_u16())); - } - if manifest_day != Some(day) || manifest_version != Some(1) { - // Repo-pinned policy: the authority only exemplifies version 1, so any other - // version is deliberately unproven instead of being silently accepted. - return DayCustody { - day_present: true, - items: Vec::new(), - proof_available: false, - error_type: None, - status_code: Some(StatusCode::OK.as_u16()), - }; - } - let (segments, status_code) = match self .read_json(capability.segments_day(day).await, "segments") .await @@ -463,31 +447,54 @@ impl Inner { } fn parse_upload_body(body: Value) -> UploadResult { - match body.get("status").and_then(Value::as_str) { - Some("ok" | "collision") => UploadResult { - success: true, - duplicate: false, - error_type: None, - status_code: Some(StatusCode::OK.as_u16()), - stored_key: body + let status = body.get("status").and_then(Value::as_str); + let file_descriptors: Option> = body + .get("file_descriptors") + .and_then(|v| serde_json::from_value(v.clone()).ok()); + + match status { + Some("ok" | "collision") => { + let stored_key = body .get("segment") .and_then(Value::as_str) - .map(str::to_owned), - }, - Some("duplicate") => UploadResult { - success: true, - duplicate: true, - error_type: None, - status_code: Some(StatusCode::OK.as_u16()), - stored_key: body + .map(str::to_owned); + UploadResult { + success: true, + error_type: None, + status_code: Some(StatusCode::OK.as_u16()), + stored_key, + file_descriptors, + reason_code: None, + is_local_failure: false, + } + } + Some("duplicate") => { + let stored_key = body .get("existing_segment") .and_then(Value::as_str) - .map(str::to_owned), - }, - Some("failed") => { - UploadResult::failure(Some(ErrorType::Client), Some(StatusCode::OK.as_u16())) + .map(str::to_owned); + UploadResult { + success: true, + error_type: None, + status_code: Some(StatusCode::OK.as_u16()), + stored_key, + file_descriptors, + reason_code: None, + is_local_failure: false, + } } - _ => UploadResult::failure(Some(ErrorType::Incompatible), Some(StatusCode::OK.as_u16())), + Some("failed") => UploadResult::failure( + Some(ErrorType::Client), + Some(StatusCode::OK.as_u16()), + body.get("reason_code") + .and_then(Value::as_str) + .map(str::to_owned), + ), + _ => UploadResult::failure( + Some(ErrorType::Incompatible), + Some(StatusCode::OK.as_u16()), + None, + ), } } @@ -644,13 +651,6 @@ fn custody_failure(error_type: ErrorType, status_code: Option) -> DayCustod } } -fn probe_failure(error_type: ErrorType, status_code: Option) -> ManifestProbe { - ManifestProbe { - error_type: Some(error_type), - status_code, - } -} - fn deserialize_lenient_option<'de, D, T>(deserializer: D) -> Result, D::Error> where D: Deserializer<'de>, @@ -978,8 +978,8 @@ mod tests { assert!(!upload.is_finished()); peer.release_request_credit(); assert!(upload.await.unwrap().success); - peer.wait_for_requests(4).await; - assert_eq!(peer.requests().len(), 4); + peer.wait_for_requests(2).await; + assert_eq!(peer.requests().len(), 2); assert!(legacy.requests().is_empty()); drop(client); session.shutdown().await.unwrap(); @@ -1156,20 +1156,14 @@ mod tests { // tests/test_upload.py::test_upload_segment_returns_stored_key #[tokio::test] async fn upload_segment_returns_stored_key() { - for (body, duplicate, key) in [ - ( - json!({"status":"ok", "segment":"120000_005"}), - false, - "120000_005", - ), + for (body, key) in [ + (json!({"status":"ok", "segment":"120000_005"}), "120000_005"), ( json!({"status":"collision", "segment":"120000_006"}), - false, "120000_006", ), ( json!({"status":"duplicate", "existing_segment":"115959_300"}), - true, "115959_300", ), ] { @@ -1181,7 +1175,6 @@ mod tests { .upload_segment("day", "segment", &[media]) .await; assert!(result.success); - assert_eq!(result.duplicate, duplicate); assert_eq!(result.stored_key.as_deref(), Some(key)); } } @@ -1384,7 +1377,7 @@ mod tests { } #[tokio::test] - async fn v3_custody_requires_the_complete_triad() { + async fn v3_custody_requires_segments_response() { let server = MockServer::new(vec![]).await; let temp = TempDir::new().unwrap(); server.enqueue_day_custody(DayCustodyFixture::new( @@ -1399,12 +1392,11 @@ mod tests { assert!(result.day_present && result.proof_available); assert_eq!(result.items[0].key.as_deref(), Some("new")); let requests = server.requests(); - assert_eq!(requests.len(), 3); - for request in requests { - assert_eq!(request.headers[OBSERVER_PROTOCOL_VERSION_HEADER], "3"); - assert!(!request.headers.contains_key("authorization")); - assert!(!request.headers.contains_key("x-solstone-observer")); - } + assert_eq!(requests.len(), 1); + assert_eq!(requests[0].uri, "/app/devices/ingest/segments/20260101"); + assert_eq!(requests[0].headers[OBSERVER_PROTOCOL_VERSION_HEADER], "3"); + assert!(!requests[0].headers.contains_key("authorization")); + assert!(!requests[0].headers.contains_key("x-solstone-observer")); } #[tokio::test] @@ -1422,41 +1414,6 @@ mod tests { assert!(result.error_type.is_none()); } - #[tokio::test] - async fn v3_day_manifest_mismatch_and_version_are_unproven() { - for fixture in [ - DayCustodyFixture::new("20260101", Vec::new()).with_day_manifest_day("20260102"), - DayCustodyFixture::new("20260101", Vec::new()).with_version(2), - ] { - let result = fetch_custody_fixture(fixture).await; - assert!(result.day_present); - assert!(!result.proof_available); - assert!(result.error_type.is_none()); - } - } - - #[tokio::test] - async fn v3_absent_manifest_day_is_reachable_without_proof() { - let result = fetch_custody_fixture(DayCustodyFixture::absent("20260101")).await; - assert!(!result.day_present); - assert!(!result.proof_available); - assert!(result.error_type.is_none()); - assert_eq!(result.status_code, Some(200)); - } - - #[tokio::test] - async fn manifest_probe_is_one_reachability_request() { - let server = MockServer::new(vec![]).await; - let temp = TempDir::new().unwrap(); - server.enqueue_manifest_probe(200, b"not-a-manifest"); - let result = client(&config(&server, &temp), &server.url) - .probe_manifest() - .await; - assert!(result.error_type.is_none()); - assert_eq!(server.requests().len(), 1); - assert_eq!(server.requests()[0].uri, "/app/devices/ingest/manifest"); - } - #[tokio::test] async fn v3_wrong_segments_protocol_and_malformed_legs_are_incompatible() { let protocol = fetch_custody_fixture( @@ -1466,21 +1423,10 @@ mod tests { assert_eq!(protocol.error_type, Some(ErrorType::Incompatible)); let malformed = fetch_custody_fixture( - DayCustodyFixture::new("20260101", Vec::new()) - .with_malformed_leg(crate::test_support::DayCustodyLeg::Segments, b"not-json"), + DayCustodyFixture::new("20260101", Vec::new()).with_malformed(b"not-json"), ) .await; assert_eq!(malformed.error_type, Some(ErrorType::Incompatible)); - - let failed = fetch_custody_fixture( - DayCustodyFixture::new("20260101", Vec::new()).with_http_failure( - crate::test_support::DayCustodyLeg::DayManifest, - 500, - b"{}", - ), - ) - .await; - assert_eq!(failed.error_type, Some(ErrorType::Transient)); } #[tokio::test] @@ -1488,8 +1434,7 @@ mod tests { let server = MockServer::new(vec![]).await; let temp = TempDir::new().unwrap(); server.enqueue_day_custody( - DayCustodyFixture::new("20260101", Vec::new()) - .with_malformed_leg(crate::test_support::DayCustodyLeg::Segments, br#"[]"#), + DayCustodyFixture::new("20260101", Vec::new()).with_malformed(br#"[]"#), ); let result = client(&config(&server, &temp), &server.url) .fetch_day_custody("20260101") @@ -1501,11 +1446,7 @@ mod tests { .iter() .map(|request| request.uri.as_str()) .collect::>(), - vec![ - "/app/devices/ingest/manifest", - "/app/devices/ingest/manifest/20260101", - "/app/devices/ingest/segments/20260101", - ] + vec!["/app/devices/ingest/segments/20260101"] ); } diff --git a/vendor/observer-client-contract/consumer-audit.json b/vendor/observer-client-contract/consumer-audit.json index b07e88d..21694c6 100644 --- a/vendor/observer-client-contract/consumer-audit.json +++ b/vendor/observer-client-contract/consumer-audit.json @@ -9,7 +9,7 @@ "consumer": "solstone-linux" }, { - "commit": "19c972c4fea775176cea6421ac8b87f3bb20ab42", + "commit": "f276076e3554c75d75cbd3616644f46d831c0d90", "consumer": "solstone-windows" } ], @@ -93,50 +93,6 @@ "source_files": [ "crates/solstone-linux/src/upload.rs" ] - }, - { - "classification": "legacy_v2_unmigrated", - "consumer": "solstone-windows", - "legacy_surface": "observer_v2_register", - "rationale": "Pinned revision calls a legacy v2 capability and is not verified against the linked-device v3 ingest surface.", - "revision": "19c972c4fea775176cea6421ac8b87f3bb20ab42", - "source_files": [ - "crates/observer-pl/src/lib.rs", - "crates/observer-pl/src/wire.rs" - ] - }, - { - "classification": "legacy_v2_unmigrated", - "consumer": "solstone-windows", - "legacy_surface": "observer_ingest_v2_upload", - "rationale": "Pinned revision calls a legacy v2 capability and is not verified against the linked-device v3 ingest surface.", - "revision": "19c972c4fea775176cea6421ac8b87f3bb20ab42", - "source_files": [ - "crates/observer-pl/src/lib.rs", - "crates/observer-pl/src/wire.rs" - ] - }, - { - "classification": "legacy_v2_unmigrated", - "consumer": "solstone-windows", - "legacy_surface": "observer_ingest_v2_event", - "rationale": "Pinned revision calls a legacy v2 capability and is not verified against the linked-device v3 ingest surface.", - "revision": "19c972c4fea775176cea6421ac8b87f3bb20ab42", - "source_files": [ - "crates/observer-pl/src/lib.rs", - "crates/observer-pl/src/wire.rs" - ] - }, - { - "classification": "legacy_v2_unmigrated", - "consumer": "solstone-windows", - "legacy_surface": "observer_ingest_v2_segments", - "rationale": "Pinned revision calls a legacy v2 capability and is not verified against the linked-device v3 ingest surface.", - "revision": "19c972c4fea775176cea6421ac8b87f3bb20ab42", - "source_files": [ - "crates/observer-pl/src/lib.rs", - "crates/observer-pl/src/wire.rs" - ] } ], "schema": "solstone.client-ingest-contract-consumer-audit.v2", @@ -155,14 +111,14 @@ }, { "consumer": "solstone-windows", - "path": "crates/observer-pl/src/lib.rs", - "revision": "19c972c4fea775176cea6421ac8b87f3bb20ab42", + "path": "crates/pl-transport-win/src/client.rs", + "revision": "f276076e3554c75d75cbd3616644f46d831c0d90", "role": "production" }, { "consumer": "solstone-windows", - "path": "crates/observer-pl/src/wire.rs", - "revision": "19c972c4fea775176cea6421ac8b87f3bb20ab42", + "path": "crates/pl-transport-win/src/coordinator.rs", + "revision": "f276076e3554c75d75cbd3616644f46d831c0d90", "role": "production" } ], diff --git a/vendor/observer-client-contract/manifest.json b/vendor/observer-client-contract/manifest.json index 1c57202..30327e4 100644 --- a/vendor/observer-client-contract/manifest.json +++ b/vendor/observer-client-contract/manifest.json @@ -2,7 +2,7 @@ "audited_consumer_revisions": [ { "consumer_identifier": "solstone-windows", - "revision": "19c972c4fea775176cea6421ac8b87f3bb20ab42" + "revision": "f276076e3554c75d75cbd3616644f46d831c0d90" }, { "consumer_identifier": "solstone-linux", @@ -14,10 +14,11 @@ } ], "bundle_schema_identity": "solstone.client-ingest-contract-bundle.schema.v1", - "bundle_semver": "10.0.0", + "bundle_semver": "12.2.0", "client_protocol_version": 3, "component_closure": [ "Error", + "FileDescriptor", "SegmentFile", "SegmentItem", "SegmentsEnvelope" @@ -30,7 +31,7 @@ "files": [ { "path": "consumer-audit.json", - "sha256": "dde0a1b0abaff079a39eca2748ce05201003436763bc9d531fe893976b6092c5" + "sha256": "9d8944fc5f873e858e3dab9f9251ded9e840cec0ef2284f510f70a560026d57b" }, { "path": "fixtures/wire-behavior.json", @@ -38,7 +39,7 @@ }, { "path": "projection.openapi.json", - "sha256": "cc3d33694f3093f706e6844e832912d52c52e0abdd32f72c631429fe6450adb5" + "sha256": "2fc55fce7f9f7cb5b5dd2da13299acd371231caad9e4d877c6846ca6269c7198" }, { "path": "vectors.json", @@ -51,7 +52,7 @@ "id": "openapi.client_ingest_authority", "path": "core/crates/solstone-core-repository-contracts/src/contracts/client_ingest_authority.json", "role": "openapi_source", - "sha256": "c27f58430f00cfbcf49f693a86033b5b5a3ec6a30c6d43778170ed6d24e2e2c5" + "sha256": "cc75cc6caa1db1d3b3c36db903f25b188158e0c162745fedb01f1ed3c09c0a1e" } ], "openapi_document_version": "1.0.0", @@ -103,12 +104,6 @@ "solstone-linux-legacy-v2-unmigrated" ], "consumer_identifier": "solstone-linux" - }, - { - "adoption_blocker_ids": [ - "solstone-windows-legacy-v2-unmigrated" - ], - "consumer_identifier": "solstone-windows" } ] } diff --git a/vendor/observer-client-contract/projection.openapi.json b/vendor/observer-client-contract/projection.openapi.json index 62e2889..deee63a 100644 --- a/vendor/observer-client-contract/projection.openapi.json +++ b/vendor/observer-client-contract/projection.openapi.json @@ -126,7 +126,6 @@ "speaker_review_unavailable", "speaker_sentence_missing", "speaker_voiceprint_busy", - "support_portal_failed", "talent_not_found", "talent_operation_failed", "talent_run_malformed", @@ -145,6 +144,41 @@ ], "type": "object" }, + "FileDescriptor": { + "additionalProperties": true, + "properties": { + "disposition": { + "enum": [ + "written", + "already_held", + "received_not_written" + ], + "type": "string" + }, + "sha256": { + "pattern": "^[0-9a-f]{64}$", + "type": "string" + }, + "size": { + "minimum": 0, + "type": "integer" + }, + "submitted": { + "type": "string" + }, + "written": { + "type": "string" + } + }, + "required": [ + "submitted", + "written", + "size", + "sha256", + "disposition" + ], + "type": "object" + }, "SegmentFile": { "additionalProperties": true, "properties": { @@ -236,7 +270,7 @@ "info": { "description": "Generated from client_ingest_authority.json. Covers only the four Rust-served linked-device devices/ingest operations.", "title": "Linked-device v3 ingest client contract", - "version": "10.0.0", + "version": "12.2.0", "x-generated": true, "x-generated-by": "solstone-core-repository-contracts" }, @@ -295,7 +329,17 @@ "summary": "Duplicate segment", "value": { "existing_segment": "143000_1", + "file_descriptors": [ + { + "disposition": "already_held", + "sha256": "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", + "size": 524288, + "submitted": "audio.m4a", + "written": "audio.m4a" + } + ], "message": "All files already received", + "meta": {}, "status": "duplicate" } }, @@ -303,9 +347,19 @@ "summary": "New segment accepted", "value": { "bytes": 524288, + "file_descriptors": [ + { + "disposition": "written", + "sha256": "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", + "size": 524288, + "submitted": "audio.m4a", + "written": "audio.m4a" + } + ], "files": [ "audio.m4a" ], + "meta": {}, "segment": "143000_1", "status": "ok" } @@ -321,6 +375,10 @@ "type": "string" }, "file_descriptors": { + "description": "One custody receipt per submitted file, returned only after the journal durably recorded the upload. A device proves custody of a file when its `sha256` and `size` equal the local file's and its `disposition` is `written` or `already_held`; `received_not_written` is never custody.", + "items": { + "$ref": "#/components/schemas/FileDescriptor" + }, "type": "array" }, "files": { @@ -402,9 +460,12 @@ } } }, - "description": "Held sidecar bytes conflict.", + "description": "Held sidecar bytes conflict, or the device's stream binding is refused.", "x-reason-codes": [ - "content_conflict" + "content_conflict", + "pairing_identity_unavailable", + "foreign_stream_binding", + "malformed_evidence_row" ] }, "413": { @@ -439,7 +500,10 @@ } } }, - "description": "Journal write or notification failed." + "description": "Journal write or notification failed, or the owner removed the requested segment (`reason_code: segment_removed`, body `status: failed`). A device that knows `segment_removed` stops re-sending that segment.", + "x-reason-codes": [ + "segment_removed" + ] } }, "summary": "Upload one capture segment from a linked device", @@ -450,7 +514,8 @@ }, "/app/devices/ingest/manifest": { "get": { - "description": "Return days with uploaded linked-device segment history.", + "deprecated": true, + "description": "Deprecated. Return days with uploaded linked-device segment history. A device proves custody from the upload response's `file_descriptors` and reads `segments/{day}` only when its own record of an upload is missing; this operation keeps answering until devices in the field no longer call it.", "operationId": "client.ingestManifest", "parameters": [ { @@ -559,7 +624,8 @@ }, "/app/devices/ingest/manifest/{day}": { "get": { - "description": "Return file names, sizes, sha256 hashes, and custody status for every segment on one day.", + "deprecated": true, + "description": "Deprecated. Return file names, sizes, sha256 hashes, and custody status for every segment on one day, the same listing `segments/{day}` returns. This operation keeps answering until devices in the field no longer call it.", "operationId": "client.ingestManifestDay", "parameters": [ { @@ -813,6 +879,17 @@ } }, "x-vocabularies": { + "FileDescriptor.disposition": { + "classification": "closed", + "id": "FileDescriptor.disposition", + "source_pointer": "/components/schemas/FileDescriptor/properties/disposition", + "unknown_value_behavior": "reject", + "values": [ + "written", + "already_held", + "received_not_written" + ] + }, "SegmentFile.status": { "classification": "closed", "id": "SegmentFile.status",