From 4ff61e1068268236e5bbd34445fa139cb980bc47 Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Fri, 24 Jul 2026 06:01:03 -0600 Subject: [PATCH] feat(native-sol): port sol import to the native client Add the sol-import/top-level-import authority beside chat and route sol import through the native sol client. Preserve the two-request import flow: save or save-path first, then start with the staged path, timestamp, and force flag. Add a ClientItemIdProvider seam so parity vectors can pin deterministic client_item_id values while production emits uuid4 hex. Keep the frozen partial-failure form, staged {path} but processing was not queued, with a nonzero exit and no queued-success output. The existing sol-call/sol-chat partition counts remain 152/2/1/1. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/bin/resolve_parity_leaves.rs | 8 +- .../solstone-core-sol-client-cli/src/lib.rs | 54 +- .../tests/parity.rs | 29 +- .../solstone-core-sol-client/src/command.rs | 6 +- .../src/generated/inventory.rs | 16 + .../src/json_format.rs | 13 + .../solstone-core-sol-client/src/seam.rs | 35 + core/crates/solstone-core-sol/src/main.rs | 53 +- core/fixtures/native-sol/applicability.json | 17 + core/fixtures/native-sol/parity/import.jsonl | 15 + scripts/build_native_sol_inventory.py | 29 +- scripts/check_native_sol_architecture.py | 3 +- scripts/check_native_sol_conformance.py | 25 +- scripts/check_native_sol_coverage.py | 96 +- solstone/apps/activities/native/command.rs | 1 + solstone/apps/support/native/command.rs | 1 + solstone/think/native/chat/command.rs | 1 + solstone/think/native/import/authority.toml | 35 + solstone/think/native/import/command.rs | 1055 +++++++++++++++++ solstone/think/tools/native/health/command.rs | 2 + 20 files changed, 1470 insertions(+), 24 deletions(-) create mode 100644 solstone/think/native/import/authority.toml create mode 100644 solstone/think/native/import/command.rs diff --git a/core/crates/solstone-core-sol-client-cli/src/bin/resolve_parity_leaves.rs b/core/crates/solstone-core-sol-client-cli/src/bin/resolve_parity_leaves.rs index 3ba3c8ad1..09306950d 100644 --- a/core/crates/solstone-core-sol-client-cli/src/bin/resolve_parity_leaves.rs +++ b/core/crates/solstone-core-sol-client-cli/src/bin/resolve_parity_leaves.rs @@ -41,10 +41,10 @@ fn main() -> Result<(), String> { .ok_or_else(|| format!("{}:{}: non-string argv", path.display(), index + 1)) }) .collect::, _>>()?; - let lookup_args = if surface == "sol-chat" { - vec!["chat".to_string()] - } else { - argv + let lookup_args = match surface { + "sol-chat" => vec!["chat".to_string()], + "sol-import" => vec!["import".to_string()], + _ => argv, }; let entry = resolve_surface_leaf(surface, &lookup_args); println!( diff --git a/core/crates/solstone-core-sol-client-cli/src/lib.rs b/core/crates/solstone-core-sol-client-cli/src/lib.rs index ce2c931b9..4c2583a5a 100644 --- a/core/crates/solstone-core-sol-client-cli/src/lib.rs +++ b/core/crates/solstone-core-sol-client-cli/src/lib.rs @@ -7,13 +7,15 @@ use std::ffi::{OsStr, OsString}; use solstone_core_sol_client::aggregate; use solstone_core_sol_client::command::{CommandContext, CommandOutput}; use solstone_core_sol_client::seam::{ - BuildIdentityProvider, ChatEventSource, Clock, FileProvider, HttpTransport, + BuildIdentityProvider, ChatEventSource, ClientItemIdProvider, Clock, FileProvider, + HttpTransport, }; #[derive(Debug, Clone, PartialEq, Eq)] pub enum Outcome { Migrated { path: Vec }, Chat { args: Vec }, + Import { args: Vec }, MovedStub { name: OsString }, Unsupported { args: Vec }, } @@ -24,6 +26,7 @@ pub struct DispatchSeams<'a> { pub chat_events: Option<&'a dyn ChatEventSource>, pub files: Option<&'a dyn FileProvider>, pub build_identity: Option<&'a dyn BuildIdentityProvider>, + pub client_item_ids: Option<&'a dyn ClientItemIdProvider>, } #[must_use] @@ -40,6 +43,16 @@ pub fn evaluate_args(args: &[OsString]) -> Outcome { }, ) } + [command, rest @ ..] if command == OsStr::new("import") => { + match_generated_surface_path("sol-import", &[String::from("import")]).map_or_else( + || Outcome::Unsupported { + args: args.to_vec(), + }, + |_entry| Outcome::Import { + args: rest.to_vec(), + }, + ) + } _ => Outcome::Unsupported { args: args.to_vec(), }, @@ -68,6 +81,33 @@ pub fn dispatch_sol_chat_with_seams( chat_events: seams.chat_events, files: seams.files, build_identity: seams.build_identity, + client_item_ids: seams.client_item_ids, + }) +} + +#[must_use] +pub fn dispatch_sol_import_with_seams( + args: &[String], + env: &BTreeMap, + stdin: &str, + today: &str, + seams: DispatchSeams<'_>, +) -> CommandOutput { + let Some((_, handler)) = match_generated_surface_path("sol-import", &[String::from("import")]) + else { + return CommandOutput::failure("Unsupported native sol command.\n", 64); + }; + handler(CommandContext { + args, + env, + stdin, + today, + transport: seams.transport, + clock: seams.clock, + chat_events: None, + files: seams.files, + build_identity: seams.build_identity, + client_item_ids: seams.client_item_ids, }) } @@ -109,6 +149,7 @@ pub fn dispatch_sol_call( chat_events: None, files: None, build_identity: None, + client_item_ids: None, }, ) } @@ -135,6 +176,7 @@ pub fn dispatch_sol_call_with_seams( chat_events: None, files: seams.files, build_identity: seams.build_identity, + client_item_ids: seams.client_item_ids, }) } @@ -238,6 +280,16 @@ mod tests { ); } + #[test] + fn routes_top_level_import_to_import_shell() { + assert_eq!( + evaluate_args(&args(&["import", "sample.txt"])), + Outcome::Import { + args: args(&["sample.txt"]) + } + ); + } + #[test] fn classifies_unported_call_as_unsupported_without_spawn_path() { assert_eq!( diff --git a/core/crates/solstone-core-sol-client-cli/tests/parity.rs b/core/crates/solstone-core-sol-client-cli/tests/parity.rs index a2dd00e02..4a3612289 100644 --- a/core/crates/solstone-core-sol-client-cli/tests/parity.rs +++ b/core/crates/solstone-core-sol-client-cli/tests/parity.rs @@ -7,8 +7,8 @@ use std::path::PathBuf; use serde_json::{Value, json}; use solstone_core_sol_client::error::ClientError; use solstone_core_sol_client::seam::{ - ChatInput, ExpectedHttpCall, FakeBuildIdentityProvider, FakeClock, FixtureFileProvider, - RecordedHttpCall, ScriptedChatEventSource, ScriptedHttpTransport, + ChatInput, ExpectedHttpCall, FakeBuildIdentityProvider, FakeClientItemIdProvider, FakeClock, + FixtureFileProvider, RecordedHttpCall, ScriptedChatEventSource, ScriptedHttpTransport, }; use solstone_core_sol_client::sse::iter_sse_events; use solstone_core_sol_client::transport::{ @@ -17,6 +17,7 @@ use solstone_core_sol_client::transport::{ }; use solstone_core_sol_client_cli::{ DispatchSeams, dispatch_sol_call_with_seams, dispatch_sol_chat_with_seams, + dispatch_sol_import_with_seams, }; const ACTIVITIES_VECTORS: &str = @@ -98,6 +99,12 @@ fn run_vector(vector: &Value) { "python": "3.test" } }))); + let client_item_ids = FakeClientItemIdProvider::new( + vector + .get("client_item_id") + .and_then(Value::as_str) + .unwrap_or("11111111111141118111111111111111"), + ); let output = if vector["surface"].as_str() == Some("sol-chat") { dispatch_sol_chat_with_seams( @@ -111,6 +118,23 @@ fn run_vector(vector: &Value) { chat_events: Some(&chat_events), files: Some(&files), build_identity: Some(&build_identity), + client_item_ids: Some(&client_item_ids), + }, + ) + } else if vector["surface"].as_str() == Some("sol-import") { + let import_args = argv.iter().skip(1).cloned().collect::>(); + dispatch_sol_import_with_seams( + &import_args, + &env, + stdin, + today, + DispatchSeams { + transport: &transport, + clock: Some(&clock), + chat_events: None, + files: Some(&files), + build_identity: Some(&build_identity), + client_item_ids: Some(&client_item_ids), }, ) } else { @@ -125,6 +149,7 @@ fn run_vector(vector: &Value) { chat_events: None, files: Some(&files), build_identity: Some(&build_identity), + client_item_ids: Some(&client_item_ids), }, ) }; diff --git a/core/crates/solstone-core-sol-client/src/command.rs b/core/crates/solstone-core-sol-client/src/command.rs index a2516d331..b78d82e96 100644 --- a/core/crates/solstone-core-sol-client/src/command.rs +++ b/core/crates/solstone-core-sol-client/src/command.rs @@ -3,7 +3,10 @@ use std::collections::BTreeMap; -use crate::seam::{BuildIdentityProvider, ChatEventSource, Clock, FileProvider, HttpTransport}; +use crate::seam::{ + BuildIdentityProvider, ChatEventSource, ClientItemIdProvider, Clock, FileProvider, + HttpTransport, +}; #[derive(Clone, Copy)] pub struct CommandContext<'a> { @@ -16,6 +19,7 @@ pub struct CommandContext<'a> { pub chat_events: Option<&'a dyn ChatEventSource>, pub files: Option<&'a dyn FileProvider>, pub build_identity: Option<&'a dyn BuildIdentityProvider>, + pub client_item_ids: Option<&'a dyn ClientItemIdProvider>, } #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/core/crates/solstone-core-sol-client/src/generated/inventory.rs b/core/crates/solstone-core-sol-client/src/generated/inventory.rs index 9747f60c2..e6a927ec0 100644 --- a/core/crates/solstone-core-sol-client/src/generated/inventory.rs +++ b/core/crates/solstone-core-sol-client/src/generated/inventory.rs @@ -33,6 +33,8 @@ mod solstone_apps_thinking_native_command_rs; mod solstone_apps_transcripts_native_command_rs; #[path = "../../../../../solstone/think/native/chat/command.rs"] mod solstone_think_native_chat_command_rs; +#[path = "../../../../../solstone/think/native/import/command.rs"] +mod solstone_think_native_import_command_rs; #[path = "../../../../../solstone/think/native/moved/command.rs"] mod solstone_think_native_moved_command_rs; #[path = "../../../../../solstone/think/tools/native/health/command.rs"] @@ -1889,6 +1891,19 @@ pub const ENTRIES: &[InventoryEntry] = &[ contract_operation_id: None, handler: "chat", }, + InventoryEntry { + surface: "sol-import", + path: &["import"], + kind: "top-level", + help: "Import media through the journal", + params_json: "[{\"count\":false,\"default\":null,\"flag_value\":null,\"hidden\":false,\"is_flag\":false,\"kind\":\"argument\",\"multiple\":false,\"name\":\"media\",\"nargs\":1,\"options\":[\"media\"],\"required\":false,\"secondary\":[],\"type\":\"text\"},{\"count\":false,\"default\":null,\"flag_value\":null,\"hidden\":true,\"is_flag\":false,\"kind\":\"argument\",\"multiple\":false,\"name\":\"extra\",\"nargs\":-1,\"options\":[\"extra\"],\"required\":false,\"secondary\":[],\"type\":\"text\"},{\"count\":false,\"default\":null,\"flag_value\":null,\"hidden\":false,\"is_flag\":false,\"kind\":\"option\",\"multiple\":false,\"name\":\"timestamp\",\"nargs\":1,\"options\":[\"--timestamp\"],\"required\":false,\"secondary\":[],\"type\":\"text\"},{\"count\":false,\"default\":null,\"flag_value\":null,\"hidden\":false,\"is_flag\":false,\"kind\":\"option\",\"multiple\":false,\"name\":\"facet\",\"nargs\":1,\"options\":[\"--facet\"],\"required\":false,\"secondary\":[],\"type\":\"text\"},{\"count\":false,\"default\":null,\"flag_value\":null,\"hidden\":false,\"is_flag\":false,\"kind\":\"option\",\"multiple\":false,\"name\":\"setting\",\"nargs\":1,\"options\":[\"--setting\"],\"required\":false,\"secondary\":[],\"type\":\"text\"},{\"count\":false,\"default\":null,\"flag_value\":null,\"hidden\":false,\"is_flag\":false,\"kind\":\"option\",\"multiple\":false,\"name\":\"source\",\"nargs\":1,\"options\":[\"--source\"],\"required\":false,\"secondary\":[],\"type\":\"text\"},{\"count\":false,\"default\":false,\"flag_value\":true,\"hidden\":false,\"is_flag\":true,\"kind\":\"option\",\"multiple\":false,\"name\":\"force\",\"nargs\":1,\"options\":[\"--force\"],\"required\":false,\"secondary\":[],\"type\":\"boolean\"},{\"count\":false,\"default\":null,\"flag_value\":null,\"hidden\":false,\"is_flag\":false,\"kind\":\"option\",\"multiple\":false,\"name\":\"auto\",\"nargs\":1,\"options\":[\"--auto\"],\"required\":false,\"secondary\":[],\"type\":\"text\"},{\"count\":false,\"default\":false,\"flag_value\":true,\"hidden\":false,\"is_flag\":true,\"kind\":\"option\",\"multiple\":false,\"name\":\"deterministic_only\",\"nargs\":1,\"options\":[\"--deterministic-only\"],\"required\":false,\"secondary\":[],\"type\":\"boolean\"},{\"count\":false,\"default\":false,\"flag_value\":true,\"hidden\":false,\"is_flag\":true,\"kind\":\"option\",\"multiple\":false,\"name\":\"dry_run\",\"nargs\":1,\"options\":[\"--dry-run\"],\"required\":false,\"secondary\":[],\"type\":\"boolean\"},{\"count\":false,\"default\":false,\"flag_value\":true,\"hidden\":false,\"is_flag\":true,\"kind\":\"option\",\"multiple\":false,\"name\":\"json\",\"nargs\":1,\"options\":[\"--json\"],\"required\":false,\"secondary\":[],\"type\":\"boolean\"},{\"count\":false,\"default\":false,\"flag_value\":true,\"hidden\":false,\"is_flag\":true,\"kind\":\"option\",\"multiple\":false,\"name\":\"verbose\",\"nargs\":1,\"options\":[\"-v\",\"--verbose\"],\"required\":false,\"secondary\":[],\"type\":\"boolean\"},{\"count\":false,\"default\":false,\"flag_value\":true,\"hidden\":false,\"is_flag\":true,\"kind\":\"option\",\"multiple\":false,\"name\":\"backends\",\"nargs\":1,\"options\":[\"--backends\"],\"required\":false,\"secondary\":[],\"type\":\"boolean\"},{\"count\":false,\"default\":null,\"flag_value\":null,\"hidden\":false,\"is_flag\":false,\"kind\":\"option\",\"multiple\":false,\"name\":\"sync\",\"nargs\":1,\"options\":[\"--sync\"],\"required\":false,\"secondary\":[],\"type\":\"text\"},{\"count\":false,\"default\":false,\"flag_value\":true,\"hidden\":false,\"is_flag\":true,\"kind\":\"option\",\"multiple\":false,\"name\":\"save\",\"nargs\":1,\"options\":[\"--save\"],\"required\":false,\"secondary\":[],\"type\":\"boolean\"},{\"count\":false,\"default\":null,\"flag_value\":null,\"hidden\":false,\"is_flag\":false,\"kind\":\"option\",\"multiple\":false,\"name\":\"path\",\"nargs\":1,\"options\":[\"--path\"],\"required\":false,\"secondary\":[],\"type\":\"text\"},{\"count\":false,\"default\":false,\"flag_value\":true,\"hidden\":false,\"is_flag\":true,\"kind\":\"option\",\"multiple\":false,\"name\":\"list_importers\",\"nargs\":1,\"options\":[\"--list-importers\"],\"required\":false,\"secondary\":[],\"type\":\"boolean\"},{\"count\":false,\"default\":null,\"flag_value\":null,\"hidden\":false,\"is_flag\":false,\"kind\":\"argument\",\"multiple\":false,\"name\":\"journal_source\",\"nargs\":1,\"options\":[\"journal-source\"],\"required\":false,\"secondary\":[],\"type\":\"text\"}]", + entry_type: "top-level-import", + operation_id: "import.top_level", + method: None, + route: None, + contract_operation_id: None, + handler: "import_top_level", + }, InventoryEntry { surface: "sol-call", path: &["identity"], @@ -2216,6 +2231,7 @@ pub const HANDLERS: &[Handler] = &[ solstone_apps_transcripts_native_command_rs::speakers, solstone_apps_transcripts_native_command_rs::stats, solstone_think_native_chat_command_rs::chat, + solstone_think_native_import_command_rs::import_top_level, solstone_think_native_moved_command_rs::identity, solstone_think_native_moved_command_rs::navigate, solstone_think_tools_native_health_command_rs::summary, diff --git a/core/crates/solstone-core-sol-client/src/json_format.rs b/core/crates/solstone-core-sol-client/src/json_format.rs index 4469cad92..9290aa0e9 100644 --- a/core/crates/solstone-core-sol-client/src/json_format.rs +++ b/core/crates/solstone-core-sol-client/src/json_format.rs @@ -62,6 +62,11 @@ pub fn json_compact_ascii(value: &Value) -> String { ensure_ascii(&json_compact(value)) } +#[must_use] +pub fn sorted_json_compact_ascii(value: &Value) -> String { + ensure_ascii(&json_compact(&sort_json(value))) +} + #[must_use] pub fn json_compact_utf8(value: &Value) -> String { json_compact(value) @@ -171,6 +176,14 @@ mod tests { ); } + #[test] + fn compact_prints_objects_with_sorted_keys() { + assert_eq!( + sorted_json_compact_ascii(&json!({"b": 2, "a": {"d": 4, "c": 3}})), + "{\"a\": {\"c\": 3, \"d\": 4}, \"b\": 2}" + ); + } + #[test] fn compact_prints_objects_with_utf8_when_requested() { assert_eq!( diff --git a/core/crates/solstone-core-sol-client/src/seam.rs b/core/crates/solstone-core-sol-client/src/seam.rs index 494a2a5cf..770557e56 100644 --- a/core/crates/solstone-core-sol-client/src/seam.rs +++ b/core/crates/solstone-core-sol-client/src/seam.rs @@ -53,10 +53,15 @@ pub trait BuildIdentityProvider { fn build_identity(&self, journal: &Path) -> Option; } +pub trait ClientItemIdProvider { + fn client_item_id(&self) -> String; +} + pub trait FileProvider { fn read(&self, path: &Path) -> IoResult>; fn read_to_string(&self, path: &Path) -> std::io::Result; fn exists(&self, path: &Path) -> bool; + fn is_file(&self, path: &Path) -> bool; fn canonicalize(&self, path: &Path) -> std::io::Result; } @@ -302,6 +307,26 @@ impl BuildIdentityProvider for FakeBuildIdentityProvider { } } +#[derive(Debug, Clone)] +pub struct FakeClientItemIdProvider { + value: String, +} + +impl FakeClientItemIdProvider { + #[must_use] + pub fn new(value: impl Into) -> Self { + Self { + value: value.into(), + } + } +} + +impl ClientItemIdProvider for FakeClientItemIdProvider { + fn client_item_id(&self) -> String { + self.value.clone() + } +} + #[derive(Debug, Clone, Default)] pub struct FixtureFileProvider { files: HashMap>, @@ -349,6 +374,10 @@ impl FileProvider for FixtureFileProvider { self.files.contains_key(path) } + fn is_file(&self, path: &Path) -> bool { + self.exists(path) + } + fn canonicalize(&self, path: &Path) -> IoResult { if self.exists(path) { Ok(path.to_path_buf()) @@ -416,6 +445,12 @@ mod tests { ); } + #[test] + fn deterministic_client_item_id_fake_returns_configured_value() { + let provider = FakeClientItemIdProvider::new("fixed-client-id"); + assert_eq!(provider.client_item_id(), "fixed-client-id"); + } + #[test] fn scripted_chat_source_opens_sse_and_advances_on_poll() { let request = SseRequest { diff --git a/core/crates/solstone-core-sol/src/main.rs b/core/crates/solstone-core-sol/src/main.rs index d15d15fef..d6c6cf1ae 100644 --- a/core/crates/solstone-core-sol/src/main.rs +++ b/core/crates/solstone-core-sol/src/main.rs @@ -8,7 +8,7 @@ use std::path::{Path, PathBuf}; use std::process::{Command, ExitCode}; use std::sync::{Mutex, mpsc}; use std::thread; -use std::time::{Duration, Instant, SystemTime}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use std::{env, fs}; use chrono::Local; @@ -19,19 +19,19 @@ use solstone_core_journal::{ use solstone_core_sol_client::command::CommandOutput; use solstone_core_sol_client::port::read_convey_port; use solstone_core_sol_client::seam::{ - BuildIdentityProvider, ChatEventSource, ChatInput, Clock, FileProvider, HttpTransport, - ProcessOutput, ProcessSpawner, + BuildIdentityProvider, ChatEventSource, ChatInput, ClientItemIdProvider, Clock, FileProvider, + HttpTransport, ProcessOutput, ProcessSpawner, }; use solstone_core_sol_client::sse::SseDecoder; use solstone_core_sol_client::transport::UreqHttpTransport; use solstone_core_sol_client_cli::{ DispatchSeams, Outcome, dispatch_sol_call_with_seams, dispatch_sol_chat_with_seams, - evaluate_args, + dispatch_sol_import_with_seams, evaluate_args, }; const EXIT_USAGE: u8 = 64; const EXIT_TEMPFAIL: u8 = 75; -const USAGE: &str = "Usage:\n solstone-core-sol --version\n solstone-core-sol help\n solstone-core-sol status\n solstone-core-sol path\n solstone-core-sol call [args...]\n solstone-core-sol chat [args...]\n"; +const USAGE: &str = "Usage:\n solstone-core-sol --version\n solstone-core-sol help\n solstone-core-sol status\n solstone-core-sol path\n solstone-core-sol call [args...]\n solstone-core-sol chat [args...]\n solstone-core-sol import [args...]\n"; fn main() -> ExitCode { let args = env::args_os().skip(1).collect::>(); @@ -52,6 +52,7 @@ fn main() -> ExitCode { [command] if command == OsStr::new("status") => run_status(), [command, rest @ ..] if command == OsStr::new("call") => run_dispatched(&args, rest), [command, rest @ ..] if command == OsStr::new("chat") => run_dispatched(&args, rest), + [command, rest @ ..] if command == OsStr::new("import") => run_dispatched(&args, rest), [flag, ..] if flag.to_string_lossy().starts_with('-') => { eprint!("{USAGE}"); ExitCode::from(EXIT_USAGE) @@ -114,6 +115,7 @@ fn run_dispatched(all_args: &[OsString], command_args: &[OsString]) -> ExitCode let clock = SystemClock::default(); let files = RealFileProvider; let build_identity = RealBuildIdentityProvider; + let client_item_ids = RealClientItemIdProvider; let chat_events = ChannelChatEventSource::default(); let output = match outcome { @@ -128,6 +130,7 @@ fn run_dispatched(all_args: &[OsString], command_args: &[OsString]) -> ExitCode chat_events: None, files: Some(&files), build_identity: Some(&build_identity), + client_item_ids: Some(&client_item_ids), }, ), Outcome::Chat { .. } => dispatch_sol_chat_with_seams( @@ -141,6 +144,21 @@ fn run_dispatched(all_args: &[OsString], command_args: &[OsString]) -> ExitCode chat_events: Some(&chat_events), files: Some(&files), build_identity: Some(&build_identity), + client_item_ids: Some(&client_item_ids), + }, + ), + Outcome::Import { .. } => dispatch_sol_import_with_seams( + &args, + &env, + &stdin, + &today, + DispatchSeams { + transport: &transport, + clock: None, + chat_events: None, + files: Some(&files), + build_identity: Some(&build_identity), + client_item_ids: Some(&client_item_ids), }, ), Outcome::Unsupported { .. } => { @@ -327,11 +345,36 @@ impl FileProvider for RealFileProvider { path.exists() } + fn is_file(&self, path: &Path) -> bool { + path.is_file() + } + fn canonicalize(&self, path: &Path) -> IoResult { fs::canonicalize(path) } } +struct RealClientItemIdProvider; + +impl ClientItemIdProvider for RealClientItemIdProvider { + fn client_item_id(&self) -> String { + let mut bytes = [0_u8; 16]; + let read = fs::File::open("/dev/urandom") + .and_then(|mut file| file.read_exact(&mut bytes)) + .is_ok(); + if !read { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + bytes.copy_from_slice(&nanos.to_be_bytes()); + } + bytes[6] = (bytes[6] & 0x0f) | 0x40; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + bytes.iter().map(|byte| format!("{byte:02x}")).collect() + } +} + #[derive(Debug)] struct RealProcessSpawner; diff --git a/core/fixtures/native-sol/applicability.json b/core/fixtures/native-sol/applicability.json index 9d4417d93..5b495f241 100644 --- a/core/fixtures/native-sol/applicability.json +++ b/core/fixtures/native-sol/applicability.json @@ -2,6 +2,23 @@ "schema": "native-sol-applicability-v1", "note": "Applicability case IDs are requirements-first: each dispatch declares required adversarial behavior here before adding vectors. The coverage gate structurally enforces later multi-request boundaries, upload missing/unreadable/later-file/rejection cases, env explicit/absent or valid-absent cases, and dry-run preview/commit coverage.", "http_count": 152, + "top_level_entries": { + "import.top_level": { + "surface": "sol-import", + "path": ["import"], + "entry_type": "top-level-import", + "backing_contract_operation_ids": ["import.save", "import.savePath", "import.start"], + "case_ids": { + "file_success": ["import-top-level-file-success"], + "save_path_success": ["import-top-level-save-path-success"], + "host_rejections": ["import-top-level-reject-dry-run", "import-top-level-reject-backends", "import-top-level-reject-list-importers", "import-top-level-reject-sync", "import-top-level-reject-save", "import-top-level-reject-path", "import-top-level-reject-auto-guidance", "import-top-level-reject-journal-source"], + "json_output": ["import-top-level-json-output"], + "malformed": ["import-top-level-malformed-save"], + "unreachable": ["import-top-level-unreachable-save"], + "partial_failure": ["import-top-level-partial-start-failure"] + } + } + }, "entries": { "activities.list": { "path": ["activities", "list"], diff --git a/core/fixtures/native-sol/parity/import.jsonl b/core/fixtures/native-sol/parity/import.jsonl index 0d6e0bff7..02224b719 100644 --- a/core/fixtures/native-sol/parity/import.jsonl +++ b/core/fixtures/native-sol/parity/import.jsonl @@ -15,3 +15,18 @@ {"surface":"sol-call","env":{},"stdin":"","files":{},"clock":{"today":"20260723"},"normalizations":[],"id":"import-resolve-staged-facet-neither","argv":["import","resolve-staged-facet","work/facet/foo.staged.json","--source","phone"],"transport":{"requests":[]},"expected":{"stdout":"","stderr":"Error: Exactly one of --apply or --skip is required.\n","exit":1,"requests":[]}} {"surface":"sol-call","env":{},"stdin":"","files":{},"clock":{"today":"20260723"},"normalizations":[],"id":"import-resolve-staged-facet-both","argv":["import","resolve-staged-facet","work/facet/foo.staged.json","--apply","--skip","--source","phone"],"transport":{"requests":[]},"expected":{"stdout":"","stderr":"Error: Exactly one of --apply or --skip is required.\n","exit":1,"requests":[]}} {"surface":"sol-call","env":{},"stdin":"","files":{},"clock":{"today":"20260723"},"normalizations":[],"id":"import-resolve-staged-facet-not-found","argv":["import","resolve-staged-facet","missing.staged.json","--apply","--source","phone"],"transport":{"requests":[{"method":"POST","path":"/app/import/api/journal-sources/phone/resolve-facet","query":[],"headers":[],"timeout_policy":"api","json":{"staged_file":"missing.staged.json","mode":"apply"},"fault":{"error":"I couldn't find that import.","status":404,"reason_code":"import_not_found","detail":"staged file not found","payload":{"reason_code":"import_not_found","detail":"staged file not found"}}}]},"expected":{"stdout":"","stderr":"Error: staged file not found\n","exit":1,"requests":[{"method":"POST","path":"/app/import/api/journal-sources/phone/resolve-facet","query":[],"json":{"staged_file":"missing.staged.json","mode":"apply"},"headers":[],"timeout_policy":"api"}]}} +{"surface":"sol-import","env":{},"stdin":"","files":{"sample.txt":"xxxxx"},"clock":{"today":"20260723"},"client_item_id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","normalizations":[],"id":"import-top-level-file-success","argv":["import","{files}/sample.txt","--facet","work","--setting","office","--source","ics","--deterministic-only","--force"],"transport":{"requests":[{"method":"UPLOAD","path":"/app/import/api/save","headers":[],"timeout_policy":"upload","multipart":{"files":[{"field_name":"file","filename":"sample.txt","content_type":"application/octet-stream","length":5}],"data":[["client_item_id","aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"],["facet","work"],["setting","office"],["source_hint","ics"],["deterministic_only","true"]]},"response":{"json":{"path":"/journal/imports/20260101_120000/sample.txt","timestamp":"20260101_120000"}}},{"method":"POST","path":"/app/import/api/start","query":[],"headers":[],"timeout_policy":"api","json":{"path":"/journal/imports/20260101_120000/sample.txt","timestamp":"20260101_120000","force":true},"response":{"json":{"status":"ok","task_id":"task-file"}}}]},"expected":{"stdout":"staged /journal/imports/20260101_120000/sample.txt\ntimestamp 20260101_120000\nqueued processing task task-file\n","stderr":"","exit":0,"requests":[{"method":"UPLOAD","path":"/app/import/api/save","multipart":{"files":[{"field_name":"file","filename":"sample.txt","content_type":"application/octet-stream","length":5}],"data":[["client_item_id","aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"],["facet","work"],["setting","office"],["source_hint","ics"],["deterministic_only","true"]]},"headers":[],"timeout_policy":"upload"},{"method":"POST","path":"/app/import/api/start","query":[],"json":{"path":"/journal/imports/20260101_120000/sample.txt","timestamp":"20260101_120000","force":true},"headers":[],"timeout_policy":"api"}]}} +{"surface":"sol-import","env":{},"stdin":"","files":{},"clock":{"today":"20260723"},"client_item_id":"bbbbbbbbbbbb4bbb8bbbbbbbbbbbbbbb","normalizations":[],"id":"import-top-level-save-path-success","argv":["import","/journal-host/media/source-dir","--timestamp","20260202_030405"],"transport":{"requests":[{"method":"POST","path":"/app/import/api/save-path","query":[],"headers":[],"timeout_policy":"api","json":{"client_item_id":"bbbbbbbbbbbb4bbb8bbbbbbbbbbbbbbb","path":"/journal-host/media/source-dir"},"response":{"json":{"path":"/journal/imports/20260101_130000/source-dir","timestamp":"20260101_130000"}}},{"method":"POST","path":"/app/import/api/start","query":[],"headers":[],"timeout_policy":"api","json":{"path":"/journal/imports/20260101_130000/source-dir","timestamp":"20260202_030405","force":false},"response":{"json":{"status":"ok","task_id":"task-path"}}}]},"expected":{"stdout":"staged /journal/imports/20260101_130000/source-dir\ntimestamp 20260101_130000\nqueued processing task task-path\n","stderr":"","exit":0,"requests":[{"method":"POST","path":"/app/import/api/save-path","query":[],"json":{"client_item_id":"bbbbbbbbbbbb4bbb8bbbbbbbbbbbbbbb","path":"/journal-host/media/source-dir"},"headers":[],"timeout_policy":"api"},{"method":"POST","path":"/app/import/api/start","query":[],"json":{"path":"/journal/imports/20260101_130000/source-dir","timestamp":"20260202_030405","force":false},"headers":[],"timeout_policy":"api"}]}} +{"surface":"sol-import","env":{},"stdin":"","files":{},"clock":{"today":"20260723"},"client_item_id":"cccccccccccc4ccc8ccccccccccccccc","normalizations":[],"id":"import-top-level-json-output","argv":["import","media.txt","--json"],"transport":{"requests":[{"method":"POST","path":"/app/import/api/save-path","query":[],"headers":[],"timeout_policy":"api","json":{"client_item_id":"cccccccccccc4ccc8ccccccccccccccc","path":"media.txt"},"response":{"json":{"path":"/journal/imports/20260101_140000/media.txt","timestamp":"20260101_140000"}}},{"method":"POST","path":"/app/import/api/start","query":[],"headers":[],"timeout_policy":"api","json":{"path":"/journal/imports/20260101_140000/media.txt","timestamp":"20260101_140000","force":false},"response":{"json":{"status":"ok","task_id":"task-json"}}}]},"expected":{"stdout":"{\"path\": \"/journal/imports/20260101_140000/media.txt\", \"save\": {\"path\": \"/journal/imports/20260101_140000/media.txt\", \"timestamp\": \"20260101_140000\"}, \"start\": {\"status\": \"ok\", \"task_id\": \"task-json\"}, \"status\": \"queued\", \"timestamp\": \"20260101_140000\"}\n","stderr":"","exit":0,"requests":[{"method":"POST","path":"/app/import/api/save-path","query":[],"json":{"client_item_id":"cccccccccccc4ccc8ccccccccccccccc","path":"media.txt"},"headers":[],"timeout_policy":"api"},{"method":"POST","path":"/app/import/api/start","query":[],"json":{"path":"/journal/imports/20260101_140000/media.txt","timestamp":"20260101_140000","force":false},"headers":[],"timeout_policy":"api"}]}} +{"surface":"sol-import","env":{},"stdin":"","files":{},"clock":{"today":"20260723"},"normalizations":[],"id":"import-top-level-reject-dry-run","argv":["import","media.txt","--dry-run"],"transport":{"requests":[]},"expected":{"stdout":"","stderr":"sol import: `--dry-run` requires the journal host. Run this on the journal host with `journal importer`.\n","exit":2,"requests":[]}} +{"surface":"sol-import","env":{},"stdin":"","files":{},"clock":{"today":"20260723"},"normalizations":[],"id":"import-top-level-reject-backends","argv":["import","--backends"],"transport":{"requests":[]},"expected":{"stdout":"","stderr":"sol import: `--backends` requires the journal host. Run this on the journal host with `journal importer`.\n","exit":2,"requests":[]}} +{"surface":"sol-import","env":{},"stdin":"","files":{},"clock":{"today":"20260723"},"normalizations":[],"id":"import-top-level-reject-list-importers","argv":["import","--list-importers"],"transport":{"requests":[]},"expected":{"stdout":"","stderr":"sol import: `--list-importers` requires the journal host. Run this on the journal host with `journal importer`.\n","exit":2,"requests":[]}} +{"surface":"sol-import","env":{},"stdin":"","files":{},"clock":{"today":"20260723"},"normalizations":[],"id":"import-top-level-reject-sync","argv":["import","--sync","plaud"],"transport":{"requests":[]},"expected":{"stdout":"","stderr":"sol import: `--sync` requires the journal host. Run this on the journal host with `journal importer`.\n","exit":2,"requests":[]}} +{"surface":"sol-import","env":{},"stdin":"","files":{},"clock":{"today":"20260723"},"normalizations":[],"id":"import-top-level-reject-save","argv":["import","media.txt","--save"],"transport":{"requests":[]},"expected":{"stdout":"","stderr":"sol import: `--save` requires the journal host. Run this on the journal host with `journal importer`.\n","exit":2,"requests":[]}} +{"surface":"sol-import","env":{},"stdin":"","files":{},"clock":{"today":"20260723"},"normalizations":[],"id":"import-top-level-reject-path","argv":["import","media.txt","--path","/tmp/source"],"transport":{"requests":[]},"expected":{"stdout":"","stderr":"sol import: `--path` requires the journal host. Run this on the journal host with `journal importer`.\n","exit":2,"requests":[]}} +{"surface":"sol-import","env":{},"stdin":"","files":{},"clock":{"today":"20260723"},"normalizations":[],"id":"import-top-level-reject-auto-guidance","argv":["import","media.txt","--auto","timestamps are Pacific"],"transport":{"requests":[]},"expected":{"stdout":"","stderr":"sol import: `--auto ` requires the journal host. Use `--timestamp` here or run `journal importer`.\n","exit":2,"requests":[]}} +{"surface":"sol-import","env":{},"stdin":"","files":{},"clock":{"today":"20260723"},"normalizations":[],"id":"import-top-level-reject-journal-source","argv":["import","journal-source","list"],"transport":{"requests":[]},"expected":{"stdout":"","stderr":"sol import: journal-source management moved to `sol call import `.\n","exit":2,"requests":[]}} +{"surface":"sol-import","env":{},"stdin":"","files":{},"clock":{"today":"20260723"},"normalizations":[],"id":"import-top-level-malformed-save","argv":["import","media.txt"],"transport":{"requests":[{"method":"POST","path":"/app/import/api/save-path","query":[],"headers":[],"timeout_policy":"api","json":{"client_item_id":"11111111111141118111111111111111","path":"media.txt"},"response":{"json":[]}}]},"expected":{"stdout":"","stderr":"sol import: couldn't read journal response\n","exit":1,"requests":[{"method":"POST","path":"/app/import/api/save-path","query":[],"json":{"client_item_id":"11111111111141118111111111111111","path":"media.txt"},"headers":[],"timeout_policy":"api"}]}} +{"surface":"sol-import","env":{},"stdin":"","files":{},"clock":{"today":"20260723"},"normalizations":[],"id":"import-top-level-unreachable-save","argv":["import","media.txt"],"transport":{"requests":[{"method":"POST","path":"/app/import/api/save-path","query":[],"headers":[],"timeout_policy":"api","json":{"client_item_id":"11111111111141118111111111111111","path":"media.txt"},"fault":{"kind":"unreachable","detail":"connection refused"}}]},"expected":{"stdout":"","stderr":"sol import: couldn't reach the journal. Start it with 'journal up' and retry.\n","exit":1,"requests":[{"method":"POST","path":"/app/import/api/save-path","query":[],"json":{"client_item_id":"11111111111141118111111111111111","path":"media.txt"},"headers":[],"timeout_policy":"api"}]}} +{"surface":"sol-import","env":{},"stdin":"","files":{},"clock":{"today":"20260723"},"normalizations":[],"id":"import-top-level-duplicate-staged","argv":["import","media.txt"],"transport":{"requests":[{"method":"POST","path":"/app/import/api/save-path","query":[],"headers":[],"timeout_policy":"api","json":{"client_item_id":"11111111111141118111111111111111","path":"media.txt"},"response":{"json":{"status":"duplicate","recommended_action":"do_not_start","path":"/journal/imports/20260101_150000/media.txt","timestamp":"20260101_150000","duplicate":{"state":"staged","import_id":"20260101_150000"}}}}]},"expected":{"stdout":"sol import: already staged as 20260101_150000; skipping\n","stderr":"","exit":0,"requests":[{"method":"POST","path":"/app/import/api/save-path","query":[],"json":{"client_item_id":"11111111111141118111111111111111","path":"media.txt"},"headers":[],"timeout_policy":"api"}]}} +{"surface":"sol-import","env":{},"stdin":"","files":{},"clock":{"today":"20260723"},"normalizations":[],"id":"import-top-level-partial-start-failure","argv":["import","media.txt"],"transport":{"requests":[{"method":"POST","path":"/app/import/api/save-path","query":[],"headers":[],"timeout_policy":"api","json":{"client_item_id":"11111111111141118111111111111111","path":"media.txt"},"response":{"json":{"path":"/journal/imports/20260101_160000/media.txt","timestamp":"20260101_160000"}}},{"method":"POST","path":"/app/import/api/start","query":[],"headers":[],"timeout_policy":"api","json":{"path":"/journal/imports/20260101_160000/media.txt","timestamp":"20260101_160000","force":false},"fault":{"error":"queue failed","status":500,"reason_code":"import_metadata_failed","detail":null,"payload":{"reason_code":"import_metadata_failed","error":"queue failed","detail":null}}}]},"expected":{"stdout":"","stderr":"sol import: staged /journal/imports/20260101_160000/media.txt but processing was not queued: queue failed\n","exit":1,"requests":[{"method":"POST","path":"/app/import/api/save-path","query":[],"json":{"client_item_id":"11111111111141118111111111111111","path":"media.txt"},"headers":[],"timeout_policy":"api"},{"method":"POST","path":"/app/import/api/start","query":[],"json":{"path":"/journal/imports/20260101_160000/media.txt","timestamp":"20260101_160000","force":false},"headers":[],"timeout_policy":"api"}]}} diff --git a/scripts/build_native_sol_inventory.py b/scripts/build_native_sol_inventory.py index 2b3f75b59..d242edd2a 100644 --- a/scripts/build_native_sol_inventory.py +++ b/scripts/build_native_sol_inventory.py @@ -36,12 +36,14 @@ PARAM_KEYS = { } PARAM_REQUIRED_KEYS = PARAM_KEYS - {"default", "flag_value"} ORACLE_PATH = REPO_ROOT / "core/fixtures/native-sol/sol-call-grammar-v1.json" -ENTRY_TYPES = {"http", "moved-stub", "top-level-chat", "local"} +ENTRY_TYPES = {"http", "moved-stub", "top-level-chat", "top-level-import", "local"} COMMAND_KINDS = {"command", "callback", "top-level"} HTTP_METHODS = {"GET", "POST", "PUT", "PATCH", "DELETE"} FINAL_ORACLE_TOTAL = 178 FINAL_HTTP_TOTAL = 152 FINAL_JOURNAL_PYTHON_COMPAT_TOTAL = 23 +FINAL_TOP_LEVEL_CHAT_TOTAL = 1 +FINAL_TOP_LEVEL_IMPORT_TOTAL = 1 FINAL_STUB_COUNTS = {"moved-stub": 2, "local": 1} FINAL_HTTP_GROUP_COUNTS = { "activities": 6, @@ -143,7 +145,7 @@ def parse_entry( raise ValueError(f"{label}: path must be a non-empty string list") command_path = tuple(raw_path) surface = raw_entry.get("surface", "sol-call") - if surface not in {"sol-call", "sol-chat"}: + if surface not in {"sol-call", "sol-chat", "sol-import"}: raise ValueError(f"{label}: unsupported surface {surface!r}") kind = require_string(raw_entry, "kind", Path(label)) if kind not in COMMAND_KINDS: @@ -467,6 +469,28 @@ def check_complete_partition( return errors +def check_top_level_partition(entries: list[AuthorityEntry]) -> list[str]: + errors: list[str] = [] + expected = { + ("sol-chat", "top-level-chat"): FINAL_TOP_LEVEL_CHAT_TOTAL, + ("sol-import", "top-level-import"): FINAL_TOP_LEVEL_IMPORT_TOTAL, + } + actual: dict[tuple[str, str], int] = {} + for entry in entries: + if entry.surface == "sol-call": + continue + key = (entry.surface, entry.entry_type) + actual[key] = actual.get(key, 0) + 1 + for key, expected_count in sorted(expected.items()): + actual_count = actual.get(key, 0) + if actual_count != expected_count: + errors.append(f"{key} authority count {actual_count} != {expected_count}") + unexpected = sorted(set(actual) - set(expected)) + if unexpected: + errors.append(f"unexpected top-level native authorities: {unexpected!r}") + return errors + + def collect_oracle_paths(oracle_path: Path) -> tuple[list[str], set[tuple[str, ...]]]: if not oracle_path.is_file(): return [f"{oracle_path} is missing"], set() @@ -513,6 +537,7 @@ def main() -> int: output = args.output.resolve() entries = discover(root) partition_errors = check_complete_partition(entries, ORACLE_PATH) + partition_errors.extend(check_top_level_partition(entries)) if partition_errors: for error in partition_errors: print(error) diff --git a/scripts/check_native_sol_architecture.py b/scripts/check_native_sol_architecture.py index 4dc5e18c3..bd72f84b8 100644 --- a/scripts/check_native_sol_architecture.py +++ b/scripts/check_native_sol_architecture.py @@ -171,7 +171,8 @@ def check_native_http_ownership() -> list[Violation]: sources = { entry.source for entry in discover(REPO_ROOT) - if entry.surface == "sol-call" and entry.entry_type == "http" + if (entry.surface == "sol-call" and entry.entry_type == "http") + or entry.entry_type == "top-level-import" } for path in sorted(sources): text = path.read_text(encoding="utf-8") diff --git a/scripts/check_native_sol_conformance.py b/scripts/check_native_sol_conformance.py index 1398529ee..06a918ae9 100644 --- a/scripts/check_native_sol_conformance.py +++ b/scripts/check_native_sol_conformance.py @@ -101,11 +101,24 @@ def check_conformance( errors.extend(check_non_http_entry(authority, contract_by_operation)) elif authority.entry_type == "top-level-chat": errors.extend( - check_top_level_chat( + check_top_level_backing_contracts( authority, raw_authority, contract_by_operation, route_map, + "top-level-chat", + "chat", + ) + ) + elif authority.entry_type == "top-level-import": + errors.extend( + check_top_level_backing_contracts( + authority, + raw_authority, + contract_by_operation, + route_map, + "top-level-import", + "import", ) ) else: @@ -193,11 +206,13 @@ def check_non_http_entry( return errors -def check_top_level_chat( +def check_top_level_backing_contracts( authority: AuthorityEntry, raw_authority: RawAuthorityEntry | None, contract_by_operation: dict[str, ContractOperation], route_map: dict[tuple[str, str], Callable[..., Any]], + entry_type: str, + label: str, ) -> list[str]: errors = check_non_http_entry(authority, contract_by_operation) operation_id = authority.operation_id @@ -207,7 +222,7 @@ def check_top_level_chat( else None ) if not isinstance(authority_ids, list) or not authority_ids: - errors.append(f"{operation_id}: top-level-chat must declare backing contracts") + errors.append(f"{operation_id}: {entry_type} must declare backing contracts") return errors for backing_id in authority_ids: if not isinstance(backing_id, str) or not backing_id: @@ -215,11 +230,11 @@ def check_top_level_chat( continue contract = contract_by_operation.get(backing_id) if contract is None: - errors.append(f"{operation_id}: missing chat backing contract {backing_id}") + errors.append(f"{operation_id}: missing {label} backing contract {backing_id}") continue if (contract.method, contract.route) not in route_map: errors.append( - f"{operation_id}: missing chat backing route " + f"{operation_id}: missing {label} backing route " f"{contract.method} {contract.route}" ) return errors diff --git a/scripts/check_native_sol_coverage.py b/scripts/check_native_sol_coverage.py index ec284d4f8..ee006f01b 100644 --- a/scripts/check_native_sol_coverage.py +++ b/scripts/check_native_sol_coverage.py @@ -13,10 +13,16 @@ from pathlib import Path from typing import Any try: - from scripts.build_native_sol_inventory import FINAL_HTTP_TOTAL, REPO_ROOT, discover + from scripts.build_native_sol_inventory import ( + FINAL_HTTP_TOTAL, + FINAL_TOP_LEVEL_IMPORT_TOTAL, + REPO_ROOT, + discover, + ) except ModuleNotFoundError: # pragma: no cover - direct script execution path. from build_native_sol_inventory import ( # type: ignore[no-redef] FINAL_HTTP_TOTAL, + FINAL_TOP_LEVEL_IMPORT_TOTAL, REPO_ROOT, discover, ) @@ -44,6 +50,11 @@ def check_coverage(root: Path = REPO_ROOT) -> list[str]: for entry in discover(root) if entry.surface == "sol-call" and entry.entry_type == "http" } + required_top_level_import = { + entry.operation_id + for entry in discover(root) + if entry.surface == "sol-import" and entry.entry_type == "top-level-import" + } vectors = load_vectors(PARITY_DIR) resolved = resolve_vectors(PARITY_DIR, vectors) applicability, applicability_errors = load_applicability(APPLICABILITY) @@ -72,10 +83,43 @@ def check_coverage(root: Path = REPO_ROOT) -> list[str]: f"authority count {len(required)}" ) errors.extend(compare_sets("applicability keys", required, keys)) + top_level_entries = applicability.get("top_level_entries", {}) + if not isinstance(top_level_entries, dict): + errors.append("applicability top_level_entries must be an object") + top_level_entries = {} + import_keys = set(top_level_entries) + errors.extend( + compare_sets( + "top-level import applicability keys", + required_top_level_import, + import_keys, + ) + ) + + if len(required_top_level_import) != FINAL_TOP_LEVEL_IMPORT_TOTAL: + errors.append( + f"current top-level import authority count {len(required_top_level_import)} " + f"!= {FINAL_TOP_LEVEL_IMPORT_TOTAL}" + ) - buckets = collect_buckets(vectors, resolved, required, errors) + buckets = collect_buckets(vectors, resolved, required, {"http"}, errors) for bucket_name in ("request_binding", "success", "failure"): errors.extend(compare_sets(bucket_name, required, buckets[bucket_name])) + import_buckets = collect_buckets( + vectors, + resolved, + required_top_level_import, + {"top-level-import"}, + errors, + ) + for bucket_name in ("request_binding", "success", "failure"): + errors.extend( + compare_sets( + f"top-level import {bucket_name}", + required_top_level_import, + import_buckets[bucket_name], + ) + ) if not applicability_errors: errors.extend( @@ -83,6 +127,13 @@ def check_coverage(root: Path = REPO_ROOT) -> list[str]: applicability["entries"], vectors, resolved, buckets ) ) + errors.extend( + check_top_level_import_cases( + applicability.get("top_level_entries", {}), + vectors, + resolved, + ) + ) return errors @@ -120,6 +171,9 @@ def load_applicability(path: Path) -> tuple[dict[str, Any], list[str]]: ): if key not in entry: errors.append(f"{operation_id}: missing applicability field {key}") + top_level_entries = payload.get("top_level_entries", {}) + if top_level_entries is not None and not isinstance(top_level_entries, dict): + errors.append("applicability top_level_entries must be an object") return payload, errors @@ -193,6 +247,7 @@ def collect_buckets( vectors: dict[str, dict[str, Any]], resolved: dict[str, dict[str, Any]], required: set[str], + entry_types: set[str], errors: list[str], ) -> dict[str, set[str]]: buckets: dict[str, set[str]] = { @@ -208,7 +263,7 @@ def collect_buckets( f"{vector_id}: argv did not resolve through production dispatch" ) continue - if entry_type != "http" or operation_id not in required: + if entry_type not in entry_types or operation_id not in required: continue expected = vector.get("expected") or {} requests = expected.get("requests") if isinstance(expected, dict) else None @@ -280,6 +335,41 @@ def compare_sets(label: str, required: set[str], actual: set[str]) -> list[str]: return errors +def check_top_level_import_cases( + entries: Any, + vectors: dict[str, dict[str, Any]], + resolved: dict[str, dict[str, Any]], +) -> list[str]: + if not isinstance(entries, dict): + return [] + errors: list[str] = [] + for operation_id, entry in sorted(entries.items()): + if not isinstance(entry, dict): + errors.append(f"{operation_id}: top-level applicability entry must be an object") + continue + case_ids = entry.get("case_ids", {}) + if not isinstance(case_ids, dict): + errors.append(f"{operation_id}: top-level case_ids must be an object") + continue + for case_name, ids in sorted(case_ids.items()): + if not isinstance(ids, list) or not ids: + errors.append(f"{operation_id}: top-level case {case_name} must be non-empty") + continue + for vector_id in ids: + if vector_id not in vectors: + errors.append( + f"{operation_id}: top-level case {case_name} unknown vector {vector_id!r}" + ) + continue + mapped = resolved[vector_id].get("operation_id") + if mapped != operation_id: + errors.append( + f"{operation_id}: top-level case {case_name} vector " + f"{vector_id!r} maps to {mapped!r}" + ) + return errors + + def check_applicability_requirements( entries: dict[str, dict[str, Any]], vectors: dict[str, dict[str, Any]], diff --git a/solstone/apps/activities/native/command.rs b/solstone/apps/activities/native/command.rs index 33ae3029e..c63892a5b 100644 --- a/solstone/apps/activities/native/command.rs +++ b/solstone/apps/activities/native/command.rs @@ -838,6 +838,7 @@ mod tests { chat_events: None, files: None, build_identity: None, + client_item_ids: None, }); assert_eq!( diff --git a/solstone/apps/support/native/command.rs b/solstone/apps/support/native/command.rs index 7ea2e30b5..83862904a 100644 --- a/solstone/apps/support/native/command.rs +++ b/solstone/apps/support/native/command.rs @@ -1325,6 +1325,7 @@ mod tests { chat_events: None, files: None, build_identity: None, + client_item_ids: None, }); assert_eq!( diff --git a/solstone/think/native/chat/command.rs b/solstone/think/native/chat/command.rs index 5ecf46cdc..1b64d6d29 100644 --- a/solstone/think/native/chat/command.rs +++ b/solstone/think/native/chat/command.rs @@ -697,6 +697,7 @@ mod tests { chat_events: Some(&events), files: None, build_identity: None, + client_item_ids: None, }); assert_eq!( diff --git a/solstone/think/native/import/authority.toml b/solstone/think/native/import/authority.toml new file mode 100644 index 000000000..038412335 --- /dev/null +++ b/solstone/think/native/import/authority.toml @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +schema = "native-sol-authority-v1" +source = "command.rs" + +[[entries]] +surface = "sol-import" +path = ["import"] +kind = "top-level" +help = "Import media through the journal" +operation_id = "import.top_level" +entry_type = "top-level-import" +handler = "import_top_level" +backing_contract_operation_ids = ["import.save", "import.savePath", "import.start"] +params = [ + { name = "media", kind = "argument", type = "text", required = false, nargs = 1, multiple = false, options = ["media"], secondary = [], hidden = false, is_flag = false, count = false }, + { name = "extra", kind = "argument", type = "text", required = false, nargs = -1, multiple = false, options = ["extra"], secondary = [], hidden = true, is_flag = false, count = false }, + { name = "timestamp", kind = "option", type = "text", required = false, nargs = 1, multiple = false, options = ["--timestamp"], secondary = [], hidden = false, is_flag = false, count = false }, + { name = "facet", kind = "option", type = "text", required = false, nargs = 1, multiple = false, options = ["--facet"], secondary = [], hidden = false, is_flag = false, count = false }, + { name = "setting", kind = "option", type = "text", required = false, nargs = 1, multiple = false, options = ["--setting"], secondary = [], hidden = false, is_flag = false, count = false }, + { name = "source", kind = "option", type = "text", required = false, nargs = 1, multiple = false, options = ["--source"], secondary = [], hidden = false, is_flag = false, count = false }, + { name = "force", kind = "option", type = "boolean", required = false, nargs = 1, multiple = false, default = false, options = ["--force"], secondary = [], hidden = false, is_flag = true, count = false, flag_value = true }, + { name = "auto", kind = "option", type = "text", required = false, nargs = 1, multiple = false, options = ["--auto"], secondary = [], hidden = false, is_flag = false, count = false }, + { name = "deterministic_only", kind = "option", type = "boolean", required = false, nargs = 1, multiple = false, default = false, options = ["--deterministic-only"], secondary = [], hidden = false, is_flag = true, count = false, flag_value = true }, + { name = "dry_run", kind = "option", type = "boolean", required = false, nargs = 1, multiple = false, default = false, options = ["--dry-run"], secondary = [], hidden = false, is_flag = true, count = false, flag_value = true }, + { name = "json", kind = "option", type = "boolean", required = false, nargs = 1, multiple = false, default = false, options = ["--json"], secondary = [], hidden = false, is_flag = true, count = false, flag_value = true }, + { name = "verbose", kind = "option", type = "boolean", required = false, nargs = 1, multiple = false, default = false, options = ["-v", "--verbose"], secondary = [], hidden = false, is_flag = true, count = false, flag_value = true }, + { name = "backends", kind = "option", type = "boolean", required = false, nargs = 1, multiple = false, default = false, options = ["--backends"], secondary = [], hidden = false, is_flag = true, count = false, flag_value = true }, + { name = "sync", kind = "option", type = "text", required = false, nargs = 1, multiple = false, options = ["--sync"], secondary = [], hidden = false, is_flag = false, count = false }, + { name = "save", kind = "option", type = "boolean", required = false, nargs = 1, multiple = false, default = false, options = ["--save"], secondary = [], hidden = false, is_flag = true, count = false, flag_value = true }, + { name = "path", kind = "option", type = "text", required = false, nargs = 1, multiple = false, options = ["--path"], secondary = [], hidden = false, is_flag = false, count = false }, + { name = "list_importers", kind = "option", type = "boolean", required = false, nargs = 1, multiple = false, default = false, options = ["--list-importers"], secondary = [], hidden = false, is_flag = true, count = false, flag_value = true }, + { name = "journal_source", kind = "argument", type = "text", required = false, nargs = 1, multiple = false, options = ["journal-source"], secondary = [], hidden = false, is_flag = false, count = false }, +] diff --git a/solstone/think/native/import/command.rs b/solstone/think/native/import/command.rs new file mode 100644 index 000000000..5e83e88c0 --- /dev/null +++ b/solstone/think/native/import/command.rs @@ -0,0 +1,1055 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright (c) 2026 sol pbc + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use serde_json::{Map, Value, json}; + +use crate::command::{CommandContext, CommandOutput}; +use crate::decode::decode_response; +use crate::error::ClientError; +use crate::json_format::sorted_json_compact_ascii; +use crate::transport::{ + ApiRequest, FormField, HttpMethod, MultipartFile, TimeoutPolicy, UploadRequest, +}; + +const IMPORT_API: &str = "/app/import/api"; +const JOURNAL_HOST_HINT: &str = "Run this on the journal host with `journal importer`."; +const HELP: &str = "usage: sol import [-h] [--timestamp TIMESTAMP] [--facet FACET] [--setting SETTING] [--source SOURCE] [--force] [--auto [AUTO]] [--deterministic-only] [--dry-run] [--backends] [--sync BACKEND] [--save] [--path PATH] [--list-importers] [--json] [-v] [media]\n\nImport media through the journal\n"; + +#[must_use] +pub fn import_top_level(ctx: CommandContext<'_>) -> CommandOutput { + let parsed = match parse_args(ctx.args) { + Ok(parsed) => parsed, + Err(error) => return argparse_error(error), + }; + if parsed.help { + return CommandOutput::success(HELP); + } + if let Some(output) = reject_unsupported_modes(&parsed) { + return output; + } + if !parsed.extra.is_empty() { + return argparse_error(format!( + "unexpected argument(s): {}", + parsed.extra.join(" ") + )); + } + if parsed.media.is_none() { + return argparse_error("the following arguments are required: media".to_string()); + } + run_import(ctx, &parsed) +} + +#[derive(Debug, Clone, Default)] +struct ParsedArgs { + media: Option, + extra: Vec, + timestamp: Option, + facet: Option, + setting: Option, + source: Option, + force: bool, + auto: AutoArg, + deterministic_only: bool, + dry_run: bool, + json: bool, + help: bool, + backends: bool, + sync: Option, + save: bool, + path: Option, + list_importers: bool, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +enum AutoArg { + #[default] + Absent, + Bare, + Guidance, +} + +fn parse_args(args: &[String]) -> Result { + let mut parsed = ParsedArgs::default(); + let mut index = 0; + while index < args.len() { + let token = &args[index]; + if token == "--" { + for value in &args[index + 1..] { + push_positional(&mut parsed, value.clone()); + } + break; + } + if token == "-h" || token == "--help" { + parsed.help = true; + } else if token == "--force" { + parsed.force = true; + } else if token == "--deterministic-only" { + parsed.deterministic_only = true; + } else if token == "--dry-run" { + parsed.dry_run = true; + } else if token == "--json" { + parsed.json = true; + } else if token == "-v" || token == "--verbose" { + } else if token == "--backends" { + parsed.backends = true; + } else if token == "--save" { + parsed.save = true; + } else if token == "--list-importers" { + parsed.list_importers = true; + } else if let Some(value) = token.strip_prefix("--timestamp=") { + parsed.timestamp = Some(value.to_string()); + } else if token == "--timestamp" { + parsed.timestamp = Some(take_value(args, &mut index, "--timestamp")?); + } else if let Some(value) = token.strip_prefix("--facet=") { + parsed.facet = Some(value.to_string()); + } else if token == "--facet" { + parsed.facet = Some(take_value(args, &mut index, "--facet")?); + } else if let Some(value) = token.strip_prefix("--setting=") { + parsed.setting = Some(value.to_string()); + } else if token == "--setting" { + parsed.setting = Some(take_value(args, &mut index, "--setting")?); + } else if let Some(value) = token.strip_prefix("--source=") { + parsed.source = Some(value.to_string()); + } else if token == "--source" { + parsed.source = Some(take_value(args, &mut index, "--source")?); + } else if let Some(value) = token.strip_prefix("--sync=") { + parsed.sync = Some(value.to_string()); + } else if token == "--sync" { + parsed.sync = Some(take_value(args, &mut index, "--sync")?); + } else if let Some(value) = token.strip_prefix("--path=") { + parsed.path = Some(value.to_string()); + } else if token == "--path" { + parsed.path = Some(take_value(args, &mut index, "--path")?); + } else if let Some(value) = token.strip_prefix("--auto=") { + let _ = value; + parsed.auto = AutoArg::Guidance; + } else if token == "--auto" { + if args + .get(index + 1) + .is_some_and(|value| !value.starts_with('-')) + { + parsed.auto = AutoArg::Guidance; + index += 1; + } else { + parsed.auto = AutoArg::Bare; + } + } else if token.starts_with('-') { + return Err(format!("unrecognized arguments: {token}")); + } else { + push_positional(&mut parsed, token.clone()); + } + index += 1; + } + Ok(parsed) +} + +fn take_value(args: &[String], index: &mut usize, option: &str) -> Result { + *index += 1; + args.get(*index) + .cloned() + .ok_or_else(|| format!("argument {option}: expected one argument")) +} + +fn push_positional(parsed: &mut ParsedArgs, value: String) { + if parsed.media.is_none() { + parsed.media = Some(value); + } else { + parsed.extra.push(value); + } +} + +fn reject_unsupported_modes(parsed: &ParsedArgs) -> Option { + if parsed.media.as_deref() == Some("journal-source") { + return Some(rejected( + "journal-source management moved to `sol call import `.", + )); + } + if parsed.dry_run { + return Some(rejected(format!( + "`--dry-run` requires the journal host. {JOURNAL_HOST_HINT}" + ))); + } + if parsed.backends { + return Some(rejected(format!( + "`--backends` requires the journal host. {JOURNAL_HOST_HINT}" + ))); + } + if parsed.list_importers { + return Some(rejected(format!( + "`--list-importers` requires the journal host. {JOURNAL_HOST_HINT}" + ))); + } + if parsed.sync.is_some() { + return Some(rejected(format!( + "`--sync` requires the journal host. {JOURNAL_HOST_HINT}" + ))); + } + if parsed.save { + return Some(rejected(format!( + "`--save` requires the journal host. {JOURNAL_HOST_HINT}" + ))); + } + if parsed + .path + .as_deref() + .is_some_and(|value| !value.is_empty()) + { + return Some(rejected(format!( + "`--path` requires the journal host. {JOURNAL_HOST_HINT}" + ))); + } + if matches!(parsed.auto, AutoArg::Guidance) { + return Some(rejected( + "`--auto ` requires the journal host. Use `--timestamp` here or run `journal importer`.", + )); + } + None +} + +fn rejected(message: impl AsRef) -> CommandOutput { + CommandOutput::failure(format!("sol import: {}\n", message.as_ref()), 2) +} + +fn argparse_error(error: String) -> CommandOutput { + CommandOutput::failure(format!("{HELP}sol import: error: {error}\n"), 2) +} + +fn run_import(ctx: CommandContext<'_>, parsed: &ParsedArgs) -> CommandOutput { + let client_item_id = ctx + .client_item_ids + .map(|provider| provider.client_item_id()) + .unwrap_or_else(|| "00000000000000000000000000000000".to_string()); + let save_response = match save_media(ctx, parsed, &client_item_id) { + Ok(response) => response, + Err(ImportError::Unreachable) => { + return CommandOutput::failure( + "sol import: couldn't reach the journal. Start it with 'journal up' and retry.\n", + 1, + ); + } + Err(error) => return print_client_error("stage import", error), + }; + if save_response.get("status").and_then(Value::as_str) == Some("duplicate") + || save_response + .get("recommended_action") + .and_then(Value::as_str) + == Some("do_not_start") + { + return print_duplicate(&save_response, parsed.json); + } + let staged_path = save_response + .get("path") + .and_then(Value::as_str) + .unwrap_or_else(|| parsed.media.as_deref().unwrap_or_default()) + .to_string(); + let start_response = match start_import(ctx, parsed, &save_response) { + Ok(response) => response, + Err(ImportError::Unreachable) => { + return CommandOutput::failure( + format!( + "sol import: staged {staged_path} but processing was not queued: couldn't reach the journal\n" + ), + 1, + ); + } + Err(error) => return print_partial_error(&staged_path, error), + }; + print_success(&save_response, &start_response, parsed.json) +} + +#[derive(Debug, Clone)] +enum ImportError { + Unreachable, + Malformed, + Client { + error: String, + detail: Option, + }, +} + +fn save_media( + ctx: CommandContext<'_>, + parsed: &ParsedArgs, + client_item_id: &str, +) -> Result, ImportError> { + let media = parsed.media.as_deref().unwrap_or_default(); + let media_path = expand_user(media, ctx.env); + let mut data = save_data(parsed, client_item_id); + let response = if ctx.files.is_some_and(|files| files.is_file(&media_path)) { + let files = ctx.files.expect("checked files provider exists"); + let body = files + .read(&media_path) + .map_err(|error| ImportError::Client { + error: error.to_string(), + detail: None, + })?; + let filename = media_path + .file_name() + .and_then(|value| value.to_str()) + .unwrap_or_default() + .to_string(); + ctx.transport + .upload(UploadRequest { + path: format!("{IMPORT_API}/save"), + files: vec![MultipartFile { + field_name: "file".to_string(), + filename, + content_type: Some("application/octet-stream".to_string()), + body, + }], + data: data + .into_iter() + .map(|(name, value)| FormField { name, value }) + .collect(), + headers: vec![], + boundary: None, + policy: TimeoutPolicy::Upload, + }) + .map_err(map_transport_error)? + } else { + let mut payload = Map::new(); + for (key, value) in data.drain(..) { + payload.insert(key, Value::String(value)); + } + payload.insert("path".to_string(), Value::String(path_string(&media_path))); + ctx.transport + .request(ApiRequest { + method: HttpMethod::Post, + path: format!("{IMPORT_API}/save-path"), + params: vec![], + json: Some(Value::Object(payload)), + headers: vec![], + policy: TimeoutPolicy::Api, + }) + .map_err(map_transport_error)? + }; + decode_object(response) +} + +fn save_data(parsed: &ParsedArgs, client_item_id: &str) -> Vec<(String, String)> { + let mut data = Vec::new(); + data.push(("client_item_id".to_string(), client_item_id.to_string())); + push_payload_value(&mut data, "facet", parsed.facet.as_deref()); + push_payload_value(&mut data, "setting", parsed.setting.as_deref()); + push_payload_value(&mut data, "source_hint", parsed.source.as_deref()); + if parsed.deterministic_only { + data.push(("deterministic_only".to_string(), "true".to_string())); + } + data +} + +fn push_payload_value(data: &mut Vec<(String, String)>, key: &str, value: Option<&str>) { + let Some(stripped) = value.map(str::trim).filter(|value| !value.is_empty()) else { + return; + }; + data.push((key.to_string(), stripped.to_string())); +} + +fn start_import( + ctx: CommandContext<'_>, + parsed: &ParsedArgs, + save_response: &Map, +) -> Result, ImportError> { + let path = save_response + .get("path") + .and_then(Value::as_str) + .filter(|value| !value.is_empty()) + .ok_or(ImportError::Malformed)?; + let timestamp = parsed + .timestamp + .as_deref() + .or_else(|| save_response.get("timestamp").and_then(Value::as_str)) + .filter(|value| !value.is_empty()) + .ok_or(ImportError::Malformed)?; + let response = ctx + .transport + .request(ApiRequest { + method: HttpMethod::Post, + path: format!("{IMPORT_API}/start"), + params: vec![], + json: Some(json!({ + "path": path, + "timestamp": timestamp, + "force": parsed.force, + })), + headers: vec![], + policy: TimeoutPolicy::Api, + }) + .map_err(map_transport_error)?; + let object = decode_object(response)?; + let task_id = object + .get("task_id") + .and_then(Value::as_str) + .filter(|value| !value.is_empty()); + if task_id.is_none() { + return Err(ImportError::Malformed); + } + Ok(object) +} + +fn decode_object( + response: crate::transport::HttpResponse, +) -> Result, ImportError> { + let value = decode_response(&response).map_err(map_transport_error)?; + value.as_object().cloned().ok_or(ImportError::Malformed) +} + +fn map_transport_error(error: ClientError) -> ImportError { + match error { + ClientError::Unreachable { .. } => ImportError::Unreachable, + ClientError::MalformedSuccess { .. } => ImportError::Malformed, + other => ImportError::Client { + error: other.message().to_string(), + detail: other.detail().map(str::to_string), + }, + } +} + +fn print_client_error(operation: &str, error: ImportError) -> CommandOutput { + match error { + ImportError::Malformed => { + CommandOutput::failure("sol import: couldn't read journal response\n", 1) + } + ImportError::Client { error, detail } => { + let mut stderr = format!("sol import: failed to {operation}: {error}\n"); + if let Some(detail) = detail { + stderr.push_str(&format!("sol import: {detail}\n")); + } + CommandOutput::failure(stderr, 1) + } + ImportError::Unreachable => CommandOutput::failure( + "sol import: couldn't reach the journal. Start it with 'journal up' and retry.\n", + 1, + ), + } +} + +fn print_partial_error(staged_path: &str, error: ImportError) -> CommandOutput { + match error { + ImportError::Malformed => CommandOutput::failure( + format!( + "sol import: staged {staged_path} but processing was not queued: couldn't read journal response\n" + ), + 1, + ), + ImportError::Client { error, detail } => { + let mut stderr = format!( + "sol import: staged {staged_path} but processing was not queued: {error}\n" + ); + if let Some(detail) = detail { + stderr.push_str(&format!("sol import: {detail}\n")); + } + CommandOutput::failure(stderr, 1) + } + ImportError::Unreachable => CommandOutput::failure( + format!( + "sol import: staged {staged_path} but processing was not queued: couldn't reach the journal\n" + ), + 1, + ), + } +} + +fn print_success( + save_response: &Map, + start_response: &Map, + json_out: bool, +) -> CommandOutput { + let timestamp = save_response + .get("timestamp") + .cloned() + .unwrap_or(Value::Null); + let path = save_response.get("path").cloned().unwrap_or(Value::Null); + if json_out { + return CommandOutput::success(format!( + "{}\n", + sorted_json_compact_ascii(&json!({ + "status": "queued", + "path": path, + "timestamp": timestamp, + "save": Value::Object(save_response.clone()), + "start": Value::Object(start_response.clone()), + })) + )); + } + let mut stdout = String::new(); + stdout.push_str(&format!("staged {}\n", display_value(&path))); + if let Some(timestamp) = timestamp.as_str().filter(|value| !value.is_empty()) { + stdout.push_str(&format!("timestamp {timestamp}\n")); + } + if let Some(task_id) = start_response.get("task_id").and_then(Value::as_str) { + stdout.push_str(&format!("queued processing task {task_id}\n")); + } else { + stdout.push_str("queued processing\n"); + } + CommandOutput::success(stdout) +} + +fn print_duplicate(save_response: &Map, json_out: bool) -> CommandOutput { + if json_out { + return CommandOutput::success(format!( + "{}\n", + sorted_json_compact_ascii(&Value::Object(save_response.clone())) + )); + } + let duplicate = save_response.get("duplicate").and_then(Value::as_object); + let Some(duplicate) = duplicate else { + return CommandOutput::success("sol import: duplicate import; skipping\n"); + }; + match duplicate.get("state").and_then(Value::as_str) { + Some("imported") => { + let imported_at = duplicate + .get("imported_at") + .and_then(Value::as_str) + .unwrap_or("unknown date"); + let entries = duplicate + .get("entry_count") + .filter(|value| !value.is_null()) + .map(|count| format!(" ({} entries)", display_value(count))) + .unwrap_or_default(); + CommandOutput::success(format!( + "sol import: already imported on {imported_at}{entries}; skipping\n" + )) + } + Some("staged") => { + let import_id = duplicate + .get("import_id") + .and_then(Value::as_str) + .unwrap_or("unknown"); + CommandOutput::success(format!( + "sol import: already staged as {import_id}; skipping\n" + )) + } + _ => CommandOutput::success("sol import: duplicate import; skipping\n"), + } +} + +fn display_value(value: &Value) -> String { + value + .as_str() + .map(str::to_string) + .unwrap_or_else(|| value.to_string()) +} + +fn expand_user(value: &str, env: &BTreeMap) -> PathBuf { + if value == "~" { + return env + .get("HOME") + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(value)); + } + if let Some(rest) = value.strip_prefix("~/") + && let Some(home) = env.get("HOME") + { + return Path::new(home).join(rest); + } + PathBuf::from(value) +} + +fn path_string(path: &Path) -> String { + path.to_string_lossy().to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + use std::collections::{BTreeMap, HashMap}; + + use serde_json::json; + + use crate::command::{CommandContext, CommandOutput}; + use crate::error::ClientError; + use crate::seam::{ + ExpectedHttpCall, FakeClientItemIdProvider, FixtureFileProvider, ScriptedHttpTransport, + }; + use crate::transport::{ + ApiRequest, FormField, HttpMethod, HttpResponse, MultipartFile, TimeoutPolicy, + UploadRequest, + }; + + fn string_args(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_string()).collect() + } + + fn json_response(value: Value, policy: TimeoutPolicy) -> HttpResponse { + HttpResponse { + status: 200, + headers: vec![], + body: serde_json::to_vec(&value).expect("json response"), + policy, + } + } + + fn run_import_case( + args: &[&str], + transport: &ScriptedHttpTransport, + files: &FixtureFileProvider, + client_item_ids: &FakeClientItemIdProvider, + ) -> CommandOutput { + let args = string_args(args); + let env = BTreeMap::new(); + import_top_level(CommandContext { + args: &args, + env: &env, + stdin: "", + today: "20260723", + transport, + clock: None, + chat_events: None, + files: Some(files), + build_identity: None, + client_item_ids: Some(client_item_ids), + }) + } + + #[test] + fn local_file_uses_multipart_upload_with_generated_client_item_id() { + let mut fixtures = HashMap::new(); + fixtures.insert(PathBuf::from("/tmp/sample.txt"), b"hello".to_vec()); + let files = FixtureFileProvider::new(fixtures); + let client_item_ids = FakeClientItemIdProvider::new("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + let transport = ScriptedHttpTransport::new(vec![ + ExpectedHttpCall::Upload { + expected: UploadRequest { + path: "/app/import/api/save".to_string(), + files: vec![MultipartFile { + field_name: "file".to_string(), + filename: "sample.txt".to_string(), + content_type: Some("application/octet-stream".to_string()), + body: b"hello".to_vec(), + }], + data: vec![ + FormField { + name: "client_item_id".to_string(), + value: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".to_string(), + }, + FormField { + name: "facet".to_string(), + value: "work".to_string(), + }, + FormField { + name: "setting".to_string(), + value: "office".to_string(), + }, + FormField { + name: "source_hint".to_string(), + value: "ics".to_string(), + }, + FormField { + name: "deterministic_only".to_string(), + value: "true".to_string(), + }, + ], + headers: vec![], + boundary: None, + policy: TimeoutPolicy::Upload, + }, + result: Ok(json_response( + json!({ + "path": "/journal/imports/20260101_120000/sample.txt", + "timestamp": "20260101_120000" + }), + TimeoutPolicy::Upload, + )), + }, + ExpectedHttpCall::Request { + expected: ApiRequest { + method: HttpMethod::Post, + path: "/app/import/api/start".to_string(), + params: vec![], + json: Some(json!({ + "path": "/journal/imports/20260101_120000/sample.txt", + "timestamp": "20260101_120000", + "force": true, + })), + headers: vec![], + policy: TimeoutPolicy::Api, + }, + result: Ok(json_response( + json!({"status": "ok", "task_id": "task-file"}), + TimeoutPolicy::Api, + )), + }, + ]); + + let output = run_import_case( + &[ + "/tmp/sample.txt", + "--facet", + " work ", + "--setting", + " office ", + "--source", + " ics ", + "--deterministic-only", + "--force", + ], + &transport, + &files, + &client_item_ids, + ); + + assert_eq!( + output, + CommandOutput { + stdout: "staged /journal/imports/20260101_120000/sample.txt\ntimestamp 20260101_120000\nqueued processing task task-file\n".to_string(), + stderr: String::new(), + exit: 0, + } + ); + transport.assert_done(); + } + + #[test] + fn host_path_uses_save_path_and_timestamp_override_for_start() { + let files = FixtureFileProvider::default(); + let client_item_ids = FakeClientItemIdProvider::new("bbbbbbbbbbbb4bbb8bbbbbbbbbbbbbbb"); + let transport = ScriptedHttpTransport::new(vec![ + ExpectedHttpCall::Request { + expected: ApiRequest { + method: HttpMethod::Post, + path: "/app/import/api/save-path".to_string(), + params: vec![], + json: Some(json!({ + "client_item_id": "bbbbbbbbbbbb4bbb8bbbbbbbbbbbbbbb", + "path": "/journal-host/media/source-dir" + })), + headers: vec![], + policy: TimeoutPolicy::Api, + }, + result: Ok(json_response( + json!({ + "path": "/journal/imports/20260101_130000/source-dir", + "timestamp": "20260101_130000" + }), + TimeoutPolicy::Api, + )), + }, + ExpectedHttpCall::Request { + expected: ApiRequest { + method: HttpMethod::Post, + path: "/app/import/api/start".to_string(), + params: vec![], + json: Some(json!({ + "path": "/journal/imports/20260101_130000/source-dir", + "timestamp": "20260202_030405", + "force": false, + })), + headers: vec![], + policy: TimeoutPolicy::Api, + }, + result: Ok(json_response( + json!({"status": "ok", "task_id": "task-path"}), + TimeoutPolicy::Api, + )), + }, + ]); + + let output = run_import_case( + &[ + "/journal-host/media/source-dir", + "--timestamp", + "20260202_030405", + ], + &transport, + &files, + &client_item_ids, + ); + + assert_eq!( + output.stdout, + "staged /journal/imports/20260101_130000/source-dir\ntimestamp 20260101_130000\nqueued processing task task-path\n" + ); + assert_eq!(output.stderr, ""); + assert_eq!(output.exit, 0); + transport.assert_done(); + } + + #[test] + fn json_output_is_sorted_and_includes_save_and_start_payloads() { + let files = FixtureFileProvider::default(); + let client_item_ids = FakeClientItemIdProvider::new("cccccccccccc4ccc8ccccccccccccccc"); + let transport = ScriptedHttpTransport::new(vec![ + ExpectedHttpCall::Request { + expected: ApiRequest { + method: HttpMethod::Post, + path: "/app/import/api/save-path".to_string(), + params: vec![], + json: Some(json!({ + "client_item_id": "cccccccccccc4ccc8ccccccccccccccc", + "path": "media.txt" + })), + headers: vec![], + policy: TimeoutPolicy::Api, + }, + result: Ok(json_response( + json!({ + "path": "/journal/imports/20260101_140000/media.txt", + "timestamp": "20260101_140000" + }), + TimeoutPolicy::Api, + )), + }, + ExpectedHttpCall::Request { + expected: ApiRequest { + method: HttpMethod::Post, + path: "/app/import/api/start".to_string(), + params: vec![], + json: Some(json!({ + "path": "/journal/imports/20260101_140000/media.txt", + "timestamp": "20260101_140000", + "force": false, + })), + headers: vec![], + policy: TimeoutPolicy::Api, + }, + result: Ok(json_response( + json!({"status": "ok", "task_id": "task-json"}), + TimeoutPolicy::Api, + )), + }, + ]); + + let output = run_import_case( + &["media.txt", "--json"], + &transport, + &files, + &client_item_ids, + ); + + assert_eq!( + output.stdout, + "{\"path\": \"/journal/imports/20260101_140000/media.txt\", \"save\": {\"path\": \"/journal/imports/20260101_140000/media.txt\", \"timestamp\": \"20260101_140000\"}, \"start\": {\"status\": \"ok\", \"task_id\": \"task-json\"}, \"status\": \"queued\", \"timestamp\": \"20260101_140000\"}\n" + ); + assert_eq!(output.stderr, ""); + assert_eq!(output.exit, 0); + transport.assert_done(); + } + + #[test] + fn host_only_modes_reject_with_frozen_messages() { + let cases: Vec<(Vec<&str>, &str)> = vec![ + ( + vec!["media.txt", "--dry-run"], + "sol import: `--dry-run` requires the journal host. Run this on the journal host with `journal importer`.\n", + ), + ( + vec!["--backends"], + "sol import: `--backends` requires the journal host. Run this on the journal host with `journal importer`.\n", + ), + ( + vec!["--list-importers"], + "sol import: `--list-importers` requires the journal host. Run this on the journal host with `journal importer`.\n", + ), + ( + vec!["--sync", "plaud"], + "sol import: `--sync` requires the journal host. Run this on the journal host with `journal importer`.\n", + ), + ( + vec!["media.txt", "--save"], + "sol import: `--save` requires the journal host. Run this on the journal host with `journal importer`.\n", + ), + ( + vec!["media.txt", "--path", "/tmp/source"], + "sol import: `--path` requires the journal host. Run this on the journal host with `journal importer`.\n", + ), + ( + vec!["media.txt", "--auto", "timestamps are Pacific"], + "sol import: `--auto ` requires the journal host. Use `--timestamp` here or run `journal importer`.\n", + ), + ( + vec!["journal-source", "list"], + "sol import: journal-source management moved to `sol call import `.\n", + ), + ]; + + for (args, stderr) in cases { + let files = FixtureFileProvider::default(); + let client_item_ids = FakeClientItemIdProvider::new("11111111111141118111111111111111"); + let transport = ScriptedHttpTransport::new(vec![]); + let output = run_import_case(&args, &transport, &files, &client_item_ids); + assert_eq!( + output, + CommandOutput { + stdout: String::new(), + stderr: stderr.to_string(), + exit: 2, + }, + "{args:?}" + ); + transport.assert_done(); + } + } + + #[test] + fn malformed_save_response_uses_frozen_error() { + let files = FixtureFileProvider::default(); + let client_item_ids = FakeClientItemIdProvider::new("11111111111141118111111111111111"); + let transport = ScriptedHttpTransport::new(vec![ExpectedHttpCall::Request { + expected: ApiRequest { + method: HttpMethod::Post, + path: "/app/import/api/save-path".to_string(), + params: vec![], + json: Some(json!({ + "client_item_id": "11111111111141118111111111111111", + "path": "media.txt" + })), + headers: vec![], + policy: TimeoutPolicy::Api, + }, + result: Ok(json_response(json!([]), TimeoutPolicy::Api)), + }]); + + let output = run_import_case(&["media.txt"], &transport, &files, &client_item_ids); + + assert_eq!( + output, + CommandOutput { + stdout: String::new(), + stderr: "sol import: couldn't read journal response\n".to_string(), + exit: 1, + } + ); + transport.assert_done(); + } + + #[test] + fn unreachable_save_response_uses_frozen_error() { + let files = FixtureFileProvider::default(); + let client_item_ids = FakeClientItemIdProvider::new("11111111111141118111111111111111"); + let transport = ScriptedHttpTransport::new(vec![ExpectedHttpCall::Request { + expected: ApiRequest { + method: HttpMethod::Post, + path: "/app/import/api/save-path".to_string(), + params: vec![], + json: Some(json!({ + "client_item_id": "11111111111141118111111111111111", + "path": "media.txt" + })), + headers: vec![], + policy: TimeoutPolicy::Api, + }, + result: Err(ClientError::unreachable(Some( + "connection refused".to_string(), + ))), + }]); + + let output = run_import_case(&["media.txt"], &transport, &files, &client_item_ids); + + assert_eq!( + output, + CommandOutput { + stdout: String::new(), + stderr: "sol import: couldn't reach the journal. Start it with 'journal up' and retry.\n".to_string(), + exit: 1, + } + ); + transport.assert_done(); + } + + #[test] + fn duplicate_save_response_short_circuits_without_start_request() { + let files = FixtureFileProvider::default(); + let client_item_ids = FakeClientItemIdProvider::new("11111111111141118111111111111111"); + let transport = ScriptedHttpTransport::new(vec![ExpectedHttpCall::Request { + expected: ApiRequest { + method: HttpMethod::Post, + path: "/app/import/api/save-path".to_string(), + params: vec![], + json: Some(json!({ + "client_item_id": "11111111111141118111111111111111", + "path": "media.txt" + })), + headers: vec![], + policy: TimeoutPolicy::Api, + }, + result: Ok(json_response( + json!({ + "status": "duplicate", + "recommended_action": "do_not_start", + "duplicate": { + "state": "staged", + "import_id": "20260101_150000" + } + }), + TimeoutPolicy::Api, + )), + }]); + + let output = run_import_case(&["media.txt"], &transport, &files, &client_item_ids); + + assert_eq!( + output, + CommandOutput { + stdout: "sol import: already staged as 20260101_150000; skipping\n".to_string(), + stderr: String::new(), + exit: 0, + } + ); + transport.assert_done(); + } + + #[test] + fn staged_but_not_queued_partial_failure_has_no_success_output() { + let files = FixtureFileProvider::default(); + let client_item_ids = FakeClientItemIdProvider::new("11111111111141118111111111111111"); + let transport = ScriptedHttpTransport::new(vec![ + ExpectedHttpCall::Request { + expected: ApiRequest { + method: HttpMethod::Post, + path: "/app/import/api/save-path".to_string(), + params: vec![], + json: Some(json!({ + "client_item_id": "11111111111141118111111111111111", + "path": "media.txt" + })), + headers: vec![], + policy: TimeoutPolicy::Api, + }, + result: Ok(json_response( + json!({ + "path": "/journal/imports/20260101_160000/media.txt", + "timestamp": "20260101_160000" + }), + TimeoutPolicy::Api, + )), + }, + ExpectedHttpCall::Request { + expected: ApiRequest { + method: HttpMethod::Post, + path: "/app/import/api/start".to_string(), + params: vec![], + json: Some(json!({ + "path": "/journal/imports/20260101_160000/media.txt", + "timestamp": "20260101_160000", + "force": false, + })), + headers: vec![], + policy: TimeoutPolicy::Api, + }, + result: Err(ClientError::ReasonRejected { + status: 500, + error: "queue failed".to_string(), + reason_code: Some("import_metadata_failed".to_string()), + detail: None, + payload: Box::new(json!({"error": "queue failed"})), + }), + }, + ]); + + let output = run_import_case(&["media.txt"], &transport, &files, &client_item_ids); + + assert_eq!(output.stdout, ""); + assert_eq!( + output.stderr, + "sol import: staged /journal/imports/20260101_160000/media.txt but processing was not queued: queue failed\n" + ); + assert_eq!(output.exit, 1); + assert!(!output.stdout.contains("queued processing")); + transport.assert_done(); + } +} diff --git a/solstone/think/tools/native/health/command.rs b/solstone/think/tools/native/health/command.rs index be4633205..72e178443 100644 --- a/solstone/think/tools/native/health/command.rs +++ b/solstone/think/tools/native/health/command.rs @@ -446,6 +446,7 @@ mod tests { chat_events: None, files: None, build_identity: None, + client_item_ids: None, }); assert_eq!( @@ -547,6 +548,7 @@ mod tests { chat_events: None, files: None, build_identity: None, + client_item_ids: None, }); assert_eq!( -- 2.51.2