diff --git a/core/Cargo.lock b/core/Cargo.lock index 5b8a4cdc4..5a8ded438 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -170,6 +176,18 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.12.1" @@ -228,6 +246,12 @@ dependencies = [ "phf", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "const-oid" version = "0.9.6" @@ -249,6 +273,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "crypto-bigint" version = "0.5.5" @@ -399,6 +432,15 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + [[package]] name = "ff" version = "0.13.1" @@ -415,6 +457,16 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "foldhash" version = "0.2.0" @@ -504,6 +556,16 @@ dependencies = [ "wasip2", ] +[[package]] +name = "gif" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "glob" version = "0.3.4" @@ -680,6 +742,34 @@ dependencies = [ "cc", ] +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "color_quant", + "gif", + "image-webp", + "moxcms", + "num-traits", + "png", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "image-webp" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" +dependencies = [ + "byteorder-lite", + "quick-error", +] + [[package]] name = "indexmap" version = "2.14.0" @@ -788,6 +878,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.2" @@ -799,6 +899,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + [[package]] name = "multer" version = "3.1.0" @@ -1007,6 +1117,19 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + [[package]] name = "portable-atomic" version = "1.14.0" @@ -1066,6 +1189,18 @@ dependencies = [ "unicase", ] +[[package]] +name = "pxfm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" + +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quote" version = "1.0.47" @@ -1366,6 +1501,12 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "siphasher" version = "1.0.3" @@ -1435,6 +1576,17 @@ dependencies = [ "tower-http", ] +[[package]] +name = "solstone-core-depict" +version = "1.0.22" +dependencies = [ + "base64", + "image", + "serde", + "serde_json", + "solstone-core-journal-io", +] + [[package]] name = "solstone-core-entity" version = "1.0.22" @@ -2177,6 +2329,12 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + [[package]] name = "windows-core" version = "0.62.2" @@ -2391,3 +2549,18 @@ name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zune-core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9" + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core", +] diff --git a/core/Cargo.toml b/core/Cargo.toml index 158ee5c44..f38188468 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -21,6 +21,7 @@ members = [ "crates/solstone-core-spl", "crates/solstone-core-speakers", "crates/solstone-core-speakers-analyze", + "crates/solstone-core-depict", "crates/solstone-core-speakers-onnx", ] resolver = "3" @@ -57,11 +58,13 @@ hyper = { version = "1.11.0", default-features = false, features = ["server", "h hyper-util = { version = "0.1.20", default-features = false, features = ["tokio", "service"] } solstone-core-speakers = { path = "crates/solstone-core-speakers" } solstone-core-speakers-analyze = { path = "crates/solstone-core-speakers-analyze" } +solstone-core-depict = { path = "crates/solstone-core-depict" } solstone-core-speakers-onnx = { path = "crates/solstone-core-speakers-onnx" } chrono = { version = "0.4.45", default-features = false, features = ["clock"] } chrono-tz = "0.10.4" glob = "0.3.3" hdbscan = { version = "0.12.0", default-features = false, features = ["serial"] } +image = { version = "0.25.9", default-features = false, features = ["gif", "jpeg", "png", "webp"] } log = "0.4.29" md5 = "0.8.1" nix = { version = "0.30.1", default-features = false, features = ["fs"] } diff --git a/core/crates/solstone-core-depict/Cargo.toml b/core/crates/solstone-core-depict/Cargo.toml new file mode 100644 index 000000000..270426c4b --- /dev/null +++ b/core/crates/solstone-core-depict/Cargo.toml @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +[package] +name = "solstone-core-depict" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false + +[dependencies] +base64 = "0.22.1" +image.workspace = true +serde.workspace = true +serde_json.workspace = true +solstone-core-journal-io.workspace = true + +[lints] +workspace = true diff --git a/core/crates/solstone-core-depict/src/lib.rs b/core/crates/solstone-core-depict/src/lib.rs new file mode 100644 index 000000000..3da8484b6 --- /dev/null +++ b/core/crates/solstone-core-depict/src/lib.rs @@ -0,0 +1,625 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright (c) 2026 sol pbc + +//! Standalone native still-image depiction handler. +//! +//! It is intentionally not wired into `journal depict` yet. The Python +//! implementation remains the differential reference until the cutover lode. + +use std::env; +use std::ffi::OsString; +use std::fs; +use std::io::{Cursor, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use base64::Engine; +use image::{DynamicImage, ImageFormat}; +use serde_json::{Map, Value, json}; +use solstone_core_journal_io::{AtomicWriteOptions, write_jsonl}; + +pub const REQUEST_SCHEMA: &str = "solstone-generate-request-v1"; +pub const RESPONSE_SCHEMA: &str = "solstone-generate-response-v1"; +pub const ERROR_SCHEMA: &str = "solstone-generate-error-v1"; +pub const DESCRIPTION_PROMPT: &str = "Describe this image in detail. Include any visible text, people, objects, setting, and notable context. Return a concise natural-language description."; +pub const USAGE: &str = "Usage: solstone-core-depict [--redo]"; +const MAX_VLM_DIM: u32 = 1920; +const ENGINE_NAME: &str = "rf-detr.cpp"; +const ENGINE_REF: &str = "65c0ffcc"; +const MODEL_NAME: &str = "rfdetr-nano-f16"; +const THRESHOLD: f64 = 0.25; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Arguments { + pub image_path: PathBuf, + pub redo: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DepictError { + Usage(String), + Image(String), + Wire(String), + Metadata(String), + Output(String), +} + +impl DepictError { + pub fn reason(&self) -> &'static str { + match self { + Self::Usage(_) => "malformed-request", + Self::Image(_) => "image-invalid", + Self::Wire(_) => "generate-wire-failed", + Self::Metadata(_) => "metadata-invalid", + Self::Output(_) => "output-unwritable", + } + } + + pub fn exit_code(&self) -> i32 { + // This is the handler namespace: all failures are ordinary non-hold-raw. + 1 + } + + pub fn detail(&self) -> &str { + match self { + Self::Usage(detail) + | Self::Image(detail) + | Self::Wire(detail) + | Self::Metadata(detail) + | Self::Output(detail) => detail, + } + } +} + +pub fn error_json_line(error: &DepictError) -> String { + json!({"schema": ERROR_SCHEMA, "reason": error.reason(), "detail": error.detail()}).to_string() +} + +pub fn parse_args(args: &[OsString]) -> Result { + match args { + [image] => Ok(Arguments { + image_path: PathBuf::from(image), + redo: false, + }), + [image, redo] if redo == "--redo" => Ok(Arguments { + image_path: PathBuf::from(image), + redo: true, + }), + _ => Err(DepictError::Usage(USAGE.to_owned())), + } +} + +#[derive(Debug, Clone, PartialEq)] +pub enum WireFailure { + NoEngine, + Failure { reason: String, detail: String }, +} + +pub trait WireClient { + fn describe(&self, image_png: &[u8]) -> Result; +} + +pub trait Detector { + fn detect(&self, full_png: &[u8]) -> Result, String>; +} + +pub struct SystemWireClient; + +impl WireClient for SystemWireClient { + fn describe(&self, image_png: &[u8]) -> Result { + let helper = sibling_executable("solstone-generate-wire").map_err(|detail| { + WireFailure::Failure { + reason: "wire-launch-failed".to_owned(), + detail, + } + })?; + let request = json!({ + "schema": REQUEST_SCHEMA, + "contents": [ + {"type": "text", "text": DESCRIPTION_PROMPT}, + {"type": "image", "data": base64::engine::general_purpose::STANDARD.encode(image_png), "mime_type": "image/png"} + ], + "context": "observe.depict" + }); + let mut child = Command::new(helper) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|error| WireFailure::Failure { + reason: "wire-launch-failed".to_owned(), + detail: error.to_string(), + })?; + if let Some(stdin) = child.stdin.as_mut() { + stdin + .write_all(request.to_string().as_bytes()) + .map_err(|error| WireFailure::Failure { + reason: "wire-write-failed".to_owned(), + detail: error.to_string(), + })?; + } + let output = child + .wait_with_output() + .map_err(|error| WireFailure::Failure { + reason: "wire-wait-failed".to_owned(), + detail: error.to_string(), + })?; + if output.status.success() { + let parsed: Value = + serde_json::from_slice(&output.stdout).map_err(|error| WireFailure::Failure { + reason: "malformed-response".to_owned(), + detail: error.to_string(), + })?; + let text = parsed + .get("result") + .and_then(Value::as_object) + .and_then(|result| result.get("text")) + .and_then(Value::as_str) + .ok_or_else(|| WireFailure::Failure { + reason: "malformed-response".to_owned(), + detail: "response has no result text".to_owned(), + })?; + if parsed.get("schema").and_then(Value::as_str) != Some(RESPONSE_SCHEMA) { + return Err(WireFailure::Failure { + reason: "malformed-response".to_owned(), + detail: "response schema is not supported".to_owned(), + }); + } + return Ok(text.to_owned()); + } + let error: Value = serde_json::from_slice(&output.stderr).unwrap_or(Value::Null); + let reason = error + .get("reason") + .and_then(Value::as_str) + .unwrap_or("wire-failed"); + let detail = error + .get("detail") + .and_then(Value::as_str) + .unwrap_or("generate wire failed"); + if reason == "no-engine-configured" && output.status.code() == Some(69) { + Err(WireFailure::NoEngine) + } else { + Err(WireFailure::Failure { + reason: reason.to_owned(), + detail: detail.to_owned(), + }) + } + } +} + +pub struct SystemDetector; + +impl Detector for SystemDetector { + fn detect(&self, full_png: &[u8]) -> Result, String> { + let query = query_rfdetr_paths()?; + if query.status != "installed" { + return Ok(None); + } + let (binary, model) = match (query.binary_path, query.model_path) { + (Some(binary), Some(model)) => (binary, model), + _ => return Err("installed RF-DETR query omitted paths".to_owned()), + }; + let temporary = DetectorTempDir::new()?; + let input = temporary.path.join("input.png"); + let output = temporary.path.join("output.json"); + fs::write(&input, full_png).map_err(|error| error.to_string())?; + let process = Command::new(binary) + .args(["detect", "--model"]) + .arg(model) + .args(["--input"]) + .arg(&input) + .args(["--output"]) + .arg(&output) + .args(["--threshold", "0.25", "--threads", "4"]) + .output() + .map_err(|error| error.to_string())?; + if !process.status.success() { + return Err("rfdetr-cli detect failed".to_owned()); + } + let parsed = serde_json::from_slice(&fs::read(output).map_err(|error| error.to_string())?) + .map_err(|error| error.to_string())?; + Ok(Some(parsed)) + } +} + +struct RfdetrPaths { + status: String, + binary_path: Option, + model_path: Option, +} + +fn query_rfdetr_paths() -> Result { + let python = sibling_python()?; + let output = Command::new(python) + .args(["-P", "-m", "solstone.observe.rfdetr_paths_query"]) + .output() + .map_err(|error| error.to_string())?; + if !output.status.success() { + return Err("RF-DETR install-state query failed".to_owned()); + } + let value: Value = serde_json::from_slice(&output.stdout).map_err(|error| error.to_string())?; + Ok(RfdetrPaths { + status: value + .get("status") + .and_then(Value::as_str) + .ok_or("RF-DETR query has no status")? + .to_owned(), + binary_path: value + .get("binary_path") + .and_then(Value::as_str) + .map(PathBuf::from), + model_path: value + .get("model_path") + .and_then(Value::as_str) + .map(PathBuf::from), + }) +} + +fn sibling_executable(name: &str) -> Result { + let current = env::current_exe().map_err(|error| error.to_string())?; + let candidate = current + .parent() + .ok_or("native executable has no parent")? + .join(name); + if candidate.is_file() { + Ok(candidate) + } else { + Err(format!( + "missing sibling executable {}", + candidate.display() + )) + } +} + +fn sibling_python() -> Result { + let current = env::current_exe().map_err(|error| error.to_string())?; + let directory = current.parent().ok_or("native executable has no parent")?; + for name in ["python3", "python"] { + let candidate = directory.join(name); + if candidate.is_file() { + return Ok(candidate); + } + } + Err("missing sibling Python interpreter".to_owned()) +} + +struct DetectorTempDir { + path: PathBuf, +} + +impl DetectorTempDir { + fn new() -> Result { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|error| error.to_string())? + .as_nanos(); + let path = env::temp_dir().join(format!("rfdetr_{}_{nonce}", std::process::id())); + fs::create_dir(&path).map_err(|error| error.to_string())?; + Ok(Self { path }) + } +} + +impl Drop for DetectorTempDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RunOutcome { + Written, + Skipped, + NoEngine, +} + +pub fn run_with_clients( + image_path: &Path, + redo: bool, + wire: &dyn WireClient, + detector: &dyn Detector, +) -> Result { + validate_image_path(image_path)?; + let output_path = image_path.with_extension("jsonl"); + if output_path.exists() && !redo { + return Ok(RunOutcome::Skipped); + } + let source = fs::read(image_path).map_err(|error| DepictError::Image(error.to_string()))?; + let image = + image::load_from_memory(&source).map_err(|error| DepictError::Image(error.to_string()))?; + let full_png = encode_png(&image)?; + let prepared = resize_for_vlm(image); + let prepared_png = encode_png(&prepared)?; + let description = match wire.describe(&prepared_png) { + Ok(description) => description.trim().to_owned(), + Err(WireFailure::NoEngine) => return Ok(RunOutcome::NoEngine), + Err(WireFailure::Failure { reason, detail }) => { + return Err(DepictError::Wire(format!("{reason}: {detail}"))); + } + }; + let header = build_header(&image_path.file_name().unwrap_or_default().to_string_lossy())?; + let mut entry = Map::new(); + entry.insert("start".to_owned(), Value::String("00:00:00".to_owned())); + entry.insert("text".to_owned(), Value::String(description)); + match detector.detect(&full_png) { + Ok(Some(result)) => match detections_block(result) { + Ok(block) => { + entry.insert("detections".to_owned(), block); + } + Err(error) => eprintln!("native depict: detection output omitted: {error}"), + }, + Ok(None) => {} + Err(error) => eprintln!("native depict: detection omitted: {error}"), + } + write_jsonl( + &output_path, + [Value::Object(header), Value::Object(entry)], + AtomicWriteOptions::default(), + ) + .map_err(|error| DepictError::Output(error.to_string()))?; + Ok(RunOutcome::Written) +} + +pub fn resize_for_vlm(image: DynamicImage) -> DynamicImage { + if image.width().max(image.height()) <= MAX_VLM_DIM { + image + } else { + image.thumbnail(MAX_VLM_DIM, MAX_VLM_DIM) + } +} + +fn encode_png(image: &DynamicImage) -> Result, DepictError> { + let mut bytes = Cursor::new(Vec::new()); + image + .write_to(&mut bytes, ImageFormat::Png) + .map_err(|error| DepictError::Image(error.to_string()))?; + Ok(bytes.into_inner()) +} + +fn validate_image_path(path: &Path) -> Result<(), DepictError> { + if !path.is_file() { + return Err(DepictError::Usage(format!( + "Image not found: {}", + path.display() + ))); + } + let parent = path + .parent() + .and_then(Path::file_name) + .and_then(|name| name.to_str()) + .unwrap_or_default(); + if !is_segment_key(parent) { + return Err(DepictError::Usage(format!( + "Image must be in a segment directory (HHMMSS_LEN/), but parent is: {parent}" + ))); + } + Ok(()) +} + +fn is_segment_key(value: &str) -> bool { + let Some((clock, suffix)) = value.split_once('_') else { + return false; + }; + if clock.len() != 6 || !clock.bytes().all(|byte| byte.is_ascii_digit()) { + return false; + } + let digit_count = suffix + .bytes() + .take_while(|byte| byte.is_ascii_digit()) + .count(); + digit_count > 0 && (digit_count == suffix.len() || suffix.as_bytes()[digit_count] == b'_') +} + +pub fn build_header(raw_name: &str) -> Result, DepictError> { + let observer = env::var("OBSERVER_NAME").ok(); + let segment_meta = env::var("SEGMENT_META").ok(); + build_header_from_values(raw_name, observer.as_deref(), segment_meta.as_deref()) +} + +fn build_header_from_values( + raw_name: &str, + observer: Option<&str>, + segment_meta: Option<&str>, +) -> Result, DepictError> { + let mut header = Map::new(); + header.insert("raw".to_owned(), Value::String(raw_name.to_owned())); + header.insert("kind".to_owned(), Value::String("image".to_owned())); + if let Some(observer) = observer.filter(|observer| !observer.is_empty()) { + header.insert("observer".to_owned(), Value::String(observer.to_owned())); + } + if let Some(segment_meta) = segment_meta.filter(|segment_meta| !segment_meta.is_empty()) { + match serde_json::from_str::(segment_meta) { + Ok(Value::Object(meta)) => header.extend(meta), + Ok(_) => { + return Err(DepictError::Metadata( + "SEGMENT_META must be an object".to_owned(), + )); + } + Err(_) => eprintln!("native depict: invalid SEGMENT_META JSON"), + } + } + Ok(header) +} + +fn detections_block(result: Value) -> Result { + let object = result + .as_object() + .ok_or("detector output is not an object")?; + let image = object + .get("image") + .ok_or("detector output has no image")? + .clone(); + let objects = object + .get("detections") + .ok_or("detector output has no detections")? + .clone(); + Ok(json!({ + "engine": ENGINE_NAME, + "engine_ref": ENGINE_REF, + "model": MODEL_NAME, + "threshold": THRESHOLD, + "source": "still", + "gate": "still", + "image": image, + "objects": objects, + })) +} + +pub fn run(arguments: Arguments) -> Result { + run_with_clients( + &arguments.image_path, + arguments.redo, + &SystemWireClient, + &SystemDetector, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use image::{ImageBuffer, Rgb}; + + struct SuccessWire; + impl WireClient for SuccessWire { + fn describe(&self, _: &[u8]) -> Result { + Ok(" detail ".to_owned()) + } + } + struct NoEngineWire; + impl WireClient for NoEngineWire { + fn describe(&self, _: &[u8]) -> Result { + Err(WireFailure::NoEngine) + } + } + struct FailingWire; + impl WireClient for FailingWire { + fn describe(&self, _: &[u8]) -> Result { + Err(WireFailure::Failure { + reason: "incomplete-text".to_owned(), + detail: "bad".to_owned(), + }) + } + } + struct NoDetector; + impl Detector for NoDetector { + fn detect(&self, _: &[u8]) -> Result, String> { + Ok(None) + } + } + struct BrokenDetector; + impl Detector for BrokenDetector { + fn detect(&self, _: &[u8]) -> Result, String> { + Err("unavailable".to_owned()) + } + } + struct CannedDetector; + impl Detector for CannedDetector { + fn detect(&self, _: &[u8]) -> Result, String> { + Ok(Some( + json!({"image":{"width":4,"height":4},"detections":[{"class_name":"bottle"}]}), + )) + } + } + + fn fixture_image() -> (PathBuf, PathBuf) { + let root = env::temp_dir().join(format!( + "depict-test-{}", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let segment = root.join("123456_300"); + fs::create_dir_all(&segment).unwrap(); + let image = segment.join("photo.png"); + ImageBuffer::, _>::from_pixel(4, 4, Rgb([255, 0, 0])) + .save(&image) + .unwrap(); + (root, image) + } + + #[test] + fn header_merges_metadata_and_rejects_non_object() { + let header = build_header_from_values( + "photo.png", + Some("camera"), + Some(r#"{"stream":"default","kind":"override"}"#), + ) + .unwrap(); + assert_eq!(header["raw"], "photo.png"); + assert_eq!(header["kind"], "override"); + assert_eq!(header["observer"], "camera"); + assert!(build_header_from_values("photo.png", None, Some("not-json")).is_ok()); + assert!(matches!( + build_header_from_values("photo.png", None, Some("[]")), + Err(DepictError::Metadata(_)) + )); + } + + #[test] + fn skip_redo_and_no_engine_preserve_output_rules() { + let (root, image) = fixture_image(); + let output = image.with_extension("jsonl"); + fs::write(&output, "old\n").unwrap(); + assert_eq!( + run_with_clients(&image, false, &SuccessWire, &NoDetector).unwrap(), + RunOutcome::Skipped + ); + assert_eq!( + run_with_clients(&image, true, &SuccessWire, &NoDetector).unwrap(), + RunOutcome::Written + ); + fs::remove_file(&output).unwrap(); + assert_eq!( + run_with_clients(&image, false, &NoEngineWire, &NoDetector).unwrap(), + RunOutcome::NoEngine + ); + assert!(!output.exists()); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn wire_failures_do_not_write_and_detection_is_fail_open() { + let (root, image) = fixture_image(); + let output = image.with_extension("jsonl"); + assert!(matches!( + run_with_clients(&image, false, &FailingWire, &NoDetector), + Err(DepictError::Wire(_)) + )); + assert!(!output.exists()); + assert_eq!( + run_with_clients(&image, false, &SuccessWire, &BrokenDetector).unwrap(), + RunOutcome::Written + ); + let rows: Vec = fs::read_to_string(&output) + .unwrap() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect(); + assert!(rows[1].get("detections").is_none()); + assert_eq!( + run_with_clients(&image, true, &SuccessWire, &CannedDetector).unwrap(), + RunOutcome::Written + ); + let rows: Vec = fs::read_to_string(&output) + .unwrap() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect(); + assert_eq!(rows[1]["detections"]["gate"], "still"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn exit_codes_are_never_hold_raw() { + for error in [ + DepictError::Usage("x".to_owned()), + DepictError::Image("x".to_owned()), + DepictError::Wire("x".to_owned()), + DepictError::Metadata("x".to_owned()), + DepictError::Output("x".to_owned()), + ] { + assert_eq!(error.exit_code(), 1); + } + } +} diff --git a/core/crates/solstone-core-depict/src/main.rs b/core/crates/solstone-core-depict/src/main.rs new file mode 100644 index 000000000..ab73595da --- /dev/null +++ b/core/crates/solstone-core-depict/src/main.rs @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright (c) 2026 sol pbc + +use std::env; +use std::process; + +use solstone_core_depict::{error_json_line, parse_args, run}; + +fn main() { + let args: Vec<_> = env::args_os().skip(1).collect(); + match parse_args(&args).and_then(run) { + Ok(_) => {} + Err(error) => { + eprintln!("{}", error_json_line(&error)); + process::exit(error.exit_code()); + } + } +} diff --git a/solstone/observe/rfdetr_paths_query.py b/solstone/observe/rfdetr_paths_query.py new file mode 100644 index 000000000..c16da59f0 --- /dev/null +++ b/solstone/observe/rfdetr_paths_query.py @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +"""Private one-record, read-only RF-DETR install-state query for native helpers.""" + +from __future__ import annotations + +import json + +from solstone.think.providers.rfdetr_install import rfdetr_paths + + +def main() -> None: + paths = rfdetr_paths() + print( + json.dumps( + { + "status": paths.status, + "binary_path": str(paths.binary_path) if paths.binary_path else None, + "model_path": str(paths.model_path) if paths.model_path else None, + } + ) + ) + + +if __name__ == "__main__": + main() diff --git a/tests/test_depict_differential.py b/tests/test_depict_differential.py new file mode 100644 index 000000000..53da9c074 --- /dev/null +++ b/tests/test_depict_differential.py @@ -0,0 +1,106 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +"""Parsed-JSON parity between Python depict and the standalone native helper.""" + +from __future__ import annotations + +import json +import os +import shutil +import stat +import subprocess +from pathlib import Path + +from PIL import Image + +from solstone.observe import depict + +ROOT = Path(__file__).parents[1] + + +def _image(root: Path, name: str) -> Path: + segment = root / "chronicle" / "20240101" / "default" / "123456_300" + segment.mkdir(parents=True) + path = segment / name + Image.new("RGB", (4, 4), "red").save(path) + return path + + +def _executable(path: Path, text: str) -> None: + path.write_text(text, encoding="utf-8") + path.chmod(path.stat().st_mode | stat.S_IXUSR) + + +def _native_copy(tmp_path: Path) -> Path: + subprocess.run(["cargo", "build", "-p", "solstone-core-depict"], cwd=ROOT / "core", check=True) + source = ROOT / "core" / "target" / "debug" / "solstone-core-depict" + if not source.exists(): + source = ROOT / "target" / "debug" / "solstone-core-depict" + target = tmp_path / "bin" / "solstone-core-depict" + target.parent.mkdir() + shutil.copy2(source, target) + target.chmod(target.stat().st_mode | stat.S_IXUSR) + return target + + +def _install_native_stubs(binary: Path, detector: Path | None = None, description: str = "A concise image description") -> None: + directory = binary.parent + _executable( + directory / "solstone-generate-wire", + f"#!/usr/bin/env python3\nimport json, sys\njson.load(sys.stdin)\nprint(json.dumps({{'schema':'solstone-generate-response-v1','result':{{'text':{description!r}}}}}))\n", + ) + if detector is None: + _executable(directory / "python3", "#!/bin/sh\nprintf '%s\\n' '{\"status\":\"not_installed\",\"binary_path\":null,\"model_path\":null}'\n") + else: + _executable( + directory / "python3", + f"#!/bin/sh\nprintf '%s\\n' '{{\"status\":\"installed\",\"binary_path\":{json.dumps(str(detector))},\"model_path\":\"model.bin\"}}'\n", + ) + + +def _rows(path: Path) -> list[dict]: + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] + + +def test_native_depict_matches_python_values_with_metadata_and_detection(tmp_path, monkeypatch): + py_image = _image(tmp_path / "python", "photo.png") + native_image = _image(tmp_path / "native", "photo.png") + monkeypatch.setenv("OBSERVER_NAME", "camera") + monkeypatch.setenv("SEGMENT_META", json.dumps({"stream": "default", "facet": "personal"})) + canned = {"image": {"width": 4, "height": 4}, "detections": [{"class_name": "bottle", "score": 0.67}]} + monkeypatch.setattr(depict, "generate", lambda **_: "A concise image description") + monkeypatch.setattr(depict, "detect_objects", lambda _: canned) + depict.run(py_image) + + detector = tmp_path / "detector" + _executable(detector, "#!/usr/bin/env python3\nimport json, sys\nout=sys.argv[sys.argv.index('--output')+1]\njson.dump({'image':{'width':4,'height':4},'detections':[{'class_name':'bottle','score':0.67}]},open(out,'w'))\n") + binary = _native_copy(tmp_path) + _install_native_stubs(binary, detector) + subprocess.run([binary, native_image], check=True, env=os.environ.copy()) + assert _rows(native_image.with_suffix(".jsonl")) == _rows(py_image.with_suffix(".jsonl")) + + +def test_native_and_python_agree_on_skip_redo_and_no_engine(tmp_path, monkeypatch): + py_image = _image(tmp_path / "python", "photo.png") + native_image = _image(tmp_path / "native", "photo.png") + py_output, native_output = py_image.with_suffix(".jsonl"), native_image.with_suffix(".jsonl") + py_output.write_text("old\n", encoding="utf-8") + native_output.write_text("old\n", encoding="utf-8") + monkeypatch.setattr(depict, "generate", lambda **_: "replacement") + assert depict.run(py_image) is None + binary = _native_copy(tmp_path) + _install_native_stubs(binary) + subprocess.run([binary, native_image], check=True) + assert py_output.read_text() == native_output.read_text() == "old\n" + depict.run(py_image, redo=True) + _install_native_stubs(binary, description="replacement") + subprocess.run([binary, native_image, "--redo"], check=True) + assert _rows(py_output) == _rows(native_output) + + py_output.unlink(); native_output.unlink() + monkeypatch.setattr(depict, "generate", lambda **_: (_ for _ in ()).throw(depict.NoBrainConfiguredError())) + assert depict.run(py_image) is None + _executable(binary.parent / "solstone-generate-wire", "#!/bin/sh\nprintf '%s\\n' '{\"schema\":\"solstone-generate-error-v1\",\"reason\":\"no-engine-configured\",\"detail\":\"none\"}' >&2\nexit 69\n") + subprocess.run([binary, native_image], check=True) + assert not py_output.exists() and not native_output.exists()