diff --git a/README.md b/README.md index 051cf77..b9877f3 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ The current durable record formats are defined in [docs/schemas.md](docs/schemas - The worker initializes `$GIT_DIR/sip/logs/jobs.sqlite` with workflow execution, job, dependency, and FTS5 log tables. When `refs/sip/orchestrator` points at a commit containing `workflow.wasm` or - `workflow.wat`, the worker gives that module the selected workflow YAML and + `workflow.wat`, the worker lets that module read selected workflow files and records declared jobs and dependencies. - `examples/wasm-modules/basic` is a Rust guest module for the first host ABI. It reads a small YAML subset with either a `jobs:` map or `tasks:` list. diff --git a/docs/schemas.md b/docs/schemas.md index e6cad38..35a2d8a 100644 --- a/docs/schemas.md +++ b/docs/schemas.md @@ -128,13 +128,13 @@ The worker creates this database before processing queued events. It contains: Version 1 records workflow execution rows when run refs are created. If `refs/sip/orchestrator` points at a commit whose tree contains `workflow.wasm` -or `workflow.wat`, and the selected workflow commit contains -`.sip/workflows/sip.yml`, `.sip/workflows/ci.yml`, `.build.yml`, or the matching -`.yaml` variants, the worker passes that YAML to the module and records jobs -declared from its `_start` function. If the orchestrator ref does not exist, no -workflow module runs. If `SIP_EXECUTOR_MODULE` points at an executor WASM -module, the worker executes ready jobs through that module and stores output -lines in `job_logs`. +or `workflow.wat`, the worker runs that module against the selected workflow +commit and records jobs declared from its `_start` function. The host exposes +workflow-file read calls; choosing whether to read YAML, which filenames to +recognize, and how to parse them is a guest module detail. If the orchestrator +ref does not exist, no workflow module runs. If `SIP_EXECUTOR_MODULE` points at +an executor WASM module, the worker executes ready jobs through that module and +stores output lines in `job_logs`. ## WASM Job Declaration ABI @@ -142,8 +142,13 @@ The initial ABI is memory-based and intentionally small. Guest modules import these functions from `sip:workflow/env`: ```text -workflow_len() -> i32 -read_workflow(ptr: i32, len: i32) -> i32 +workflow_file_len(path_ptr: i32, path_len: i32) -> i32 +read_workflow_file( + path_ptr: i32, + path_len: i32, + dst_ptr: i32, + dst_len: i32, +) -> i32 declare_job( name_ptr: i32, name_len: i32, @@ -158,13 +163,18 @@ declare_job( ) -> i32 ``` -The guest exports `_start`. During `_start`, it reads the workflow YAML through -`workflow_len` and `read_workflow`, parses the subset it understands, and calls -`declare_job` once per job. `parent` and `requires` are job names in this first -slice; the host translates them to durable job IDs before writing SQLite rows. -`requires` is encoded as newline-delimited UTF-8 names. The basic module uses -`unsafe-host-shell` as the executor and passes the parsed command/script text as -UTF-8. +The guest exports `_start`. During `_start`, it may read files from the selected +workflow commit with `workflow_file_len` and `read_workflow_file`, parse the +format it understands, and call `declare_job` once per job. Version 1 serves +paths under `.sip/workflows/` plus the legacy root `.build.yml` and +`.build.yaml` paths. Missing or disallowed paths return `-1`. `parent` and +`requires` are job names in this first slice; the host translates them to +durable job IDs before writing SQLite rows. `requires` is encoded as +newline-delimited UTF-8 names. The basic module looks for +`.sip/workflows/sip.yml`, `.sip/workflows/sip.yaml`, `.sip/workflows/ci.yml`, +`.sip/workflows/ci.yaml`, `.build.yml`, and `.build.yaml`, uses +`unsafe-host-shell` as the executor, and passes the parsed command/script text +as UTF-8. ## WASM Executor ABI diff --git a/examples/wasm-modules/basic/src/lib.rs b/examples/wasm-modules/basic/src/lib.rs index 8f4b4ba..f842c66 100644 --- a/examples/wasm-modules/basic/src/lib.rs +++ b/examples/wasm-modules/basic/src/lib.rs @@ -1,9 +1,14 @@ #[link(wasm_import_module = "sip:workflow/env")] extern "C" { - #[link_name = "workflow_len"] - fn workflow_len() -> i32; - #[link_name = "read_workflow"] - fn read_workflow(ptr: *mut u8, len: i32) -> i32; + #[link_name = "workflow_file_len"] + fn workflow_file_len(path_ptr: *const u8, path_len: i32) -> i32; + #[link_name = "read_workflow_file"] + fn read_workflow_file( + path_ptr: *const u8, + path_len: i32, + dst_ptr: *mut u8, + dst_len: i32, + ) -> i32; #[link_name = "declare_job"] fn declare_job( name_ptr: *const u8, @@ -28,16 +33,32 @@ pub extern "C" fn _start() { } fn read_yaml() -> String { - let len = unsafe { workflow_len() }; - if len <= 0 { - return String::new(); - } - let mut bytes = vec![0; len as usize]; - let read = unsafe { read_workflow(bytes.as_mut_ptr(), len) }; - if read != len { - return String::new(); + for path in [ + ".sip/workflows/sip.yml", + ".sip/workflows/sip.yaml", + ".sip/workflows/ci.yml", + ".sip/workflows/ci.yaml", + ".build.yml", + ".build.yaml", + ] { + let len = unsafe { workflow_file_len(path.as_ptr(), path.len() as i32) }; + if len <= 0 { + continue; + } + let mut bytes = vec![0; len as usize]; + let read = unsafe { + read_workflow_file( + path.as_ptr(), + path.len() as i32, + bytes.as_mut_ptr(), + len, + ) + }; + if read == len { + return String::from_utf8(bytes).unwrap_or_default(); + } } - String::from_utf8(bytes).unwrap_or_default() + String::new() } fn declare_jobs(yaml: &str) { diff --git a/src/main.rs b/src/main.rs index 222e7be..67ec9a8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,7 +2,7 @@ use std::env; use std::ffi::OsStr; use std::fs; use std::io::{self, Write}; -use std::path::{Path, PathBuf}; +use std::path::{Component, Path, PathBuf}; use std::process::{Command, Stdio}; use wasmtime::{Caller, Engine, Linker, Module, Store}; @@ -87,7 +87,8 @@ struct DeclaredJob { #[derive(Debug, Default)] struct WorkflowHost { - yaml: Vec, + repo: PathBuf, + workflow_oid: String, jobs: Vec, } @@ -360,17 +361,12 @@ fn process_event( if workflow.oid != "none" { if let Some(module_path) = orchestrator_module_path(repo, dirs, name)? { let module_path = TempPath::new(module_path); - if let Some(workflow_yaml) = read_workflow_yaml(repo, &workflow)? { - let jobs = run_workflow_module( - module_path.as_ref(), - workflow_yaml.as_bytes().to_vec(), - )?; - workflow_ran = true; - let jobs = persist_declared_jobs(&db, &workflow_execution_id, &jobs)?; - if let Some(executor_path) = executor_module_path()? { - let checkout = checkout_workdir(repo, dirs, name, &workflow)?; - execute_declared_jobs(&executor_path, &db, &checkout, &jobs)?; - } + let jobs = run_workflow_module(module_path.as_ref(), repo, &workflow)?; + workflow_ran = true; + let jobs = persist_declared_jobs(&db, &workflow_execution_id, &jobs)?; + if let Some(executor_path) = executor_module_path()? { + let checkout = checkout_workdir(repo, dirs, name, &workflow)?; + execute_declared_jobs(&executor_path, &db, &checkout, &jobs)?; } } } @@ -514,23 +510,6 @@ fn orchestrator_module_path( Ok(None) } -fn read_workflow_yaml(repo: &Path, workflow: &Workflow) -> Result, String> { - for path in [ - ".sip/workflows/sip.yml", - ".sip/workflows/sip.yaml", - ".sip/workflows/ci.yml", - ".sip/workflows/ci.yaml", - ".build.yml", - ".build.yaml", - ] { - if let Some(body) = git_output(repo, ["show", format!("{}:{path}", workflow.oid).as_str()])? - { - return Ok(Some(body)); - } - } - Ok(None) -} - struct TempPath { path: PathBuf, } @@ -555,7 +534,8 @@ impl Drop for TempPath { fn run_workflow_module( module_path: &Path, - workflow_yaml: Vec, + repo: &Path, + workflow: &Workflow, ) -> Result, String> { let engine = Engine::default(); let module = Module::from_file(&engine, module_path).map_err(|err| { @@ -569,20 +549,45 @@ fn run_workflow_module( linker .func_wrap( "sip:workflow/env", - "workflow_len", - |caller: Caller<'_, WorkflowHost>| -> i32 { caller.data().yaml.len() as i32 }, + "workflow_file_len", + |mut caller: Caller<'_, WorkflowHost>, path_ptr: i32, path_len: i32| -> i32 { + let Some(path) = read_guest_string(&mut caller, path_ptr, path_len) else { + return -1; + }; + let repo = caller.data().repo.clone(); + let workflow_oid = caller.data().workflow_oid.clone(); + match workflow_file_bytes(&repo, &workflow_oid, &path) { + Ok(Some(bytes)) => bytes.len() as i32, + Ok(None) => -1, + Err(_) => -1, + } + }, ) - .map_err(|err| format!("could not define workflow_len host function: {err}"))?; + .map_err(|err| format!("could not define workflow_file_len host function: {err}"))?; linker .func_wrap( "sip:workflow/env", - "read_workflow", - |mut caller: Caller<'_, WorkflowHost>, ptr: i32, len: i32| -> i32 { - let yaml = caller.data().yaml.clone(); - write_guest_memory(&mut caller, ptr, len, &yaml) + "read_workflow_file", + |mut caller: Caller<'_, WorkflowHost>, + path_ptr: i32, + path_len: i32, + dst_ptr: i32, + dst_len: i32| + -> i32 { + let Some(path) = read_guest_string(&mut caller, path_ptr, path_len) else { + return -1; + }; + let repo = caller.data().repo.clone(); + let workflow_oid = caller.data().workflow_oid.clone(); + let bytes = match workflow_file_bytes(&repo, &workflow_oid, &path) { + Ok(Some(bytes)) => bytes, + Ok(None) => return -1, + Err(_) => return -1, + }; + write_guest_memory(&mut caller, dst_ptr, dst_len, &bytes) }, ) - .map_err(|err| format!("could not define read_workflow host function: {err}"))?; + .map_err(|err| format!("could not define read_workflow_file host function: {err}"))?; linker .func_wrap( "sip:workflow/env", @@ -619,7 +624,8 @@ fn run_workflow_module( let mut store = Store::new( &engine, WorkflowHost { - yaml: workflow_yaml, + repo: repo.to_path_buf(), + workflow_oid: workflow.oid.clone(), jobs: Vec::new(), }, ); @@ -636,6 +642,39 @@ fn run_workflow_module( Ok(std::mem::take(&mut store.data_mut().jobs)) } +fn workflow_file_bytes( + repo: &Path, + workflow_oid: &str, + path: &str, +) -> Result>, String> { + if !is_allowed_workflow_file(path) { + return Ok(None); + } + git_bytes(repo, ["show", format!("{workflow_oid}:{path}").as_str()]) +} + +fn is_allowed_workflow_file(path: &str) -> bool { + if path == ".build.yml" || path == ".build.yaml" { + return true; + } + let path = Path::new(path); + let mut components = path.components(); + if components.next() != Some(Component::Normal(OsStr::new(".sip"))) { + return false; + } + if components.next() != Some(Component::Normal(OsStr::new("workflows"))) { + return false; + } + let mut has_file_component = false; + for component in components { + match component { + Component::Normal(_) => has_file_component = true, + _ => return false, + } + } + has_file_component +} + fn write_guest_memory( caller: &mut Caller<'_, WorkflowHost>, ptr: i32,