From 76edc5c8bfa3b7fb14397ccf06cae2ebb8b2f77a Mon Sep 17 00:00:00 2001 From: dietrich ayala Date: Mon, 28 Sep 2026 16:51:16 +0200 Subject: [PATCH] Kimi as a declared provider: alias provider in folknet.json, pi compat from configuration The local end-to-end environment runs jobs on a Kimi coding-plan key while the rig is down. Three things kept that from working by configuration alone; each is now a configuration value that folknet leaves unset, so folknet and the development box run one code path: - jobs/src/inventory.rs used_providers() counted every pi-backed model as the self-hosted `gateway`. It now takes a named model's provider from its alias's `provider` in bench/folknet.json (alias_provider(), default `gateway`), so a Kimi job is checked and recorded under its own provider. The `gateway` service is needed only when a `gateway` model is named. - bench/folknet.json declares `kimi-for-coding` (provider `kimi`) and the `kimi` provider, billed by subscription because the key is a coding-plan key. - Kimi refuses pi's `developer` role. jobs/scripts/pi-session.sh passes NEV_MODEL_COMPAT, a JSON object from the environment or .env, to pi as the model's `compat`; the box's ~/.lane-model-env sets it. The test comparing allowed providers with manifest PROVIDERS leaves out providers only an alias names. clear_agent_env() removes NEV_MODEL_COMPAT so a lane that exports it does not change what the jobs tests see. Verified through the development box's lane: cargo test -p jobs, and the harness folknet and wording tests. The fixture Dagu job passed on kimi-for-coding. --- bench/folknet.json | 4 +++- docs/folknet-inventory.md | 12 ++++++++++ jobs/scripts/pi-session.sh | 31 +++++++++++++++++++------ jobs/src/inventory.rs | 46 ++++++++++++++++++++++++++++---------- jobs/tests/common/job.rs | 6 +++-- jobs/tests/inventory.rs | 44 +++++++++++++++++++++++++++++++++--- jobs/tests/pi_review.rs | 17 ++++++++++++++ 7 files changed, 135 insertions(+), 25 deletions(-) diff --git a/bench/folknet.json b/bench/folknet.json index 704da36d..102dafff 100644 --- a/bench/folknet.json +++ b/bench/folknet.json @@ -76,11 +76,13 @@ "glm-5.3-flash-exl3": { "pool": "pool:glm-arm-pairs", "max_in_flight": null, "state": "retired", "retired_on": "2026-09-22", "removed_from_gateway_on": "2026-09-25" - } + }, + "kimi-for-coding": { "pool": null, "provider": "kimi", "max_in_flight": null, "state": "serving" } }, "providers": { "gateway": { "billing": "self-hosted", "allowed": true, "rule": "the project's own rig, through the gateway key" }, "claude-code": { "billing": "subscription", "allowed": true, "rule": "subscription agent logged in on the lane" }, + "kimi": { "billing": "subscription", "allowed": true, "rule": "Kimi coding plan through KIMI_TOKEN, reached at the lane's NEV_LANE_MODEL_URL; for development and testing while the rig is down" }, "anthropic-api": { "billing": "metered", "allowed": false, "rule": "no metered Anthropic API without a person's approval" } } } diff --git a/docs/folknet-inventory.md b/docs/folknet-inventory.md index 6e68cbea..a0723c91 100644 --- a/docs/folknet-inventory.md +++ b/docs/folknet-inventory.md @@ -272,6 +272,18 @@ that the allowed set declared in the file equals that list, so the code and the declaration cannot drift apart. There is no path by which a job obtains approval: a use that needs it happens outside jobs, by a person. +A model served by a remote OpenAI-compatible endpoint rather than the rig (Kimi, for +development and testing while the rig is down) is declared as an alias like the rig's, +with a `provider` naming its own `providers` entry (`kimi-for-coding` names `kimi`, billed +by subscription). The check counts a named model under its alias's `provider`, `gateway` +when there is none (`jobs/src/inventory.rs` `alias_provider()`), so a job on such a model +is checked and recorded as that provider, not as the self-hosted rig. Such providers are +reached only through an alias, never named by a manifest, so the test comparing the +allowed set with `PROVIDERS` leaves them out. Which endpoint a lane's jobs reach is the +lane's `NEV_LANE_MODEL_URL` in `~/.lane-model-env`; one that pi talks to differently from +what it guesses from the URL also sets `NEV_MODEL_COMPAT` there +(`jobs/scripts/pi-session.sh`). + Availability of a subscription provider is probed without spending a model call: the harness's own login check on the lane, hourly. Its usage limits are not tracked; a stage that hits one fails like any other stage failure. diff --git a/jobs/scripts/pi-session.sh b/jobs/scripts/pi-session.sh index 0983c75f..77fd18de 100755 --- a/jobs/scripts/pi-session.sh +++ b/jobs/scripts/pi-session.sh @@ -18,6 +18,12 @@ # rewritten .env's NEV_MODEL_URL to the gateway's address as the lane reaches it, and the # key arrived in the .env bench/remote-lane.sh shipped. # +# NEV_MODEL_COMPAT (else GENERA_MODEL_COMPAT; environment, else ./.env), when set, is a +# JSON object given to pi as the model's `compat`, pi's settings for an OpenAI-compatible +# endpoint that differs from what pi guesses from the URL. The rig's gateway needs none; an +# endpoint that refuses the `developer` role takes '{"supportsDeveloperRole":false}'. On a +# lane it comes from ~/.lane-model-env, which the Dagu steps source. +# # pi gets a configuration directory of its own (PI_CODING_AGENT_DIR), in # $JOB_STATE_DIR/pi/ (else a temporary directory), holding a models.json (mode # 600) with one provider, `folknet`, of api `openai-completions` at /v1 and one model. @@ -87,6 +93,7 @@ setting() { } URL="$(setting MODEL_URL)" KEY="$(setting MODEL_API_KEY)" +COMPAT="$(setting MODEL_COMPAT)" [ -n "$URL" ] || { echo "pi-session: no NEV_MODEL_URL in the environment or ./.env" >&2; exit 2; } URL="${URL%/}" case "$URL" in */v1) ;; *) URL="$URL/v1" ;; esac @@ -117,18 +124,28 @@ fi chmod 700 "$DIR" ( umask 077 - python3 - "$DIR/models.json" "$URL" "$MODEL" "$CONTEXT_WINDOW" "$MAX_TOKENS" <<'PY' + python3 - "$DIR/models.json" "$URL" "$MODEL" "$CONTEXT_WINDOW" "$MAX_TOKENS" "$COMPAT" <<'PY' import json, sys -path, url, model, window, max_tokens = sys.argv[1:] +path, url, model, window, max_tokens, compat = sys.argv[1:] +entry = { + "id": model, "name": model, "reasoning": True, "input": ["text"], + "contextWindow": int(window), "maxTokens": int(max_tokens), + "cost": {"input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0}, +} +if compat: + try: + compat = json.loads(compat) + except ValueError: + compat = None + if not isinstance(compat, dict): + print("pi-session: NEV_MODEL_COMPAT is not a JSON object", file=sys.stderr) + sys.exit(2) + entry["compat"] = compat config = {"providers": {"folknet": { "baseUrl": url, "api": "openai-completions", "apiKey": "$JOB_GATEWAY_API_KEY", - "models": [{ - "id": model, "name": model, "reasoning": True, "input": ["text"], - "contextWindow": int(window), "maxTokens": int(max_tokens), - "cost": {"input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0}, - }], + "models": [entry], }}} with open(path, "w") as f: json.dump(config, f, indent=2) diff --git a/jobs/src/inventory.rs b/jobs/src/inventory.rs index 706ce686..be5f1875 100644 --- a/jobs/src/inventory.rs +++ b/jobs/src/inventory.rs @@ -20,10 +20,12 @@ //! `model` names a model of the subscription, not a gateway alias, and is covered by the //! provider rule instead; //! - a provider an agent uses that the declared file does not list or has `allowed: false`, -//! or whose snapshot entry (when the snapshot has a `providers` section) is not `ready`; +//! or whose snapshot entry (when the snapshot has a `providers` section) is not `ready`. +//! A named model's provider is its alias's `provider` in the declared file, `gateway` +//! when it gives none ([`alias_provider`]); //! - a service the stages need, when the snapshot has a `services` section, that is missing -//! there or not `ready`: `gateway` when any gateway model is named, `nas` when a stage has -//! a `ledger` check. +//! there or not `ready`: `gateway` when a model whose provider is `gateway` is named, +//! `nas` when a stage has a `ledger` check. //! //! The probe writes `models` only today; `providers` and `services` are checked once a //! snapshot carries them, so a snapshot from today's probe is not refused for lacking @@ -128,24 +130,44 @@ pub fn named_models(manifest: &Manifest) -> Vec { out } -/// The providers the manifest's agents use: `gateway` for a gateway agent and for a -/// command agent that names a model (a coding-agent session through the gateway), -/// `claude-code` for a claude-code agent, and the top-level `provider`. -pub fn used_providers(manifest: &Manifest) -> Vec { +/// The provider that bills for `alias`: its `provider` in the declared file's `aliases`, +/// else `gateway` (the rig's own models, reached through the gateway key). An alias served +/// by a remote endpoint (Kimi, while the rig is down) names its own provider there, so the +/// check and the record say who bills for it rather than counting it as self-hosted. +pub fn alias_provider<'a>(declared: &'a Value, alias: &str) -> &'a str { + declared + .get("aliases") + .and_then(|a| a.get(alias)) + .and_then(|a| a.get("provider")) + .and_then(Value::as_str) + .unwrap_or("gateway") +} + +/// The providers the manifest's agents use: for every model [`named_models`] finds, the +/// provider the declared file gives its alias ([`alias_provider`]); `gateway` for a gateway +/// agent that names no models; `claude-code` for a claude-code agent; and the top-level +/// `provider`, except that a top-level `gateway` whose models are named is replaced by those +/// models' providers. +pub fn used_providers(manifest: &Manifest, declared: &Value) -> Vec { let mut out: Vec = Vec::new(); let mut add = |p: &str| { if !out.iter().any(|x| x == p) { out.push(p.to_string()); } }; + let top_models_named = !manifest.agents.contains_key(CANDIDATES_ROLE) && manifest.models.is_some(); if let Some(p) = &manifest.provider { - add(p); + if !(p == "gateway" && top_models_named) { + add(p); + } + } + for m in named_models(manifest) { + add(alias_provider(declared, &m.alias)); } for agent in manifest.agents.values() { match agent.kind.as_str() { - "gateway" => add("gateway"), + "gateway" if agent.models.is_none() => add("gateway"), "claude-code" => add("claude-code"), - "command" if agent.model.is_some() => add("gateway"), _ => {} } } @@ -285,7 +307,7 @@ pub fn check(manifest: &Manifest, declared: &Value, snapshot: &Value, now: i64) } let snap_providers = snapshot.get("providers").and_then(Value::as_object); - for p in used_providers(manifest) { + for p in used_providers(manifest, declared) { match providers.get(&p) { None => permanent(&mut report, format!("provider `{p}` is not declared in bench/folknet.json")), Some(entry) if entry.get("allowed").and_then(Value::as_bool) != Some(true) => { @@ -304,7 +326,7 @@ pub fn check(manifest: &Manifest, declared: &Value, snapshot: &Value, now: i64) } let mut needed: BTreeMap<&str, &str> = BTreeMap::new(); - if !named.is_empty() { + if named.iter().any(|m| alias_provider(declared, &m.alias) == "gateway") { needed.insert("gateway", "the job names a gateway model"); } if manifest.stages.iter().any(|s| s.checks.iter().any(|c| c.check == "ledger")) { diff --git a/jobs/tests/common/job.rs b/jobs/tests/common/job.rs index d7e5e2ce..8b3272f5 100644 --- a/jobs/tests/common/job.rs +++ b/jobs/tests/common/job.rs @@ -174,12 +174,14 @@ pub use jobs::runner::AGENT_VARS; /// would outside any job. `job-run` also removes them from every stage it runs; this keeps /// the tests independent of that. Also removes the job-events settings: a test never posts /// to a real job-events store (a writer running the workspace tests inside a real job has -/// them set), and picks its own attempt id. +/// them set), and picks its own attempt id. And the lane's pi settings for its model +/// endpoint (`~/.lane-model-env` exports `NEV_MODEL_COMPAT` where the endpoint needs it), +/// so what pi is given under test does not depend on the lane. pub fn clear_agent_env(cmd: &mut Command) { for var in AGENT_VARS { cmd.env_remove(var); } - for var in ["NEV_JOB_EVENTS_URL", "NEV_JOB_EVENTS_TOKEN", "DAG_RUN_ID"] { + for var in ["NEV_JOB_EVENTS_URL", "NEV_JOB_EVENTS_TOKEN", "DAG_RUN_ID", "NEV_MODEL_COMPAT", "GENERA_MODEL_COMPAT"] { cmd.env_remove(var); } } diff --git a/jobs/tests/inventory.rs b/jobs/tests/inventory.rs index 43e0bb61..6dbeebf4 100644 --- a/jobs/tests/inventory.rs +++ b/jobs/tests/inventory.rs @@ -187,6 +187,33 @@ fn a_provider_not_allowed_is_refused() { assert_fail(o, "provider `gateway` is not allowed: the project's own rig"); } +/// A model whose alias names its own `provider` in the declared file (Kimi) is checked +/// under that provider, not as the self-hosted `gateway`: disallowing `kimi` refuses the +/// job, disallowing `gateway` does not, and a failing `gateway` service does not matter. +#[test] +fn an_alias_provider_is_the_provider_checked() { + let dir = TempDir::new("inventory-alias-provider"); + let manifest = fixture_job(dir.path(), "kimi-for-coding"); + let mut snap: Value = serde_json::from_str(&std::fs::read_to_string(snapshot("models-only")).unwrap()).unwrap(); + snap["models"]["kimi-for-coding"] = serde_json::json!({ + "state": "serving", "listed": true, "status": "ready", + "observed_at": "2026-09-25T17:59:46Z", "since": "2026-09-25T09:00:00Z", + "last_ready_at": "2026-09-25T17:59:46Z", "max_in_flight": null + }); + snap["services"] = serde_json::json!({ "gateway": { "status": "failing" } }); + let snap_path = dir.path().join("kimi.inventory.json"); + std::fs::write(&snap_path, serde_json::to_string(&snap).unwrap()).unwrap(); + let with = |provider: &str, allowed: bool| { + let mut decl: Value = serde_json::from_str(&std::fs::read_to_string(declared()).unwrap()).unwrap(); + decl["providers"][provider]["allowed"] = Value::Bool(allowed); + let path = dir.path().join(format!("folknet-{provider}-{allowed}.json")); + std::fs::write(&path, serde_json::to_string(&decl).unwrap()).unwrap(); + checks::inventory(&manifest, &snap_path, &path, now()) + }; + assert_pass(&with("gateway", false), "kimi-for-coding ready"); + assert_fail(with("kimi", false), "provider `kimi` is not allowed"); +} + #[test] fn unreadable_snapshot_cannot_run() { let dir = TempDir::new("inventory-unreadable"); @@ -203,15 +230,26 @@ fn unreadable_snapshot_cannot_run() { assert!(o.message().contains("format is not 1"), "{o:?}"); } -/// The providers `bench/folknet.json` marks allowed are exactly the ones a manifest may -/// name, so the code and the declaration cannot drift apart. +/// The providers `bench/folknet.json` marks allowed, less those only an alias names (a +/// remote endpoint's models, reached as gateway models), are exactly the ones a manifest may +/// name, so the code and the declaration cannot drift apart. Every provider an alias names +/// is declared. #[test] fn declared_allowed_providers_equal_manifest_providers() { let decl: Value = serde_json::from_str(&std::fs::read_to_string(declared()).unwrap()).unwrap(); let providers = decl["providers"].as_object().expect("bench/folknet.json has a providers object"); + let by_alias: BTreeSet<&str> = decl["aliases"] + .as_object() + .expect("bench/folknet.json has an aliases object") + .values() + .filter_map(|a| a["provider"].as_str()) + .collect(); + for p in &by_alias { + assert!(providers.contains_key(*p), "alias provider `{p}` is not declared under providers"); + } let allowed: BTreeSet<&str> = providers .iter() - .filter(|(_, p)| p["allowed"].as_bool() == Some(true)) + .filter(|(name, p)| p["allowed"].as_bool() == Some(true) && !by_alias.contains(name.as_str())) .map(|(name, _)| name.as_str()) .collect(); let code: BTreeSet<&str> = jobs::manifest::PROVIDERS.into_iter().collect(); diff --git a/jobs/tests/pi_review.rs b/jobs/tests/pi_review.rs index d821a198..8801da4a 100644 --- a/jobs/tests/pi_review.rs +++ b/jobs/tests/pi_review.rs @@ -142,6 +142,7 @@ fn a_verdict_line_becomes_review_json_and_the_job_passes() { let models: Value = serde_json::from_str(&job.read_state_file("stub-pi-models.json")).unwrap(); assert_eq!(models["providers"]["folknet"]["baseUrl"], format!("{}/v1", common::stub_gateway_url())); assert!(!job.read_state_file("stub-pi-models.json").contains("stub-key"), "the key is not written"); + assert!(models["providers"]["folknet"]["models"][0].get("compat").is_none(), "{models}"); // The prompt carries the spec's path, the commit, the diff and the earlier stage. let prompt = job.read_state_file("stub-pi-stdin"); @@ -152,6 +153,22 @@ fn a_verdict_line_becomes_review_json_and_the_job_passes() { assert!(prompt.contains(&base), "{prompt}"); } +/// NEV_MODEL_COMPAT reaches pi as the model's `compat`, for an endpoint that differs from +/// what pi guesses from its URL (Kimi refuses the `developer` role). +#[test] +fn model_compat_settings_reach_pi() { + let (job, base) = job("pi-review-compat", "fixture-model-reviewer"); + let reply = job.tmp.path().join("reply"); + std::fs::write(&reply, "VERDICT: pass\n- Scope: in scope.\n").unwrap(); + let out = run(&job, &base, |c| { + with_stub(c, &reply); + c.env("NEV_MODEL_COMPAT", r#"{"supportsDeveloperRole":false}"#); + }); + assert_eq!(out.status.code(), Some(0), "{}", stderr(&out)); + let models: Value = serde_json::from_str(&job.read_state_file("stub-pi-models.json")).unwrap(); + assert_eq!(models["providers"]["folknet"]["models"][0]["compat"], serde_json::json!({"supportsDeveloperRole": false})); +} + #[test] fn a_fail_verdict_fails_the_job_at_review() { let (job, base) = job("pi-review-fail", "fixture-model-reviewer"); -- 2.54.0