diff --git a/Makefile b/Makefile index bd345a8e0..7af4f0eb2 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ export TMPDIR := /var/tmp PYTEST_BASETEMP_INIT := BASETEMP=$$(mktemp -d /var/tmp/solstone-pytest-XXXXXX); trap 'rm -rf "$$BASETEMP"' EXIT INT TERM; PYTEST_BASETEMP_FLAG := --basetemp "$$BASETEMP" -.PHONY: install uninstall test test-cov test-app test-only format format-check install-checks ci clean clean-install coverage watch versions update update-prices preflight pre-commit skills render-packaging openapi check-openapi contract check-contract dev all sandbox sandbox-stop install-models parakeet-helper parakeet-helper-clean wheel-macos wheel-macos-clean verify verify-api update-api-baselines service-logs check-layer-hygiene check-api-conventions check-journal-io-access check-journal-io-mechanic check-call-http-only check-tools-http-only check-access-imports-clean check-convey-bind-imports-clean check-thin-base-install check-cogitate-prompts smoke-cogitate release release-test FORCE +.PHONY: install uninstall test test-cov test-app test-only format format-check install-checks ci clean clean-install coverage watch versions update update-prices preflight pre-commit skills render-packaging openapi check-openapi contract check-contract dev all sandbox sandbox-stop install-models parakeet-helper parakeet-helper-clean wheel-macos wheel-macos-clean verify verify-api update-api-baselines eval-schemas service-logs check-layer-hygiene check-api-conventions check-journal-io-access check-journal-io-mechanic check-call-http-only check-tools-http-only check-access-imports-clean check-convey-bind-imports-clean check-schema-bounds check-thin-base-install check-cogitate-prompts smoke-cogitate release release-test FORCE # Default target - install package in editable mode all: install @@ -230,6 +230,9 @@ verify-api: .installed $(MAKE) sandbox-stop; \ exit $$RESULT +eval-schemas: .installed + $(VENV_BIN)/python tests/eval_schemas.py + # Regenerate API baseline files. By default uses the deterministic Flask # test-client path (frozen time). For sandbox-only endpoints (graph, search, # badge-count, updated-days), pass SANDBOX=1 to regenerate from the live @@ -410,6 +413,9 @@ install-checks: .installed @echo "=== Running call-http-only check ===" @$(MAKE) check-call-http-only @echo "" + @echo "=== Running schema-bounds check ===" + @$(MAKE) check-schema-bounds + @echo "" @echo "=== Running tools-http-only check ===" @$(MAKE) check-tools-http-only @echo "" @@ -512,6 +518,10 @@ check-journal-io-mechanic: .installed check-call-http-only: .installed $(VENV_BIN)/python scripts/check_call_http_only.py +# Generation schema bounds ratchet +check-schema-bounds: .installed + $(VENV_BIN)/python scripts/check_schema_bounds.py + # Built-in sol call tools HTTP-only gate check-tools-http-only: .installed $(VENV_BIN)/python scripts/check_tools_http_only.py diff --git a/docs/PROVIDERS.md b/docs/PROVIDERS.md index aeb67196c..b0fe44a7d 100644 --- a/docs/PROVIDERS.md +++ b/docs/PROVIDERS.md @@ -102,6 +102,20 @@ class GenerateResult(TypedDict, total=False): **Important:** Providers should gracefully ignore unsupported parameters rather than raising errors. +### Structured-output schema preparation + +Canonical generation schemas may contain local grammar bounds such as +`maxItems` and `maxLength`. The wrapper in `solstone/think/models.py` prepares a +provider-facing copy with `solstone/think/schema_prep.py` before calling a +provider. `STRICT_UNSUPPORTED_KEYWORDS` is the single support matrix for strict +cloud providers; local receives a canonical copy so llama.cpp can turn bounds +into grammar constraints. Response validation still uses the canonical schema. + +Use `make check-schema-bounds` to run the bounds ratchet for canonical schemas. +Use `make eval-schemas` to run the opt-in local llama.cpp structured-output +eval harness; it requires `journal install-provider local` and a running local +server via `journal start` or `journal service start`. + ## run_cogitate() Handles tool-calling execution. diff --git a/scripts/check_schema_bounds.py b/scripts/check_schema_bounds.py new file mode 100644 index 000000000..23e12747a --- /dev/null +++ b/scripts/check_schema_bounds.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +"""Ratcheting guard for generation-schema array and free-text bounds.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parent.parent + +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from solstone.apps.timeline.rollup import build_rollup_schema # noqa: E402 +from solstone.think.schema_bounds import unbounded_nodes # noqa: E402 + +ALLOWLIST: dict[str, str] = { + "build_rollup_schema(3)": "documents follow-on lode", + "solstone/apps/entities/talent/detection.schema.json": ( + "entity_observer follow-on lode" + ), + "solstone/apps/entities/talent/entities_review.schema.json": ( + "entity_observer follow-on lode" + ), + "solstone/apps/entities/talent/entity_observer.schema.json": ( + "entity_observer follow-on lode" + ), + "solstone/apps/timeline/talent/segment_summary.schema.json": ( + "documents follow-on lode" + ), + "solstone/observe/categories/meeting.schema.json": "screen follow-on lode", + "solstone/observe/describe.schema.json": "screen follow-on lode", + "solstone/observe/enrich.schema.json": ( + "unbounded pending KG schema enrichment arc" + ), + "solstone/observe/extract.schema.json": "screen follow-on lode", + "solstone/observe/transcribe/gemini.schema.json": ( + "unbounded pending KG schema enrichment arc" + ), + "solstone/talent/chat.schema.json": "messaging follow-on lode", + "solstone/talent/participation.schema.json": "calendar follow-on lode", + "solstone/talent/participation_entry.schema.json": "calendar follow-on lode", + "solstone/talent/pulse.schema.json": "morning_briefing follow-on lode", + "solstone/talent/schedule.schema.json": "calendar follow-on lode", + "solstone/talent/sense.schema.json": ("unbounded pending KG schema enrichment arc"), + "solstone/talent/speaker_attribution.schema.json": ( + "unbounded pending KG schema enrichment arc" + ), + "solstone/talent/steward.schema.json": "morning_briefing follow-on lode", + "solstone/talent/story.schema.json": "story follow-on lode", + "solstone/think/detect_created.schema.json": ( + "unbounded pending KG schema enrichment arc" + ), + "solstone/think/detect_transcript_json.schema.json": ( + "unbounded pending KG schema enrichment arc" + ), + "solstone/think/detect_transcript_segment.schema.json": ( + "unbounded pending KG schema enrichment arc" + ), +} + + +def discover_schemas(root: Path) -> dict[str, dict[str, Any]]: + """Return generation schemas keyed by stable schema id.""" + discovered: dict[str, dict[str, Any]] = {} + for path in sorted((root / "solstone").glob("**/*.schema.json")): + schema = json.loads(path.read_text(encoding="utf-8")) + if isinstance(schema.get("x-journal-contract"), dict): + continue + discovered[path.relative_to(root).as_posix()] = schema + discovered["build_rollup_schema(3)"] = build_rollup_schema(3) + return discovered + + +def evaluate( + root: Path, allowlist: dict[str, str] +) -> tuple[list[str], list[str], list[str]]: + """Return ``(new, stale, tracked)`` human-readable lines.""" + schemas = discover_schemas(root) + live = {} + for schema_id, schema in schemas.items(): + hits = unbounded_nodes(schema) + if hits: + live[schema_id] = hits + + new: list[str] = [] + stale: list[str] = [] + tracked: list[str] = [] + + for schema_id in sorted(set(live) | set(allowlist)): + hits = live.get(schema_id, []) + reason = allowlist.get(schema_id) + if hits and reason is None: + joined = ", ".join(hits) + new.append( + f"{schema_id}: {len(hits)} unbounded node(s): {joined} - " + "add generation bounds or add a temporary allowlist reason." + ) + elif not hits and reason is not None: + stale.append( + f"{schema_id}: allowlisted but now clean - delete the entry " + "from check_schema_bounds.py." + ) + elif hits and reason is not None: + tracked.append( + f"{schema_id}: {len(hits)} unbounded node(s) ({reason}; allowlisted)" + ) + + return new, stale, tracked + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="generation schema bounds lint") + parser.add_argument( + "--root", + type=Path, + default=ROOT, + help="Repository root to scan (defaults to the checkout root).", + ) + args = parser.parse_args(argv) + + new, stale, tracked = evaluate(args.root, ALLOWLIST) + + if tracked: + print("schema-bounds: known unbounded schemas (allowlisted):") + for line in tracked: + print(f" {line}") + print() + + if new or stale: + if new: + print("schema-bounds: NEW violations:", file=sys.stderr) + for line in new: + print(f" {line}", file=sys.stderr) + print(file=sys.stderr) + if stale: + print("schema-bounds: STALE allowlist entries:", file=sys.stderr) + for line in stale: + print(f" {line}", file=sys.stderr) + print(file=sys.stderr) + print( + "Generation schemas need maxItems on arrays and maxLength on " + "free-text strings; remove stale allowlist entries as schemas are " + "bounded.", + file=sys.stderr, + ) + return 1 + + print("schema-bounds: pass") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/solstone/think/models.py b/solstone/think/models.py index 27f6c142a..ccd77cacc 100644 --- a/solstone/think/models.py +++ b/solstone/think/models.py @@ -15,6 +15,7 @@ from typing import Any, Callable, Dict, List, NamedTuple, Optional, Union import frontmatter from jsonschema import Draft202012Validator +from solstone.think.schema_prep import prepare_provider_schema from solstone.think.utils import get_config, get_journal logger = logging.getLogger(__name__) @@ -1298,6 +1299,7 @@ def generate( # Get provider module via registry (raises ValueError for unknown providers) provider_mod = get_provider_module(provider) + provider_schema = prepare_provider_schema(json_schema, provider) timeout_s = DEFAULT_PROVIDER_TIMEOUT_S if timeout_s is None else timeout_s @@ -1310,7 +1312,7 @@ def generate( max_output_tokens=max_output_tokens, system_instruction=system_instruction, json_output=json_output, - json_schema=json_schema, + json_schema=provider_schema, thinking_budget=thinking_budget, timeout_s=timeout_s, **kwargs, @@ -1511,6 +1513,7 @@ def generate_with_result( _reject_local_cloud_model_override(provider, model_override) provider_mod = get_provider_module(provider) + provider_schema = prepare_provider_schema(json_schema, provider) timeout_s = DEFAULT_PROVIDER_TIMEOUT_S if timeout_s is None else timeout_s @@ -1522,7 +1525,7 @@ def generate_with_result( max_output_tokens=max_output_tokens, system_instruction=system_instruction, json_output=json_output, - json_schema=json_schema, + json_schema=provider_schema, thinking_budget=thinking_budget, timeout_s=timeout_s, **kwargs, @@ -1619,6 +1622,7 @@ async def agenerate( # Get provider module via registry (raises ValueError for unknown providers) provider_mod = get_provider_module(provider) + provider_schema = prepare_provider_schema(json_schema, provider) timeout_s = DEFAULT_PROVIDER_TIMEOUT_S if timeout_s is None else timeout_s @@ -1631,7 +1635,7 @@ async def agenerate( max_output_tokens=max_output_tokens, system_instruction=system_instruction, json_output=json_output, - json_schema=json_schema, + json_schema=provider_schema, thinking_budget=thinking_budget, timeout_s=timeout_s, **kwargs, diff --git a/solstone/think/schema_bounds.py b/solstone/think/schema_bounds.py new file mode 100644 index 000000000..7724eaf25 --- /dev/null +++ b/solstone/think/schema_bounds.py @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +"""Guard helpers for generation-schema bounds.""" + +from __future__ import annotations + +from typing import Any + + +def _has_type(node: dict[str, Any], schema_type: str) -> bool: + value = node.get("type") + if value == schema_type: + return True + return isinstance(value, list) and schema_type in value + + +def unbounded_nodes(schema: dict[str, Any]) -> list[str]: + """Return paths for generation nodes missing local grammar bounds. + + Violations: + - array nodes (``type == "array"`` or nullable/list-valued type containing + ``"array"``) without ``maxItems`` + - free-text string nodes (``"string"`` in type) that have none of + ``enum``, ``const``, ``pattern``, or ``format`` and no ``maxLength`` + """ + found: list[str] = [] + + def walk(node: Any, path: str) -> None: + if isinstance(node, dict): + if _has_type(node, "array") and "maxItems" not in node: + found.append(path) + if ( + _has_type(node, "string") + and "maxLength" not in node + and not {"enum", "const", "pattern", "format"} & set(node) + ): + found.append(path) + for key, value in node.items(): + walk(value, f"{path}/{key}") + elif isinstance(node, list): + for index, value in enumerate(node): + walk(value, f"{path}[{index}]") + + walk(schema, "$") + return found diff --git a/solstone/think/schema_eval.py b/solstone/think/schema_eval.py new file mode 100644 index 000000000..ad0f432c8 --- /dev/null +++ b/solstone/think/schema_eval.py @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +"""Pure scoring helpers for structured-output schema evals.""" + +from __future__ import annotations + +import json +from typing import Any, Sequence + +from jsonschema import Draft202012Validator + + +def schema_validity(text: str, schema: dict[str, Any]) -> dict[str, Any]: + """Validate response text against ``schema`` and return a serializable result.""" + try: + parsed = json.loads(text) + except json.JSONDecodeError as exc: + return { + "valid": False, + "errors": [ + { + "path": "$", + "message": f"Invalid JSON: {exc.msg}", + } + ], + } + + errors = [] + for error in sorted( + Draft202012Validator(schema).iter_errors(parsed), + key=lambda item: list(item.path), + ): + path = "$" + for part in error.path: + if isinstance(part, int): + path += f"[{part}]" + else: + path += f"/{part}" + errors.append({"path": path, "message": error.message}) + + return {"valid": not errors, "errors": errors} + + +def content_preservation(text: str, expect_contains: Sequence[str]) -> dict[str, Any]: + """Score case-insensitive substring preservation in response text.""" + needles = list(expect_contains) + if not needles: + return {"fraction": 1.0, "found": [], "missing": []} + + haystack = text.lower() + found = [needle for needle in needles if needle.lower() in haystack] + missing = [needle for needle in needles if needle.lower() not in haystack] + return { + "fraction": len(found) / len(needles), + "found": found, + "missing": missing, + } diff --git a/solstone/think/schema_prep.py b/solstone/think/schema_prep.py new file mode 100644 index 000000000..d1a1aa544 --- /dev/null +++ b/solstone/think/schema_prep.py @@ -0,0 +1,109 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +"""Provider-facing JSON Schema preparation for strict structured output.""" + +from __future__ import annotations + +import copy +import logging +from typing import Any + +logger = logging.getLogger(__name__) + +# Provider structured-output support references: +# - OpenAI: https://developers.openai.com/api/docs/guides/structured-outputs +# supports pattern/format/minimum/maximum/minItems/maxItems; minLength and +# maxLength are not listed in the supported subset. +# - Anthropic: https://platform.claude.com/docs/en/build-with-claude/structured-outputs +# rejects numeric/string/array constraints beyond minItems values 0 or 1; +# strip minItems wholesale here for deterministic simplicity. Pattern is +# documented as supported. +# - Google: https://ai.google.dev/gemini-api/docs/structured-output +# supports string format, number minimum/maximum, and array minItems/maxItems. +STRICT_UNSUPPORTED_KEYWORDS: dict[str, frozenset[str]] = { + "openai": frozenset({"$schema", "$comment", "minLength", "maxLength"}), + "google": frozenset({"$schema", "$comment", "minLength", "maxLength"}), + "anthropic": frozenset( + { + "$schema", + "$comment", + "minLength", + "maxLength", + "minItems", + "maxItems", + "minimum", + "maximum", + } + ), +} + +# Hazard: cloud-provider reductions are request-only. Canonical response +# validation in models.py still enforces stripped bounds, so an Anthropic +# response that overruns future canonical maxItems/maxLength bounds will fail +# loudly with SchemaValidationError. + + +def unsupported_keyword_hits(schema: dict[str, Any] | None, provider: str) -> list[str]: + """Return JSON-pointer-ish paths for keywords unsupported by ``provider``.""" + if schema is None: + return [] + + unsupported = STRICT_UNSUPPORTED_KEYWORDS.get(provider, frozenset()) + if not unsupported: + return [] + + found: list[str] = [] + + def walk(node: Any, path: str) -> None: + if isinstance(node, dict): + for key, value in node.items(): + child_path = f"{path}/{key}" + if key in unsupported: + found.append(child_path) + walk(value, child_path) + elif isinstance(node, list): + for index, value in enumerate(node): + walk(value, f"{path}[{index}]") + + walk(schema, "$") + return found + + +def prepare_provider_schema( + schema: dict[str, Any] | None, provider: str +) -> dict[str, Any] | None: + """Return a provider-facing copy of ``schema`` with unsupported keys removed.""" + if schema is None: + return None + + prepared = copy.deepcopy(schema) + unsupported = STRICT_UNSUPPORTED_KEYWORDS.get(provider, frozenset()) + if not unsupported: + return prepared + + removed: list[str] = [] + + def walk(node: Any, path: str) -> None: + if isinstance(node, dict): + for key in list(node): + child_path = f"{path}/{key}" + if key in unsupported: + node.pop(key) + removed.append(child_path) + continue + walk(node[key], child_path) + elif isinstance(node, list): + for index, value in enumerate(node): + walk(value, f"{path}[{index}]") + + walk(prepared, "$") + + if removed: + logger.debug( + "Removed %d unsupported JSON Schema keyword(s) for provider %s", + len(removed), + provider, + ) + + return prepared diff --git a/tests/eval_schemas.py b/tests/eval_schemas.py new file mode 100644 index 000000000..0b7dd220b --- /dev/null +++ b/tests/eval_schemas.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +"""Opt-in local structured-output schema eval harness.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] + +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from solstone.think.models import generate_with_result # noqa: E402 +from solstone.think.providers.local import LocalProviderError # noqa: E402 +from solstone.think.schema_eval import ( # noqa: E402 + content_preservation, + schema_validity, +) + +DEFAULT_CASES = ROOT / "tests" / "fixtures" / "schema_eval" / "cases.jsonl" +DEFAULT_OUT = ROOT / "tmp" / "schema-eval" +LOCAL_NOT_READY = ( + "Local schema eval requires the bundled local provider. Run " + "`journal install-provider local`, then start it with `journal start` " + "(or `journal service start` for an installed service)." +) + + +def _resolve_path(path: Path) -> Path: + if path.is_absolute(): + return path + return ROOT / path + + +def load_cases(path: Path) -> list[dict[str, Any]]: + cases: list[dict[str, Any]] = [] + with path.open(encoding="utf-8") as handle: + for lineno, line in enumerate(handle, start=1): + stripped = line.strip() + if not stripped: + continue + case = json.loads(stripped) + if "schema_path" in case: + schema_path = _resolve_path(Path(case["schema_path"])) + case["schema"] = json.loads(schema_path.read_text(encoding="utf-8")) + cases.append(case) + return cases + + +def run_case(case: dict[str, Any]) -> dict[str, Any]: + result = generate_with_result( + contents=case["input"], + context="schema.eval", + provider="local", + temperature=0.0, + max_output_tokens=512, + system_instruction=case["system_instruction"], + json_output=True, + json_schema=case["schema"], + ) + text = result["text"] + return { + "name": case["name"], + "text": text, + "schema_validity": schema_validity(text, case["schema"]), + "content_preservation": content_preservation( + text, case.get("expect_contains", []) + ), + "finish_reason": result.get("finish_reason"), + "model": result.get("model"), + "usage": result.get("usage"), + } + + +def _atomic_write_text(path: Path, text: str) -> None: + tmp = path.with_suffix(path.suffix + ".tmp") + tmp.write_text(text, encoding="utf-8") + tmp.replace(path) + + +def write_outputs(out_dir: Path, results: list[dict[str, Any]]) -> None: + out_dir.mkdir(parents=True, exist_ok=True) + jsonl = "".join(json.dumps(result, sort_keys=True) + "\n" for result in results) + _atomic_write_text(out_dir / "results.jsonl", jsonl) + + valid_count = sum(1 for result in results if result["schema_validity"]["valid"]) + average_content = ( + sum(result["content_preservation"]["fraction"] for result in results) + / len(results) + if results + else 1.0 + ) + summary = ( + f"cases: {len(results)}\n" + f"schema_valid: {valid_count}/{len(results)}\n" + f"average_content_preservation: {average_content:.3f}\n" + ) + _atomic_write_text(out_dir / "summary.txt", summary) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="local structured schema eval") + parser.add_argument("--cases", type=Path, default=DEFAULT_CASES) + parser.add_argument("--out", type=Path, default=DEFAULT_OUT) + args = parser.parse_args(argv) + + cases = load_cases(_resolve_path(args.cases)) + results: list[dict[str, Any]] = [] + for case in cases: + try: + results.append(run_case(case)) + except LocalProviderError as exc: + if exc.reason_code == "local_model_not_ready": + print(LOCAL_NOT_READY, file=sys.stderr) + return 2 + raise + + out_dir = _resolve_path(args.out) + write_outputs(out_dir, results) + + valid_count = sum(1 for result in results if result["schema_validity"]["valid"]) + average_content = ( + sum(result["content_preservation"]["fraction"] for result in results) + / len(results) + if results + else 1.0 + ) + print(f"schema eval wrote {out_dir}") + print(f"schema-valid: {valid_count}/{len(results)}") + print(f"average content preservation: {average_content:.3f}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/fixtures/schema_eval/cases.jsonl b/tests/fixtures/schema_eval/cases.jsonl new file mode 100644 index 000000000..d69f2525f --- /dev/null +++ b/tests/fixtures/schema_eval/cases.jsonl @@ -0,0 +1,3 @@ +{"name":"bounded_snacks","schema":{"type":"object","properties":{"snacks":{"type":"array","maxItems":2,"items":{"type":"string","maxLength":20}}},"required":["snacks"],"additionalProperties":false},"system_instruction":"Return only JSON matching the schema. Extract the requested facts from the input.","input":"Extract up to two snack names from this list: apple, banana, cherry, date, elderberry. Prefer apple and banana.","expect_contains":["apple","banana"]} +{"name":"contact_card","schema":{"type":"object","properties":{"name":{"type":"string","maxLength":60},"role":{"type":"string","maxLength":80},"summary":{"type":"string","maxLength":160}},"required":["name","role","summary"],"additionalProperties":false},"system_instruction":"Return only JSON matching the schema. Preserve the named person and role.","input":"Mira Patel is the incident coordinator. She is tracking database failover status for the evening handoff.","expect_contains":["Mira Patel","incident coordinator","database failover"]} +{"name":"decision_flags","schema":{"type":"object","properties":{"decision":{"type":"string","maxLength":120},"approved":{"type":"boolean"},"tags":{"type":"array","maxItems":3,"items":{"type":"string","enum":["billing","security","launch","support"]}}},"required":["decision","approved","tags"],"additionalProperties":false},"system_instruction":"Return only JSON matching the schema. Capture the decision and select applicable tags.","input":"The team approved delaying the launch until the security review finishes. Billing and support are not involved.","expect_contains":["delaying","security","launch"]} diff --git a/tests/test_schema_bounds.py b/tests/test_schema_bounds.py new file mode 100644 index 000000000..e657d6383 --- /dev/null +++ b/tests/test_schema_bounds.py @@ -0,0 +1,79 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +from __future__ import annotations + +from typing import Any + +import pytest + +from solstone.think.schema_bounds import unbounded_nodes + + +def _schema(property_schema: dict[str, Any]) -> dict[str, Any]: + return { + "type": "object", + "properties": {"field": property_schema}, + "required": ["field"], + "additionalProperties": False, + } + + +def test_bounded_array_passes() -> None: + assert unbounded_nodes(_schema({"type": "array", "maxItems": 3})) == [] + + +def test_unbounded_array_fails() -> None: + assert unbounded_nodes(_schema({"type": "array"})) == ["$/properties/field"] + + +@pytest.mark.parametrize( + "property_schema", + [ + {"type": "string", "enum": ["a"]}, + {"type": "string", "const": "a"}, + {"type": "string", "pattern": "^[a-z]+$"}, + {"type": "string", "format": "date-time"}, + ], +) +def test_constrained_strings_pass(property_schema: dict[str, Any]) -> None: + assert unbounded_nodes(_schema(property_schema)) == [] + + +def test_bare_string_fails() -> None: + assert unbounded_nodes(_schema({"type": "string"})) == ["$/properties/field"] + + +def test_bounded_string_passes() -> None: + assert unbounded_nodes(_schema({"type": "string", "maxLength": 80})) == [] + + +def test_nested_arrays_and_objects_report_paths() -> None: + schema = { + "type": "object", + "properties": { + "groups": { + "type": "array", + "items": { + "type": "object", + "properties": { + "title": {"type": ["string", "null"]}, + "codes": { + "type": ["array", "null"], + "maxItems": 3, + "items": {"type": "string", "pattern": "^[A-Z]+$"}, + }, + }, + "required": ["title", "codes"], + "additionalProperties": False, + }, + } + }, + "required": ["groups"], + "additionalProperties": False, + } + + assert unbounded_nodes(schema) == [ + "$/properties/groups", + "$/properties/groups/items/properties/title", + ] diff --git a/tests/test_schema_eval.py b/tests/test_schema_eval.py new file mode 100644 index 000000000..3fa9ed28b --- /dev/null +++ b/tests/test_schema_eval.py @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +from __future__ import annotations + +from solstone.think.schema_eval import content_preservation, schema_validity + + +def test_schema_validity_accepts_valid_response() -> None: + result = schema_validity( + '{"items": ["alpha"]}', + { + "type": "object", + "properties": { + "items": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["items"], + "additionalProperties": False, + }, + ) + + assert result == {"valid": True, "errors": []} + + +def test_schema_validity_reports_invalid_json() -> None: + result = schema_validity("{", {"type": "object"}) + + assert result["valid"] is False + assert result["errors"][0]["path"] == "$" + + +def test_schema_validity_reports_schema_errors() -> None: + result = schema_validity( + '{"items": ["alpha", "beta"]}', + { + "type": "object", + "properties": { + "items": { + "type": "array", + "maxItems": 1, + "items": {"type": "string"}, + }, + }, + "required": ["items"], + "additionalProperties": False, + }, + ) + + assert result["valid"] is False + assert result["errors"][0]["path"] == "$/items" + + +def test_content_preservation_scores_case_insensitive_matches() -> None: + result = content_preservation( + '{"summary": "Alpha and beta are present."}', + ["alpha", "BETA", "gamma"], + ) + + assert result == { + "fraction": 2 / 3, + "found": ["alpha", "BETA"], + "missing": ["gamma"], + } + + +def test_content_preservation_empty_needles_passes() -> None: + assert content_preservation("anything", []) == { + "fraction": 1.0, + "found": [], + "missing": [], + } diff --git a/tests/test_schema_prep.py b/tests/test_schema_prep.py new file mode 100644 index 000000000..01a230e42 --- /dev/null +++ b/tests/test_schema_prep.py @@ -0,0 +1,168 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +from __future__ import annotations + +import copy +import json +from pathlib import Path +from types import SimpleNamespace +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest + +from solstone.apps.timeline.rollup import build_rollup_schema +from solstone.think.models import SchemaValidationError, generate +from solstone.think.schema_prep import prepare_provider_schema + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +@pytest.fixture +def bounded_schema() -> dict[str, Any]: + return { + "type": "object", + "properties": { + "labels": { + "type": "array", + "maxItems": 2, + "items": { + "type": "string", + "maxLength": 12, + "pattern": "^[a-z]+$", + "enum": ["alpha", "beta"], + }, + } + }, + "required": ["labels"], + "additionalProperties": False, + } + + +def _discover_schemas() -> tuple[dict[str, Any], ...]: + discovered: list[dict[str, Any]] = [] + for path in sorted((REPO_ROOT / "solstone").glob("**/*.schema.json")): + schema = json.loads(path.read_text(encoding="utf-8")) + if isinstance(schema.get("x-journal-contract"), dict): + continue + discovered.append(schema) + discovered.append(build_rollup_schema(3)) + return tuple(discovered) + + +def test_local_receives_canonical_copy(bounded_schema: dict[str, Any]) -> None: + prepared = prepare_provider_schema(bounded_schema, "local") + + assert prepared == bounded_schema + assert prepared is not bounded_schema + + +@pytest.mark.parametrize("provider", ["openai", "google"]) +def test_openai_and_google_keep_array_bounds_and_strip_length_bounds( + bounded_schema: dict[str, Any], provider: str +) -> None: + prepared = prepare_provider_schema(bounded_schema, provider) + + labels = prepared["properties"]["labels"] # type: ignore[index] + item = labels["items"] + assert labels["maxItems"] == 2 + assert "maxLength" not in item + assert item["pattern"] == "^[a-z]+$" + assert item["enum"] == ["alpha", "beta"] + + +def test_anthropic_strips_array_and_length_bounds( + bounded_schema: dict[str, Any], +) -> None: + prepared = prepare_provider_schema(bounded_schema, "anthropic") + + labels = prepared["properties"]["labels"] # type: ignore[index] + item = labels["items"] + assert "maxItems" not in labels + assert "maxLength" not in item + assert item["pattern"] == "^[a-z]+$" + assert item["enum"] == ["alpha", "beta"] + + +@pytest.mark.parametrize("provider", ["local", "openai", "google", "anthropic", "fake"]) +def test_prepare_provider_schema_is_pure_and_idempotent( + bounded_schema: dict[str, Any], provider: str +) -> None: + original = copy.deepcopy(bounded_schema) + prepared = prepare_provider_schema(bounded_schema, provider) + + assert bounded_schema == original + assert prepare_provider_schema(prepared, provider) == prepared + + +def test_none_and_unknown_provider_passthrough( + bounded_schema: dict[str, Any], +) -> None: + assert prepare_provider_schema(None, "openai") is None + + prepared = prepare_provider_schema(bounded_schema, "fake") + assert prepared == bounded_schema + assert prepared is not bounded_schema + + +@pytest.mark.parametrize("provider", ["local", "openai", "google", "anthropic"]) +@pytest.mark.parametrize("schema", _discover_schemas()) +def test_current_shipped_schemas_are_byte_identical_after_prep( + schema: dict[str, Any], provider: str +) -> None: + assert prepare_provider_schema(schema, provider) == schema + + +def _patched_generate( + provider: str, schema: dict[str, Any], response_text: str +) -> MagicMock: + """Run ``generate`` against a stubbed provider module; return its mock.""" + run_generate = MagicMock( + return_value={"text": response_text, "finish_reason": "stop"} + ) + provider_module = SimpleNamespace(run_generate=run_generate) + + with ( + patch( + "solstone.think.models.resolve_provider", + return_value=(provider, "model"), + ), + patch( + "solstone.think.providers.get_provider_module", + return_value=provider_module, + ), + ): + generate("hello", "test.context", json_schema=schema) + + return run_generate + + +def test_generate_sends_reduced_schema_to_anthropic( + bounded_schema: dict[str, Any], +) -> None: + run_generate = _patched_generate("anthropic", bounded_schema, '{"labels": []}') + + sent = run_generate.call_args.kwargs["json_schema"] + assert sent == prepare_provider_schema(bounded_schema, "anthropic") + assert "maxItems" not in sent["properties"]["labels"] + assert bounded_schema["properties"]["labels"]["maxItems"] == 2 + + +def test_generate_sends_canonical_schema_to_local( + bounded_schema: dict[str, Any], +) -> None: + run_generate = _patched_generate("local", bounded_schema, '{"labels": []}') + + assert run_generate.call_args.kwargs["json_schema"] == bounded_schema + + +def test_generate_validates_response_against_canonical_schema( + bounded_schema: dict[str, Any], +) -> None: + """D4: bounds stripped from the anthropic request are still enforced on the + response. A 3-item answer overruns the canonical maxItems:2 and fails loudly.""" + overrun = '{"labels": ["alpha", "beta", "alpha"]}' + + with pytest.raises(SchemaValidationError): + _patched_generate("anthropic", bounded_schema, overrun) diff --git a/tests/test_schema_strict_portability.py b/tests/test_schema_strict_portability.py index f384b7ee8..036c7d545 100644 --- a/tests/test_schema_strict_portability.py +++ b/tests/test_schema_strict_portability.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright (c) 2026 sol pbc -"""Offline CI gate for req_bfbdbux6 strict schema portability.""" +"""Offline CI gate for strict structured-output schema portability.""" from __future__ import annotations @@ -12,21 +12,14 @@ from typing import Any import pytest from solstone.apps.timeline.rollup import build_rollup_schema +from solstone.think.schema_prep import ( + prepare_provider_schema, + unsupported_keyword_hits, +) from solstone.think.talent import hydrate_runtime_enums REPO_ROOT = Path(__file__).resolve().parents[1] -BANNED_KEYS = frozenset( - { - "$schema", - "$comment", - "minLength", - "maxLength", - "minItems", - "maxItems", - "minimum", - "maximum", - } -) +STRICT_PROVIDERS = ("openai", "anthropic", "google") def _discover_schemas() -> tuple[tuple[str, dict[str, Any]], ...]: @@ -56,8 +49,6 @@ def violations(schema: dict[str, Any]) -> list[str]: def walk(node: Any, path: str) -> None: if isinstance(node, dict): for key in node: - if key in BANNED_KEYS: - found.append(f"{path}: banned key {key!r}") if key == "oneOf": found.append(f"{path}: banned key 'oneOf'") @@ -80,23 +71,6 @@ def violations(schema: dict[str, Any]) -> list[str]: return found -def banned_key_hits(schema: dict[str, Any]) -> list[str]: - found: list[str] = [] - - def walk(node: Any, path: str) -> None: - if isinstance(node, dict): - for key, value in node.items(): - if key in BANNED_KEYS: - found.append(f"{path}: banned key {key!r}") - walk(value, f"{path}/{key}") - elif isinstance(node, list): - for index, value in enumerate(node): - walk(value, f"{path}[{index}]") - - walk(schema, "$") - return found - - @pytest.mark.parametrize( ("schema_id", "schema"), [pytest.param(schema_id, schema, id=schema_id) for schema_id, schema in SCHEMAS], @@ -108,6 +82,20 @@ def test_all_discovered_schemas_are_strict_portable( assert schema_violations == [], f"{schema_id}: {schema_violations}" +@pytest.mark.parametrize("provider", STRICT_PROVIDERS) +@pytest.mark.parametrize( + ("schema_id", "schema"), + [pytest.param(schema_id, schema, id=schema_id) for schema_id, schema in SCHEMAS], +) +def test_prepared_schemas_have_no_provider_unsupported_keywords( + schema_id: str, schema: dict[str, Any], provider: str +) -> None: + prepared = prepare_provider_schema(schema, provider) + + assert prepared is not None + assert unsupported_keyword_hits(prepared, provider) == [], schema_id + + @pytest.mark.parametrize( "schema_path", [ @@ -123,7 +111,10 @@ def test_zero_facet_runtime_hydration_of_shipped_schemas_has_no_banned_keys( hydrated = hydrate_runtime_enums(schema) - assert banned_key_hits(hydrated) == [] + for provider in STRICT_PROVIDERS: + prepared = prepare_provider_schema(hydrated, provider) + assert prepared is not None + assert unsupported_keyword_hits(prepared, provider) == [] @pytest.mark.parametrize( @@ -131,15 +122,20 @@ def test_zero_facet_runtime_hydration_of_shipped_schemas_has_no_banned_keys( [ { "type": "object", - "$comment": "bad", "properties": { - "a": {"type": "array", "minItems": 1}, + "a": {"type": "array"}, "b": {"type": "string"}, + "c": {"oneOf": [{"type": "string"}, {"type": "integer"}]}, }, "required": ["a"], - "additionalProperties": False, } ], ) def test_strict_portability_guard_rejects_bad_schema(schema: dict[str, Any]) -> None: - assert violations(schema) + schema_violations = violations(schema) + + assert any( + "object missing additionalProperties:false" in v for v in schema_violations + ) + assert any("properties not required" in v for v in schema_violations) + assert any("banned key 'oneOf'" in v for v in schema_violations)