diff --git a/Makefile b/Makefile index 46ca1d8c..30e23a6b 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,7 @@ BUILD ?= XCODEBUILD_FLAGS ?= BUILD_BENCHMARK_SCENARIO ?= ci BUILD_BENCHMARK_SAMPLES ?= 1 +CLI_INTEGRATION_TEST_FILTER ?= ProwlCLIIntegrationTests FORMAT_BASE_REF ?= origin/main BUILD_SETTINGS_CACHE := $(CURRENT_MAKEFILE_DIR)/.build_settings_cache.json PBXPROJ_PATH := $(CURRENT_MAKEFILE_DIR)/supacode.xcodeproj/project.pbxproj @@ -336,8 +337,9 @@ test-app: ensure-ghostty # Run app/unit tests via xcodebuild set -e; \ if [ "$$xcodebuild_status" -ne 0 ]; then \ bash "$(CURRENT_MAKEFILE_DIR)/scripts/print-xcresult-failures.sh" "$$result_bundle" || true; \ + exit "$$xcodebuild_status"; \ fi; \ - exit "$$xcodebuild_status" + bash "$(CURRENT_MAKEFILE_DIR)/scripts/assert-xcresult-tests.sh" "$$result_bundle" test-cli-smoke: build-cli # Smoke test CLI executable @set -euo pipefail; \ @@ -351,7 +353,14 @@ test-cli-smoke: build-cli # Smoke test CLI executable jq -e '.error.code == "APP_NOT_RUNNING"' /tmp/prowl-cli-smoke.json >/dev/null test-cli-integration: # Run CLI integration tests via SwiftPM - swift test --filter ProwlCLIIntegrationTests + @test_list="$$(swift test list)"; \ + matching_test_count="$$(printf '%s\n' "$$test_list" | grep -Ec '$(CLI_INTEGRATION_TEST_FILTER)' || true)"; \ + if [ "$$matching_test_count" -eq 0 ]; then \ + echo "error: CLI integration filter matched zero tests: $(CLI_INTEGRATION_TEST_FILTER)" >&2; \ + exit 1; \ + fi; \ + echo "CLI integration filter matched $$matching_test_count test(s)."; \ + swift test --skip-build --filter '$(CLI_INTEGRATION_TEST_FILTER)' benchmark-build: ensure-ghostty embed-cli-debug embed-docs # Benchmark clean and compilation-cache build/test time @BUILD_BENCHMARK_ROOT="$(CURRENT_MAKEFILE_DIR)/.build-benchmark/build-time" \ diff --git a/docs-ai/016-dev-build-and-ci-workflow/005-build-test-time-optimization.md b/docs-ai/016-dev-build-and-ci-workflow/005-build-test-time-optimization.md index 9c22a48d..3fa7d117 100644 --- a/docs-ai/016-dev-build-and-ci-workflow/005-build-test-time-optimization.md +++ b/docs-ai/016-dev-build-and-ci-workflow/005-build-test-time-optimization.md @@ -134,6 +134,9 @@ PR #678 implemented the planned build graph and cache changes: scenarios with JSONL history and retained raw/xcsift logs. - `ShellClientStreamingTests.swift` and `AgentProfileTests.swift` move expensive collection inference outside `#expect`; rerunning the 500 ms diagnostics reported neither hotspot. +- `test-app` validates the successful `.xcresult` contains a passing, non-empty test run; + `test-cli-integration` lists tests first and rejects a filter matching zero tests. These + guards close SwiftPM/Xcode's otherwise-successful empty-test false-negative path. Local M2 Pro / Xcode 26.6 verification after implementation: @@ -144,7 +147,9 @@ Local M2 Pro / Xcode 26.6 verification after implementation: | Integrated test, warm CAS sample 2 | 48.443 s | All three benchmark runs passed 2,263 App tests. `make check`, `make build-app`, `make test`, -`make build-cli`, `make test-cli-smoke`, and `make test-cli-integration` also passed. +`make build-cli`, `make test-cli-smoke`, and `make test-cli-integration` also passed. Negative-path +checks injected exit 23 from both `xcodebuild` and `swift test`, and used a zero-match CLI filter; +all three made their Make target fail, while the workflow's parallel wait harness returned 1. GitHub Actions run `30925147131` supplied one cache-population and two warm samples on the same Xcode 26.6 runner image family: diff --git a/scripts/assert-xcresult-tests.sh b/scripts/assert-xcresult-tests.sh new file mode 100755 index 00000000..f8fd6760 --- /dev/null +++ b/scripts/assert-xcresult-tests.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ "$#" -ne 1 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +result_bundle="$1" + +if [[ ! -d "$result_bundle" ]]; then + echo "error: xcresult bundle not found: $result_bundle" >&2 + exit 1 +fi + +if ! command -v jq >/dev/null 2>&1; then + echo "error: jq is required to validate xcresult test counts" >&2 + exit 1 +fi + +summary_json="$(mktemp)" +cleanup() { + rm -f "$summary_json" +} +trap cleanup EXIT + +if ! xcrun xcresulttool get test-results summary --path "$result_bundle" --compact >"$summary_json"; then + echo "error: failed to parse xcresult test summary: $result_bundle" >&2 + exit 1 +fi + +read -r result total_tests failed_tests < <( + jq -er ' + def numeric: + if type == "number" then . + elif type == "string" then tonumber + else error("expected numeric test count") + end; + [ + (.result // "Unknown"), + ((.totalTestCount // 0) | numeric), + ((.failedTests // 0) | numeric) + ] | @tsv + ' "$summary_json" +) + +if [[ "$result" != "Passed" ]]; then + echo "error: xcresult test result is $result, expected Passed" >&2 + exit 1 +fi + +if (( failed_tests != 0 )); then + echo "error: xcresult reports $failed_tests failed test(s)" >&2 + exit 1 +fi + +if (( total_tests <= 0 )); then + echo "error: xcresult reports zero tests; refusing a false-success test run" >&2 + exit 1 +fi + +echo "Verified xcresult: $total_tests test(s), zero failures."