From d5ec6caf25e97932f491df8cc64dd3326dd4e6b0 Mon Sep 17 00:00:00 2001 From: Marcel van Lohuizen Date: Mon, 30 Mar 2026 14:51:05 +0200 Subject: [PATCH] internal/cuetxtar: add inline test runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add @test(...) attribute support to cuetxtar, replacing golden-file comparison with assertions written directly in txtar .cue source. Two syntactic forms are supported: - Inline form: @test(...) as a field attribute makes that field a self-contained sub-test. - Structural form: @test(...) as a decl attribute inside a struct marks it as a test container with in:/eq:/leq: sub-fields. Directives: eq, leq, err (with code/contains/any), kind, closed, skip, desc, permute, debugCheck, file, and CUE_UPDATE writeback. (ignore and shareID to be introduced later.) The TxTarTest.Run dispatcher detects inline archives automatically; archives without @test attributes continue to use the existing golden-file path. Add reference txtar suites under cue/testdata/inlinetest/ covering all directives, structural form, permute, shareId (todo), and debugCheck. Add specs under doc/specs/ for the inline test attributes, runner, and migration-tool design. Signed-off-by: Marcel van Lohuizen Change-Id: Ie7c700213fe4d6b8d2210ad6d9aea407c324fa64 Reviewed-on: https://cue.gerrithub.io/c/cue-lang/cue/+/1234432 Reviewed-by: Daniel Martí Unity-Result: CUE porcuepine TryBot-Result: CUEcueckoo --- cue/testdata/basicrewrite/001_regexp.txtar | 89 +- .../basicrewrite/aliases/aliases.txtar | 24 +- .../015_list_comprehension.txtar | 55 +- .../fulleval/012_disjunctions_of_lists.txtar | 39 +- .../fulleval/051_detectIncompleteYAML.txtar | 110 +- .../fulleval/052_detectIncompleteJSON.txtar | 52 +- cue/testdata/inlinetest/debug.txtar | 66 + cue/testdata/inlinetest/decl_eq.txtar | 29 + cue/testdata/inlinetest/decl_eq_mixed.txtar | 14 + cue/testdata/inlinetest/decl_eq_pattern.txtar | 17 + cue/testdata/inlinetest/hidden_fields.txtar | 55 + cue/testdata/inlinetest/permute.txtar | 105 ++ cue/testdata/inlinetest/sharing.txtar | 72 + cue/testdata/inlinetest/struct_eq.txtar | 14 + cue/testdata/inlinetest/structural.txtar | 90 + cue/testdata/readme.md | 287 ++- cue/testdata/resolve/001.txtar | 22 +- doc/specs/inline-test-attributes/spec.md | 513 ++++++ doc/specs/test-migration-tool/spec.md | 59 + internal/core/adt/eval_test.go | 29 + internal/core/runtime/testdata/nullinit.txtar | 2 +- internal/cuetxtar/astcmp.go | 901 ++++++++++ internal/cuetxtar/astcmp_test.go | 793 +++++++++ internal/cuetxtar/inline.go | 1550 +++++++++++++++++ internal/cuetxtar/inline_test.go | 380 ++++ internal/cuetxtar/inlinerunner_test.go | 152 ++ internal/cuetxtar/permute.go | 296 ++++ internal/cuetxtar/txtar.go | 27 + internal/cuetxtar/writeback.go | 99 ++ 29 files changed, 5615 insertions(+), 326 deletions(-) create mode 100644 cue/testdata/inlinetest/debug.txtar create mode 100644 cue/testdata/inlinetest/decl_eq.txtar create mode 100644 cue/testdata/inlinetest/decl_eq_mixed.txtar create mode 100644 cue/testdata/inlinetest/decl_eq_pattern.txtar create mode 100644 cue/testdata/inlinetest/hidden_fields.txtar create mode 100644 cue/testdata/inlinetest/permute.txtar create mode 100644 cue/testdata/inlinetest/sharing.txtar create mode 100644 cue/testdata/inlinetest/struct_eq.txtar create mode 100644 cue/testdata/inlinetest/structural.txtar create mode 100644 doc/specs/inline-test-attributes/spec.md create mode 100644 doc/specs/test-migration-tool/spec.md create mode 100644 internal/cuetxtar/astcmp.go create mode 100644 internal/cuetxtar/astcmp_test.go create mode 100644 internal/cuetxtar/inline.go create mode 100644 internal/cuetxtar/inline_test.go create mode 100644 internal/cuetxtar/inlinerunner_test.go create mode 100644 internal/cuetxtar/permute.go create mode 100644 internal/cuetxtar/writeback.go diff --git a/cue/testdata/basicrewrite/001_regexp.txtar b/cue/testdata/basicrewrite/001_regexp.txtar index 3a0065cb1..d87b2a04e 100644 --- a/cue/testdata/basicrewrite/001_regexp.txtar +++ b/cue/testdata/basicrewrite/001_regexp.txtar @@ -1,28 +1,37 @@ #name: regexp #evalPartial -- in.cue -- -c1: "a" =~ "a" -c2: "foo" =~ "[a-z]{3}" -c3: "foo" =~ "[a-z]{4}" -c4: "foo" !~ "[a-z]{4}" +c1: "a" =~ "a" @test(eq, true) +c2: "foo" =~ "[a-z]{3}" @test(eq, true) +c3: "foo" =~ "[a-z]{4}" @test(eq, false) +c4: "foo" !~ "[a-z]{4}" @test(eq, true) -b1: =~"a" +b1: =~"a" @test(eq, "a") b1: "a" -b2: =~"[a-z]{3}" +b2: =~"[a-z]{3}" @test(eq, "foo") b2: "foo" -b3: =~"[a-z]{4}" +b3: =~"[a-z]{4}" @test(err, code=eval, contains="out of bound", pos=[0:5 1:5]) b3: "foo" -b4: !~"[a-z]{4}" +b4: !~"[a-z]{4}" @test(eq, "foo") b4: "foo" -s1: !="b" & =~"c" // =~"c" -s2: =~"c" & !="b" // =~"c" -s3: !="b" & =~"[a-z]" // != "b" & =~"[a-z]" -s4: =~"[a-z]" & !="b" // != "b" & =~"[a-z]" +s1: !="b" & =~"c" @test(eq, =~"c") +s2: =~"c" & !="b" @test(eq, =~"c") +s3: !="b" & =~"[a-z]" @test(eq, !="b" & =~"[a-z]") +s4: =~"[a-z]" & !="b" @test(eq, =~"[a-z]" & !="b") // TODO: allow different order -e1: "foo" =~ 1 -e2: "foo" !~ true -e3: !="a" & <5 +e1: "foo" =~ 1 @test(err, code=eval, + contains="cannot use 1", pos=[ + 0:5 + 0:14 + ]) +e2: "foo" !~ true @test(err, code=eval, + contains="cannot use true", pos=[ + 0:5 + 0:14 + ]) +e3: !="a" & <5 @test(err, code=eval, + contains="conflicting values", pos=[0:5 0:13]) -- out/compile -- --- in.cue { @@ -56,53 +65,3 @@ Retain: 0 Unifications: 16 Conjuncts: 25 Disjuncts: 16 --- out/evalalpha -- -Errors: -e3: conflicting values !="a" and <5 (mismatched types string and number): - ./in.cue:22:5 - ./in.cue:22:13 -b3: invalid value "foo" (out of bound =~"[a-z]{4}"): - ./in.cue:10:5 - ./in.cue:11:5 -e1: cannot use 1 (type int) as type (string|bytes): - ./in.cue:20:5 - ./in.cue:20:14 -e2: cannot use true (type bool) as type (string|bytes): - ./in.cue:21:5 - ./in.cue:21:14 - -Result: -(_|_){ - // [eval] - c1: (bool){ true } - c2: (bool){ true } - c3: (bool){ false } - c4: (bool){ true } - b1: (string){ "a" } - b2: (string){ "foo" } - b3: (_|_){ - // [eval] b3: invalid value "foo" (out of bound =~"[a-z]{4}"): - // ./in.cue:10:5 - // ./in.cue:11:5 - } - b4: (string){ "foo" } - s1: (string){ =~"c" } - s2: (string){ =~"c" } - s3: (string){ &(!="b", =~"[a-z]") } - s4: (string){ &(=~"[a-z]", !="b") } - e1: (_|_){ - // [eval] e1: cannot use 1 (type int) as type (string|bytes): - // ./in.cue:20:5 - // ./in.cue:20:14 - } - e2: (_|_){ - // [eval] e2: cannot use true (type bool) as type (string|bytes): - // ./in.cue:21:5 - // ./in.cue:21:14 - } - e3: (_|_){ - // [eval] e3: conflicting values !="a" and <5 (mismatched types string and number): - // ./in.cue:22:5 - // ./in.cue:22:13 - } -} diff --git a/cue/testdata/basicrewrite/aliases/aliases.txtar b/cue/testdata/basicrewrite/aliases/aliases.txtar index 5d170e962..1e647a9c4 100644 --- a/cue/testdata/basicrewrite/aliases/aliases.txtar +++ b/cue/testdata/basicrewrite/aliases/aliases.txtar @@ -3,12 +3,21 @@ t0: { a=_a: _ let _b = a _out: _b + @test(eq, { + _a: _ + let _b = _ + _out: _ + }) } t1: { _a: b let b = c c=d: 3 -} +} @test(eq, { + _a: 3 + let b = 3 + d: 3 +}) -- out/compile -- --- in.cue { @@ -33,16 +42,3 @@ Retain: 3 Unifications: 9 Conjuncts: 14 Disjuncts: 10 --- out/evalalpha -- -(struct){ - t0: (struct){ - _a: (_){ _ } - let _b#1 = (_){ _ } - _out: (_){ _ } - } - t1: (struct){ - _a: (int){ 3 } - let b#2 = (int){ 3 } - d: (int){ 3 } - } -} diff --git a/cue/testdata/comprehensions/015_list_comprehension.txtar b/cue/testdata/comprehensions/015_list_comprehension.txtar index 767d8f7e0..22e3540e0 100644 --- a/cue/testdata/comprehensions/015_list_comprehension.txtar +++ b/cue/testdata/comprehensions/015_list_comprehension.txtar @@ -1,34 +1,38 @@ -#name: list comprehension -#evalFull -- in.cue -- -a: [for k, v in b if k < "d" if v > b.a {k}] b: { a: 1 b: 2 c: 3 d: 4 } -c: [for _, x in b for _, y in b if x < y {x}] -d: [for x, _ in a {x}] + +a: [for k, v in b if k < "d" if v > b.a {k}] @test(eq, ["b", "c"]) +c: { + [for _, x in b for _, y in b if x < y {x}] + @test(eq, [1, 1, 1, 2, 2, 3]) +} +d: [for x, _ in a {x}] @test(eq, [0, 1]) -- out/compile -- --- in.cue { - a: [ - for k, v in 〈1;b〉 if (〈0;k〉 < "d") if (〈0;v〉 > 〈2;b〉.a) { - 〈1;k〉 - }, - ] b: { a: 1 b: 2 c: 3 d: 4 } - c: [ - for _, x in 〈1;b〉 for _, y in 〈2;b〉 if (〈1;x〉 < 〈0;y〉) { - 〈2;x〉 + a: [ + for k, v in 〈1;b〉 if (〈0;k〉 < "d") if (〈0;v〉 > 〈2;b〉.a) { + 〈1;k〉 }, ] + c: { + [ + for _, x in 〈2;b〉 for _, y in 〈3;b〉 if (〈1;x〉 < 〈0;y〉) { + 〈2;x〉 + }, + ] + } d: [ for x, _ in 〈1;a〉 { 〈1;x〉 @@ -45,28 +49,3 @@ Retain: 10 Unifications: 19 Conjuncts: 36 Disjuncts: 23 --- out/evalalpha -- -(struct){ - a: (#list){ - 0: (string){ "b" } - 1: (string){ "c" } - } - b: (struct){ - a: (int){ 1 } - b: (int){ 2 } - c: (int){ 3 } - d: (int){ 4 } - } - c: (#list){ - 0: (int){ 1 } - 1: (int){ 1 } - 2: (int){ 1 } - 3: (int){ 2 } - 4: (int){ 2 } - 5: (int){ 3 } - } - d: (#list){ - 0: (int){ 0 } - 1: (int){ 1 } - } -} diff --git a/cue/testdata/fulleval/012_disjunctions_of_lists.txtar b/cue/testdata/fulleval/012_disjunctions_of_lists.txtar index b9f5f937e..93449420c 100644 --- a/cue/testdata/fulleval/012_disjunctions_of_lists.txtar +++ b/cue/testdata/fulleval/012_disjunctions_of_lists.txtar @@ -1,10 +1,11 @@ -#name: disjunctions of lists -#evalFull -- in.cue -- -l: *[ int, int] | [ string, string] - +l: *[int, int] | [string, string] l1: ["a", "b"] -l2: l & ["c", "d"] + +t1: { + l & ["c", "d"] + @test(eq, ["c", "d"]) +} -- out/compile -- --- in.cue { @@ -19,10 +20,12 @@ l2: l & ["c", "d"] "a", "b", ] - l2: (〈0;l〉 & [ - "c", - "d", - ]) + t1: { + (〈1;l〉 & [ + "c", + "d", + ]) + } } -- out/eval/stats -- Leaks: 0 @@ -34,21 +37,3 @@ Retain: 0 Unifications: 14 Conjuncts: 24 Disjuncts: 18 --- out/evalalpha -- -(struct){ - l: (list){ |(*(#list){ - 0: (int){ int } - 1: (int){ int } - }, (#list){ - 0: (string){ string } - 1: (string){ string } - }) } - l1: (#list){ - 0: (string){ "a" } - 1: (string){ "b" } - } - l2: (#list){ - 0: (string){ "c" } - 1: (string){ "d" } - } -} diff --git a/cue/testdata/fulleval/051_detectIncompleteYAML.txtar b/cue/testdata/fulleval/051_detectIncompleteYAML.txtar index 9383a2c00..466d2eb18 100644 --- a/cue/testdata/fulleval/051_detectIncompleteYAML.txtar +++ b/cue/testdata/fulleval/051_detectIncompleteYAML.txtar @@ -11,13 +11,27 @@ import yaml "encoding/yaml" #foo: { use: _vars.something } - baz: yaml.Marshal(_vars.something) - foobar: yaml.Marshal(#foo) + baz: yaml.Marshal(_vars.something) @test(err, code=incomplete, pos=[0:11]) + foobar: yaml.Marshal(#foo) @test(err, code=incomplete, pos=[0:11 -6:21 -3:9]) } } Val: #Spec & { _vars: something: "var-string" -} +} @test(eq, { + _vars$foobar: something: "var-string" + data: { + #foo: use: "var-string" + baz: "var-string\n" + foobar: "use: var-string\n" + } +}) +-- out/errors.txt -- +[incomplete] #Spec.data.baz: non-concrete argument 0: + in.cue:11:11 +[incomplete] #Spec.data.foobar: error in call to encoding/yaml.Marshal: incomplete value string: + in.cue:12:11 + in.cue:6:21 + in.cue:9:9 -- out/eval/stats -- Leaks: 0 Freed: 17 @@ -28,96 +42,6 @@ Retain: 0 Unifications: 17 Conjuncts: 32 Disjuncts: 17 --- out/evalalpha -- -(struct){ - #Spec: (#struct){ - _vars(:foobar): (#struct){ - something: (string){ string } - } - data: (#struct){ - #foo: (#struct){ - use: (string){ string } - } - baz: (_|_){ - // [incomplete] #Spec.data.baz: error in call to encoding/yaml.Marshal: non-concrete argument 0: - // ./in.cue:11:11 - } - foobar: (_|_){ - // [incomplete] #Spec.data.foobar: error in call to encoding/yaml.Marshal: incomplete value string: - // ./in.cue:12:11 - // ./in.cue:6:21 - // ./in.cue:9:9 - } - } - } - Val: (#struct){ - _vars(:foobar): (#struct){ - something: (string){ "var-string" } - } - data: (#struct){ - #foo: (#struct){ - use: (string){ "var-string" } - } - baz: (string){ "var-string\n" } - foobar: (string){ "use: var-string\n" } - } - } -} --- diff/-out/evalalpha<==>+out/eval -- -diff old new ---- old -+++ new -@@ -8,7 +8,7 @@ - use: (string){ string } - } - baz: (_|_){ -- // [incomplete] #Spec.data.baz: non-concrete argument 0: -+ // [incomplete] #Spec.data.baz: error in call to encoding/yaml.Marshal: non-concrete argument 0: - // ./in.cue:11:11 - } - foobar: (_|_){ -@@ -15,6 +15,7 @@ - // [incomplete] #Spec.data.foobar: error in call to encoding/yaml.Marshal: incomplete value string: - // ./in.cue:12:11 - // ./in.cue:6:21 -+ // ./in.cue:9:9 - } - } - } --- out/eval -- -(struct){ - #Spec: (#struct){ - _vars(:foobar): (#struct){ - something: (string){ string } - } - data: (#struct){ - #foo: (#struct){ - use: (string){ string } - } - baz: (_|_){ - // [incomplete] #Spec.data.baz: non-concrete argument 0: - // ./in.cue:11:11 - } - foobar: (_|_){ - // [incomplete] #Spec.data.foobar: error in call to encoding/yaml.Marshal: incomplete value string: - // ./in.cue:12:11 - // ./in.cue:6:21 - } - } - } - Val: (#struct){ - _vars(:foobar): (#struct){ - something: (string){ "var-string" } - } - data: (#struct){ - #foo: (#struct){ - use: (string){ "var-string" } - } - baz: (string){ "var-string\n" } - foobar: (string){ "use: var-string\n" } - } - } -} -- out/compile -- --- in.cue { diff --git a/cue/testdata/fulleval/052_detectIncompleteJSON.txtar b/cue/testdata/fulleval/052_detectIncompleteJSON.txtar index 1bb89a16c..ce5ffa901 100644 --- a/cue/testdata/fulleval/052_detectIncompleteJSON.txtar +++ b/cue/testdata/fulleval/052_detectIncompleteJSON.txtar @@ -11,13 +11,25 @@ import "encoding/json" #foo: { use: _vars.something } - baz: json.Marshal(_vars.something) - foobar: json.Marshal(#foo) + baz: json.Marshal(_vars.something) @test(err, code=incomplete) + foobar: json.Marshal(#foo) @test(err, code=incomplete) } } Val: #Spec & { _vars: something: "var-string" -} +} @test(eq, { + _vars$foobar: something: "var-string" + data: { + #foo: use: "var-string" + baz: "\"var-string\"" + foobar: "{\"use\":\"var-string\"}" + } +}) +-- out/errors.txt -- +[incomplete] #Spec.data.baz: non-concrete argument 0: + in.cue:11:11 +[incomplete] cannot convert incomplete value "string" to JSON: + in.cue:6:21 -- out/eval/stats -- Leaks: 0 Freed: 17 @@ -28,40 +40,6 @@ Retain: 0 Unifications: 17 Conjuncts: 32 Disjuncts: 17 --- out/evalalpha -- -(struct){ - #Spec: (#struct){ - _vars(:foobar): (#struct){ - something: (string){ string } - } - data: (#struct){ - #foo: (#struct){ - use: (string){ string } - } - baz: (_|_){ - // [incomplete] #Spec.data.baz: error in call to encoding/json.Marshal: non-concrete argument 0: - // ./in.cue:11:11 - } - foobar: (_|_){ - // [incomplete] #Spec.data.foobar: error in call to encoding/json.Marshal: cannot convert incomplete value "string" to JSON: - // ./in.cue:12:11 - // ./in.cue:6:21 - } - } - } - Val: (#struct){ - _vars(:foobar): (#struct){ - something: (string){ "var-string" } - } - data: (#struct){ - #foo: (#struct){ - use: (string){ "var-string" } - } - baz: (string){ "\"var-string\"" } - foobar: (string){ "{\"use\":\"var-string\"}" } - } - } -} -- out/compile -- --- in.cue { diff --git a/cue/testdata/inlinetest/debug.txtar b/cue/testdata/inlinetest/debug.txtar new file mode 100644 index 000000000..6e50d6e8b --- /dev/null +++ b/cue/testdata/inlinetest/debug.txtar @@ -0,0 +1,66 @@ +-- in.cue -- +// debugCheck: compare debug-printer output via @test(debugCheck, ...) attributes. + +// ── scalar value ───────────────────────── + +debugCheckScalar: 42 @test(debugCheck, "(int){ 42 }") + +// ── struct value ───────────────────────── + +debugCheckStruct: { + @test(debugCheck, """ + (struct){ + a: (int){ 1 } + b: (int){ 2 } + } + """) + a: 1 + b: 2 +} + +// ── computed struct ─────────────────────── + +debugCheckComputed: { + @test(debugCheck, """ + (struct){ + x: (int){ 10 } + y: (int){ 20 } + } + """) + x: 10 + y: x * 2 +} + +// ── version override: v3 debugCheck takes precedence ── +// Both default and v3 have the same expected output here, +// verifying that versioned attrs are accepted and applied correctly. + +debugCheckVersioned: { + @test(debugCheck, """ + (struct){ + z: (int){ 99 } + } + """) + @test(debugCheck:v3, """ + (struct){ + z: (int){ 99 } + } + """) + z: 99 +} +-- out/compile -- +--- in.cue +{ + debugCheckScalar: 42 + debugCheckStruct: { + a: 1 + b: 2 + } + debugCheckComputed: { + x: 10 + y: (〈0;x〉 * 2) + } + debugCheckVersioned: { + z: 99 + } +} diff --git a/cue/testdata/inlinetest/decl_eq.txtar b/cue/testdata/inlinetest/decl_eq.txtar new file mode 100644 index 000000000..d718143d2 --- /dev/null +++ b/cue/testdata/inlinetest/decl_eq.txtar @@ -0,0 +1,29 @@ +-- in.cue -- +// File-level @test(eq, VALUE) as a decl attribute: checks the entire +// file's evaluated value against VALUE. All fields are covered +// implicitly — no per-field @test is required. + +a: 1 +b: a + 1 +c: { + x: "hello" + y: b +} +@test(eq, { + a: 1 + b: 2 + c: { + x: "hello" + y: 2 + } +}) +-- out/compile -- +--- in.cue +{ + a: 1 + b: (〈0;a〉 + 1) + c: { + x: "hello" + y: 〈1;b〉 + } +} diff --git a/cue/testdata/inlinetest/decl_eq_mixed.txtar b/cue/testdata/inlinetest/decl_eq_mixed.txtar new file mode 100644 index 000000000..dcad78a2a --- /dev/null +++ b/cue/testdata/inlinetest/decl_eq_mixed.txtar @@ -0,0 +1,14 @@ +-- in.cue -- +// File-level @test(eq, ...) coexists with field-level @test attributes. +// Both assertions run. The file-level @test covers all fields, so per-field +// coverage is not required, but individual fields may still carry @test. + +x: 1 @test(eq, 1) +y: x + 1 +@test(eq, {x: 1, y: 2}) +-- out/compile -- +--- in.cue +{ + x: 1 + y: (〈0;x〉 + 1) +} diff --git a/cue/testdata/inlinetest/decl_eq_pattern.txtar b/cue/testdata/inlinetest/decl_eq_pattern.txtar new file mode 100644 index 000000000..6c7d062b5 --- /dev/null +++ b/cue/testdata/inlinetest/decl_eq_pattern.txtar @@ -0,0 +1,17 @@ +-- in.cue -- +// File-level @test(eq, VALUE) where a pattern constraint contributes +// to the output — the eq check applies to the whole file. + +{[X=string]: baz: X} +bar: {} +@test(eq, {bar: baz: "bar"}) +-- out/compile -- +--- in.cue +{ + { + [string]: { + baz: 〈1;-〉 + } + } + bar: {} +} diff --git a/cue/testdata/inlinetest/hidden_fields.txtar b/cue/testdata/inlinetest/hidden_fields.txtar new file mode 100644 index 000000000..5678bb5f2 --- /dev/null +++ b/cue/testdata/inlinetest/hidden_fields.txtar @@ -0,0 +1,55 @@ +-- in.cue -- +// Tests that hidden fields in @test(eq, {...}) must be fully specified. +// A hidden field present in the value but absent from the expected struct +// causes a test failure. A hidden field in the expected struct that is absent +// from the value also causes a failure. + +package foo + +// -- hidden field present in both value and expected -- +// Field-attribute form: hidden field in both value and expected passes. +hiddenInBoth: { + _secret: 42 + a: _secret + 1 +} @test(eq, {_secret$foo: 42, a: 43}) + +// Decl-attribute form. +hiddenDeclForm: { + @test(eq, {_secret$foo: "x", b: "xx"}) + _secret: "x" + b: _secret + _secret +} + +// Hidden definition field in both value and expected. +hiddenDefInBoth: { + _#Meta: {info: "internal"} + a: 1 +} @test(eq, {_#Meta$foo: {info: "internal"}, a: 1}) + +// -- no hidden fields -- + +// No hidden fields on either side: passes trivially. +noHidden: {a: 1, b: 2} @test(eq, {a: 1, b: 2}) + +-- out/compile -- +--- in.cue +{ + hiddenInBoth: { + _secret(:foo): 42 + a: (〈0;_secret(:foo)〉 + 1) + } + hiddenDeclForm: { + _secret(:foo): "x" + b: (〈0;_secret(:foo)〉 + 〈0;_secret(:foo)〉) + } + hiddenDefInBoth: { + _#Meta(:foo): { + info: "internal" + } + a: 1 + } + noHidden: { + a: 1 + b: 2 + } +} diff --git a/cue/testdata/inlinetest/permute.txtar b/cue/testdata/inlinetest/permute.txtar new file mode 100644 index 000000000..edf8b7030 --- /dev/null +++ b/cue/testdata/inlinetest/permute.txtar @@ -0,0 +1,105 @@ +-- in.cue -- +// Structural permute tests: @test(permute) placed on the struct whose fields +// should be permuted — either as a decl attr on the container, or as a decl +// attr directly on the 'in' struct. + +// ── Container-level @test(permute): permutes all fields of `in` ── + +threeFieldPermute: { + @test(permute) + a: b + c + b: 1 + c: 2 + @test(eq, { + a: 3 + b: 1 + c: 2 + }) + @test(permuteCount, 6) // 3 fields = 3! = 6 permutations +} + +twoFieldPermute: { + @test(permute) + x: y + 1 + y: 5 + @test(eq, { + x: 6 + y: 5 + }) + @test(permuteCount, 2) // 2 fields = 2! = 2 permutations +} + +// ── Decl attr on `in` directly: equivalent syntax ── + +inDeclPermute: { + @test(permute) + p: q + r + q: 3 + r: 7 + @test(eq, { + p: 10 + q: 3 + r: 7 + }) + @test(permuteCount, 6) // 3 fields = 3! = 6 permutations +} + +// ── Self-contained concrete struct: trivially order-invariant ── + +concretePermute: { + x: { + @test(permute) + alpha: 1 + beta: 2 + gamma: 3 + } + y: { + @test(permute) + alpha: 1 + beta: 2 + gamma: 3 + } + @test(eq, { + x: { + alpha: 1 + beta: 2 + gamma: 3 + } + y: { + alpha: 1 + beta: 2 + gamma: 3 + } + }) + @test(permuteCount, 12) // 2 structs * 3! = 12 permutations +} +-- out/compile -- +--- in.cue +{ + threeFieldPermute: { + a: (〈0;b〉 + 〈0;c〉) + b: 1 + c: 2 + } + twoFieldPermute: { + x: (〈0;y〉 + 1) + y: 5 + } + inDeclPermute: { + p: (〈0;q〉 + 〈0;r〉) + q: 3 + r: 7 + } + concretePermute: { + x: { + alpha: 1 + beta: 2 + gamma: 3 + } + y: { + alpha: 1 + beta: 2 + gamma: 3 + } + } +} diff --git a/cue/testdata/inlinetest/sharing.txtar b/cue/testdata/inlinetest/sharing.txtar new file mode 100644 index 000000000..e10857b14 --- /dev/null +++ b/cue/testdata/inlinetest/sharing.txtar @@ -0,0 +1,72 @@ +-- in.cue -- +// shareID assertions on eq sub-fields in structural form. +// @test(shareID=name) on a field within an eq struct asserts that the +// corresponding path in evaluate(in) shares the same canonical adt.Vertex as +// every other field carrying the same name in this test case. + +// ── basic: field reference creates vertex sharing ── +// In CUE, b: a makes b an alias of a at the ADT level. After evaluation, +// in.b.DerefValue() == in.a.DerefValue(), so the shareID assertion passes. + +basicSharing: { + @test(skip, why="sharing checks not implemented yet") + a: {x: 1, y: 2} + b: a + @test(eq, { + a: { x: 1, y: 2 } @test(shareID=A) + b: a @test(shareID=A) + }) +} + +// ── three-way sharing ───────────────────── +// g is the canonical node; p and q are both references to g. + +threeWaySharing: { + @test(skip, why="sharing checks not implemented yet") + g: {n: 42} + p: g + q: g + @test(eq, { + g: { n: 42 } @test(shareID=G) + p: g @test(shareID=G) + q: g @test(shareID=G) + }) +} + +// ── versioned shareID: @test(shareID:v3=name) applies only under v3 ── +// The unversioned group "common" covers all evaluator versions; the versioned +// group "v3group" is only checked when running under v3. + +versionedSharing: { + @test(skip, why="sharing checks not implemented yet") + a: {z: 1} + b: a + @test(eq, { + a: { z: 1 } @test(shareID=common) + b: a @test(shareID=common) + }) +} +-- out/compile -- +--- in.cue +{ + basicSharing: { + a: { + x: 1 + y: 2 + } + b: 〈0;a〉 + } + threeWaySharing: { + g: { + n: 42 + } + p: 〈0;g〉 + q: 〈0;g〉 + } + versionedSharing: { + a: { + z: 1 + } + b: 〈0;a〉 + } +} diff --git a/cue/testdata/inlinetest/struct_eq.txtar b/cue/testdata/inlinetest/struct_eq.txtar new file mode 100644 index 000000000..fb222a013 --- /dev/null +++ b/cue/testdata/inlinetest/struct_eq.txtar @@ -0,0 +1,14 @@ +-- in.cue -- +myStruct: { + @test(eq, {a: 1, b: 2}) + a: 1 + b: a + 1 +} +-- out/compile -- +--- in.cue +{ + myStruct: { + a: 1 + b: (〈0;a〉 + 1) + } +} diff --git a/cue/testdata/inlinetest/structural.txtar b/cue/testdata/inlinetest/structural.txtar new file mode 100644 index 000000000..85cab272f --- /dev/null +++ b/cue/testdata/inlinetest/structural.txtar @@ -0,0 +1,90 @@ +-- in.cue -- +// Tests converted from structural form to inline form. + +// ── error-only ───────────────────────── + +errOnly: 1 & 2 @test(err) @test(debugCheck, """ + (_|_){ + // [eval] errOnly: conflicting values 2 and 1: + // in.cue:5:10 + // in.cue:5:14 + } + """) + +errWithCode: string & 1 @test(err, code=eval) @test(debugOutput, """ + (_|_){ + // [eval] errWithCode: conflicting values string and 1 (mismatched types string and int): + // in.cue:7:14 + // in.cue:7:23 + } + """) + +// ── eq: computed struct ──────────────── + +eqBasic: {a: 1, b: 2} @test(eq, {a: 1, b: 2}) + +eqComputed: { + @test(eq, {x: 10, y: 20}) + x: 10 + y: x * 2 +} + +// ── leq: subsumption ────────────────── + +leqBasic: {x: 42} @test(leq, {x: int}) @test(eq, {x: 42}) + +// ── skip ────────────────────────────── + +skippedTest: {a: 1} @test(skip, why="demonstrating skip") + +// ── desc ────────────────────────────── + +descTest: {v: "ok"} @test(desc="description") @test(eq, {v: "ok"}) + +// ── version-specific ────────────────── + +versionOverride: {x: 1} @test(eq, {x: 1}) @test(eq:unknown, {x: 3}) + +// ── plain fixture field + references ────────────────── + +// A plain field with no @test is accessible but not a sub-test. +plainFixture: { + x: 42 +} + +refsFixture: plainFixture.x @test(eq, 42) + +// ── mixed inline ────────────────────── + +mixedInline: 42 @test(eq, 42) +-- out/compile -- +--- in.cue +{ + errOnly: (1 & 2) + errWithCode: (string & 1) + eqBasic: { + a: 1 + b: 2 + } + eqComputed: { + x: 10 + y: (〈0;x〉 * 2) + } + leqBasic: { + x: 42 + } + skippedTest: { + a: 1 + } + descTest: { + v: "ok" + } + versionOverride: { + x: 1 + } + plainFixture: { + x: 42 + } + refsFixture: 〈0;plainFixture〉.x + mixedInline: 42 +} diff --git a/cue/testdata/readme.md b/cue/testdata/readme.md index 73a4b5817..ade7badca 100644 --- a/cue/testdata/readme.md +++ b/cue/testdata/readme.md @@ -1,49 +1,272 @@ -# CUE Test Suite +# Inline Test Attributes — `cue/testdata/inlinetest` -This directory contains a test suite for testing the CUE language. This is only -intended to test language evaluation and exporting. Eventually it will also -contains tests for parsing and formatting. It is not intended to cover -testing of the API itself. +This directory contains the reference test suite for the **inline assertion** +system implemented in `internal/cuetxtar/inline.go`. Each `.txtar` archive +here exercises a distinct feature area. + +--- ## Overview -### Work in progress +Inline assertion mode replaces golden-file comparison with `@test(...)` +attributes written directly in the `.cue` source of a txtar archive. The +runner (`cuetxtar.RunInlineTests`, called from `TestEvalV3`) detects inline +mode automatically: if any `.cue` file in the archive contains at least one +`@test(...)` attribute, the archive is processed as an inline test. Archives +with no `@test` attributes continue to use golden-file comparison unchanged. + +There are two syntactic positions for `@test(...)` attributes: + +--- + +## Inline form + +A `@test(...)` *field attribute* placed on a top-level field makes that field a +self-contained test case. The field name becomes the sub-test name. + +```cue +// basic.txtar — in.cue + +simpleInt: 42 @test(eq, 42) +simpleStr: "hello" @test(eq, "hello") +errField: 1 & 2 @test(err) +kindInt: int @test(kind=int) +``` + +--- + +## File-level form + +A `@test(eq, VALUE)` *decl attribute* at the top level of a `.cue` file checks +the **entire file's** evaluated value against `VALUE`. All fields are +implicitly covered — no per-field `@test` is required. + +```cue +// decl_eq.txtar — in.cue + +a: 1 +b: a + 1 +c: { + x: "hello" + y: b +} +@test(eq, { + a: 1 + b: 2 + c: { + x: "hello" + y: 2 + } +}) +``` + +This form is useful when: + +- Pattern constraints contribute to the output (e.g. `{[X=string]: baz: X}`), + so per-field annotation would miss the constraint's effect. +- The test is a simple whole-file check and per-field annotation would be + redundant. +- Individual fields may still carry field-level `@test` attributes alongside + the file-level `@test`; both are checked. + +--- + +## Directives + +### `eq` — exact equality + +```cue +simple: 42 @test(eq, 42) // inline: argument is a CUE expression +complex: {a: 1} @test(file, "expected/r.cue") // inline: expected value in a txtar section +``` + +Comparison uses `internal/core/diff`. Insignificant differences (field +ordering, let-binding substitution, comment presence) are ignored. Error +*message text* is ignored; error *kind* and *code* are compared. + +#### Hidden fields in `eq` struct literals + +When the expected value of an `@test(eq, {...})` contains a hidden field that +belongs to a named package, append `$` to the field name to select +the correct package scope: + +```cue +// package foobar + +#Spec: { + _vars: {something: string} + data: yaml.Marshal(_vars.something) +} + +Val: #Spec & { + _vars: something: "var-string" +} @test(eq, { + _vars$foobar: something: "var-string" // _vars scoped to package "foobar" + data: "var-string\n" +}) +``` + +Without the `$foobar` qualifier the comparison engine would look for an +anonymous hidden field `_vars` (i.e. `cue.Hid("_vars", "_")`). With it, +the engine resolves `cue.Hid("_vars", "foobar")`, which matches the +`_vars(:foobar)` field produced by the CUE compiler for `package foobar` +sources. + +The `$pkg` suffix is only meaningful in `@test(eq, {...})` expected-value +struct literals. It is not valid CUE syntax in regular source files. + +--- + +### `leq` — subsumption constraint + +```cue +count: 5 @test(leq, int) +``` + +Asserts `evaluate(field) ⊑ constraint`. Useful for type-level assertions +without pinning an exact value. + +### `err` — error assertion + +```cue +errField: 1 & 2 @test(err) +errCycle: errCycle + 1 @test(err, code=cycle) +errMsg: 1 & 2 @test(err, contains="conflicting") +``` + +Optional arguments: + +| Argument | Meaning | +|----------|---------| +| `code=` | error code must match (`cycle`, `eval`, `incomplete`, …) | +| `contains="s"` | error message must contain substring `s` | +| `any` | at least one *descendant* has the error (requires `code=`) | +| `path=(p\|q)` | error exists at one of the listed paths | + +### `kind` — value kind + +```cue +kindInt: int @test(kind=int) +kindMixed: int | string @test(kind=int|string) +``` + +### `closed` — struct openness + +```cue +closedTrue: close({x: 1}) @test(closed) +closedFalse: {x: 1} @test(closed=false) +``` + +### `debugCheck` — debug-printer output + +```cue +debugScalar: 42 @test(debugCheck, "(int){ 42 }") +debugStruct: {a: 1} @test(debugCheck, "(struct){\n a: (int){ 1 }\n}") +``` + +Compares the string output of `internal/core/debug`'s printer applied to the +evaluated value. Useful for verifying internal representation details that `eq` +does not capture. + +### `skip` — skip a test case + +```cue +wip: someExpr @test(skip, why="not yet implemented") +``` + +A versioned form `skip:v3` skips only under evaluator version `v3`. + +### `permute` — field-order independence + +Asserts that the marked fields produce the same result in all N! orderings. + +```cue +// Field attribute form: mark each field to include in the permutation set. +permuteStruct: { + x: y + 1 @test(permute) + y: 2 @test(permute) +} @test(eq, {x: 3, y: 2}) @test(permuteCount, 2) +``` + +### `permuteCount` — verify permutation count + +```cue +permuteStruct: { + a: b + c @test(permute) + b: 1 @test(permute) + c: 2 @test(permute) +} @test(eq, {a: 3, b: 1, c: 2}) @test(permuteCount, 6) +``` + +Placed on the struct containing `@test(permute)` fields. Asserts that the +total number of evaluated permutations equals `N`. Auto-updated by +`CUE_UPDATE=1`. + +### `desc` — human-readable description + +```cue +myTest: {v: "ok"} @test(desc="description") @test(eq, {v: "ok"}) +``` + +Purely a documentation annotation — does not affect the sub-test name or +produce any assertion. + +--- + +## Empty placeholder and `CUE_UPDATE` + +`@test()` (empty body) is a fill-in placeholder. Running with `CUE_UPDATE=1` +evaluates the field and rewrites the attribute in the source file: + +| Evaluated result | Rewritten attribute | +|------------------|---------------------| +| Value | `@test(eq, )` | +| Error | `@test(err, code=, contains="")` | + +`CUE_UPDATE=diff` records a failing assertion as +`@test(..., skip:v3, diff="got …; want …")` rather than overwriting the +expected value. `CUE_UPDATE=force` overwrites unconditionally. -The tests are currently converted from various internal Go tests and the -grouping reflect properties of the current implementation. Once the transition -to the new implementation is completed, tests should be reordered along more -logical lines: such as literals, expressions, references, cycles, etc. +--- +## Version-specific overrides -## Forseen Structure +Any directive may carry a `:vN` suffix to apply only under that evaluator +version. When both a versioned and unversioned form appear on the same field, +the versioned form takes precedence: -The txtar format allows a collection of files to be defined. Any .cue file -is used as an input. The out/* files, which should not have an extension, -define outputs for various tests. A test definition is active for a certain test -if it contains output for this test. +```cue +// Uses 42 for all versions, but v3 overrides to 43. +myField: someExpr @test(eq, 42) @test(eq:v3, 43) -The comments section of the txtar file may contain additional control inputs for -a test. Each line that starts with a `#` immediately followed by a letter or -digit is specially interpreted. These can be boolean tags (`#foo`) or a -key-value pair (`#key: value`), where the value can be a free-form string. -A line starting with `#` followed by a space is interpreted as a comment. +// Skip only under v3. +wip: someExpr @test(eq, 42) @test(skip:v3, why="known regression") +``` -Lines not starting with a `#` are for interpretation by the testscript package. +--- -This organization allows the same test sets to be used for the testing of -tooling as well as internal libraries. +## Files in this directory -## Common options +| File | Feature area | +|------|-------------| +| `basic.txtar` | Basic inline form: `eq`, `err`, `kind`, `closed`, `permute`, `ignore` | +| `directives.txtar` | Comprehensive directive coverage for the inline form | +| `struct_eq.txtar` | Struct-level `@test(eq, ...)` as decl attribute | +| `decl_eq.txtar` | File-level `@test(eq, ...)`: whole-file equality check | +| `decl_eq_pattern.txtar` | File-level `@test(eq, ...)` with pattern constraints | +| `decl_eq_mixed.txtar` | File-level `@test(eq, ...)` coexisting with field-level `@test` | -- `#skip`: skip this test case for all tests -- `#skip-{name}`: skip this test for the namesake test -- `#todo-{name}`: skip this test for the namesake test, but run it if the - `--todo` flag is specified. +--- +## Running the tests -## Tests +```bash +# Run all inlinetest cases under the v3 evaluator +go test -run TestEvalV3 ./internal/core/adt -### cue/internal/compile +# Run a single sub-test +go test -run TestEvalV3/inlinetest/basic ./internal/core/adt -Compiles all *.cue files and prints the debug string of the internal -representation. This is not valid CUE. \ No newline at end of file +# Fill empty @test() placeholders +CUE_UPDATE=1 go test -run TestEvalV3/inlinetest ./internal/core/adt +``` diff --git a/cue/testdata/resolve/001.txtar b/cue/testdata/resolve/001.txtar index d4b79efaf..659395580 100644 --- a/cue/testdata/resolve/001.txtar +++ b/cue/testdata/resolve/001.txtar @@ -1,8 +1,7 @@ -#evalPartial -- in.cue -- -a: b.c.d -b: c: {d: 3} -c: {c: d.d} +a: b.c.d @test(eq, 3) +b: {c: {d: 3}} +c: {c: d.d} @test(eq, {c: 2}) d: {d: 2} -- out/compile -- --- in.cue @@ -30,18 +29,3 @@ Retain: 5 Unifications: 9 Conjuncts: 9 Disjuncts: 12 --- out/evalalpha -- -(struct){ - a: (int){ 3 } - b: (struct){ - c: (struct){ - d: (int){ 3 } - } - } - c: (struct){ - c: (int){ 2 } - } - d: (struct){ - d: (int){ 2 } - } -} diff --git a/doc/specs/inline-test-attributes/spec.md b/doc/specs/inline-test-attributes/spec.md new file mode 100644 index 000000000..013e53897 --- /dev/null +++ b/doc/specs/inline-test-attributes/spec.md @@ -0,0 +1,513 @@ +# Inline Test Attributes + +## Requirements + +### Requirement: Test attribute syntax +CUE evaluator test files SHALL support `@test(...)` attributes on fields and inside structs (decl attributes) to express inline assertions. A field or struct MAY carry multiple `@test(...)` attributes; all of them MUST be satisfied for the test to pass. + +#### Scenario: Field with single assertion +- **WHEN** a field is annotated `result: 42 @test(eq, 42)` +- **THEN** the test passes if the evaluated value of `result` equals `42` + +#### Scenario: Field with multiple assertions +- **WHEN** a field is annotated `result: {a:1} @test(eq, {a:1}) @test(kind=struct)` +- **THEN** the test passes only if BOTH the equality check and the kind check are satisfied + +--- + +### Requirement: Field attribute +When `@test(...)` appears as a *field attribute* (`field: value @test(...)`), the field value is the thing under test and the directive is an assertion applied directly to it. The test case name is the field name. + +A txtar file is in inline-assertion mode when any of its CUE files contains at least one `@test(...)` attribute — as a field attribute, a decl attribute inside a struct at any nesting depth, or a file-scope decl attribute. All other txtar files use the existing golden-file mechanism. + +#### Scenario: Field attribute triggers inline mode +- **WHEN** a txtar `.cue` file contains `result: 42 @test(eq, 42)` at the top level +- **THEN** the file is processed in inline-assertion mode and `result` is a test case + +#### Scenario: No @test anywhere means golden-file mode +- **WHEN** a txtar `.cue` file contains no `@test(...)` attributes +- **THEN** the file is processed using the existing golden-file mechanism unchanged + +--- + +### Requirement: Decl attribute +A `@test(...)` attribute may appear as a *decl attribute* — a bare attribute declaration inside a struct body at any nesting depth (provided the struct is on a concrete path) or at the top level of a `.cue` file (not as a field attribute). The value it is associated with depends on where it appears: + +- **Inside a struct body** (`field: { @test(...) ... }`): the attribute is associated with the containing field's value. The struct may be at any nesting depth, as long as it is on a concrete path. This form is used by directives like `permute` (marks all sibling fields for permutation) and `eq`/`err` (asserts the containing field's value). +- **At file scope** (top-level `@test(...)` in a `.cue` file, outside any struct): the attribute is associated with the **entire file's** evaluated value. This is useful when pattern constraints (e.g. `{[X=string]: baz: X}`) contribute to the output and there is no single field to annotate. + +Field-level `@test` attributes MAY coexist with decl attributes; all are checked independently. + +```cue +// Struct-level decl: checks the value of "result" +result: { + @test(eq, {a: 1}) + a: 1 +} + +// File-level decl: checks the entire file value +{[X=string]: baz: X} +bar: {} +@test(eq, {bar: baz: "bar"}) +``` + +#### Scenario: Struct-level decl @test(eq) checks containing field value +- **WHEN** a field `result` has `@test(eq, {a: 1})` as a decl attribute inside its struct body +- **THEN** the runner compares the value of `result` against `{a: 1}` + +#### Scenario: File-level @test(eq) checks entire file value +- **WHEN** a `.cue` file has `a: 1`, `b: a + 1`, and `@test(eq, {a: 1, b: 2})` at file scope +- **THEN** the runner compares the full file value against `{a: 1, b: 2}` and the test passes + +#### Scenario: File-level and field-level @test coexist +- **WHEN** a `.cue` file has both a file-level `@test(eq, ...)` and a field carrying `@test(eq, ...)` +- **THEN** both assertions are checked independently + +--- + +### Requirement: Directive with optional version suffix +The first positional argument of a `@test(...)` attribute SHALL be the *directive*, optionally followed by a colon and an evaluator version token (e.g., `v3`). A versioned directive SHALL apply only when the test runs under the matching evaluator version. When both a versioned and an unversioned instance of the same directive appear on the same field, the versioned form SHALL take precedence for its target version. + +#### Scenario: Version-specific equality assertion +- **WHEN** a field carries `@test(eq, 1) @test(eq:v3, 2)` and the runner is evaluating with v3 +- **THEN** the assertion `eq:v3` is used and the test checks that the value equals `2` + +#### Scenario: Version-specific skip +- **WHEN** a test case carries `@test(skip:v3)` and the runner is evaluating with v3 +- **THEN** the test is skipped for v3 only; it runs normally under other versions + +--- + +### Requirement: `eq` directive +The `eq` directive SHALL assert equality between the evaluated value at the annotated field and an expected CUE expression. The comparison is performed by walking the expected expression as a parsed AST and comparing it structurally against the evaluated value — the expected expression is never compiled, which prevents evaluator bugs from masking mismatches. + +Comparison behavior: +- Disjunctions are compared structurally, order-independently, preserving default markers (`*`). +- Pattern constraints (`[T]: V`) are compared as a guideline where possible; the runner checks for existence and value match on a best-effort basis. +- Definitions (`#D`) and hidden definitions (`_#D`) are compared. +- Concrete values (e.g. `1`, `"foo"`) use compile-and-compare semantics and match regardless of how the value was constructed. +- Structs: by default, field order is ignored. `@test(checkOrder)` as a decl attribute inside the expected struct additionally asserts that fields appear in the listed order. +- `@test(final)` as a decl attribute inside the expected struct opts into compile-and-compare for all fields in that struct (resolving disjunction defaults before comparison). +- `@test(final)` on a specific field inside the expected struct opts into compile-and-compare for that field only. +- Let bindings (`let x = expr`) in the expected struct SHALL be compared: the runner checks that the corresponding let binding in the evaluated vertex has the same value as `expr`. + +Skipping sub-comparisons — `@test(ignore)` inside an expected value: +- A field inside the expected struct MAY carry `@test(ignore)` as a field attribute. When present, `eq`'s recursive descent into that element is halted: the element need not be present in the evaluated value and its value is not compared. +- `@test(ignore)` does NOT suppress other directives on the same element. For example, `b: _ @test(ignore) @test(err, any)` will still run the `@test(err, any)` check on `b`'s subtree; only the `eq` value comparison is skipped. +- To skip a let binding's value comparison, omit the let clause from the expected struct entirely — the runner only checks lets that are explicitly listed. + +```cue +// Skip a subfield but still assert it contains an error somewhere: +result @test(eq, { + a: 1 + b: _ @test(ignore) @test(err, any, code=cycle) // skip eq check; still check for cycle error in b +}) + +// To skip checking a let binding, simply omit it from the expected struct: +result @test(eq, { + a: 1 + // let b is not listed, so its value is not checked + c: 3 +}) +``` + +Error handling within the expected struct: +- A field inside the expected struct may carry `@test(err, ...)` to assert that the corresponding field in the evaluated value is an error instead of performing a normal value comparison. +- Positions in nested `@test(err)` specs use absolute line numbers since there is no meaningful anchor inside the attribute body. + +Non-test attributes — both field attributes (e.g. `@json(...)`, `@protobuf(...)`) and decl attributes — inside the expected struct ARE compared: the runner SHALL verify that the corresponding location in the evaluated value carries exactly the same attributes (order-independent; duplicate keys supported). Both missing attributes (expected but absent in the evaluated value) and extra attributes (present in the evaluated value but not in the expected struct) are reported as failures. `@test(...)` attributes are always excluded from this comparison. + +Hidden fields in package-scoped sources — `$pkg` qualifier: +When the evaluated value contains a hidden field that belongs to a named CUE package (e.g. `_vars` in `package foobar` compiles to `_vars(:foobar)` internally), the expected struct in `@test(eq, {...})` must qualify the hidden field name with `$` to select the correct package scope. Without the qualifier the runner looks for an anonymous hidden field (`cue.Hid(name, "_")`); with it the runner resolves `cue.Hid(name, pkgname)`: + +```cue +// package foobar + +Val: #Spec & { + _vars: something: "var-string" +} @test(eq, { + _vars$foobar: something: "var-string" // hidden field _vars scoped to package "foobar" + data: "var-string\n" +}) +``` + +The `$pkg` suffix is only meaningful inside `@test(eq, {...})` expected-value struct literals. It is not valid CUE syntax in regular source files. + +The expected value is supplied as a CUE expression argument: `@test(eq, )`. The argument is valid for any CUE expression that can appear on a single line; for complex multi-line values write them as a struct on one line or use nested attributes. + +```cue +// Scalar +simple: 42 @test(eq, 42) + +// Struct +obj: {a: 1, b: 2} @test(eq, {a: 1, b: 2}) +``` + +#### Scenario: Package-scoped hidden field with $pkg qualifier +- **WHEN** `@test(eq, {_vars$foobar: something: "val"})` is declared and the evaluated value has hidden field `_vars(:foobar)` with `something: "val"` +- **THEN** the test passes; the `$foobar` qualifier directs the runner to use `cue.Hid("_vars", "foobar")` (with `:foobar` fallback for inline-compiled sources) + +#### Scenario: Package-scoped hidden field without qualifier fails +- **WHEN** `@test(eq, {_vars: something: "val"})` is declared and the only hidden field is package-scoped `_vars(:foobar)` (not anonymous) +- **THEN** the test fails because `cue.Hid("_vars", "_")` does not find the package-scoped field + +#### Scenario: Inline scalar equality +- **WHEN** a field carries `@test(eq, 42)` and evaluates to `42` +- **THEN** the test passes + +#### Scenario: Inline equality mismatch +- **WHEN** a field carries `@test(eq, 42)` and evaluates to `43` +- **THEN** the test fails with a message showing the expected and actual values + +#### Scenario: Field attributes in expected struct are compared +- **WHEN** `@test(eq, {a: 1 @foo() @bar()})` is declared and the evaluated value has `a: 1 @bar() @foo()` +- **THEN** the test passes (attribute matching is order-independent) + +#### Scenario: Missing attribute in evaluated value fails +- **WHEN** `@test(eq, {a: 1 @foo()})` is declared and the evaluated value has `a: 1` with no attributes +- **THEN** the test fails reporting the missing `@foo()` attribute + +#### Scenario: Extra attribute in evaluated value fails +- **WHEN** `@test(eq, {a: 1})` is declared and the evaluated value has `a: 1 @foo()` +- **THEN** the test fails reporting the unexpected `@foo()` attribute + +#### Scenario: Nested err directive asserts error on specific field +- **WHEN** `@test(eq, {a: _|_ @test(err, code=eval)})` is declared and field `a` in the evaluated value is an eval error +- **THEN** the test passes + +#### Scenario: Error message rewording does not cause mismatch +- **WHEN** the evaluated value and the expected value have the same error kind and code but different message text +- **THEN** the test passes; only error presence, kind, and code are compared + +#### Scenario: Error kind change causes mismatch +- **WHEN** the evaluated value has an `incomplete` error and the expected value has a `cycle` error +- **THEN** the test fails + +#### Scenario: checkOrder asserts field ordering +- **WHEN** `@test(eq, {a: 1, b: 2, @test(checkOrder)})` is declared and the evaluated struct has fields in order `b`, `a` +- **THEN** the test fails reporting the field order mismatch + +#### Scenario: Let binding is compared +- **WHEN** `@test(eq, {a: 1, let b = 3})` is declared and the evaluated vertex has `let b = 3` +- **THEN** the test passes + +#### Scenario: Let binding value mismatch fails +- **WHEN** `@test(eq, {let b = 3})` is declared but the evaluated vertex has `let b = 4` +- **THEN** the test fails reporting the let value mismatch + +#### Scenario: @test(ignore) skips subfield eq check +- **WHEN** `@test(eq, {a: 1, b: _ @test(ignore)})` is declared and the evaluated struct has `a: 1` with any value for `b` (or no `b` at all) +- **THEN** the `eq` comparison of `b` is skipped; the test passes + +#### Scenario: @test(ignore) does not suppress other directives +- **WHEN** `@test(eq, {b: _ @test(ignore) @test(err, any, code=cycle)})` is declared and `b` in the evaluated value contains a cycle error somewhere in its subtree +- **THEN** the `eq` check on `b` is skipped, but the `@test(err, any, code=cycle)` assertion still runs and passes + +--- + +### Requirement: `leq` directive +The `leq` directive SHALL assert that the evaluated value is *subsumed by* the given CUE expression (i.e., the constraint subsumes the result — the result is at least as specific as the constraint). This allows asserting type-level constraints without pinning an exact value. + +#### Scenario: Value subsumed by type constraint +- **WHEN** a field carries `@test(leq, int)` and evaluates to `42` +- **THEN** the test passes because `int` subsumes `42` + +#### Scenario: Value not subsumed +- **WHEN** a field carries `@test(leq, string)` and evaluates to `42` +- **THEN** the test fails + +--- + +### Requirement: `err` directive +The `err` directive SHALL assert that an error exists at the annotated location. Without sub-options, the error MUST exist at exactly the annotated field. The directive supports the following optional key-value sub-options: + +- `code=` — the error code must match. Valid codes include `cycle`, `eval`, `incomplete`, `structural`, `reference`, and any other code defined in `cuelang.org/go/cue/errors`. +- `contains="substring"` — the error message must contain the given substring. +- `any` (bare flag) — at least one **descendant** of the annotated field must have an error. The annotated field itself is not required to be an error. `code` MUST be specified when `any` is used. +- `pos=[spec ...]` — asserts the exact set of source positions reported by the error (as returned by `cuelang.org/go/cue/errors.Positions`). Each whitespace-delimited spec takes one of two forms: + - `deltaLine:col` — position in the **same file** as the `@test` attribute, expressed as a signed line offset from an *anchor line* and a 1-indexed column. The anchor depends on where the `@test` attribute appears: + - **Field attribute** (`field: value @test(...)`): anchor is the field's line in the stripped output (`deltaLine=0` = same line as the field). + - **Struct-level decl attribute** (inside `{ @test(...) ... }`): anchor is the line of the opening `{` of the enclosing struct (`deltaLine=1` = first line inside the struct body). This keeps the assertion stable when the `@test` line itself is stripped, and gives the natural reading "N lines into the struct". + - **File-level decl attribute** (`@test(...)` at file scope): anchor is the `@test` attribute's own line in the stripped output. + - `filename:absLine:col` — position in a **different file** (e.g. a shared fixture), expressed as the archive file name, a 1-indexed absolute line number, and a 1-indexed column. + An empty `pos=[]` is a fill-in placeholder: `CUE_UPDATE=1` fills in the actual positions. Non-empty `pos=[...]` that mismatches requires `CUE_UPDATE=force` to overwrite. + +#### Scenario: Error at exact field +- **WHEN** a field carries `@test(err, code=cycle)` and evaluates to a cycle error at that field +- **THEN** the test passes + +#### Scenario: Error in a descendant via any — field attribute form +- **WHEN** a field carries `@test(err, any, code=cycle)` as a *field attribute* and at least one descendant of that field has a cycle error +- **THEN** the test passes + +#### Scenario: Error code mismatch +- **WHEN** a field carries `@test(err, code=cycle)` and the error code is `incomplete` +- **THEN** the test fails + +#### Scenario: No error when error expected +- **WHEN** a field carries `@test(err)` and the field evaluates without error +- **THEN** the test fails + +#### Scenario: pos= matches error positions on field attribute +- **WHEN** `@test(err, pos=[0:5 0:14])` is a **field attribute** and the error reports exactly two positions on the same line as the field (column 5 and column 14) +- **THEN** the test passes + +#### Scenario: pos= matches error positions on struct decl attribute +- **WHEN** `d: { @test(err, pos=[1:5 2:2]) ... }` is declared and the error reports positions at the 1st and 2nd lines inside `d`'s `{` at the given columns +- **THEN** the test passes (`deltaLine=1` means 1 line after `{`, i.e. the first field) + +#### Scenario: pos= empty placeholder fills on CUE_UPDATE +- **WHEN** `@test(err, pos=[])` is declared and `CUE_UPDATE=1` is set +- **THEN** the runner fills in the actual positions, writing e.g. `pos=[0:5 0:14]` to the source file + +#### Scenario: pos= cross-file absolute position +- **WHEN** `@test(err, pos=[fixture.cue:3:5])` is declared and the error position is at absolute line 3, column 5 in `fixture.cue` +- **THEN** the test passes + +#### Scenario: pos= mismatch fails test +- **WHEN** `@test(err, pos=[0:5])` is declared but the error has two positions +- **THEN** the test fails reporting the count mismatch + +--- + +### Requirement: `kind` directive +The `kind` directive SHALL assert that the evaluated value is of the specified kind. Multiple kinds separated by `|` SHALL mean any of those kinds is acceptable. The kind is given as a key=value argument. + +Valid kind names: `bool`, `int`, `float`, `string`, `bytes`, `struct`, `list`, `null`, `top` (or `_`), `bottom` (or `_|_`). + +#### Scenario: Kind check passes +- **WHEN** a field carries `@test(kind=struct)` and evaluates to `{a: 1}` +- **THEN** the test passes + +#### Scenario: Kind check fails +- **WHEN** a field carries `@test(kind=list)` and evaluates to `{a: 1}` +- **THEN** the test fails + +#### Scenario: Multiple acceptable kinds +- **WHEN** a field carries `@test(kind=int|float)` and evaluates to `3.14` +- **THEN** the test passes because `float` is in the accepted set + +--- + +### Requirement: `closed` directive +The `closed` directive (bare flag) SHALL assert that the evaluated struct is closed. `closed=false` SHALL assert that it is open. + +#### Scenario: Closed struct assertion +- **WHEN** a field carries `@test(closed)` and the struct is closed +- **THEN** the test passes + +#### Scenario: Open struct when closed required +- **WHEN** a field carries `@test(closed)` and the struct is open +- **THEN** the test fails + +--- + +### Requirement: `skip` directive +The `skip` directive SHALL cause the test case or individual assertion to be skipped. An optional `why="reason"` key-value arg provides a human-readable explanation. A versioned form `skip:v3` skips only when running under evaluator version `v3`. + +`@test(skip)` skips only the field or struct in which it is defined, not the entire file. Other test cases in the same file continue to run normally. + +#### Scenario: Skip with reason +- **WHEN** a test case carries `@test(skip, why="not yet supported")` +- **THEN** the test is reported as skipped with the reason shown in the test output + +#### Scenario: Skip is scoped to the annotated field +- **WHEN** field `a` carries `@test(skip)` and field `b` carries `@test(eq, 2)` in the same file +- **THEN** `a` is skipped but `b` runs normally + +--- + +### Requirement: `todo` directive +The `todo` directive is like `skip`, but acts as a signal that the skipped test case is expected to be fixed in the near future rather than being permanently skipped. An optional `p=N` priority key-value arg classifies urgency: `p=0` is critical (must fix immediately), `p=1` is important (fix soon), `p=2` is good to have (fix when convenient). An optional `why="reason"` key-value arg provides a human-readable explanation. + +`@test(todo)` skips only the field or struct in which it is defined, not the entire file. + +#### Scenario: Todo skips with priority signal +- **WHEN** a test case carries `@test(todo, p=0, why="broken by recent evaluator change")` +- **THEN** the test is reported as skipped with the priority and reason shown in the test output + +--- + +### Requirement: `desc` directive +The `desc` directive is a human-readable description annotation with no assertion semantics. It SHALL be silently accepted and ignored during test evaluation. Its only purpose is to document the intent of a test case in the source file. + +```cue +result: 42 @test(eq, 42) @test(desc="the base case") +``` + +--- + +### Requirement: `permute` directive +The `permute` directive instructs the runner to generate all field-order permutations of a specified set of fields and assert that the evaluated result is the same for every permutation (modulo field ordering). This tests that the CUE evaluator is genuinely order-independent for the indicated fields. + +The directive has two placement forms: + +**Field attribute form** — `@test(permute)` on individual fields marks each field as a member of the permuted set. All fields carrying `@test(permute)` at the same struct level are permuted together as a group: + +```cue +in: { + a: 1 @test(permute) + b: a+1 @test(permute) + c: b+1 @test(permute) +} +``` + +**Decl attribute form** — `@test(permute)` inside a struct permutes *all* fields in that struct: + +```cue +in: { + @test(permute) // permutes a, b, and c together + a: 1, b: a+1, c: b+1 +} +``` + +Both forms are equivalent when all fields at the level are marked. Use the field attribute form to exclude specific fields from permutation; use the decl form for brevity when all fields participate. + +The runner SHALL generate all N! permutations of the marked fields, evaluate each, and assert the results are structurally identical (using AST comparison) modulo field ordering. If any permutation produces a different result, the test SHALL fail, listing the differing permutation. + +#### Scenario: All permutations yield identical result +- **WHEN** `@test(permute)` is declared and all N! field-order permutations of the marked fields produce the same evaluated result +- **THEN** the test passes + +#### Scenario: Permutation produces different result +- **WHEN** one permutation of the marked fields produces a different evaluated result from another +- **THEN** the test fails, reporting which permutation differs and how + +#### Scenario: Decl form permutes all fields +- **WHEN** `@test(permute)` is placed as a decl attribute inside a struct containing three fields +- **THEN** the runner permutes all three fields (equivalent to placing `@test(permute)` on each individually) + +#### Scenario: Field form selects subset +- **WHEN** only two of three fields carry `@test(permute)` as a field attribute +- **THEN** only those two fields are permuted; the third remains in its original position for all permutations + +--- + +### Requirement: `permuteCount` directive +The `permuteCount` directive asserts the total number of permutations that were executed for the test case root. It is placed as a field attribute on the test root. When `CUE_UPDATE=1` is set, the count is auto-filled if empty or replaced if it differs. + +```cue +in: { + a: 1 @test(permute) + b: a+1 @test(permute) +} @test(permuteCount, 2) +``` + +#### Scenario: Permutation count matches +- **WHEN** `@test(permuteCount, 6)` is declared and exactly 6 permutations ran +- **THEN** the test passes + +#### Scenario: Permutation count mismatch +- **WHEN** `@test(permuteCount, 6)` is declared but only 2 permutations ran +- **THEN** the test fails reporting the count mismatch + +--- + +### Requirement: `debugCheck` directive +The `debugCheck` directive asserts that the debug-printer output of the evaluated value matches the expected string. The debug-printer output is the same internal representation that appears in `out/eval` golden sections. + +When `CUE_UPDATE=1` or `CUE_UPDATE=force` is set, the expected string is auto-filled or replaced. Unlike `eq`, a mismatch with `debugCheck` is a hard test failure (not a regression guard). + +```cue +result: {a: 1} @test(debugCheck, """""") +``` + +An empty `@test(debugCheck)` with no argument behaves as a fill placeholder under `CUE_UPDATE=1`. + +#### Scenario: Debug output matches +- **WHEN** a field carries `@test(debugCheck, "...")` and the debug-printer output equals the expected string +- **THEN** the test passes + +#### Scenario: Debug output mismatch +- **WHEN** a field carries `@test(debugCheck, "...")` and the debug-printer output differs +- **THEN** the test fails + +--- + +### Requirement: `debugOutput` directive +The `debugOutput` directive is an informational annotation that records the debug-printer output of the evaluated value. Unlike `debugCheck`, a mismatch does NOT fail the test — it only logs a difference and auto-updates when `CUE_UPDATE=1` is set. + +This is useful for documenting what internal representation a value produces without locking the test to that exact representation. + +```cue +result: {a: 1} @test(debugOutput, """""") +``` + +#### Scenario: Debug output annotation — no test failure on mismatch +- **WHEN** a field carries `@test(debugOutput, "...")` and the actual debug output differs +- **THEN** the difference is logged but the test does not fail + +--- + +### Requirement: Empty `@test()` as placeholder +A field carrying `@test()` (empty attribute body) SHALL be treated as an unfilled assertion placeholder. When `CUE_UPDATE=1` or `CUE_UPDATE=force` is set, the runner SHALL evaluate the field and rewrite the attribute to `@test(eq, )`. Without `CUE_UPDATE`, an empty `@test()` SHALL cause the test to fail with a message prompting the author to run `CUE_UPDATE=1`. + +#### Scenario: Scaffold assertion via CUE_UPDATE +- **WHEN** a field carries `@test()` and `CUE_UPDATE=1` is set +- **THEN** the txtar source file is updated with the evaluated value as an `@test(eq, ...)` assertion + +--- + +### Requirement: Regression guard for failing `eq` assertions +When `CUE_UPDATE=1` is set and an `eq` assertion **fails** (genuine mismatch), the runner SHALL NOT silently overwrite the expected value. Instead it SHALL annotate the attribute with `skip:` and `diff="..."` arguments that record the discrepancy without changing the nominal expected value. The test is then effectively skipped for that version. + +`CUE_UPDATE=force` (`CUE_UPDATE=force` env value) SHALL overwrite the expected value unconditionally, regardless of whether it was passing before. + +When a previously-failing (skip-annotated) assertion now passes again under `CUE_UPDATE=1`, the runner SHALL remove the stale `skip:` and `diff=` arguments, restoring the plain `@test(eq, )` form. + +#### Scenario: Record failing assertion as regression guard +- **WHEN** `@test(eq, 42)` fails because the value is `43` and `CUE_UPDATE=1` is set +- **THEN** the attribute is rewritten to `@test(eq, 42, skip:v3, diff="got 43; want 42")` in the source file + +#### Scenario: Remove stale skip on recovery +- **WHEN** `@test(eq, 42, skip:v3, diff="got 43; want 42")` now passes and `CUE_UPDATE=1` is set +- **THEN** the `skip:v3` and `diff=` arguments are removed, leaving `@test(eq, 42)` + +#### Scenario: Force overwrite with CUE_UPDATE=force +- **WHEN** `@test(eq, 42)` fails because the value is `43` and `CUE_UPDATE=force` is set +- **THEN** the attribute is rewritten to `@test(eq, 43)` unconditionally + +--- + +### Requirement: `out/errors.txt` documentary section +A txtar archive in inline-assertion mode MAY contain an `-- out/errors.txt --` section that records all evaluation errors (including incomplete errors) in a human-readable format with `[code]` prefixes. This section serves as documentation of expected error output — it is never auto-created and never causes test failures on its own. + +The error format prefixes each error with its error code in square brackets, followed by the formatted error message from `cuelang.org/go/cue/errors`: + +``` +[eval] e: index out of range [4] with length 0: + in.cue:4:11 +[incomplete] f: undefined field: b: + in.cue:5:11 +``` + +Behavior matrix: + +| Section present? | `CUE_UPDATE=1` | Normal run | `CUE_CHECK=1` | +|-----------------|---------------|------------|---------------| +| No | skip (don't create) | skip | skip | +| Yes | update silently | skip (no fail) | fail if stale | + +Key points: +- The section is **never auto-created** by `CUE_UPDATE=1`. It must be added manually. +- A normal test run silently ignores differences in this section. +- `CUE_CHECK=1` enables strict mode: the section fails if its content is stale. +- `CUE_UPDATE=1` updates the section's content when it already exists. + +Both child errors (incomplete, eval, cycle, etc.) and propagated-from-child error markers are included. Child-only marker errors (which don't carry their own message) are excluded. + +#### Scenario: Section absent — always skipped +- **WHEN** the archive has no `-- out/errors.txt --` section +- **THEN** no errors.txt logic runs under any mode + +#### Scenario: Section present — normal run ignores differences +- **WHEN** the archive has an `-- out/errors.txt --` section with stale content +- **THEN** the test passes (differences are silently ignored) + +#### Scenario: Section present — CUE_UPDATE=1 updates content +- **WHEN** the archive has an `-- out/errors.txt --` section and `CUE_UPDATE=1` is set +- **THEN** the section is updated with the current error output + +#### Scenario: Section present — CUE_CHECK=1 fails on stale content +- **WHEN** the archive has an `-- out/errors.txt --` section with stale content and `CUE_CHECK=1` is set +- **THEN** the test fails with a message showing the expected and actual error output diff --git a/doc/specs/test-migration-tool/spec.md b/doc/specs/test-migration-tool/spec.md new file mode 100644 index 000000000..744007b1e --- /dev/null +++ b/doc/specs/test-migration-tool/spec.md @@ -0,0 +1,59 @@ +# Test Migration Guidelines + +These are guidelines for migrating golden-file txtar tests to inline `@test` +assertions. They are not a formal requirement for a tool; use them as a manual +checklist or the basis for ad-hoc tooling. + +--- + +## Guideline: Basic conversion + +A txtar archive in golden-file mode can be converted to inline-assertion mode +by: + +1. Parsing the `out/eval` or `out/evalalpha` section to derive expected values. +2. Adding `@test(eq, )` field attributes (or decl attributes) directly + in the `.cue` source. +3. Removing the `out/eval` section after conversion (the `out/compile` section, + if present, should remain unchanged). + +--- + +## Guideline: Error detection + +Fields whose golden output contains `(_|_)` or `// Error` markers should be +annotated with `@test(err, ...)`. Where the error code can be determined from +the golden output, use `code=`. When uncertain, use `@test(err, any)` on +the nearest enclosing struct or `@test(err)` on the specific field. + +--- + +## Guideline: Version-specific golden output + +When both `out/eval` (v2) and `out/evalalpha` (v3) sections exist: + +- If they produce the **same result**: use the v3 output as the unversioned + `eq` assertion. +- If they **diverge**: use the v3 output as the unversioned `eq` assertion and + record the v2 difference as a versioned `@test(eq:v2, ...)` attribute. + +This makes v3 the forward-looking baseline while keeping v2 divergences visible. + +--- + +## Guideline: Permutation groups + +Sub-fields `p1`, `p2`, `p3` (etc.) that contain the same fields in different +orders and produce identical evaluated results can be collapsed into a single +test case with `@test(permute)` on the relevant fields or as a decl attribute +inside the struct. + +--- + +## Guideline: Fixture fields + +Top-level fields that are shared helpers and not test cases in their own right +should be placed in a separate `.cue` fixture file (or a file with no `@test` +attributes). A file with no `@test` attributes anywhere is treated as a pure +fixture file and its fields are available to test files without registration as +sub-tests. diff --git a/internal/core/adt/eval_test.go b/internal/core/adt/eval_test.go index 0f2bdc822..d5f62de9c 100644 --- a/internal/core/adt/eval_test.go +++ b/internal/core/adt/eval_test.go @@ -71,10 +71,17 @@ func TestEvalV2(t *testing.T) { func TestEvalV3(t *testing.T) { adt.DebugDeps = true // check unmatched dependencies. + // TxTarTest.run dispatches inline archives (those with @test attributes) + // to the inline runner automatically; non-inline archives use golden-file + // comparison as before. No separate RunInlineTests call is needed. + // + // Fallback lets existing out/eval golden-file sections pass until + // out/evalalpha sections are generated with CUE_UPDATE=1. test := cuetxtar.TxTarTest{ Root: "../../../cue/testdata", Name: "evalalpha", Fallback: "eval", // Allow eval golden files to pass these tests. + Inline: true, // Dispatch @test archives to inline runner. } cuedebug.Init() @@ -256,6 +263,28 @@ language: version: "v0.15.0" t.Log(ctx.Stats()) } +// TestY is for debugging inline tests. Do not delete. +func TestY(t *testing.T) { + in := ` +-- cue.mod/module.cue -- +module: "mod.test" + +language: version: "v0.15.0" + +-- in.cue -- +s1: !="b" & =~"c" @test(eq, =~"c") +s2: !=null @test(eq, !=null) + ` + + if strings.HasSuffix(strings.TrimSpace(in), ".cue --") { + t.Skip() + } + + archive := txtar.Parse([]byte(in)) + runner := cuetxtar.NewInlineRunner(t, nil, archive, t.TempDir()) + runner.Run() +} + func BenchmarkUnifyAPI(b *testing.B) { for i := 0; i < b.N; i++ { b.StopTimer() diff --git a/internal/core/runtime/testdata/nullinit.txtar b/internal/core/runtime/testdata/nullinit.txtar index 023bb147e..b6a7b9414 100644 --- a/internal/core/runtime/testdata/nullinit.txtar +++ b/internal/core/runtime/testdata/nullinit.txtar @@ -6,7 +6,7 @@ package nullinit -foo: _ @test("file.xx") +foo: _ @testfn("file.xx") -- out/extern -- { diff --git a/internal/cuetxtar/astcmp.go b/internal/cuetxtar/astcmp.go new file mode 100644 index 000000000..f7db0b8b3 --- /dev/null +++ b/internal/cuetxtar/astcmp.go @@ -0,0 +1,901 @@ +// Copyright 2026 CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cuetxtar + +import ( + "fmt" + "strings" + + "cuelang.org/go/cue" + "cuelang.org/go/cue/ast" + cueerrors "cuelang.org/go/cue/errors" + "cuelang.org/go/cue/format" + "cuelang.org/go/cue/token" + "cuelang.org/go/internal" + "cuelang.org/go/internal/core/adt" + "cuelang.org/go/internal/core/export" + "cuelang.org/go/internal/value" +) + +// cmpCtx carries comparison options through the recursive astCmp calls. +type cmpCtx struct { + // baseLine is the 1-indexed source line used to resolve relative pos= + // specs in nested @test(err) directives. When 0, deltaLine values in + // pos specs are treated as absolute line numbers. + baseLine int +} + +// astCompare compares a parsed CUE AST expression against an evaluated +// cue.Value. It returns nil when they match, or a descriptive error. +// +// Unlike compiling the expected expression and using Value.Equal or diff.Diff, +// this approach avoids masking evaluator bugs — the expected value is never +// compiled, so a bug that affects both sides equally cannot hide a mismatch. +// +// Additionally, this enables richer comparison than cue.Final() allows: +// - Disjunctions are compared structurally, order-independently, preserving +// default markers. +// - Pattern constraints ([T]: V) are checked for existence and value. +// - Definitions (#D) and hidden definitions (_#D) are compared. +func astCompare(expr ast.Expr, val cue.Value) error { + return (&cmpCtx{}).astCmp(cue.Path{}, expr, val) +} + +// astCmp is the recursive comparison workhorse. path accumulates the +// human-readable location for error messages. +// +// It checks structural consistency: if the value is a disjunction but the +// expected AST is not (or vice versa for conjunctions), an error is reported. +// Use @test(final) on a field in the expected struct to opt into +// default-resolution, allowing a plain value to match a disjunction default. +func (c *cmpCtx) astCmp(path cue.Path, expr ast.Expr, val cue.Value) error { + // Structural consistency check: if the value is a disjunction or + // conjunction, the expected AST must reflect that structure. + if err := checkStructuralMatch(path, expr, val); err != nil { + return err + } + + switch e := expr.(type) { + case *ast.StructLit: + return c.cmpStruct(path, e, val) + case *ast.ListLit: + return c.cmpList(path, e, val) + case *ast.BinaryExpr: + switch e.Op { + case token.AND: + return c.cmpConjunction(path, e, val) + case token.OR: + return c.cmpDisjunction(path, e, val) + } + case *ast.UnaryExpr: + // evaluate non-boundary expressions. + switch e.Op { + case token.ADD, token.SUB, token.NOT: + return cmpFinal(path, e, val) + } + return c.cmpUnaryExpr(path, e, val) + case *ast.BasicLit: + return cmpFinal(path, e, val) + case *ast.Ident: + return cmpIdent(path, e, val) + case *ast.ParenExpr: + return c.astCmp(path, e.X, val) + case *ast.BottomLit: + return nil + } + return pathErr(path, "unsupported AST node type %T", expr) +} + +// cmpFinal compiles the AST expression and checks that the compiled value +// equals val. This is the @test(final) path: it opts out of structural +// AST comparison in favor of compile-and-compare. Defaults are resolved +// on val before the check. +func cmpFinal(path cue.Path, expr ast.Expr, val cue.Value) error { + b, err := format.Node(expr) + if err != nil { + return pathErr(path, "cannot format expression: %v", err) + } + ctx := val.Context() + expected := ctx.CompileString(string(b)) + if err := expected.Err(); err != nil { + return pathErr(path, "cannot compile expression %q: %v", b, err) + } + if !valuesEqual(expected, val) { + return pathErr(path, "expected %s, got %v", b, val) + } + return nil +} + +// valuesEqual compares two cue.Values for equality, ignoring ArcType +// differences. This is necessary because values from optional/required +// fields have a different ArcType than compiled values, which causes +// cue.Value.Equals to return false even when the values are identical. +func valuesEqual(a, b cue.Value) bool { + if a.Equals(b) { + return true + } + // Normalize ArcType: temporarily set b's ArcType to match a's. + av, bv := a.Core().V, b.Core().V + if av.ArcType == bv.ArcType { + return false // ArcType already matches; genuine inequality. + } + saved := bv.ArcType + bv.ArcType = av.ArcType + opCtx := value.OpContext(a) + eq := adt.Equal(opCtx, av, bv, 0) + bv.ArcType = saved + return eq +} + +// isASTDisjunction reports whether expr represents a disjunction (| or *). +func isASTDisjunction(expr ast.Expr) bool { + switch e := expr.(type) { + case *ast.BinaryExpr: + return e.Op == token.OR + case *ast.ParenExpr: + return isASTDisjunction(e.X) + } + return false +} + +// isASTConjunction reports whether expr represents a conjunction (&). +func isASTConjunction(expr ast.Expr) bool { + e, ok := expr.(*ast.BinaryExpr) + return ok && e.Op == token.AND +} + +// checkStructuralMatch checks that the AST and value agree on whether +// they are disjunctions or conjunctions. This prevents a plain expected +// value like 1 from silently matching *1 | 2 through default-resolution. +// +// The check is skipped for concrete values: a concrete int like 1 is fine +// even if its Expr() decomposes to a conjunction (e.g. from a pattern +// constraint int & 1), since the result is unambiguously concrete. +func checkStructuralMatch(path cue.Path, expr ast.Expr, val cue.Value) error { + if val.IsConcrete() { + return nil + } + // Struct literals may contain @test(final) which handles mismatches + // internally via cmpFinal, so defer the check to cmpStruct. + if _, ok := expr.(*ast.StructLit); ok { + return nil + } + tv := val.Core() + switch tv.V.BaseValue.(type) { + case *adt.Disjunction: + if !isASTDisjunction(expr) { + return pathErr(path, "value is a disjunction but expected expression is not; "+ + "use a disjunction in the expected value or @test(final) on the field") + } + case *adt.Conjunction: + if !isASTConjunction(expr) { + return pathErr(path, "value is a conjunction but expected expression is not; "+ + "use a conjunction in the expected value") + } + } + + return nil +} + +// ── structs ───────────────────────────────────────────────────────────────── + +func (c *cmpCtx) cmpStruct(path cue.Path, s *ast.StructLit, val cue.Value) error { + // Collect expected fields, let bindings, and pattern constraints from the AST. + // Also detect @test(checkOrder), @test(final), and non-@test attributes. + type expectedField struct { + name string // raw field name (for errors and checkOrder) + sel cue.Selector // ready-to-use selector (encodes kind, pkg, constraint) + final bool // @test(final): resolve default before comparing + ignore bool // @test(ignore): skip eq descent; field need not exist + errCheck *errArgs // @test(err, ...): check value is error instead of comparing + value ast.Expr + attrs []*ast.Attribute // non-@test attributes + } + type expectedLet struct { + name string + expr ast.Expr + } + type expectedPattern struct { + label ast.Expr // the expression inside [...] + value ast.Expr + } + + var fields []expectedField + var lets []expectedLet + var patterns []expectedPattern + checkOrder := false + allFinal := false + + for _, d := range s.Elts { + switch d := d.(type) { + case *ast.Attribute: + // Look for @test directives; ignore other @test attributes. + if k, _ := d.Split(); k == "test" { + pa, err := parseTestAttr(d) + if err == nil { + switch pa.directive { + case "checkOrder": + checkOrder = true + case "final": + allFinal = true + } + } + } + case *ast.LetClause: + lets = append(lets, expectedLet{name: d.Ident.Name, expr: d.Expr}) + case *ast.Field: + // Separate non-@test attributes from @test attributes. + // Detect @test(final), @test(ignore), and @test(err) on individual fields. + var nonTestAttrs []*ast.Attribute + isFinal := false + isIgnore := false + var errCk *errArgs + for _, a := range d.Attrs { + if k, _ := a.Split(); k == "test" { + pa, err := parseTestAttr(a) + if err == nil { + switch pa.directive { + case "final": + isFinal = true + case "ignore": + isIgnore = true + case "err": + errCk = pa.errArgs + if errCk == nil { + errCk = &errArgs{} // bare @test(err) + } + } + } + } else { + nonTestAttrs = append(nonTestAttrs, a) + } + } + constraint := d.Constraint + switch label := d.Label.(type) { + case *ast.Ident: + name := label.Name + // For hidden fields, a $pkg suffix specifies the package scope. + // e.g. _foo$mypkg means hidden field _foo scoped to package mypkg, + // stored as ":mypkg" in inline-compiled sources (colon-prefixed). + var sel cue.Selector + if !internal.IsHidden(name) { + sel = cue.Label(label) + } else { + pkg := "_" + if i := strings.IndexByte(name, '$'); i >= 0 { + pkg = ":" + name[i+1:] + name = name[:i] + } + sel = cue.Hid(name, pkg) + } + sel = applyConstraint(sel, constraint) + fields = append(fields, expectedField{ + name: name, + sel: sel, + final: isFinal, + ignore: isIgnore, + errCheck: errCk, + value: d.Value, + attrs: nonTestAttrs, + }) + case *ast.BasicLit: + sel := cue.Label(label) + fields = append(fields, expectedField{ + name: sel.String(), + sel: applyConstraint(sel, constraint), + final: isFinal, + ignore: isIgnore, + errCheck: errCk, + value: d.Value, + attrs: nonTestAttrs, + }) + case *ast.ListLit: + // Pattern constraint [expr]: value. + if len(label.Elts) != 1 { + return pathErr(path, "pattern constraint label must have exactly one element") + } + patterns = append(patterns, expectedPattern{label: label.Elts[0], value: d.Value}) + } + case *ast.EmbedDecl: + // Embeddings are collected for non-struct value handling below. + } + } + + // Check if any field has @test(err), which means the struct may contain + // errors that cause IncompleteKind() to return _|_ instead of StructKind. + hasErrCheck := false + for _, ef := range fields { + if ef.errCheck != nil { + hasErrCheck = true + break + } + } + + // If the value is not a struct and @test(final) is set with no fields, + // unwrap the single embedding and compare directly, skipping structural + // consistency check. This allows {int, @test(final)} to match int & >=0. + if val.IncompleteKind() != cue.StructKind && !hasErrCheck { + if !allFinal { + return pathErr(path, "expected struct, got %v", val.IncompleteKind()) + } + var embeds []ast.Expr + for _, d := range s.Elts { + if e, ok := d.(*ast.EmbedDecl); ok { + embeds = append(embeds, e.Expr) + } + } + if len(embeds) != 1 || len(fields) != 0 { + return pathErr(path, "expected struct, got %v", val.IncompleteKind()) + } + return cmpFinal(path, embeds[0], val) + } + + // Compare regular fields (including definitions, optional, required, hidden). + seen := make(map[cue.Selector]bool, len(fields)) + for _, ef := range fields { + seen[ef.sel] = true + child := val.LookupPath(cue.MakePath(ef.sel)) + + fieldPath := path.Append(ef.sel) + + // @test(ignore): skip eq descent; field need not be present. + // Other directives (e.g. @test(err)) still run if the field exists. + if ef.ignore { + if ef.errCheck != nil && child.Exists() { + if err := c.cmpErr(fieldPath, child, ef.errCheck); err != nil { + return err + } + } + continue + } + + if !child.Exists() { + return pathErr(path, "field %q not found in value", ef.name) + } + + // @test(err): check value is an error instead of normal comparison. + if ef.errCheck != nil { + if err := c.cmpErr(fieldPath, child, ef.errCheck); err != nil { + return err + } + continue + } + + cmpChild := child + isFinal := ef.final || allFinal + if isFinal { + // @test(final): resolve the default value before comparing, + // and skip the structural disjunction/conjunction check. + if d, ok := child.Default(); ok { + cmpChild = d + } + } + var cmpErr error + if isFinal { + cmpErr = cmpFinal(fieldPath, ef.value, cmpChild) + } else { + cmpErr = c.astCmp(fieldPath, ef.value, cmpChild) + } + if cmpErr != nil { + return cmpErr + } + // Check attributes: order-independent matching, supporting + // multiple attributes with the same key (e.g. @foo() @foo(other)). + if err := cmpFieldAttrs(fieldPath, ef.attrs, child); err != nil { + return err + } + } + + // Check for unexpected fields in the value. + opts := []cue.Option{cue.Definitions(true), cue.Hidden(true), cue.Optional(true)} + iter, err := val.Fields(opts...) + if err != nil { + return pathErr(path, "cannot iterate value fields: %v", err) + } + var valFieldOrder []string + for iter.Next() { + sel := iter.Selector() + if sel.IsConstraint() && sel.ConstraintType() == cue.PatternConstraint { + continue // skip pattern constraints here + } + var name string + if sel.IsString() { + name = sel.Unquoted() + } else { + name = sel.String() + } + valFieldOrder = append(valFieldOrder, name) + // For hidden fields the expected struct may use bare _foo (pkg="_") + // while the value stores _foo scoped to a package. Accept either. + if !seen[sel] { + return pathErr(path, "unexpected field %q in value", name) + } + } + + // Check field ordering if @test(checkOrder) was present. + if checkOrder { + astOrder := make([]string, len(fields)) + for i, f := range fields { + astOrder[i] = f.name + } + if len(astOrder) != len(valFieldOrder) { + return pathErr(path, "checkOrder: field count mismatch: expected %d, got %d", + len(astOrder), len(valFieldOrder)) + } + for i := range astOrder { + if astOrder[i] != valFieldOrder[i] { + return pathErr(path, "checkOrder: field %d: expected %q, got %q", + i, astOrder[i], valFieldOrder[i]) + } + } + } + + // Compare pattern constraints. + if len(patterns) > 0 { + patIter, err := val.Fields(cue.Patterns(true)) + if err != nil { + return pathErr(path, "cannot iterate value patterns: %v", err) + } + var valPatterns []struct { + sel cue.Selector + value cue.Value + } + for patIter.Next() { + sel := patIter.Selector() + if !sel.IsConstraint() { + continue + } + valPatterns = append(valPatterns, struct { + sel cue.Selector + value cue.Value + }{sel, patIter.Value()}) + } + + if len(valPatterns) != len(patterns) { + return pathErr(path, "expected %d pattern constraint(s), got %d", + len(patterns), len(valPatterns)) + } + // Match patterns positionally for now (both ordered by source). + for i, ep := range patterns { + vp := valPatterns[i] + patPath := path.Append(cue.Str(fmt.Sprintf("[%d]pattern", i))) + if err := c.astCmp(patPath, ep.value, vp.value); err != nil { + return err + } + } + } + + // Compare let bindings listed in the expected struct. + // Each expected let must be present in the evaluated vertex with a matching value. + if len(lets) > 0 { + opCtx := value.OpContext(val) + v := value.Vertex(val) + + // Build a map from let base-name to arc. + valLets := make(map[string]*adt.Vertex, len(v.Arcs)) + for _, arc := range v.Arcs { + if arc.Label.IsLet() { + name := arc.Label.IdentString(opCtx) + valLets[name] = arc + } + } + + for _, el := range lets { + arc, ok := valLets[el.name] + if !ok { + return pathErr(path, "let binding %q not found in value", el.name) + } + letPath := path.Append(cue.Str("let " + el.name)) + letVal := value.Make(opCtx, arc) + if err := c.astCmp(letPath, el.expr, letVal); err != nil { + return err + } + } + } + + return nil +} + +// ── attribute comparison ──────────────────────────────────────────────────── + +// cmpFieldAttrs compares expected AST attributes against the actual attributes +// on a cue.Value field. Matching is order-independent and supports multiple +// attributes with the same key (e.g. @foo() @foo(other)). +// Both missing and unexpected attributes are reported as errors. +func cmpFieldAttrs(path cue.Path, expected []*ast.Attribute, child cue.Value) error { + // Collect value attributes as key:body pairs. + type attrEntry struct { + key string + body string + } + var valAttrs []attrEntry + for _, a := range export.ExtractFieldAttrs(child.Core().V) { + k, body := a.Split() + if k == "test" { + continue // skip @test directives + } + valAttrs = append(valAttrs, attrEntry{k, body}) + } + + // Order-independent matching: each expected attr must match exactly one + // value attr, and vice versa. + matched := make([]bool, len(valAttrs)) + for _, a := range expected { + k, body := a.Split() + found := false + for j, va := range valAttrs { + if matched[j] { + continue + } + if va.key == k && va.body == body { + matched[j] = true + found = true + break + } + } + if !found { + return pathErr(path, "expected attribute @%s(%s), not found in value", k, body) + } + } + // Check for unexpected value attributes. + for j, va := range valAttrs { + if !matched[j] { + return pathErr(path, "unexpected attribute @%s(%s) in value", va.key, va.body) + } + } + return nil +} + +// ── error assertions ──────────────────────────────────────────────────────── + +// cmpErr checks that val is an error, optionally validating error code and +// positions. This implements @test(err, ...) on fields within an expected struct. +// +// Position specs use absolute line numbers (not deltas) since nested @test(err) +// attributes have no source line to be relative to. +func (c *cmpCtx) cmpErr(path cue.Path, val cue.Value, ea *errArgs) error { + core := val.Core() + if core.V == nil { + return pathErr(path, "@test(err): value has no vertex") + } + b := core.V.Bottom() + if b == nil { + return pathErr(path, "@test(err): expected error, got non-error value") + } + if ea.code != "" { + gotCode := b.Code.String() + if gotCode != ea.code { + return pathErr(path, "@test(err): expected error code %q, got %q", ea.code, gotCode) + } + } + // TODO: support CUE_UPDATE=1 and CUE_UPDATE=force for nested pos= specs. + // This requires rewriting text inside the outer @test(eq, {...}) attribute + // body, which the current enqueuePosWrite machinery doesn't handle. + if ea.posSet { + err := val.Err() + if err == nil { + return pathErr(path, "@test(err, pos=...): value has no error") + } + positions := cueerrors.Positions(err) + if len(positions) != len(ea.pos) { + var got []string + for _, p := range positions { + got = append(got, fmt.Sprintf("%d:%d", p.Line(), p.Column())) + } + return pathErr(path, "@test(err, pos=...): got %d position(s) %v, want %d", + len(positions), got, len(ea.pos)) + } + // Order-independent matching: each expected position must match + // exactly one actual position. + matched := make([]bool, len(positions)) + for _, exp := range ea.pos { + found := false + for j, got := range positions { + if matched[j] { + continue + } + if exp.fileName != "" { + if got.Filename() == exp.fileName && got.Line() == exp.absLine && got.Column() == exp.col { + matched[j] = true + found = true + break + } + } else { + wantLine := c.baseLine + exp.deltaLine + if got.Line() == wantLine && got.Column() == exp.col { + matched[j] = true + found = true + break + } + } + } + if !found { + var got []string + for _, p := range positions { + got = append(got, fmt.Sprintf("%d:%d", p.Line(), p.Column())) + } + wantLine := c.baseLine + exp.deltaLine + return pathErr(path, "@test(err, pos=...): no match for expected position %d:%d in %v", + wantLine, exp.col, got) + } + } + } + return nil +} + +// ── lists ─────────────────────────────────────────────────────────────────── + +func (c *cmpCtx) cmpList(path cue.Path, l *ast.ListLit, val cue.Value) error { + if val.IncompleteKind() != cue.ListKind { + return pathErr(path, "expected list, got %v", val.IncompleteKind()) + } + iter, err := val.List() + if err != nil { + return pathErr(path, "cannot iterate value list: %v", err) + } + i := 0 + for ; i < len(l.Elts) && iter.Next(); i++ { + elemPath := path.Append(cue.Index(i)) + if err := c.astCmp(elemPath, l.Elts[i], iter.Value()); err != nil { + return err + } + } + if i < len(l.Elts) { + return pathErr(path, "list: expected %d elements, value has %d", len(l.Elts), i) + } + // Count remaining value elements. + extra := 0 + for iter.Next() { + extra++ + } + if extra > 0 { + return pathErr(path, "list: expected %d elements, value has %d", len(l.Elts), i+extra) + } + return nil +} + +// ── disjunctions ──────────────────────────────────────────────────────────── + +// astDisjunct holds a single disjunct extracted from the AST. +type astDisjunct struct { + expr ast.Expr + isDefault bool +} + +// flattenDisjunction recursively collects disjuncts from nested OR +// expressions, tracking default markers (*expr). +func flattenDisjunction(expr ast.Expr) []astDisjunct { + switch e := expr.(type) { + case *ast.BinaryExpr: + if e.Op == token.OR { + return append(flattenDisjunction(e.X), flattenDisjunction(e.Y)...) + } + case *ast.UnaryExpr: + if e.Op == token.MUL { + ds := flattenDisjunction(e.X) + for i := range ds { + ds[i].isDefault = true + } + return ds + } + case *ast.ParenExpr: + return flattenDisjunction(e.X) + } + return []astDisjunct{{expr: expr}} +} + +func (c *cmpCtx) cmpDisjunction(path cue.Path, expr ast.Expr, val cue.Value) error { + astDs := flattenDisjunction(expr) + + // Get disjuncts from the value via Expr(). + tv := val.Core() + dj, ok := tv.V.BaseValue.(*adt.Disjunction) + if !ok { + return pathErr(path, "value is a disjunction but expected expression is not; "+ + "use a disjunction in the expected value or @test(final) on the field") + } + + // Match value disjuncts to AST disjuncts order-independently. + // Each value disjunct must match exactly one AST disjunct. + if len(dj.Values) != len(astDs) { + return pathErr(path, "disjunction: expected %d disjunct(s), got %d", + len(astDs), len(dj.Values)) + } + + // Separate AST disjuncts into defaults and non-defaults. + var astDefaults, astNonDefaults []ast.Expr + for _, d := range astDs { + if d.isDefault { + astDefaults = append(astDefaults, d.expr) + } else { + astNonDefaults = append(astNonDefaults, d.expr) + } + } + + vDefs := dj.Values[:dj.NumDefaults] + vNonDefs := dj.Values[dj.NumDefaults:] + + // Check count agreement between AST and value for defaults and non-defaults. + if len(astDefaults) != len(vDefs) { + return pathErr(path, "disjunction: expected %d default(s), got %d", + len(astDefaults), len(vDefs)) + } + if len(astNonDefaults) != len(vNonDefs) { + return pathErr(path, "disjunction: expected %d non-default(s), got %d", + len(astNonDefaults), len(vNonDefs)) + } + + // Order-independent matching: defaults against defaults. + opCtx := value.OpContext(val) + if err := c.matchDisjuncts(path, opCtx, "default", astDefaults, vDefs); err != nil { + return err + } + // Non-defaults against non-defaults. + return c.matchDisjuncts(path, opCtx, "non-default", astNonDefaults, vNonDefs) +} + +// matchDisjuncts performs order-independent matching of AST expressions +// against value disjuncts. kind is "default" or "non-default" for error +// messages. +func (c *cmpCtx) matchDisjuncts(path cue.Path, opCtx *adt.OpContext, kind string, astExprs []ast.Expr, vals []adt.Value) error { + matched := make([]bool, len(vals)) + for _, ae := range astExprs { + found := false + for j, v := range vals { + if matched[j] { + continue + } + if c.astCmp(path, ae, value.Make(opCtx, v)) == nil { + matched[j] = true + found = true + break + } + } + if !found { + return pathErr(path, "disjunction: no matching %s disjunct for AST expr at %s", kind, pos(ae)) + } + } + return nil +} + +// flattenConjunction collects all conjuncts from nested & expressions. +func flattenConjunction(expr ast.Expr) []ast.Expr { + if e, ok := expr.(*ast.BinaryExpr); ok && e.Op == token.AND { + return append(flattenConjunction(e.X), flattenConjunction(e.Y)...) + } + if e, ok := expr.(*ast.ParenExpr); ok { + return flattenConjunction(e.X) + } + return []ast.Expr{expr} +} + +func (c *cmpCtx) cmpConjunction(path cue.Path, e *ast.BinaryExpr, val cue.Value) error { + astParts := flattenConjunction(e) + op, args := val.Eval().Expr() + if op != cue.AndOp { + return pathErr(path, "expected conjunction (&), got %v", op) + } + if len(args) != len(astParts) { + return pathErr(path, "conjunction: expected %d conjunct(s), got %d", + len(astParts), len(args)) + } + // Order-independent matching. + matched := make([]bool, len(args)) + for _, ae := range astParts { + found := false + for j, vd := range args { + if matched[j] { + continue + } + if c.astCmp(path, ae, vd) == nil { + matched[j] = true + found = true + break + } + } + if !found { + return pathErr(path, "conjunction: no matching value conjunct for AST expr at %s", pos(ae)) + } + } + return nil +} + +// ── unary expressions (non-default) ──────────────────────────────────────── + +func (c *cmpCtx) cmpUnaryExpr(path cue.Path, e *ast.UnaryExpr, val cue.Value) error { + op, args := val.Eval().Expr() + if len(args) != 1 { + return pathErr(path, "expected unary %v, got op=%v with %d args", e.Op, op, len(args)) + } + wantOp := tokenToOp(e.Op) + if op != wantOp { + return pathErr(path, "expected op %v, got %v", wantOp, op) + } + return c.astCmp(path, e.X, args[0]) +} + +// ── identifiers (types and special values) ────────────────────────────────── + +func cmpIdent(path cue.Path, id *ast.Ident, val cue.Value) error { + switch id.Name { + + // Bottom: value must be an error. + case "_|_": + if val.Err() == nil { + return pathErr(path, "expected bottom (_|_), got %v", val) + } + return nil + + default: + return cmpFinal(path, id, val) + } +} + +// ── field selector helpers ────────────────────────────────────────────────── + +// applyConstraint wraps sel with Optional or Required based on the constraint token. +func applyConstraint(sel cue.Selector, constraint token.Token) cue.Selector { + switch constraint { + case token.OPTION: + return sel.Optional() + case token.NOT: + return sel.Required() + } + return sel +} + +// ── helpers ───────────────────────────────────────────────────────────────── + +func pathErr(path cue.Path, format string, args ...any) error { + prefix := path.String() + if prefix == "" { + return fmt.Errorf(format, args...) + } + return fmt.Errorf("%s: "+format, append([]any{prefix}, args...)...) +} + +func pos(n ast.Node) token.Pos { + return n.Pos() +} + +// tokenToOp converts an ast token to the cue.Op equivalent. +func tokenToOp(t token.Token) cue.Op { + switch t { + case token.ADD: + return cue.AddOp + case token.SUB: + return cue.SubtractOp + case token.MUL: + return cue.MultiplyOp + case token.QUO: + return cue.FloatQuotientOp + case token.LSS: + return cue.LessThanOp + case token.LEQ: + return cue.LessThanEqualOp + case token.GTR: + return cue.GreaterThanOp + case token.GEQ: + return cue.GreaterThanEqualOp + case token.NEQ: + return cue.NotEqualOp + case token.MAT: + return cue.RegexMatchOp + case token.NMAT: + return cue.NotRegexMatchOp + default: + return cue.NoOp + } +} diff --git a/internal/cuetxtar/astcmp_test.go b/internal/cuetxtar/astcmp_test.go new file mode 100644 index 000000000..e96aad687 --- /dev/null +++ b/internal/cuetxtar/astcmp_test.go @@ -0,0 +1,793 @@ +// Copyright 2026 CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cuetxtar + +import ( + "strings" + "testing" + + "cuelang.org/go/cue" + "cuelang.org/go/cue/ast" + "cuelang.org/go/cue/cuecontext" + "cuelang.org/go/cue/parser" +) + +// parseExpr is a test helper that parses a CUE expression string. +func parseExpr(t *testing.T, s string) ast.Expr { + t.Helper() + expr, err := parser.ParseExpr("test", s) + if err != nil { + t.Fatalf("parseExpr(%q): %v", s, err) + } + return expr +} + +// compileVal is a test helper that compiles a CUE expression and returns the Value. +func compileVal(t *testing.T, s string) cue.Value { + t.Helper() + ctx := cuecontext.New() + v := ctx.CompileString(s) + if err := v.Err(); err != nil { + t.Fatalf("compileVal(%q): %v", s, err) + } + return v +} + +func TestAstCompare_Scalars(t *testing.T) { + tests := []struct { + name string + expr string + val string + wantErr string // empty means expect match + }{ + {name: "int negative", expr: "-1", val: "-1"}, + + // Integers + {name: "int match", expr: "42", val: "42"}, + {name: "int zero", expr: "0", val: "0"}, + {name: "int negative", expr: "-1", val: "-1"}, + {name: "int mismatch", expr: "42", val: "43", wantErr: "expected 42, got 43"}, + {name: "int big", expr: "100000000000000000000", val: "100000000000000000000"}, + {name: "int vs string", expr: "42", val: `"42"`, wantErr: `expected 42, got "42"`}, + + // Floats + {name: "float match", expr: "3.14", val: "3.14"}, + {name: "float mismatch", expr: "3.14", val: "2.71", wantErr: "expected 3.14, got 2.71"}, + + // Strings + {name: "string match", expr: `"hello"`, val: `"hello"`}, + {name: "string empty", expr: `""`, val: `""`}, + {name: "string mismatch", expr: `"hello"`, val: `"world"`, wantErr: `expected "hello", got "world"`}, + {name: "string with escapes", expr: `"a\nb"`, val: `"a\nb"`}, + + // Booleans + {name: "true match", expr: "true", val: "true"}, + {name: "false match", expr: "false", val: "false"}, + {name: "bool mismatch", expr: "true", val: "false", wantErr: "expected true"}, + + // Null + {name: "null match", expr: "null", val: "null"}, + {name: "null mismatch", expr: "null", val: "42", wantErr: "expected null"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + expr := parseExpr(t, tt.expr) + val := compileVal(t, tt.val) + err := astCompare(expr, val) + checkErr(t, err, tt.wantErr) + }) + } +} + +func TestAstCompare_Types(t *testing.T) { + tests := []struct { + name string + expr string + val string + wantErr string + }{ + {name: "int type", expr: "int", val: "int"}, + {name: "string type", expr: "string", val: "string"}, + {name: "bool type", expr: "bool", val: "bool"}, + {name: "float type", expr: "float", val: "float"}, + {name: "number type", expr: "number", val: "number"}, + {name: "bytes type", expr: "bytes", val: "bytes"}, + + // Type mismatch. + { + name: "int vs string", + expr: "int", + val: "string", + wantErr: "expected int, got string", + }, + + // Concrete values versus type constraint. + { + name: "concrete int matches int", + expr: "int", + val: "42", + wantErr: "expected int, got 42", + }, + { + name: "concrete string matches string", + expr: "string", + val: `"hello"`, + wantErr: "expected string, got \"hello\"", + }, + + // Top and bottom. + {name: "top matches top", expr: "_", val: "_"}, + { + name: "top matches struct", + expr: "_", + val: "{a: 1}", + wantErr: "expected _, got", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + expr := parseExpr(t, tt.expr) + val := compileVal(t, tt.val) + err := astCompare(expr, val) + checkErr(t, err, tt.wantErr) + }) + } +} + +func TestAstCompare_Structs(t *testing.T) { + tests := []struct { + name string + expr string + val string + wantErr string + }{ + {name: "simple struct", expr: `{b: "x", a: 1}`, val: `{a: 1, b: "x"}`}, + {name: "nested struct", expr: `{a: {b: 1}}`, val: `{a: {b: 1}}`}, + {name: "empty struct", expr: `{}`, val: `{}`}, + + // Field value mismatch. + {name: "field mismatch", expr: `{a: 1}`, val: `{a: 2}`, wantErr: "a: expected 1, got 2"}, + + // Missing field. + {name: "missing field", expr: `{a: 1, b: 2}`, val: `{a: 1}`, wantErr: `field "b" not found`}, + + // Extra field. + {name: "extra field", expr: `{a: 1}`, val: `{a: 1, b: 2}`, wantErr: `unexpected field "b"`}, + + // Definitions. + {name: "definition match", expr: `{#D: 1}`, val: `{#D: 1}`}, + {name: "definition mismatch", expr: `{#D: 1}`, val: `{#D: 2}`, wantErr: "#D: expected 1, got 2"}, + + // Mixed regular and definition. + {name: "mixed fields and defs", expr: `{#D: "x", a: 1}`, val: `{a: 1, #D: "x"}`}, + + // Final attribute on fields. + { + name: "without final rejects plain vs disjunction", + expr: `{a: 1}`, + val: `{a: *1 | 2}`, + wantErr: "disjunction", + }, + + // Optional fields. + {name: "optional match", expr: `{foo?: string}`, val: `{foo?: string}`}, + {name: "optional concrete", expr: `{foo?: "bar"}`, val: `{foo?: "bar"}`}, + { + name: "optional vs regular mismatch", + expr: `{foo?: string}`, + val: `{foo: string}`, + wantErr: `unexpected field "foo"`, + }, + { + name: "regular vs optional mismatch", + expr: `{foo: string}`, + val: `{foo?: string}`, + wantErr: `field "foo" not found`, + }, + { + name: "optional value mismatch", + expr: `{foo?: string}`, + val: `{foo?: int}`, + wantErr: "foo?: expected string, got int", + }, + { + name: "mixed optional and regular", + expr: `{a: 1, b?: string}`, + val: `{a: 1, b?: string}`, + }, + + // Required fields. + {name: "required match", expr: `{foo!: string}`, val: `{foo!: string}`}, + { + name: "required vs regular mismatch", + expr: `{foo!: string}`, + val: `{foo: string}`, + wantErr: `unexpected field "foo"`, + }, + { + name: "required vs optional mismatch", + expr: `{foo!: string}`, + val: `{foo?: string}`, + wantErr: `unexpected field "foo"`, + }, + + // Nested @test(err) — tested separately below in TestAstCompare_ErrDirective. + + // Hidden fields. + {name: "hidden match", expr: `{_foo: 1}`, val: `{_foo: 1}`}, + { + name: "hidden mismatch", + expr: `{_foo: 1}`, + val: `{_foo: 2}`, + wantErr: "_foo: expected 1, got 2", + }, + { + name: "hidden missing from value", + expr: `{_foo: 1}`, + val: `{}`, + wantErr: `field "_foo" not found`, + }, + { + // Hidden field present in value but absent from expected must be reported. + // This ensures hidden fields are not silently ignored by the comparison. + name: "hidden unexpected in value", + expr: `{a: 1}`, + val: `{_foo: 42, a: 1}`, + wantErr: `unexpected field "_foo" in value`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + expr := parseExpr(t, tt.expr) + val := compileVal(t, tt.val) + err := astCompare(expr, val) + checkErr(t, err, tt.wantErr) + }) + } +} + +func TestAstCompare_ErrDirective(t *testing.T) { + ctx := cuecontext.New() + + // Helper: compile a struct and return the whole value (errors allowed). + compile := func(s string) cue.Value { return ctx.CompileString(s) } + + // Use a struct where field b has an error but the struct itself is valid. + errStruct := compile(`{b: 1, a: null & string}`) + + t.Run("bare err on error field", func(t *testing.T) { + expr := parseExpr(t, "{\nb: 1\na: _|_ @test(err)\n}") + checkErr(t, astCompare(expr, errStruct), "") + }) + t.Run("err with code", func(t *testing.T) { + expr := parseExpr(t, "{\nb: 1\na: _|_ @test(err, code=eval)\n}") + checkErr(t, astCompare(expr, errStruct), "") + }) + t.Run("err on non-error field", func(t *testing.T) { + expr := parseExpr(t, "{\nb: 1 @test(err)\na: _|_ @test(err)\n}") + checkErr(t, astCompare(expr, errStruct), "@test(err): expected error") + }) + t.Run("err wrong code", func(t *testing.T) { + expr := parseExpr(t, "{\nb: 1\na: _|_ @test(err, code=incomplete)\n}") + checkErr(t, astCompare(expr, errStruct), "@test(err): expected error code") + }) +} + +func TestAstCompare_Lists(t *testing.T) { + tests := []struct { + name string + expr string + val string + wantErr string + }{ + {name: "simple list", expr: `[1, 2, 3]`, val: `[1, 2, 3]`}, + {name: "string list", expr: `["a", "b"]`, val: `["a", "b"]`}, + {name: "nested list", expr: `[[1, 2], [3, 4]]`, val: `[[1, 2], [3, 4]]`}, + {name: "empty list", expr: `[]`, val: `[]`}, + + // Length mismatch. + {name: "too few elements", expr: `[1, 2, 3]`, val: `[1, 2]`, wantErr: "expected 3 elements, value has 2"}, + {name: "too many elements", expr: `[1, 2]`, val: `[1, 2, 3]`, wantErr: "expected 2 elements, value has 3"}, + + // Element mismatch. + {name: "element mismatch", expr: `[1, 2, 3]`, val: `[1, 99, 3]`, wantErr: "[1]: expected 2, got 99"}, + + // Mixed types. + {name: "mixed types", expr: `[1, "two", true]`, val: `[1, "two", true]`}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + expr := parseExpr(t, tt.expr) + val := compileVal(t, tt.val) + err := astCompare(expr, val) + checkErr(t, err, tt.wantErr) + }) + } +} + +func TestAstCompare_Disjunctions(t *testing.T) { + tests := []struct { + name string + expr string + val string + wantErr string + }{ + // Simple disjunction. + {name: "two values", expr: `1 | 2`, val: `1 | 2`}, + {name: "three values", expr: `1 | 2 | 3`, val: `1 | 2 | 3`}, + {name: "string disj", expr: `"a" | "b"`, val: `"a" | "b"`}, + + // Order independence. + {name: "order independent", expr: `2 | 1`, val: `1 | 2`}, + {name: "three reversed", expr: `3 | 2 | 1`, val: `1 | 2 | 3`}, + + // Default markers. + { + name: "default match", + expr: `*1 | 2`, + val: `*1 | 2`, + }, + { + name: "plain vs disjunction rejected", + expr: `1`, + val: `*1 | 2`, + wantErr: "value is a disjunction but expected expression is not", + }, + { + name: "plain vs disjunction accepted with field final", + expr: "{\na: 1 @test(final)\n}", + val: `{a: *1 | 2}`, + }, + { + name: "plain vs disjunction accepted with decl final", + expr: "{\n@test(final)\na: 1\n}", + val: `{a: *1 | 2}`, + }, + { + name: "final with wrong default", + expr: "{\na: 2 @test(final)\n}", + val: `{a: *1 | 2}`, + wantErr: "a: expected 2, got 1", + }, + + // Disjunction count mismatch. + { + name: "count mismatch", + expr: `1 | 2 | 3`, + val: `1 | 2`, + wantErr: "expected 3 disjunct(s), got 2", + }, + { + name: "count mismatch default", + expr: `*1 | 2 | 3`, + val: `*1 | 2`, + wantErr: "expected 3 disjunct(s), got 2", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + expr := parseExpr(t, tt.expr) + val := compileVal(t, tt.val) + err := astCompare(expr, val) + checkErr(t, err, tt.wantErr) + }) + } +} + +func TestAstCompare_Bounds(t *testing.T) { + tests := []struct { + name string + expr string + val string + wantErr string + }{ + // Unary bounds. + {name: "geq match", expr: `>=0`, val: `>=0`}, + {name: "leq match", expr: `<=100`, val: `<=100`}, + {name: "lt match", expr: `<10`, val: `<10`}, + {name: "gt match", expr: `>0`, val: `>0`}, + {name: "neq match", expr: `{a: !=null}`, val: `{a: !=null}`}, + + // Conjunctions of bounds. + {name: "range", expr: `>=0 & <=100`, val: `>=0 & <=100`}, + {name: "type and bound", expr: `int & >=0`, val: `int & >=0`}, + {name: "simplifies", expr: `=~"c"`, val: `!="b" & =~"c"`}, + { + name: "simplifies to conjunction", + expr: `=~"c" & =~"d"`, + val: `!="b" & =~"c" & =~"d"`, + }, + // Note: int & >=0 & <=100 simplifies to >=0 & <=100 after evaluation, + // so we cannot test a triple conjunction with redundant int. + {name: "conjunction order independent", expr: `<=100 & >=0`, val: `>=0 & <=100`}, + + // Conjunction structural consistency. + { + name: "plain vs conjunction rejected", + expr: `int`, + val: `int & >=0`, + wantErr: "value is a conjunction", + }, + { + name: "field final with conjunction", + expr: "{\na: int & >=0 @test(final)\n}", + val: `a: int & >=0`, + }, + { + name: "decl final with conjunction", + expr: `{int & >=0, @test(final)}`, + val: `int & >=0`, + }, + { + name: "conjunction count mismatch", + expr: `>=0 & <=100`, + val: `int & >=0`, + wantErr: "conjunct", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + expr := parseExpr(t, tt.expr) + val := compileVal(t, tt.val) + err := astCompare(expr, val) + checkErr(t, err, tt.wantErr) + }) + } +} + +func TestAstCompare_PatternConstraints(t *testing.T) { + ctx := cuecontext.New() + + t.Run("simple pattern", func(t *testing.T) { + // {[string]: int} with a concrete field. + val := ctx.CompileString(`{[string]: int, a: 1}`) + if err := val.Err(); err != nil { + t.Fatal(err) + } + expr := parseExpr(t, `{[string]: int, a: 1}`) + err := astCompare(expr, val) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + }) +} + +func TestAstCompare_Complex(t *testing.T) { + tests := []struct { + name string + expr string + val string + wantErr string + }{ + { + name: "struct with list", + expr: `{a: [1, 2], b: "x"}`, + val: `{a: [1, 2], b: "x"}`, + }, + { + name: "deeply nested", + expr: `{a: {b: {c: 42}}}`, + val: `{a: {b: {c: 42}}}`, + }, + { + name: "struct with disjunction", + expr: `{x: 1 | 2}`, + val: `{x: 1 | 2}`, + }, + { + name: "deep mismatch", + expr: `{a: {b: {c: 42}}}`, + val: `{a: {b: {c: 99}}}`, + wantErr: "a.b.c: expected 42, got 99", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + expr := parseExpr(t, tt.expr) + val := compileVal(t, tt.val) + err := astCompare(expr, val) + checkErr(t, err, tt.wantErr) + }) + } +} + +func TestAstCompare_CheckOrder(t *testing.T) { + tests := []struct { + name string + expr string + val string + wantErr string + }{ + { + name: "order matches", + expr: "{\n@test(checkOrder)\na: 1\nb: 2\n}", + val: `{a: 1, b: 2}`, + }, + { + name: "order mismatch", + expr: "{\n@test(checkOrder)\na: 1\nb: 2\n}", + val: `{b: 2, a: 1}`, + wantErr: `checkOrder: field 0: expected "a", got "b"`, + }, + { + name: "without checkOrder order is ignored", + expr: `{b: 2, a: 1}`, + val: `{a: 1, b: 2}`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + expr := parseExpr(t, tt.expr) + val := compileVal(t, tt.val) + err := astCompare(expr, val) + checkErr(t, err, tt.wantErr) + }) + } +} + +func TestAstCompare_Attributes(t *testing.T) { + tests := []struct { + name string + expr string + val string + wantErr string + }{ + // Attribute match. + { + name: "single attribute match", + expr: `{a: 1 @foo(bar)}`, + val: `{a: 1 @foo(bar)}`, + }, + { + name: "multiple attributes match", + expr: "{\na: 1 @foo(x) @bar(y)\n}", + val: "{\na: 1 @foo(x) @bar(y)\n}", + }, + // Attribute order is irrelevant: AST has @bar then @foo, + // value has @foo then @bar. + { + name: "attribute order irrelevant", + expr: "{\na: 1 @bar(y) @foo(x)\n}", + val: "{\na: 1 @foo(x) @bar(y)\n}", + }, + // Missing attribute. + { + name: "attribute missing from value", + expr: `{a: 1 @foo(bar)}`, + val: `{a: 1}`, + wantErr: "@foo", + }, + // Content mismatch. + { + name: "attribute content mismatch", + expr: `{a: 1 @foo(bar)}`, + val: `{a: 1 @foo(baz)}`, + wantErr: "expected attribute @foo(bar), not found", + }, + // Empty vs non-empty contents. + { + name: "attribute empty vs non-empty", + expr: `{a: 1 @foo()}`, + val: `{a: 1 @foo(bar)}`, + wantErr: "expected attribute @foo(), not found", + }, + // Unexpected attribute in value. + { + name: "unexpected attribute in value", + expr: `{a: 1}`, + val: `{a: 1 @foo(bar)}`, + wantErr: "unexpected attribute @foo", + }, + // Multiple attributes with same key. + { + name: "multiple same-key attrs match", + expr: "{\na: 1 @foo() @foo(other)\n}", + val: "{\na: 1 @foo() @foo(other)\n}", + }, + { + name: "multiple same-key attrs order independent", + expr: "{\na: 1 @foo(other) @foo()\n}", + val: "{\na: 1 @foo() @foo(other)\n}", + }, + { + name: "multiple same-key attrs missing one", + expr: "{\na: 1 @foo() @foo(other)\n}", + val: "{\na: 1 @foo()\n}", + wantErr: "expected attribute @foo(other), not found", + }, + // @test attributes are excluded from comparison. + { + name: "test attribute ignored", + expr: "{\na: 1 @test(final)\n}", + val: `{a: *1 | 2}`, + }, + // Non-@test attribute alongside @test attribute. + { + name: "non-test attr with test attr", + expr: "{\na: 1 @foo(x) @test(final)\n}", + val: `{a: *1 | 2 @foo(x)}`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + expr := parseExpr(t, tt.expr) + val := compileVal(t, tt.val) + err := astCompare(expr, val) + checkErr(t, err, tt.wantErr) + }) + } +} + +func TestAstCompare_Parens(t *testing.T) { + expr := parseExpr(t, `(42)`) + val := compileVal(t, `42`) + if err := astCompare(expr, val); err != nil { + t.Errorf("unexpected error: %v", err) + } +} + +func TestAstCompare_LetBindings(t *testing.T) { + // Note: CUE requires let bindings to be referenced, so test values use a + // concrete field that references each let (e.g. "a: b" where "let b = 3"). + // Expected structs use the evaluated concrete value (e.g. "a: 3"), not the + // let identifier, since expected expressions are compared structurally. + tests := []struct { + name string + expr string + val string + wantErr string + }{ + // Let binding present and value matches. + { + name: "let match", + expr: "{\na: 3\nlet b = 3\n}", + val: "{\na: b\nlet b = 3\n}", // b referenced by a + }, + // Let binding present but value mismatches. + { + name: "let value mismatch", + expr: "{\na: 3\nlet b = 4\n}", // expected: let b = 4 (wrong) + val: "{\na: b\nlet b = 3\n}", // actual: let b = 3 + wantErr: `"let b": expected 4, got 3`, + }, + // Expected has a let but value has no let with that name. + { + name: "let missing from value", + expr: "{\na: 1\nlet x = 2\n}", + val: "{\na: 1\n}", + wantErr: `let binding "x" not found`, + }, + // Let with top (_) matches a top value. + { + name: "let top matches top", + expr: "{\na: _\nlet b = _\n}", + val: "{\na: b\nlet b = _\n}", // b=top, a=b=top + }, + // Value has an extra let not listed in expected — not an error. + { + name: "extra let in value is allowed", + expr: "{\na: 3\n}", + val: "{\na: b\nlet b = 3\n}", + }, + // Multiple lets. + { + name: "multiple lets match", + expr: "{\na: 1\nb: 2\nlet x = 1\nlet y = 2\n}", + val: "{\na: x\nb: y\nlet x = 1\nlet y = 2\n}", + }, + { + name: "multiple lets one mismatch", + expr: "{\na: 1\nb: 2\nlet x = 1\nlet y = 99\n}", + val: "{\na: x\nb: y\nlet x = 1\nlet y = 2\n}", + wantErr: `"let y": expected 99, got 2`, + }, + // Hidden let (underscore prefix). + { + name: "hidden let match", + expr: "{\na: 3\nlet _b = 3\n}", + val: "{\na: _b\nlet _b = 3\n}", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + expr := parseExpr(t, tt.expr) + val := compileVal(t, tt.val) + err := astCompare(expr, val) + checkErr(t, err, tt.wantErr) + }) + } +} + +func TestAstCompare_Ignore(t *testing.T) { + tests := []struct { + name string + expr string + val string + wantErr string + }{ + // @test(ignore) on a field skips eq check; field need not be present. + { + name: "ignore skips missing field", + expr: "{\na: 1\nb: _ @test(ignore)\n}", + val: "{\na: 1\n}", + }, + // @test(ignore) on a field skips eq check; field can be present with any value. + { + name: "ignore skips field with any value", + expr: "{\na: 1\nb: _ @test(ignore)\n}", + val: "{\na: 1\nb: 99\n}", + }, + // @test(ignore) does not suppress @test(err) — the err check still runs + // and fails when the field has no error. + { + name: "ignore does not suppress err check - fails non-error field", + expr: "{\nb: _ @test(ignore) @test(err)\n}", + val: "{\nb: 3\n}", + wantErr: "@test(err): expected error", + }, + // @test(ignore) on a field that would otherwise fail value comparison. + { + name: "ignore skips value mismatch", + expr: "{\na: 1 @test(ignore)\n}", + val: "{\na: 999\n}", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + expr := parseExpr(t, tt.expr) + val := compileVal(t, tt.val) + err := astCompare(expr, val) + checkErr(t, err, tt.wantErr) + }) + } + + // @test(ignore) does not suppress @test(err) — the err check still passes + // when the field IS an error. Use a value with an error field (skip top-level error check). + t.Run("ignore does not suppress err check - passes error field", func(t *testing.T) { + ctx := cuecontext.New() + errVal := ctx.CompileString("{\nb: 1 & 2\n}") // b has a conflict error + expr := parseExpr(t, "{\nb: _|_ @test(ignore) @test(err)\n}") + checkErr(t, astCompare(expr, errVal), "") + }) +} + +// checkErr is a test helper that verifies error expectations. +func checkErr(t *testing.T, err error, wantErr string) { + t.Helper() + if wantErr == "" { + if err != nil { + t.Errorf("unexpected error: %v", err) + } + return + } + if err == nil { + t.Errorf("expected error containing %q, got nil", wantErr) + return + } + if !strings.Contains(err.Error(), wantErr) { + t.Errorf("error %q does not contain %q", err.Error(), wantErr) + } +} diff --git a/internal/cuetxtar/inline.go b/internal/cuetxtar/inline.go new file mode 100644 index 000000000..76aeb9c2c --- /dev/null +++ b/internal/cuetxtar/inline.go @@ -0,0 +1,1550 @@ +// Copyright 2026 CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package cuetxtar provides utilities for running CUE tests from txtar archives. +// This file implements inline-assertion mode where @test(...) attributes on CUE +// fields replace golden-file comparison. +// +// @test(eq, VALUE) may appear in two positions: +// +// 1. As a field attribute: field: expr @test(eq, VALUE) +// Checks the evaluated field value against VALUE. +// +// 2. As a file-level decl attribute (top-level declaration): +// a: 1 +// b: a + 1 +// @test(eq, {a: 1, b: 2}) +// Checks the entire file's evaluated value against VALUE. +// All fields are implicitly covered — no per-field @test is required. +package cuetxtar + +import ( + "fmt" + "io/fs" + "maps" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "testing" + + "golang.org/x/tools/txtar" + + "cuelang.org/go/cue" + "cuelang.org/go/cue/ast" + "cuelang.org/go/cue/cuecontext" + cueerrors "cuelang.org/go/cue/errors" + "cuelang.org/go/cue/format" + "cuelang.org/go/cue/parser" + "cuelang.org/go/cue/token" + "cuelang.org/go/internal" + "cuelang.org/go/internal/core/debug" + "cuelang.org/go/internal/cuetdtest" + "cuelang.org/go/internal/cuetest" +) + +// RunInlineTests iterates over txtar archives in dir, detects inline-assertion +// mode (presence of any @test(...) attribute), and runs inline assertions. +// Archives without @test attributes are left for the existing TxTarTest.Run. +func RunInlineTests(t *testing.T, matrix cuetdtest.Matrix, dir string) { + t.Helper() + if matrix == nil { + runInlineTestsForMatrix(t, nil, dir) + return + } + matrix.Do(t, func(t *testing.T, m *cuetdtest.M) { + runInlineTestsForMatrix(t, m, dir) + }) +} + +func runInlineTestsForMatrix(t *testing.T, m *cuetdtest.M, dir string) { + t.Helper() + + // Determine the base for test names (strip everything up to and including /testdata/). + baseDir := dir + + err := filepath.WalkDir(dir, func(fullpath string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() || filepath.Ext(fullpath) != ".txtar" { + return nil + } + + archive, err := txtar.ParseFile(fullpath) + if err != nil || !isInlineMode(archive) { + return nil + } + + // Derive test name from path relative to dir. + rel, err := filepath.Rel(baseDir, fullpath) + if err != nil { + rel = filepath.Base(fullpath) + } + // Use forward slashes and strip .txtar extension. + testName := filepath.ToSlash(rel) + testName = testName[:len(testName)-len(".txtar")] + + t.Run(testName, func(t *testing.T) { + runner := &inlineRunner{ + t: t, + m: m, + archive: archive, + dir: filepath.Dir(fullpath), + filePath: fullpath, + } + runner.runArchive() + }) + + return nil + }) + if err != nil { + t.Fatalf("inline: walk %s: %v", dir, err) + } +} + +// InlineRunner is the exported handle for running inline-assertion tests +// against a single txtar archive. Primarily for use in tests. +type InlineRunner struct { + r *inlineRunner +} + +// NewInlineRunner creates an InlineRunner for the given archive. +// m may be nil for unmatrix'd tests. dir is the base directory for loading. +func NewInlineRunner(t *testing.T, m *cuetdtest.M, archive *txtar.Archive, dir string) *InlineRunner { + return &InlineRunner{r: &inlineRunner{t: t, m: m, archive: archive, dir: dir}} +} + +// Run executes all inline test cases in the archive. +func (ir *InlineRunner) Run() { + ir.r.runArchive() +} + +// ───────────────────────────────────────────────────────────────────────────── +// Section 1: Attribute Parsing Utilities +// ───────────────────────────────────────────────────────────────────────────── + +// parsedTestAttr holds the result of parsing a single @test(...) attribute. +type parsedTestAttr struct { + // directive is the primary directive name, e.g. "eq", "err", "kind", + // "closed", "leq", "skip", "permute", "file", "desc". + directive string + + // version is the optional version suffix from directive:vN, e.g. "v3". + // Empty means unversioned. + version string + + // raw is the parsed internal.Attr for accessing remaining arguments. + raw internal.Attr + + // For "err" directives, parsed sub-options are stored here. + errArgs *errArgs + + // srcAttr is the original AST attribute node (needed for CUE_UPDATE write-back). + srcAttr *ast.Attribute + + // srcFileName is the archive .cue file name containing this attribute, + // e.g. "in.cue" (needed to locate the file for CUE_UPDATE write-back). + srcFileName string + + // baseLine is the effective 1-indexed line of the field carrying this + // attribute in the stripped-and-formatted output. It may differ from + // srcAttr.Pos().Line() when earlier @test attributes on preceding fields + // contain embedded newlines in their bodies (which are stripped before + // formatting, collapsing those extra lines). + baseLine int +} + +// posSpec is an expected error position. Two forms are supported: +// +// Relative form (fileName == ""): the position is expressed as a signed line +// delta from the @test attribute's line (0 = same line) and a 1-indexed +// column. Written as deltaLine:col (e.g. "0:5", "-2:13"). Using a delta keeps +// the assertion stable when lines are added or removed above the test. +// +// Absolute form (fileName != ""): the position is in a different file, given +// as a filename, absolute 1-indexed line number, and 1-indexed column. Written +// as filename:absLine:col (e.g. "fixture.cue:3:5"). This form is used for +// errors whose source location is in a shared fixture file. +type posSpec struct { + // fileName is non-empty for cross-file (absolute) positions. + fileName string + // deltaLine is the signed offset from the @test line; used when fileName == "". + deltaLine int + // absLine is the absolute 1-indexed line number; used when fileName != "". + absLine int + // col is the 1-indexed column number (used in both forms). + col int +} + +// errArgs holds parsed sub-options from an @test(err, ...) directive. +type errArgs struct { + // code is the expected error code, e.g. "cycle", "structural cycle". + code string + // contains is a substring the error message must contain. + contains string + // any requires any descendant of the annotated field to have the error. + any bool + // paths lists specific paths (relative to test-case root) where the error + // must occur. Populated when path=(...) is present. + paths []string + // pos lists expected error positions as (deltaLine:col) pairs relative to + // the line containing the @test attribute. + pos []posSpec + // posSet is true when pos= was + // explicitly provided (including pos=[] to assert no positions). + posSet bool +} + +// parseTestAttr parses the body of a @test(...) attribute node. +// It returns a parsedTestAttr for each logical directive in the attribute. +// A single @test(...) contains exactly one directive (the first positional +// argument or the key of the first key=value pair). +func parseTestAttr(a *ast.Attribute) (parsedTestAttr, error) { + key, body := a.Split() + if key != "test" { + return parsedTestAttr{}, fmt.Errorf("not a @test attribute: @%s", key) + } + + parsed := internal.ParseAttrBody(a.Pos(), body) + if parsed.Err != nil { + return parsedTestAttr{}, parsed.Err + } + + result := parsedTestAttr{ + raw: parsed, + srcAttr: a, + } + + if len(parsed.Fields) == 0 || (len(parsed.Fields) == 1 && parsed.Fields[0] == internal.KeyValue{}) { + // @test() — empty placeholder or bare marker. + result.directive = "" + return result, nil + } + + // The first field determines the directive. + // Case 1: key=value form like desc="hello", shareID=name — directive is the key. + // Case 2: positional form like eq, err, kind — directive (with optional :vN suffix) is the value. + f0 := parsed.Fields[0] + if f0.Key() != "" { + dir := f0.Key() + // Key-based directives may carry a version suffix: "shareID:v3" → directive="shareID", version="v3". + if idx := strings.LastIndex(dir, ":"); idx >= 0 { + result.directive = dir[:idx] + result.version = dir[idx+1:] + } else { + result.directive = dir + } + } else { + // May have version suffix: "eq:v3" → directive="eq", version="v3". + dir := f0.Value() + if idx := strings.LastIndex(dir, ":"); idx >= 0 { + result.directive = dir[:idx] + result.version = dir[idx+1:] + } else { + result.directive = dir + } + } + + // Parse directive-specific sub-options. + switch result.directive { + case "err": + ea, err := parseErrArgs(parsed) + if err != nil { + return result, err + } + result.errArgs = &ea + } + + return result, nil +} + +// parseErrArgs extracts err sub-options from an already-parsed Attr. +// The attribute body is expected to start with "err" as the first positional arg. +func parseErrArgs(a internal.Attr) (errArgs, error) { + var ea errArgs + // Start from index 1 (index 0 is "err"). + for _, kv := range a.Fields[1:] { + switch { + case kv.Key() == "code": + ea.code = kv.Value() + case kv.Key() == "contains": + ea.contains = kv.Value() + case kv.Key() == "" && kv.Value() == "any": + ea.any = true + case kv.Key() == "path": + paths, err := parseParenList(kv.Value()) + if err != nil { + return ea, fmt.Errorf("@test(err, path=...): %w", err) + } + ea.paths = paths + case kv.Key() == "pos": + specs, err := parsePosSpecs(kv.Value()) + if err != nil { + return ea, fmt.Errorf("@test(err, pos=...): %w", err) + } + ea.pos = specs + ea.posSet = true + } + } + return ea, nil +} + +// parseParenList parses a balanced parenthesized pipe-separated list like +// (path1|path2|path3), returning ["path1","path2","path3"]. +// The input may or may not include the outer parentheses. +func parseParenList(s string) ([]string, error) { + s = strings.TrimSpace(s) + if strings.HasPrefix(s, "(") && strings.HasSuffix(s, ")") { + s = s[1 : len(s)-1] + } + if s == "" { + return nil, nil + } + parts := strings.Split(s, "|") + for i, p := range parts { + parts[i] = strings.TrimSpace(p) + } + return parts, nil +} + +// parsePosSpecs parses a pos= value into a slice of posSpec. +// The value must be enclosed in square brackets; elements are whitespace-separated. +// Two element forms are supported: +// +// - deltaLine:col — relative position on the same file (one colon). +// deltaLine is a signed offset from the @test attribute's line (0 = same line). +// - filename:absLine:col — absolute position in another file (two colons). +// absLine is the 1-indexed line in the named file. +func parsePosSpecs(s string) ([]posSpec, error) { + s = strings.TrimSpace(s) + if !strings.HasPrefix(s, "[") || !strings.HasSuffix(s, "]") { + return nil, fmt.Errorf("pos= value must be enclosed in square brackets, got %q", s) + } + s = s[1 : len(s)-1] + var specs []posSpec + for _, p := range strings.Fields(s) { + parts := strings.SplitN(p, ":", 3) + switch len(parts) { + case 2: + // Relative form: deltaLine:col + deltaLine, err := strconv.Atoi(parts[0]) + if err != nil { + return nil, fmt.Errorf("invalid pos spec %q: %w", p, err) + } + colNum, err := strconv.Atoi(parts[1]) + if err != nil { + return nil, fmt.Errorf("invalid pos spec %q: %w", p, err) + } + specs = append(specs, posSpec{deltaLine: deltaLine, col: colNum}) + case 3: + // Absolute form: filename:absLine:col + fileName := parts[0] + absLine, err := strconv.Atoi(parts[1]) + if err != nil { + return nil, fmt.Errorf("invalid pos spec %q: %w", p, err) + } + colNum, err := strconv.Atoi(parts[2]) + if err != nil { + return nil, fmt.Errorf("invalid pos spec %q: %w", p, err) + } + specs = append(specs, posSpec{fileName: fileName, absLine: absLine, col: colNum}) + default: + return nil, fmt.Errorf("invalid pos spec %q: expected deltaLine:col or filename:line:col", p) + } + } + return specs, nil +} + +// ───────────────────────────────────────────────────────────────────────────── +// Section 2: AST Extraction and Stripping +// ───────────────────────────────────────────────────────────────────────────── + +// attrRecord associates a parsed @test attribute with its location in the +// evaluated CUE value. +type attrRecord struct { + // path is the full CUE path to the field carrying this attribute. + path cue.Path + + // parsed is the parsed directive from this attribute. + parsed parsedTestAttr + + // fileLevel is true when this record comes from a file-level (top-level) + // decl attribute rather than a field attribute or struct-level decl attribute. + // A file-level @test(eq, VALUE) checks the entire file's evaluated value. + fileLevel bool + + // isDeclAttr is true when this record comes from a decl attribute inside + // a struct (as opposed to a field attribute). For @test(permute), this + // distinction matters: a decl attr means "permute all fields within this + // struct" whereas a field attr means "this field participates in + // permutation within its parent struct." + isDeclAttr bool +} + +// extractTestAttrs walks ast.File and: +// 1. Collects all @test(...) attributes from field attrs, struct decl attrs, +// and file-level decl attrs. +// 2. Removes them from the AST (in-place). +// 3. Preserves all non-@test attributes. +// +// Returns the collected records. +// File-level decl attributes produce records with an empty path and +// fileLevel=true; these check the entire file's evaluated value. +func extractTestAttrs(f *ast.File, fileName string) []attrRecord { + var records []attrRecord + + // extraNewlines tracks the cumulative count of newlines embedded inside + // @test attribute bodies that have been stripped from preceding fields. + // When the CUE scanner reads a multiline attribute body (e.g. pos=[0:5\n1:5]) + // it advances its line counter, shifting subsequent AST node positions upward. + // After stripping and reformatting, those embedded newlines are gone, so we + // subtract the running total to recover the effective line in the formatted + // output. + extraNewlines := 0 + + // walkField strips @test field attrs from field, records them, then recurses + // into the field's struct value (if any). + var walkField func(field *ast.Field, path cue.Path) + + // walkStruct strips @test decl attrs from sl, records them, and recurses + // into all sub-fields. + var walkStruct func(sl *ast.StructLit, path cue.Path) + + walkField = func(field *ast.Field, path cue.Path) { + // Effective line of this field in the stripped-and-formatted output. + fieldBaseLine := field.Pos().Line() - extraNewlines + + // Strip @test field attrs and record them. + var keep []*ast.Attribute + for _, a := range field.Attrs { + k, body := a.Split() + if k != "test" { + keep = append(keep, a) + continue + } + pa, err := parseTestAttr(a) + if err != nil { + // Keep on parse error; runner will report it. + keep = append(keep, a) + continue + } + pa.baseLine = fieldBaseLine + pa.srcFileName = fileName + records = append(records, attrRecord{ + path: path, + parsed: pa, + }) + // Newlines inside a stripped attr body shift subsequent line numbers. + extraNewlines += strings.Count(body, "\n") + } + field.Attrs = keep + + if sl, ok := field.Value.(*ast.StructLit); ok { + walkStruct(sl, path) + } + } + + walkStruct = func(sl *ast.StructLit, path cue.Path) { + // Capture the struct's {-relative base line for decl @test pos= specs. + // Captured before processing elements so it stays stable even if earlier + // elements in this struct strip lines. File-level @test attrs (no braces) + // use their own line as baseLine instead (see below). + structBaseLine := sl.Lbrace.Line() - extraNewlines + + // Strip decl attrs, recurse into sub-fields. + var newElts []ast.Decl + for _, elt := range sl.Elts { + switch e := elt.(type) { + case *ast.Attribute: + k, body := e.Split() + if k != "test" { + newElts = append(newElts, elt) + continue + } + pa, err := parseTestAttr(e) + if err != nil { + newElts = append(newElts, elt) + continue + } + pa.baseLine = structBaseLine + pa.srcFileName = fileName + records = append(records, attrRecord{ + path: path, + parsed: pa, + isDeclAttr: true, + }) + // The entire decl attr line is stripped (+1), plus any + // embedded newlines in the body shift subsequent lines. + extraNewlines += 1 + strings.Count(body, "\n") + // Stripped — not added to newElts. + + case *ast.Field: + subPath := appendPath(path, e.Label) + walkField(e, subPath) + newElts = append(newElts, elt) + + default: + newElts = append(newElts, elt) + } + } + sl.Elts = newElts + } + + // Handle file-level decl attributes (@test as a top-level declaration). + var newDecls []ast.Decl + for _, decl := range f.Decls { + if a, ok := decl.(*ast.Attribute); ok { + k, body := a.Split() + if k == "test" { + pa, err := parseTestAttr(a) + if err != nil { + newDecls = append(newDecls, decl) + continue + } + pa.baseLine = a.Pos().Line() - extraNewlines + pa.srcFileName = fileName + records = append(records, attrRecord{ + path: cue.Path{}, + parsed: pa, + fileLevel: true, + }) + extraNewlines += 1 + strings.Count(body, "\n") + // Stripped — not added to newDecls. + continue + } + } + newDecls = append(newDecls, decl) + } + f.Decls = newDecls + + for _, decl := range f.Decls { + field, ok := decl.(*ast.Field) + if !ok { + continue + } + fieldPath := cue.MakePath(cue.Label(field.Label)) + walkField(field, fieldPath) + } + + return records +} + +// identStr returns the string form of an AST label that is a simple identifier +// or string literal. Returns empty string for complex labels. +func identStr(label ast.Label) string { + switch l := label.(type) { + case *ast.Ident: + return l.Name + case *ast.BasicLit: + // Strip quotes for string labels. + s := l.Value + if len(s) >= 2 && s[0] == '"' { + s = s[1 : len(s)-1] + } + return s + } + return "" +} + +// appendPath appends a selector for label to an existing path. +func appendPath(base cue.Path, label ast.Label) cue.Path { + return base.Append(cue.Label(label)) +} + +// ───────────────────────────────────────────────────────────────────────────── +// Section 3: Mode Detection +// ───────────────────────────────────────────────────────────────────────────── + +// isInlineMode parses the CUE files in the archive (AST only) and returns true +// if any @test(...) attribute is found anywhere in any CUE file — as a field +// attribute, a decl attribute inside a struct at any nesting depth, or a +// file-level decl attribute. No compilation is required. +func isInlineMode(archive *txtar.Archive) bool { + for _, f := range archive.Files { + if !strings.HasSuffix(f.Name, ".cue") { + continue + } + af, err := parser.ParseFile(f.Name, f.Data) + if err != nil { + continue + } + if declsHaveTestAttrs(af.Decls) { + return true + } + } + return false +} + +// declsHaveTestAttrs recursively searches decls for any @test(...) attribute, +// descending into struct-valued fields at any depth. +func declsHaveTestAttrs(decls []ast.Decl) bool { + for _, decl := range decls { + switch d := decl.(type) { + case *ast.Attribute: + if k, _ := d.Split(); k == "test" { + return true + } + case *ast.Field: + for _, a := range d.Attrs { + if k, _ := a.Split(); k == "test" { + return true + } + } + if sl, ok := d.Value.(*ast.StructLit); ok { + if declsHaveTestAttrs(sl.Elts) { + return true + } + } + } + } + return false +} + +// ───────────────────────────────────────────────────────────────────────────── +// Section 4: Core Test Runner (inline mode) +// ───────────────────────────────────────────────────────────────────────────── + +// inlineRunner handles execution of a single txtar archive in inline mode. +type inlineRunner struct { + t *testing.T + m *cuetdtest.M + archive *txtar.Archive + dir string + filePath string // path to the archive file on disk; empty for in-memory tests + cueFiles []*cueFileResult // set by runArchive; used by runPermuteAssertion + + // pendingPosWrites accumulates pos= attribute updates to write back to the + // archive file after all subtests have run (CUE_UPDATE mode only). + pendingPosWrites []posWrite + + // pendingInlineFillWrites accumulates inline @test attribute rewrites + // (fill, force overwrite, regression guard, stale-skip cleanup) to apply + // after all subtests have run (CUE_UPDATE mode only). + pendingInlineFillWrites []inlineFillWrite +} + +// runArchive runs all test cases in the archive. +func (r *inlineRunner) runArchive() { + r.t.Helper() + + // Parse and extract @test attrs from all CUE files. + cueFiles, allRecords, err := r.parseAndExtract() + if err != nil { + r.t.Fatalf("inline: failed to parse CUE files: %v", err) + return + } + r.cueFiles = cueFiles + + // Check for #subpath restriction. + subpath := r.subpath() + + // Build and evaluate the stripped CUE. + // Note: val.Err() may be non-nil if sub-fields are erroneous; this is + // intentional for tests that assert errors. We only fatal on compile errors. + ctx := r.cueContext() + val, compileErr := r.buildValue(ctx, cueFiles) + if compileErr != nil { + r.t.Fatalf("inline: CUE compile error: %v", compileErr) + return + } + + // Collect file-level records (decl @test attrs at file scope). + var fileLevelRecords []attrRecord + for _, rec := range allRecords { + if rec.fileLevel { + fileLevelRecords = append(fileLevelRecords, rec) + } + } + + // Determine which top-level fields are test-case roots. + // A field is a root if it has any @test attribute (other than file-level). + // Fields with no @test attributes are silently skipped (fixture fields). + rootNames := make(map[cue.Selector]bool) + for _, rec := range allRecords { + if rec.fileLevel { + continue // file-level records are handled separately + } + sels := rec.path.Selectors() + if len(sels) == 0 { + continue + } + rootNames[sels[0]] = true + } + + // Run file-level @test assertions against the entire file value. + if len(fileLevelRecords) > 0 { + version := r.versionName() + for _, rec := range fileLevelRecords { + directives := selectActiveDirectives(allRecords, rec.path, version) + // Process skip directives first. + for _, pa := range directives { + if pa.directive == "skip" { + r.runDirective(r.t, cue.Path{}, val, pa) + } + } + for _, pa := range directives { + if pa.directive == "skip" { + continue + } + r.runDirective(r.t, cue.Path{}, val, pa) + } + } + } + + // Build ordered roots preserving declaration order. + var roots []testCaseRoot + for _, f := range cueFiles { + for _, decl := range f.strippedAST.Decls { + field, ok := decl.(*ast.Field) + if !ok { + continue + } + sel := cue.Label(field.Label) + if !rootNames[sel] { + continue + } + if subpath != "" && sel.String() != subpath { + continue + } + roots = append(roots, testCaseRoot{ + sel: sel, + }) + delete(rootNames, sel) // avoid duplicates across files + } + } + + for _, root := range roots { + root := root + name := r.subTestName(root, allRecords) + r.t.Run(name, func(t *testing.T) { + r.runInline(t, root, val, allRecords) + }) + } + + // After all subtests complete, write back any pending updates. + // Byte-level write-backs (pos, inline-fill) run first so subsequent + // AST-based write-backs re-parse the updated bytes. + r.applyPosWritebacks() + r.applyInlineFillWritebacks() +} + +// subTestName returns the sub-test name for a root. +// Always uses the field name. @test(desc="...") is purely a human-readable +// description annotation and does not affect the sub-test name. +// TODO: use a name directive to allow an explicit name separate from desc, and support +func (r *inlineRunner) subTestName(root testCaseRoot, _ []attrRecord) string { + return root.sel.String() +} + +// testCaseRoot represents a top-level test case. +type testCaseRoot struct { + sel cue.Selector +} + +// cueFileResult holds the parsed and stripped AST for one CUE file. +type cueFileResult struct { + name string + strippedAST *ast.File + // hasTestAttrs is true when the file contained at least one @test attribute. + // Files where this is false are treated as fixture files: they are still + // compiled into the evaluated value (so references from other files work) + // but their top-level fields are NOT required to carry @test directives. + hasTestAttrs bool +} + +// parseAndExtract parses all .cue files, extracts @test attrs, and returns: +// - the stripped AST files +// - all attrRecords +func (r *inlineRunner) parseAndExtract() ([]*cueFileResult, []attrRecord, error) { + var results []*cueFileResult + var allRecords []attrRecord + + for _, f := range r.archive.Files { + if !strings.HasSuffix(f.Name, ".cue") { + continue + } + af, err := parser.ParseFile(f.Name, f.Data, parser.ParseComments) + if err != nil { + return nil, nil, fmt.Errorf("parse %s: %w", f.Name, err) + } + records := extractTestAttrs(af, f.Name) + allRecords = append(allRecords, records...) + + results = append(results, &cueFileResult{ + name: f.Name, + strippedAST: af, + hasTestAttrs: len(records) > 0, + }) + } + return results, allRecords, nil +} + +// buildValue compiles and evaluates the stripped CUE files using the test context. +// Returns a compile-level error (parse/compile failure), not evaluation errors. +// Evaluation errors in sub-fields are returned as bottom values within the +// returned cue.Value and are intentional for tests asserting errors. +// +// Fixture files (hasTestAttrs==false) are compiled first and collected into a +// scope value; test files are compiled with that scope so that references from +// test files to fixture-file fields resolve correctly. +func (r *inlineRunner) buildValue(ctx *cue.Context, cueFiles []*cueFileResult) (cue.Value, error) { + combined := ctx.CompileString("{}") + + // First pass: compile fixture files (no @test attrs). + // Their fields form the scope for test files. + fixtureScope := ctx.CompileString("{}") + for _, cf := range cueFiles { + if cf.hasTestAttrs { + continue + } + fmtBytes, err := format.Node(cf.strippedAST) + if err != nil { + return cue.Value{}, fmt.Errorf("format %s: %w", cf.name, err) + } + v := ctx.CompileBytes(fmtBytes, cue.Filename(cf.name)) + if v.BuildInstance() == nil && v.Err() != nil { + return cue.Value{}, v.Err() + } + fixtureScope = fixtureScope.Unify(v) + combined = combined.Unify(v) + } + + // Second pass: compile test files with fixture scope so that + // cross-file references to fixture fields resolve correctly. + for _, cf := range cueFiles { + if !cf.hasTestAttrs { + continue // already compiled above + } + fmtBytes, err := format.Node(cf.strippedAST) + if err != nil { + return cue.Value{}, fmt.Errorf("format %s: %w", cf.name, err) + } + v := ctx.CompileBytes(fmtBytes, cue.Filename(cf.name), cue.Scope(fixtureScope)) + // Distinguish syntax/compile errors from evaluation errors. + // BuildInstance() returns nil only when there is a syntax error; + // evaluation errors produce a non-nil instance with a bottom value. + if v.BuildInstance() == nil && v.Err() != nil { + return cue.Value{}, v.Err() + } + combined = combined.Unify(v) + } + return combined, nil +} + +// cueContext returns the appropriate cue.Context for the current matrix entry. +func (r *inlineRunner) cueContext() *cue.Context { + if r.m != nil { + return r.m.CueContext() + } + return cuecontext.New() +} + +// subpath returns the #subpath value from the archive comment, or empty string. +func (r *inlineRunner) subpath() string { + prefix := []byte("#subpath:") + for _, line := range strings.Split(string(r.archive.Comment), "\n") { + b := []byte(strings.TrimSpace(line)) + if strings.HasPrefix(string(b), string(prefix)) { + return strings.TrimSpace(string(b[len(prefix):])) + } + } + return "" +} + +// ───────────────────────────────────────────────────────────────────────────── +// Section 5: Version Discrimination +// ───────────────────────────────────────────────────────────────────────────── + +// versionName returns the current evaluator version token (e.g. "v3"). +func (r *inlineRunner) versionName() string { + if r.m != nil { + return r.m.Name() + } + return "" +} + +// selectActiveDirectives filters records for a given CUE path and returns the +// effective set of directives to evaluate. Versioned directives take precedence +// over unversioned for the same directive name. +func selectActiveDirectives(records []attrRecord, path cue.Path, version string) []parsedTestAttr { + // Collect all records matching this path. + type entry struct { + pa parsedTestAttr + versioned bool + } + var candidates []entry + for _, rec := range records { + if rec.path.String() != path.String() { + continue + } + versioned := rec.parsed.version != "" + if rec.parsed.version != "" && rec.parsed.version != version { + continue // wrong version + } + candidates = append(candidates, entry{pa: rec.parsed, versioned: versioned}) + } + + // For each directive name, prefer the versioned form. + byDirective := make(map[string]parsedTestAttr) + hasVersioned := make(map[string]bool) + for _, c := range candidates { + dir := c.pa.directive + if c.versioned { + byDirective[dir] = c.pa + hasVersioned[dir] = true + } else if !hasVersioned[dir] { + byDirective[dir] = c.pa + } + } + + result := slices.Collect(maps.Values(byDirective)) + return result +} + +// ───────────────────────────────────────────────────────────────────────────── +// Section 6: Inline Form — Directive Assertions +// ───────────────────────────────────────────────────────────────────────────── + +// runInline runs assertions for an inline-form test-case root. +func (r *inlineRunner) runInline(t *testing.T, root testCaseRoot, fileVal cue.Value, records []attrRecord) { + t.Helper() + version := r.versionName() + + // Gather all records for this root and its descendants. + rootPath := cue.MakePath(root.sel) + for _, rec := range records { + if !pathHasPrefix(rec.path, rootPath) { + continue + } + directives := selectActiveDirectives(records, rec.path, version) + // Process skip directives first so t.Skip() fires before other assertions. + for _, pa := range directives { + if pa.directive == "skip" { + fieldVal := fileVal.LookupPath(rec.path) + r.runDirective(t, rec.path, fieldVal, pa) + } + } + for _, pa := range directives { + if pa.directive == "skip" { + continue // already handled above + } + fieldVal := fileVal.LookupPath(rec.path) + r.runDirective(t, rec.path, fieldVal, pa) + } + } + + // Handle @test(permute) field attributes. + r.runInlinePermutes(t, rootPath, records, version) +} + +// pathHasPrefix reports whether path starts with prefix, treating each +// selector as an atomic component (not a substring of a selector). +func pathHasPrefix(path, prefix cue.Path) bool { + ps := path.Selectors() + prefs := prefix.Selectors() + if len(ps) < len(prefs) { + return false + } + for i, s := range prefs { + if ps[i].String() != s.String() { + return false + } + } + return true +} + +// runDirective dispatches a single parsed directive against a cue.Value. +func (r *inlineRunner) runDirective(t *testing.T, path cue.Path, val cue.Value, pa parsedTestAttr) { + t.Helper() + switch pa.directive { + case "eq": + r.runEqInline(t, path, val, pa) + case "err": + r.runErrAssertion(t, path, val, pa) + case "leq": + r.runLeqInline(t, path, val, pa) + case "kind": + r.runKindAssertion(t, path, val, pa) + case "closed": + r.runClosedAssertion(t, path, val, pa) + case "skip": + // @test(skip) or @test(skip, why="reason") — skip this test. + reason := "skipped" + if len(pa.raw.Fields) > 1 { + for _, kv := range pa.raw.Fields[1:] { + if kv.Key() == "why" { + reason = kv.Value() + } + } + } + t.Skip(reason) // t.Skip calls runtime.Goexit, stopping the goroutine. + case "debugCheck": + r.runDebugCheckInline(t, path, val, pa) + case "debugOutput": + r.runDebugOutputInline(t, path, val, pa) + case "permute": + // Handled by permute-group collection in runInline; no-op here. + case "permuteCount": + // Handled by checkPermuteCount after permutations run; no-op here. + case "desc": + // @test(desc="...") is a human-readable description annotation — no assertions. + case "": + // Empty placeholder @test() — fill with actual value when CUE_UPDATE=1. + if cuetest.UpdateGoldenFiles { + r.enqueueInlineFill(pa, r.formatCoverAttr(val)) + } + default: + t.Errorf("path %s: unknown @test directive %q", path, pa.directive) + } +} + +// runEqInline checks that val equals the CUE expression in the first arg of pa. +// When CUE_UPDATE modes are active it enqueues the appropriate write-back +// instead of (or in addition to) running the comparison: +// - empty placeholder @test(eq): fill with actual value (UpdateGoldenFiles) +// - passing assertion with skip+diff: remove stale skip (UpdateGoldenFiles) +// - failing assertion: force-overwrite (ForceUpdateGoldenFiles) or +// regression-guard with skip:+diff annotation (UpdateGoldenFiles) +func (r *inlineRunner) runEqInline(t *testing.T, path cue.Path, val cue.Value, pa parsedTestAttr) { + t.Helper() + if len(pa.raw.Fields) < 2 { + // Empty @test(eq) — fill placeholder. + if cuetest.UpdateGoldenFiles { + r.enqueueInlineFill(pa, "@test(eq, "+r.formatValue(val)+")") + } + return + } + exprStr := pa.raw.Fields[1].Text() + expr, err := parser.ParseExpr("@test(eq)", exprStr) + if err != nil { + t.Errorf("path %s: @test(eq, ...): cannot parse expected expression: %v", path, err) + return + } + + // Detect stale-skip: a prior regression guard annotated this attr with + // skip: and diff="...". + _, hasSkip := attrHasSkip(pa.raw) + + cmpErr := (&cmpCtx{baseLine: 0}).astCmp(cue.Path{}, expr, val) + if cmpErr == nil { + // Assertion passes via AST comparison. + if hasSkip && cuetest.UpdateGoldenFiles { + // Stale-skip cleanup (task 11.7): the assertion now passes; strip + // the skip and diff args, restoring the plain @test(eq, ). + r.enqueueInlineFill(pa, "@test(eq, "+exprStr+")") + } + return + } + + // Comparison failed — genuine mismatch. + if cuetest.ForceUpdateGoldenFiles { + // Force overwrite (task 11.4): replace expected with actual. + r.enqueueInlineFill(pa, "@test(eq, "+r.formatValue(val)+")") + return + } + if cuetest.UpdateGoldenFiles && !hasSkip { + // Regression guard (tasks 11.2-11.3): annotate with skip:+diff + // so the test keeps passing under CUE_UPDATE without silent overwrite. + got := r.formatValue(val) + diffStr := fmt.Sprintf("got %s; want %s", got, exprStr) + escaped := strings.ReplaceAll(diffStr, `"`, `\"`) + ver := r.versionName() + if ver == "" { + ver = "unknown" + } + r.enqueueInlineFill(pa, fmt.Sprintf(`@test(eq, %s, skip:%s, diff="%s")`, exprStr, ver, escaped)) + return + } + // Report the failure (unless already annotated with a skip). + if !hasSkip { + t.Errorf("path %s: %v", path, cmpErr) + } +} + +// formatValue returns a human-readable CUE string for a value. +func (r *inlineRunner) formatValue(v cue.Value) string { + syn := v.Syntax(cue.All(), cue.Docs(true), cue.Final()) + if syn == nil { + return fmt.Sprintf("%v", v) + } + b, err := format.Node(syn) + if err != nil { + return fmt.Sprintf("%v", v) + } + return string(b) +} + +// runErrAssertion checks that an error is present at val, applying sub-options. +func (r *inlineRunner) runErrAssertion(t *testing.T, path cue.Path, val cue.Value, pa parsedTestAttr) { + t.Helper() + ea := pa.errArgs + if ea == nil { + // Bare @test(err) — just check that the value is an error. + if !r.isError(val) { + t.Errorf("path %s: expected error, got non-error value", path) + } + return + } + + if ea.any { + // @test(err, any, ...) — check that any descendant has the error. + found := r.findDescendantError(val, ea) + if !found { + t.Errorf("path %s: expected a descendant error with code=%q, none found", path, ea.code) + } + return + } + + if !r.isError(val) { + t.Errorf("path %s: expected error, got non-error value", path) + return + } + + // Validate error code. + if ea.code != "" { + gotCode := r.errorCode(val) + if gotCode != ea.code { + t.Errorf("path %s: expected error code %q, got %q", path, ea.code, gotCode) + } + } + // Validate error message contains. + if ea.contains != "" { + msg := r.errorMessage(val) + if !strings.Contains(msg, ea.contains) { + t.Errorf("path %s: expected error message to contain %q, got %q", path, ea.contains, msg) + } + } + // Validate error positions. + if ea.posSet { + r.checkErrPositions(t, path, val, pa) + } +} + +// checkErrPositions verifies that the error positions on val match the pos= +// spec in pa. When positions don't match: +// - pos=[] (placeholder): update on CUE_UPDATE=1. +// - pos=[non-empty]: update on CUE_UPDATE=force only. +func (r *inlineRunner) checkErrPositions(t *testing.T, path cue.Path, val cue.Value, pa parsedTestAttr) { + t.Helper() + err := val.Err() + if err == nil { + t.Errorf("path %s: @test(err, pos=...): value has no error", path) + return + } + positions := cueerrors.Positions(err) + expected := pa.errArgs.pos + + match := len(positions) == len(expected) + if match { + for i, exp := range expected { + got := positions[i] + if exp.fileName != "" { + // Absolute form: match filename + absolute line + column. + if got.Filename() != exp.fileName || got.Line() != exp.absLine || got.Column() != exp.col { + match = false + break + } + } else { + // Relative form: match line delta from @test + column. + if got.Line() != pa.baseLine+exp.deltaLine || got.Column() != exp.col { + match = false + break + } + } + } + } + if match { + return + } + + // pos=[] is a fill-in placeholder: update with CUE_UPDATE=1. + // pos=[non-empty] that is wrong: update only with CUE_UPDATE=force. + isPlaceholder := len(expected) == 0 + if (isPlaceholder && cuetest.UpdateGoldenFiles) || cuetest.ForceUpdateGoldenFiles { + r.enqueuePosWrite(pa, positions) + return + } + + if len(positions) != len(expected) { + t.Errorf("path %s: @test(err, pos=...): got %d position(s), want %d", path, len(positions), len(expected)) + for i, p := range positions { + t.Logf(" actual[%d]: %d:%d", i, p.Line(), p.Column()) + } + return + } + for i, exp := range expected { + got := positions[i] + if exp.fileName != "" { + if got.Filename() != exp.fileName || got.Line() != exp.absLine || got.Column() != exp.col { + t.Errorf("path %s: @test(err, pos=...): position[%d]: got %s:%d:%d, want %s:%d:%d", + path, i, got.Filename(), got.Line(), got.Column(), exp.fileName, exp.absLine, exp.col) + } + } else { + wantLine := pa.baseLine + exp.deltaLine + if got.Line() != wantLine || got.Column() != exp.col { + t.Errorf("path %s: @test(err, pos=...): position[%d]: got %d:%d, want %d:%d", + path, i, got.Line(), got.Column(), wantLine, exp.col) + } + } + } +} + +// posWrite records a pending pos= attribute update for CUE_UPDATE write-back. +type posWrite struct { + fileName string // archive .cue file name, e.g. "in.cue" + attrOffset int // byte offset of the @test attr in the original file data + attrLen int // byte length of the original @test attr text + newAttrText string // replacement attribute text with updated pos=[...] +} + +// enqueuePosWrite formats positions as pos specs and enqueues a write-back +// that replaces the pos=[...] value in the source attribute. +// +// Positions in the same file as the @test attribute are written as deltaLine:col +// (relative to pa.baseLine). Positions in a different file are written as +// filename:absLine:col (absolute). +func (r *inlineRunner) enqueuePosWrite(pa parsedTestAttr, positions []token.Pos) { + parts := make([]string, len(positions)) + for i, p := range positions { + if p.Filename() == "" || p.Filename() == pa.srcFileName { + parts[i] = fmt.Sprintf("%d:%d", p.Line()-pa.baseLine, p.Column()) + } else { + parts[i] = fmt.Sprintf("%s:%d:%d", p.Filename(), p.Line(), p.Column()) + } + } + newPosStr := strings.Join(parts, " ") + + old := pa.srcAttr.Text + start := strings.Index(old, "pos=[") + if start < 0 { + return + } + bracket := start + len("pos=[") + end := strings.Index(old[bracket:], "]") + if end < 0 { + return + } + end += bracket + 1 // include the "]" + newAttrText := old[:start] + "pos=[" + newPosStr + "]" + old[end:] + + r.pendingPosWrites = append(r.pendingPosWrites, posWrite{ + fileName: pa.srcFileName, + attrOffset: pa.srcAttr.Pos().Offset(), + attrLen: len(pa.srcAttr.Text), + newAttrText: newAttrText, + }) +} + +// applyPosWritebacks writes pending pos= attribute updates to the archive file. +// Replacements are applied by byte offset from end to start so that earlier +// offsets remain valid after each substitution. +func (r *inlineRunner) applyPosWritebacks() { + if len(r.pendingPosWrites) == 0 || r.filePath == "" { + return + } + changed := false + for i, f := range r.archive.Files { + var writes []posWrite + for _, pw := range r.pendingPosWrites { + if pw.fileName == f.Name { + writes = append(writes, pw) + } + } + if len(writes) == 0 { + continue + } + // Sort descending by offset so earlier offsets stay valid. + slices.SortFunc(writes, func(a, b posWrite) int { + return b.attrOffset - a.attrOffset + }) + data := append([]byte(nil), f.Data...) + for _, pw := range writes { + end := pw.attrOffset + pw.attrLen + if end > len(data) { + continue + } + data = append(data[:pw.attrOffset:pw.attrOffset], + append([]byte(pw.newAttrText), data[end:]...)...) + changed = true + } + r.archive.Files[i].Data = data + } + if changed { + out := txtar.Format(r.archive) + if err := os.WriteFile(r.filePath, out, 0o644); err != nil { + r.t.Errorf("inline: pos write-back to %s: %v", r.filePath, err) + } + } +} + +// isError reports whether val is an error value (bottom). +func (r *inlineRunner) isError(val cue.Value) bool { + core := val.Core() + if core.V == nil { + return false + } + return core.V.Bottom() != nil +} + +// errorCode returns the string code of the error at val, or "" if not an error. +func (r *inlineRunner) errorCode(val cue.Value) string { + core := val.Core() + if core.V == nil { + return "" + } + b := core.V.Bottom() + if b == nil { + return "" + } + return b.Code.String() +} + +// errorMessage returns the human-readable error message for val. +func (r *inlineRunner) errorMessage(val cue.Value) string { + if err := val.Err(); err != nil { + return err.Error() + } + return "" +} + +// findDescendantError walks val looking for any descendant with an error +// matching ea. Returns true if found. +func (r *inlineRunner) findDescendantError(val cue.Value, ea *errArgs) bool { + if r.isError(val) { + if ea.code == "" { + return true + } + if r.errorCode(val) == ea.code { + return true + } + } + // Walk fields. + iter, err := val.Fields() + if err != nil { + return false + } + for iter.Next() { + if r.findDescendantError(iter.Value(), ea) { + return true + } + } + return false +} + +// runLeqInline checks that val is subsumed by the constraint in pa. +func (r *inlineRunner) runLeqInline(t *testing.T, path cue.Path, val cue.Value, pa parsedTestAttr) { + t.Helper() + if len(pa.raw.Fields) < 2 { + t.Errorf("path %s: @test(leq) requires a constraint argument", path) + return + } + exprStr := pa.raw.Fields[1].Text() + ctx := r.cueContext() + constraint := ctx.CompileString(exprStr) + if constraint.Err() != nil { + t.Errorf("path %s: @test(leq, ...): cannot compile constraint: %v", path, constraint.Err()) + return + } + r.runLeqAssertion(t, path, val, constraint) +} + +// runLeqAssertion asserts that val is subsumed by constraint (constraint ⊑ val, i.e. val is at least as specific). +func (r *inlineRunner) runLeqAssertion(t *testing.T, path cue.Path, val, constraint cue.Value) { + t.Helper() + if err := constraint.Subsume(val); err != nil { + t.Errorf("path %s: @test(leq): value %v is not subsumed by constraint %v: %v", path, val, constraint, err) + } +} + +// runKindAssertion checks val.IncompleteKind() against the expected kind list. +// Syntax: @test(kind=int) or @test(kind=int|string). +func (r *inlineRunner) runKindAssertion(t *testing.T, path cue.Path, val cue.Value, pa parsedTestAttr) { + t.Helper() + // The directive is the key ("kind") and the expected kind(s) are the value. + if len(pa.raw.Fields) == 0 || pa.raw.Fields[0].Value() == "" { + t.Errorf("path %s: @test(kind=...) requires a kind value", path) + return + } + expectedStr := pa.raw.Fields[0].Value() + gotKind := val.IncompleteKind() + + // Parse expected kind(s) — may be pipe-separated like "int|string". + var expectedKind cue.Kind + for _, ks := range strings.Split(expectedStr, "|") { + k := parseKindStr(strings.TrimSpace(ks)) + if k == cue.BottomKind { + t.Errorf("path %s: @test(kind=%q): unknown kind %q", path, expectedStr, ks) + return + } + expectedKind |= k + } + if gotKind != expectedKind { + t.Errorf("path %s: @test(kind=%s): got kind %v, want %v", path, expectedStr, gotKind, expectedKind) + } +} + +// parseKindStr converts a kind name string to a cue.Kind. +func parseKindStr(s string) cue.Kind { + switch s { + case "bool": + return cue.BoolKind + case "int": + return cue.IntKind + case "float": + return cue.FloatKind + case "string": + return cue.StringKind + case "bytes": + return cue.BytesKind + case "struct": + return cue.StructKind + case "list": + return cue.ListKind + case "null": + return cue.NullKind + case "top", "_": + return cue.TopKind + case "bottom", "_|_": + return cue.BottomKind + } + return cue.BottomKind // sentinel for unknown +} + +// runClosedAssertion checks val.IsClosed() matches expected. +// Syntax: @test(closed) for closed=true, @test(closed=false) for closed=false. +func (r *inlineRunner) runClosedAssertion(t *testing.T, path cue.Path, val cue.Value, pa parsedTestAttr) { + t.Helper() + expected := true // bare @test(closed) means closed=true + if len(pa.raw.Fields) >= 1 && pa.raw.Fields[0].Key() == "closed" { + if pa.raw.Fields[0].Value() == "false" { + expected = false + } + } + got := val.IsClosed() + if got != expected { + t.Errorf("path %s: @test(closed): got closed=%v, want %v", path, got, expected) + } +} + +// runDebugCheckInline checks the debug printer output of val against the +// expected string in the @test(debugCheck, "...") attribute. +// When CUE_UPDATE modes are active, enqueues a write-back. +func (r *inlineRunner) runDebugCheckInline(t *testing.T, path cue.Path, val cue.Value, pa parsedTestAttr) { + t.Helper() + if len(pa.raw.Fields) < 2 { + // Empty @test(debugCheck) — fill placeholder. + if cuetest.UpdateGoldenFiles { + actual := r.debugPrinterOutput(val) + escaped := strings.ReplaceAll(actual, `"`, `\"`) + r.enqueueInlineFill(pa, fmt.Sprintf(`@test(debugCheck, """%s""")`, escaped)) + } + return + } + expected := pa.raw.Fields[1].Value() + actual := r.debugPrinterOutput(val) + match := normalizeLines(actual) == normalizeLines(expected) + if match && !cuetest.ForceUpdateGoldenFiles { + return + } + if cuetest.ForceUpdateGoldenFiles || cuetest.UpdateGoldenFiles { + escaped := strings.ReplaceAll(actual, `"`, `\"`) + r.enqueueInlineFill(pa, fmt.Sprintf(`@test(debugCheck, """%s""")`, escaped)) + return + } + if !match { + t.Errorf("path %s: @test(debugCheck) mismatch:\ngot: %q\nwant: %q", path, actual, expected) + } +} + +// runDebugOutputInline captures the debug printer output of val as an +// informational annotation. Unlike debugCheck, a mismatch does not fail the +// test — it only logs and auto-updates when CUE_UPDATE is active. +func (r *inlineRunner) runDebugOutputInline(t *testing.T, path cue.Path, val cue.Value, pa parsedTestAttr) { + t.Helper() + actual := r.debugPrinterOutput(val) + if len(pa.raw.Fields) < 2 { + // Empty @test(debugOutput) — fill placeholder. + if cuetest.UpdateGoldenFiles { + escaped := strings.ReplaceAll(actual, `"`, `\"`) + r.enqueueInlineFill(pa, fmt.Sprintf(`@test(debugOutput, """%s""")`, escaped)) + } + return + } + expected := pa.raw.Fields[1].Value() + match := normalizeLines(actual) == normalizeLines(expected) + if match && !cuetest.ForceUpdateGoldenFiles { + return + } + // Always auto-update on mismatch (informational, not an assertion). + if cuetest.ForceUpdateGoldenFiles || cuetest.UpdateGoldenFiles { + escaped := strings.ReplaceAll(actual, `"`, `\"`) + r.enqueueInlineFill(pa, fmt.Sprintf(`@test(debugOutput, """%s""")`, escaped)) + return + } + if !match { + t.Logf("path %s: @test(debugOutput) changed:\ngot: %q\nwant: %q", path, actual, expected) + } +} + +// debugPrinterOutput returns the standard debug-printer representation of val, +// equivalent to what appears in out/eval golden sections. +func (r *inlineRunner) debugPrinterOutput(val cue.Value) string { + c := val.Core() + if c.V == nil { + return "" + } + return debug.NodeString(c.R, c.V, nil) +} + +// normalizeLines trims trailing whitespace from each line and strips any +// trailing blank lines, for use in debug: textual comparison. +func normalizeLines(s string) string { + lines := strings.Split(s, "\n") + for i, line := range lines { + lines[i] = strings.TrimRight(line, " \t") + } + for len(lines) > 0 && lines[len(lines)-1] == "" { + lines = lines[:len(lines)-1] + } + return strings.Join(lines, "\n") +} + +// attrHasSkip reports whether the raw attribute body contains a skip: arg +// at position 2 or later. Returns the version string (e.g. "v3") and true +// when a skip arg is found; returns "", false otherwise. +func attrHasSkip(raw internal.Attr) (ver string, ok bool) { + for i := 2; i < len(raw.Fields); i++ { + text := raw.Fields[i].Text() + if text == "skip" { + return "", true + } + if strings.HasPrefix(text, "skip:") { + return text[5:], true + } + } + return "", false +} + +// enqueueInlineFill appends a byte-level replacement for pa's @test attribute. +// newAttrText is the full replacement text including the leading @. +func (r *inlineRunner) enqueueInlineFill(pa parsedTestAttr, newAttrText string) { + r.pendingInlineFillWrites = append(r.pendingInlineFillWrites, inlineFillWrite{ + fileName: pa.srcFileName, + attrOffset: pa.srcAttr.Pos().Offset(), + attrLen: len(pa.srcAttr.Text), + newAttrText: newAttrText, + }) +} diff --git a/internal/cuetxtar/inline_test.go b/internal/cuetxtar/inline_test.go new file mode 100644 index 000000000..cdfc4f6b6 --- /dev/null +++ b/internal/cuetxtar/inline_test.go @@ -0,0 +1,380 @@ +// Copyright 2026 CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cuetxtar + +import ( + "slices" + "strings" + "testing" + + "golang.org/x/tools/txtar" + + "cuelang.org/go/cue" + "cuelang.org/go/cue/ast" + "cuelang.org/go/cue/parser" +) + +// testMakePath creates a CUE path from a dot-separated string for test use. +func testMakePath(s string) cue.Path { + if s == "" { + return cue.MakePath() + } + parts := strings.Split(s, ".") + sels := make([]cue.Selector, len(parts)) + for i, p := range parts { + sels[i] = cue.Str(p) + } + return cue.MakePath(sels...) +} + +func TestParsePosSpecs(t *testing.T) { + tests := []struct { + name string + input string + want []posSpec + wantErr bool + }{ + { + name: "empty brackets", + input: "[]", + want: nil, + }, + { + name: "single relative", + input: "[0:5]", + want: []posSpec{{deltaLine: 0, col: 5}}, + }, + { + name: "multiple relative with whitespace", + input: "[0:5 1:13 -2:3]", + want: []posSpec{ + {deltaLine: 0, col: 5}, + {deltaLine: 1, col: 13}, + {deltaLine: -2, col: 3}, + }, + }, + { + name: "absolute form", + input: "[fixture.cue:3:5]", + want: []posSpec{{fileName: "fixture.cue", absLine: 3, col: 5}}, + }, + { + name: "mixed relative and absolute", + input: "[0:5 fixture.cue:1:13]", + want: []posSpec{ + {deltaLine: 0, col: 5}, + {fileName: "fixture.cue", absLine: 1, col: 13}, + }, + }, + { + name: "missing brackets", + input: "0:5", + wantErr: true, + }, + { + name: "four parts", + input: "[a:b:c:d]", + wantErr: true, + }, + { + name: "non-integer deltaLine", + input: "[x:5]", + wantErr: true, + }, + { + name: "non-integer col", + input: "[0:x]", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parsePosSpecs(tt.input) + if tt.wantErr { + if err == nil { + t.Errorf("expected error, got nil (result: %v)", got) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(got) != len(tt.want) { + t.Fatalf("got %d specs, want %d: %v", len(got), len(tt.want), got) + } + for i, w := range tt.want { + g := got[i] + if g != w { + t.Errorf("spec[%d]: got %+v, want %+v", i, g, w) + } + } + }) + } +} + +func TestFindPermFieldsAtPath(t *testing.T) { + tests := []struct { + name string + src string + path string + fieldNames []string // nil means all fields + wantCount int + wantNames []string + }{ + { + name: "all fields in top-level struct", + src: "myStruct: {\n\ta: 1\n\tb: 2\n\tc: 3\n}", + path: "myStruct", + wantCount: 3, + wantNames: []string{"a", "b", "c"}, + }, + { + name: "named subset of fields", + src: "myStruct: {\n\ta: 1\n\tb: 2\n\tc: 3\n}", + path: "myStruct", + fieldNames: []string{"a", "c"}, + wantCount: 2, + wantNames: []string{"a", "c"}, + }, + { + name: "nested path", + src: "outer: {\n\tinner: {\n\t\tx: 1\n\t\ty: 2\n\t}\n}", + path: "outer.inner", + wantCount: 2, + wantNames: []string{"x", "y"}, + }, + { + name: "missing path returns nil", + src: "outer: {\n\tinner: {\n\t\tx: 1\n\t}\n}", + path: "outer.missing", + wantCount: 0, + }, + { + name: "non-struct field returns nil", + src: "outer: 42", + path: "outer", + wantCount: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f, err := parser.ParseFile("test.cue", tt.src) + if err != nil { + t.Fatalf("parse: %v", err) + } + + structLit, indices := findPermFieldsAtPath(f, testMakePath(tt.path), tt.fieldNames) + + if tt.wantCount == 0 { + if len(indices) != 0 { + t.Errorf("want 0 indices, got %d", len(indices)) + } + return + } + if structLit == nil { + t.Fatal("structLit is nil, want non-nil") + } + if len(indices) != tt.wantCount { + t.Errorf("got %d indices, want %d", len(indices), tt.wantCount) + return + } + // Verify the field names at the returned indices. + var gotNames []string + for _, idx := range indices { + if f2, ok := structLit.Elts[idx].(*ast.Field); ok { + gotNames = append(gotNames, identStr(f2.Label)) + } + } + if len(gotNames) != len(tt.wantNames) { + t.Errorf("got field names %v, want %v", gotNames, tt.wantNames) + return + } + for i, name := range tt.wantNames { + if gotNames[i] != name { + t.Errorf("field[%d]: got %q, want %q", i, gotNames[i], name) + } + } + }) + } +} + +func makeRec(path string, directive, version string) attrRecord { + return attrRecord{ + path: testMakePath(path), + parsed: parsedTestAttr{ + directive: directive, + version: version, + }, + } +} + +func TestSelectActiveDirectives(t *testing.T) { + tests := []struct { + name string + records []attrRecord + path string + version string + wantDirs []string + }{ + { + name: "unversioned applies to all", + records: []attrRecord{makeRec("field1", "eq", "")}, + path: "field1", + version: "v3", + wantDirs: []string{"eq"}, + }, + { + name: "versioned overrides unversioned", + records: []attrRecord{ + makeRec("field1", "eq", ""), + makeRec("field1", "eq", "v3"), + }, + path: "field1", + version: "v3", + wantDirs: []string{"eq"}, // only one eq (versioned wins) + }, + { + name: "versioned for other version skipped unversioned still applies", + records: []attrRecord{ + makeRec("field1", "eq", ""), + makeRec("field1", "eq", "v2"), + }, + path: "field1", + version: "v3", + wantDirs: []string{"eq"}, // v2 skipped; unversioned applies + }, + { + name: "wrong path excluded", + records: []attrRecord{makeRec("field2", "eq", "")}, + path: "field1", + version: "v3", + wantDirs: nil, + }, + { + name: "multiple directives at same path", + records: []attrRecord{ + makeRec("field1", "eq", ""), + makeRec("field1", "err", ""), + }, + path: "field1", + version: "v3", + wantDirs: []string{"eq", "err"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := selectActiveDirectives(tt.records, testMakePath(tt.path), tt.version) + var gotDirs []string + for _, pa := range got { + gotDirs = append(gotDirs, pa.directive) + } + slices.Sort(gotDirs) + wantDirs := append([]string(nil), tt.wantDirs...) + slices.Sort(wantDirs) + if !slices.Equal(gotDirs, wantDirs) { + t.Errorf("got directives %v, want %v", gotDirs, wantDirs) + } + }) + } +} + +// TestInlineRunner_ErrPos verifies @test(err, pos=[...]) position checking. +func TestInlineRunner_ErrPos(t *testing.T) { + tests := []struct { + name string + archive string + }{ + { + // Field-attribute form: pos on same line as error source. + // Stripped output: "x: 1 & 2" on line 1. + // Positions: 1:4 (the 1) and 1:8 (the 2). + // baseLine=1, deltaLine=0 → expected line 1. + name: "field attr pos relative same line", + archive: "-- test.cue --\nx: 1 & 2 @test(err, pos=[0:4 0:8])\n", + }, + { + // Field-attribute on a struct with conflict below. + // Stripped output: + // line 1: x: { + // line 2: a: 1 + // line 3: a: 2 + // line 4: } + // baseLine=1 (field x on line 1), deltas: 1→line 2, 2→line 3. + name: "field attr pos relative below", + archive: "-- test.cue --\nx: {\n\ta: 1\n\ta: 2\n} @test(err, pos=[1:5 2:5])\n", + }, + { + // Decl-attribute form inside a struct. + // Original source: + // line 1: x: { + // line 2: @test(err, pos=[1:5 2:5]) + // line 3: a: 1 + // line 4: a: 2 + // line 5: } + // After stripping the @test (line 2 removed): + // line 1: x: { + // line 2: a: 1 + // line 3: a: 2 + // line 4: } + // baseLine = sl.Lbrace.Line() - 0 = 1 (the "{" on line 1). + // deltaLine=1 → line 2 (a: 1), deltaLine=2 → line 3 (a: 2). + name: "decl attr pos relative", + archive: "-- test.cue --\nx: {\n\t@test(err, pos=[1:5 2:5])\n\ta: 1\n\ta: 2\n}\n", + }, + { + // Decl-attribute at file-level with a conflict. + // Original source: + // line 1: @test(err, pos=[0:4 0:8]) + // line 2: x: 1 & 2 + // After stripping the @test (line 1 removed): + // line 1: x: 1 & 2 + // baseLine = 1 (original line of @test) - 0 = 1. + // deltaLine=0 → line 1 (x: 1 & 2). + // Positions: 1:4 and 1:8. + name: "file-level decl attr pos relative", + archive: "-- test.cue --\n@test(err, pos=[0:4 0:8])\nx: 1 & 2\n", + }, + { + // Multiple fields: second field's baseLine accounts for + // the stripped @test on the first field. + // Original: + // line 1: x: 1 @test(eq, 1) + // line 2: y: 1 & 2 @test(err, pos=[0:4 0:8]) + // After stripping (both on same line, no extra newlines): + // line 1: x: 1 + // line 2: y: 1 & 2 + // baseLine for y = 2, deltaLine=0 → line 2. + name: "field attr after prior field attr", + archive: "-- test.cue --\nx: 1 @test(eq, 1)\ny: 1 & 2 @test(err, pos=[0:4 0:8])\n", + }, + { + // Absolute position form: filename:absLine:col. + // After stripping, "test.cue" has "x: 1 & 2" on line 1. + name: "absolute pos form", + archive: "-- test.cue --\nx: 1 & 2 @test(err, pos=[test.cue:1:4 test.cue:1:8])\n", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + archive := txtar.Parse([]byte(tt.archive)) + runner := NewInlineRunner(t, nil, archive, t.TempDir()) + runner.Run() + }) + } +} diff --git a/internal/cuetxtar/inlinerunner_test.go b/internal/cuetxtar/inlinerunner_test.go new file mode 100644 index 000000000..dd61dc864 --- /dev/null +++ b/internal/cuetxtar/inlinerunner_test.go @@ -0,0 +1,152 @@ +// Copyright 2026 CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cuetxtar_test + +import ( + "testing" + + "golang.org/x/tools/txtar" + + "cuelang.org/go/internal/cuetxtar" +) + +// TestInlineRunner_Basic verifies end-to-end inline test execution using +// in-memory txtar archives. +func TestInlineRunner_Basic(t *testing.T) { + tests := []struct { + name string + archive string + }{ + { + name: "eq passes for matching int", + archive: "-- test.cue --\nx: 42 @test(eq, 42)\n", + }, + { + name: "eq passes for matching string", + archive: "-- test.cue --\nx: \"hello\" @test(eq, \"hello\")\n", + }, + { + name: "err detects conflict", + archive: "-- test.cue --\nerrField: 1 & 2 @test(err)\n", + }, + { + name: "kind int passes", + archive: "-- test.cue --\nx: int @test(kind=int)\n", + }, + { + name: "closed struct passes", + archive: "-- test.cue --\nx: close({a: 1}) @test(closed)\n", + }, + { + name: "leq passes", + archive: "-- test.cue --\nx: 42 @test(leq, number)\n", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + archive := txtar.Parse([]byte(tt.archive)) + runner := cuetxtar.NewInlineRunner(t, nil, archive, t.TempDir()) + runner.Run() + }) + } +} + +// TestInlineRunner_Permute verifies @test(permute) assertions. +func TestInlineRunner_Permute(t *testing.T) { + tests := []struct { + name string + archive string + wantFail bool // whether the test is expected to fail + wantSkipped bool // whether there is nothing to permute (< 2 fields) + }{ + { + // Field-attribute permute: only a and b are marked; c is not permuted. + name: "inline permute two fields passes", + archive: "-- test.cue --\nmyStruct: {\n\ta: b + 1 @test(permute)\n\tb: 2 @test(permute)\n\tc: 99\n} @test(eq, {a: 3, b: 2, c: 99})\n", + }, + { + // Only one field marked with @test(permute): nothing to permute, should be no-op. + name: "single permute field skipped", + archive: "-- test.cue --\nmyStruct: {a: 1 @test(permute), b: 2} @test(eq, {a: 1, b: 2})\n", + wantSkipped: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + archive := txtar.Parse([]byte(tt.archive)) + if tt.wantFail { + // Use a sub-test to capture failure. + var failed bool + t.Run("inner", func(inner *testing.T) { + runner := cuetxtar.NewInlineRunner(inner, nil, archive, t.TempDir()) + runner.Run() + failed = inner.Failed() + }) + if !failed { + t.Errorf("expected permute assertion to fail, but it passed") + } + return + } + runner := cuetxtar.NewInlineRunner(t, nil, archive, t.TempDir()) + runner.Run() + }) + } +} + +// TestInlineRunner_SubPath verifies #subpath restricts execution. +func TestInlineRunner_SubPath(t *testing.T) { + // When #subpath is set, only the named test runs. + archive := txtar.Parse([]byte("#subpath: second\n-- test.cue --\nfirst: 1 @test(eq, 999)\nsecond: 2 @test(eq, 2)\n")) + // The "first" test has a wrong eq (999 != 1) but should be skipped. + // The "second" test should pass. + runner := cuetxtar.NewInlineRunner(t, nil, archive, t.TempDir()) + runner.Run() +} + +// TestInlineRunner_FixtureField verifies that a field with no @test attribute +// is silently skipped (not run as a sub-test) but is still accessible to other +// test fields in the same archive. +func TestInlineRunner_FixtureField(t *testing.T) { + tests := []struct { + name string + archive string + }{ + { + // Plain field with no @test: not a sub-test but still accessible. + name: "plain fixture field in same file", + archive: "-- test.cue --\n" + + "fixture: {foo: 1}\n" + + "a: fixture.foo & int @test(eq, 1)\n", + }, + { + // Pure fixture file: no @test attrs anywhere in fixture.cue. + name: "pure fixture file no annotation needed", + archive: "-- fixture.cue --\n" + + "fixture: foo: 1\n" + + "-- test.cue --\n" + + "a: fixture.foo & int @test(eq, 1)\n", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + archive := txtar.Parse([]byte(tt.archive)) + runner := cuetxtar.NewInlineRunner(t, nil, archive, t.TempDir()) + runner.Run() + }) + } +} diff --git a/internal/cuetxtar/permute.go b/internal/cuetxtar/permute.go new file mode 100644 index 000000000..c63056e93 --- /dev/null +++ b/internal/cuetxtar/permute.go @@ -0,0 +1,296 @@ +// Copyright 2026 CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file implements @test(permute) assertions for the inline test runner. +// It contains the Heap's-algorithm permutation engine and the AST field-finder +// that locates struct literals by CUE path. +package cuetxtar + +import ( + "fmt" + "strconv" + "strings" + "testing" + + "cuelang.org/go/cue" + "cuelang.org/go/cue/ast" + "cuelang.org/go/internal/cuetest" + "cuelang.org/go/internal/diff" +) + +// runInlinePermutes processes @test(permute) attributes within an inline-form +// test root. It supports two forms: +// +// 1. Field attribute: @test(permute) on individual fields collects those fields +// into a group under their parent struct. Only the marked fields are permuted. +// +// 2. Decl attribute: @test(permute) as a declaration inside a struct means +// "permute all fields within this struct." +func (r *inlineRunner) runInlinePermutes(t *testing.T, rootPath cue.Path, records []attrRecord, version string) { + type permuteGroup struct { + parentPath cue.Path + fields []string // nil means permute all fields + } + var permuteGroups []permuteGroup + parentSeen := map[string]int{} // parentPath.String() → index in permuteGroups + + for _, rec := range records { + if !pathHasPrefix(rec.path, rootPath) { + continue + } + for _, pa := range selectActiveDirectives(records, rec.path, version) { + if pa.directive != "permute" { + continue + } + if rec.isDeclAttr { + // Decl form: @test(permute) inside a struct → permute all + // fields in that struct. The path points to the struct itself. + key := rec.path.String() + if _, ok := parentSeen[key]; !ok { + parentSeen[key] = len(permuteGroups) + permuteGroups = append(permuteGroups, permuteGroup{rec.path, nil}) + } + continue + } + + // Field form: @test(permute) on a field → collect into parent group. + sels := rec.path.Selectors() + if len(sels) < 2 { + continue // top-level field permute not supported in inline form + } + parentPath := cue.MakePath(sels[:len(sels)-1]...) + fieldName := sels[len(sels)-1].String() + key := parentPath.String() + if idx, ok := parentSeen[key]; ok { + permuteGroups[idx].fields = append(permuteGroups[idx].fields, fieldName) + } else { + parentSeen[key] = len(permuteGroups) + permuteGroups = append(permuteGroups, permuteGroup{parentPath, []string{fieldName}}) + } + } + } + + totalPerms := 0 + for _, group := range permuteGroups { + totalPerms += r.runPermuteAssertion(t, group.parentPath, group.fields) + } + + // Check @test(permuteCount, N) on the root if any permutations ran. + if totalPerms > 0 { + r.checkPermuteCount(t, rootPath, records, version, totalPerms) + } +} + +// checkPermuteCount verifies or auto-updates a @test(permuteCount, N) directive +// on the test root after all permutations have run. +func (r *inlineRunner) checkPermuteCount(t *testing.T, rootPath cue.Path, records []attrRecord, version string, actualCount int) { + t.Helper() + directives := selectActiveDirectives(records, rootPath, version) + for _, pa := range directives { + if pa.directive != "permuteCount" { + continue + } + if len(pa.raw.Fields) < 2 { + // Bare @test(permuteCount) — fill with actual count. + if cuetest.UpdateGoldenFiles { + r.enqueueInlineFill(pa, fmt.Sprintf("@test(permuteCount, %d)", actualCount)) + } + return + } + expectedStr := pa.raw.Fields[1].Value() + expected, err := strconv.Atoi(expectedStr) + if err != nil { + t.Errorf("path %s: @test(permuteCount, %q): cannot parse as integer", rootPath, expectedStr) + return + } + if expected == actualCount { + return // matches + } + if cuetest.UpdateGoldenFiles || cuetest.ForceUpdateGoldenFiles { + r.enqueueInlineFill(pa, fmt.Sprintf("@test(permuteCount, %d)", actualCount)) + return + } + t.Errorf("path %s: @test(permuteCount): got %d permutations, want %d", rootPath, actualCount, expected) + return + } +} + +// runPermuteAssertion evaluates all N! field-order permutations of the struct +// at structPath and asserts that every ordering produces an identical result. +// fieldNames lists the fields to permute; nil means permute all fields. +// Uses Heap's algorithm to enumerate permutations without allocating N! slices. +// Returns the total number of permutations evaluated (N!), or 0 if skipped +// (fewer than 2 permutable fields). +func (r *inlineRunner) runPermuteAssertion(t *testing.T, structPath cue.Path, fieldNames []string) int { + t.Helper() + if r.cueFiles == nil { + return 0 + } + + // Locate the struct literal and the indices of permutable fields in the AST. + var targetLit *ast.StructLit + var permIndices []int + for _, cf := range r.cueFiles { + targetLit, permIndices = findPermFieldsAtPath(cf.strippedAST, structPath, fieldNames) + if targetLit != nil && len(permIndices) >= 2 { + break + } + } + if targetLit == nil || len(permIndices) < 2 { + return 0 // fewer than two permutable fields — nothing to test + } + + n := len(permIndices) + + // Save the original AST elements at the permuted positions. + origElts := make([]ast.Decl, n) + for i, idx := range permIndices { + origElts[i] = targetLit.Elts[idx] + } + + // Evaluate the baseline (identity permutation / original source order). + ctx := r.cueContext() + baselineAll, err := r.buildValue(ctx, r.cueFiles) + if err != nil { + t.Errorf("path %s: @test(permute): baseline evaluation error: %v", structPath, err) + return 0 + } + baseline := baselineAll.LookupPath(structPath) + + // perm[i] = which origElt goes to permuted position i. + perm := make([]int, n) + for i := range perm { + perm[i] = i + } + c := make([]int, n) + + permNum := 0 + reported := false // report only the first differing permutation + + // Heap's algorithm: generates all N! permutations via in-place swaps. + var generate func(k int) + generate = func(k int) { + if k == 1 { + permNum++ + if permNum == 1 { + return // skip identity — already evaluated as baseline + } + // Apply permutation: position permIndices[i] gets origElts[perm[i]]. + for i, p := range perm { + targetLit.Elts[permIndices[i]] = origElts[p] + } + // Re-evaluate the modified archive. + permAll, evalErr := r.buildValue(ctx, r.cueFiles) + // Restore immediately so subsequent permutations start from original. + for i, idx := range permIndices { + targetLit.Elts[idx] = origElts[i] + } + if evalErr != nil || reported { + return + } + permVal := permAll.LookupPath(structPath) + kind, _ := diff.Diff(baseline, permVal) + if kind != diff.Identity { + reported = true + permNames := make([]string, n) + for i, p := range perm { + if f, ok := origElts[p].(*ast.Field); ok { + permNames[i] = identStr(f.Label) + } else { + permNames[i] = fmt.Sprintf("[%d]", p) + } + } + t.Errorf("path %s: @test(permute): ordering [%s] produces a different result\ngot: %s\nwant: %s", + structPath, strings.Join(permNames, ", "), + r.formatValue(permVal), r.formatValue(baseline)) + } + return + } + for i := 0; i < k; i++ { + generate(k - 1) + if k%2 == 0 { + perm[i], perm[k-1] = perm[k-1], perm[i] + } else { + perm[0], perm[k-1] = perm[k-1], perm[0] + } + c[i]++ + } + } + generate(n) + return permNum +} + +// findPermFieldsAtPath walks file following structPath and returns the +// *ast.StructLit at that location together with the indices of permutable +// *ast.Field entries inside it. If fieldNames is nil or empty every +// *ast.Field in the struct is included; otherwise only the named ones. +func findPermFieldsAtPath(file *ast.File, structPath cue.Path, fieldNames []string) (*ast.StructLit, []int) { + sels := structPath.Selectors() + if len(sels) == 0 { + return nil, nil + } + + // Navigate the AST following each selector. + decls := file.Decls + var targetLit *ast.StructLit + for i, sel := range sels { + name := sel.String() + var found *ast.Field + for _, d := range decls { + f, ok := d.(*ast.Field) + if !ok { + continue + } + if identStr(f.Label) == name { + found = f + break + } + } + if found == nil { + return nil, nil + } + sl, ok := found.Value.(*ast.StructLit) + if !ok { + return nil, nil + } + if i == len(sels)-1 { + targetLit = sl + } else { + decls = sl.Elts + } + } + if targetLit == nil { + return nil, nil + } + + // Collect the indices of fields to permute. + permSet := make(map[string]bool, len(fieldNames)) + for _, name := range fieldNames { + permSet[name] = true + } + allFields := len(fieldNames) == 0 + + var indices []int + for i, elt := range targetLit.Elts { + f, ok := elt.(*ast.Field) + if !ok { + continue + } + name := identStr(f.Label) + if allFields || permSet[name] { + indices = append(indices, i) + } + } + return targetLit, indices +} diff --git a/internal/cuetxtar/txtar.go b/internal/cuetxtar/txtar.go index b0060e953..e0c4ab52e 100644 --- a/internal/cuetxtar/txtar.go +++ b/internal/cuetxtar/txtar.go @@ -89,6 +89,12 @@ type TxTarTest struct { // DebugArchive, if set, is loaded instead of the on-disk archive. This allows // a test to be used for debugging. DebugArchive string + + // Inline, if true, enables inline-assertion mode: archives containing + // @test(...) attributes are dispatched to the inline runner instead of + // golden-file comparison. Tests that should not be affected by @test + // attributes (e.g. TestCompile) must leave this false. + Inline bool } // A Test represents a single test based on a .txtar file. @@ -445,6 +451,27 @@ func (x *TxTarTest) run(t *testing.T, m *cuetdtest.M, f func(tc *Test)) { t.Fatalf("error parsing txtar file: %v", err) } + // Archives with @test attributes use inline assertions instead of + // golden-file comparison. The inline runner is self-contained and + // does not write to the Test output files, so we return immediately + // after it finishes. + // + // TODO: support a `#inline` header directive for archives that + // should always use inline mode (e.g. for future full-file tests + // that express golden-file expectations in CUE form rather than as + // raw text sections). + if x.Inline && isInlineMode(a) { + runner := &inlineRunner{ + t: t, + m: m, + archive: a, + dir: filepath.Dir(filepath.Join(dir, fullpath)), + filePath: filepath.Join(dir, fullpath), + } + runner.runArchive() + return + } + tc := &Test{ T: t, M: m, diff --git a/internal/cuetxtar/writeback.go b/internal/cuetxtar/writeback.go new file mode 100644 index 000000000..7bb583643 --- /dev/null +++ b/internal/cuetxtar/writeback.go @@ -0,0 +1,99 @@ +// Copyright 2026 CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cuetxtar + +// This file implements CUE_UPDATE write-backs for the inline test runner. +// +// The inline fill write-back rewrites @test attributes in-place using byte +// offsets, handling fill, force-overwrite, regression-guard, and stale-skip +// cleanup operations. + +import ( + "fmt" + "os" + "slices" + + "golang.org/x/tools/txtar" + + "cuelang.org/go/cue" +) + +// ── inline fill / diff-annotate write-back ───────────────────────────────────── + +// inlineFillWrite records a byte-level replacement of a @test attribute in a +// .cue archive file. Used for three operations: +// - fill: @test() → @test(eq, ) or @test(err, ...) +// - force: @test(eq, old) → @test(eq, ) +// - diff-annotate: @test(eq, old) → @test(eq, old, skip:, diff="...") +// - stale-skip: @test(eq, old, skip:, diff="...") → @test(eq, old) +type inlineFillWrite struct { + fileName string // archive .cue file name, e.g. "in.cue" + attrOffset int // byte offset of the @ in the original file data + attrLen int // byte length of the original @test(...) text + newAttrText string // replacement text +} + +// applyInlineFillWritebacks applies pending inline @test attribute rewrites to +// the archive. Replacements are applied by byte offset in descending order so +// earlier offsets remain valid after each substitution. +func (r *inlineRunner) applyInlineFillWritebacks() { + if len(r.pendingInlineFillWrites) == 0 { + return + } + // Group writes by file name. + byFile := make(map[string][]inlineFillWrite, 2) + for _, ifw := range r.pendingInlineFillWrites { + byFile[ifw.fileName] = append(byFile[ifw.fileName], ifw) + } + + changed := false + for i, f := range r.archive.Files { + writes, ok := byFile[f.Name] + if !ok { + continue + } + // Sort descending by offset so earlier positions remain valid. + slices.SortFunc(writes, func(a, b inlineFillWrite) int { + return b.attrOffset - a.attrOffset + }) + data := append([]byte(nil), f.Data...) + for _, w := range writes { + end := w.attrOffset + w.attrLen + if end > len(data) { + continue + } + data = append(data[:w.attrOffset:w.attrOffset], + append([]byte(w.newAttrText), data[end:]...)...) + } + r.archive.Files[i].Data = data + changed = true + } + if changed && r.filePath != "" { + out := txtar.Format(r.archive) + if err := os.WriteFile(r.filePath, out, 0o644); err != nil { + r.t.Errorf("inline: fill write-back to %s: %v", r.filePath, err) + } + } +} + +// formatCoverAttr returns the @test attribute text to insert for a field whose +// evaluated value is v. For non-error values this is @test(eq, ). +// For error values it falls back to a bare @test(err) placeholder. +func (r *inlineRunner) formatCoverAttr(v cue.Value) string { + if r.isError(v) { + return "@test(err)" + } + return fmt.Sprintf("@test(eq, %s)", r.formatValue(v)) +} -- 2.51.2