diff --git a/cue/testdata/basicrewrite/012_selecting.txtar b/cue/testdata/basicrewrite/012_selecting.txtar index fedd5ed26..76c1025bd 100644 --- a/cue/testdata/basicrewrite/012_selecting.txtar +++ b/cue/testdata/basicrewrite/012_selecting.txtar @@ -1,13 +1,23 @@ -#name: selecting -#evalPartial -- in.cue -- -obj: {a: 1, b: 2} -index: {a: 1, b: 2}["b"] -mulidx: {a: 1, b: {a: 1, b: 3}}["b"]["b"] -e: {a: 1}[4] -f: {a: 1}.b -g: {a: 1}["b"] -h: [3].b +obj: {a: 1, b: 2} @test(eq, {a: 1, b: 2}) +index: {a: 1, b: 2}["b"] @test(eq, 2) +mulidx: {a: 1, b: {a: 1, b: 3}}["b"]["b"] @test(eq, 3) +e: {a: 1}[4] @test(err, code=eval, pos=[0:11]) +f: {a: 1}.b @test(err, code=incomplete, pos=[0:11]) +g: {a: 1}["b"] @test(err, code=incomplete, pos=[0:11]) +h: [3].b @test(err, code=eval, pos=[0:8]) +-- out/errors.txt -- +[eval] e: index out of range [4] with length 0: + in.cue:4:11 +[incomplete] f: undefined field: b: + in.cue:5:11 +[incomplete] g: undefined field: b: + in.cue:6:11 +[eval] h: undefined field: b: + in.cue:7:8 +-- todo.txt -- +Some error messages could use improvement. Address after query +extension is implemented. -- out/compile -- --- in.cue { @@ -49,104 +59,3 @@ Retain: 9 Unifications: 19 Conjuncts: 19 Disjuncts: 17 --- out/eval -- -Errors: -e: invalid struct selector 4 (type int): - ./in.cue:4:16 -h: invalid list index b (type string): - ./in.cue:7:13 - -Result: -(_|_){ - // [eval] - obj: (struct){ - a: (int){ 1 } - b: (int){ 2 } - } - index: (int){ 2 } - mulidx: (int){ 3 } - e: (_|_){ - // [eval] e: invalid struct selector 4 (type int): - // ./in.cue:4:16 - } - f: (_|_){ - // [incomplete] f: undefined field: b: - // ./in.cue:5:16 - } - g: (_|_){ - // [incomplete] g: undefined field: b: - // ./in.cue:6:16 - } - h: (_|_){ - // [eval] h: invalid list index b (type string): - // ./in.cue:7:13 - } -} --- out/evalalpha -- -Errors: -e: index out of range [4] with length 0: - ./in.cue:4:16 -h: undefined field: b: - ./in.cue:7:13 - -Result: -(_|_){ - // [eval] - obj: (struct){ - a: (int){ 1 } - b: (int){ 2 } - } - index: (int){ 2 } - mulidx: (int){ 3 } - e: (_|_){ - // [eval] e: index out of range [4] with length 0: - // ./in.cue:4:16 - } - f: (_|_){ - // [incomplete] f: undefined field: b: - // ./in.cue:5:16 - } - g: (_|_){ - // [incomplete] g: undefined field: b: - // ./in.cue:6:16 - } - h: (_|_){ - // [eval] h: undefined field: b: - // ./in.cue:7:13 - } -} --- diff/-out/evalalpha<==>+out/eval -- -diff old new ---- old -+++ new -@@ -1,7 +1,7 @@ - Errors: --e: invalid struct selector 4 (type int): -+e: index out of range [4] with length 0: - ./in.cue:4:16 --h: invalid list index b (type string): -+h: undefined field: b: - ./in.cue:7:13 - - Result: -@@ -14,7 +14,7 @@ - index: (int){ 2 } - mulidx: (int){ 3 } - e: (_|_){ -- // [eval] e: invalid struct selector 4 (type int): -+ // [eval] e: index out of range [4] with length 0: - // ./in.cue:4:16 - } - f: (_|_){ -@@ -26,7 +26,7 @@ - // ./in.cue:6:16 - } - h: (_|_){ -- // [eval] h: invalid list index b (type string): -+ // [eval] h: undefined field: b: - // ./in.cue:7:13 - } - } --- diff/explanation -- -Better error message for undefined field in new. -Other error got worse, but this is covered elsewhere. diff --git a/cue/testdata/fulleval/051_detectIncompleteYAML.txtar b/cue/testdata/fulleval/051_detectIncompleteYAML.txtar index 466d2eb18..4d726fe4b 100644 --- a/cue/testdata/fulleval/051_detectIncompleteYAML.txtar +++ b/cue/testdata/fulleval/051_detectIncompleteYAML.txtar @@ -26,7 +26,7 @@ Val: #Spec & { } }) -- out/errors.txt -- -[incomplete] #Spec.data.baz: non-concrete argument 0: +[incomplete] #Spec.data.baz: error in call to encoding/yaml.Marshal: non-concrete argument 0: in.cue:11:11 [incomplete] #Spec.data.foobar: error in call to encoding/yaml.Marshal: incomplete value string: in.cue:12:11 diff --git a/cue/testdata/fulleval/052_detectIncompleteJSON.txtar b/cue/testdata/fulleval/052_detectIncompleteJSON.txtar index ce5ffa901..55c68ee79 100644 --- a/cue/testdata/fulleval/052_detectIncompleteJSON.txtar +++ b/cue/testdata/fulleval/052_detectIncompleteJSON.txtar @@ -26,9 +26,10 @@ Val: #Spec & { } }) -- out/errors.txt -- -[incomplete] #Spec.data.baz: non-concrete argument 0: +[incomplete] #Spec.data.baz: error in call to encoding/json.Marshal: non-concrete argument 0: in.cue:11:11 -[incomplete] cannot convert incomplete value "string" to JSON: +[incomplete] #Spec.data.foobar: error in call to encoding/json.Marshal: cannot convert incomplete value "string" to JSON: + in.cue:12:11 in.cue:6:21 -- out/eval/stats -- Leaks: 0 diff --git a/internal/core/adt/eval_test.go b/internal/core/adt/eval_test.go index d5f62de9c..e1d7f9b74 100644 --- a/internal/core/adt/eval_test.go +++ b/internal/core/adt/eval_test.go @@ -182,6 +182,13 @@ func runEvalTest(t *cuetxtar.Test, version internal.EvaluatorVersion, dbg cuedeb fmt.Fprintln(t, "Result:") } + // Write all errors (including incomplete) with [code] prefixes to the + // documentary errors.txt section (only updated if the section exists). + cuetxtar.PrintErrors(t.WriterDoc("errors.txt"), v, &errors.Config{ + Cwd: t.Dir, + ToSlash: true, + }) + if v == nil { return } diff --git a/internal/cuetest/cuetest.go b/internal/cuetest/cuetest.go index 237f96a0d..9916c625a 100644 --- a/internal/cuetest/cuetest.go +++ b/internal/cuetest/cuetest.go @@ -61,18 +61,28 @@ var ( // // In some cases, tests might refuse to perform some updates by default. // The special value "force" can be used to force updates in that situation. -var UpdateGoldenFiles = os.Getenv(envUpdate) != "" +// +// The special value "diff" does not update files but shows a diff of the +// changes that would be applied; see [DiffGoldenFiles]. +var UpdateGoldenFiles = os.Getenv(envUpdate) != "" && os.Getenv(envUpdate) != "diff" // ForceUpdateGoldenFiles determines whether tests should update // expected output in test files even when they would not be updated // usually (for example when there are test regressions). var ForceUpdateGoldenFiles = os.Getenv(envUpdate) == "force" +// DiffGoldenFiles determines whether tests should display a diff of changes +// that would be applied by CUE_UPDATE=1, without actually writing any files. +// It is controlled by setting CUE_UPDATE=diff. +// Documentary sections (e.g. out/errors.txt) are also checked in this mode. +var DiffGoldenFiles = os.Getenv(envUpdate) == "diff" + // FormatTxtar ensures that .cue files in txtar test archives are well // formatted, updating the archive as required prior to running a test. // It is controlled by setting CUE_FORMAT_TXTAR to a non-empty string like "true". var FormatTxtar = os.Getenv(envFormatTxtar) != "" + // Condition adds support for CUE-specific testscript conditions within // testscript scripts. Supported conditions include: // diff --git a/internal/cuetxtar/errors.go b/internal/cuetxtar/errors.go new file mode 100644 index 000000000..20a106353 --- /dev/null +++ b/internal/cuetxtar/errors.go @@ -0,0 +1,53 @@ +// Copyright 2026 CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file implements helpers for the out/errors.txt documentary section. + +package cuetxtar + +import ( + "fmt" + "io" + + cueerrors "cuelang.org/go/cue/errors" + "cuelang.org/go/internal/core/adt" +) + +// PrintErrors writes all errors found in the vertex tree to w, each +// prefixed with [code]. Child-error markers (errors propagated from children) +// are suppressed; only the originating errors are reported. +// +// Used to populate the out/errors.txt (inline tests) and out//errors.txt +// (golden-file tests) documentary sections. +func PrintErrors(w io.Writer, v *adt.Vertex, cfg *cueerrors.Config) { + printVertexErrorsRec(w, v, cfg) +} + +func printVertexErrorsRec(w io.Writer, v *adt.Vertex, cfg *cueerrors.Config) { + if b := v.Bottom(); b != nil { + if !b.ChildError && b.Err != nil { + fmt.Fprintf(w, "[%s] ", b.Code) + cueerrors.Print(w, b.Err, cfg) + } + if !b.HasRecursive { + return + } + } + for _, a := range v.Arcs { + if a.Label.IsLet() { + continue + } + printVertexErrorsRec(w, a, cfg) + } +} diff --git a/internal/cuetxtar/inline.go b/internal/cuetxtar/inline.go index 0827830b7..0553c09fa 100644 --- a/internal/cuetxtar/inline.go +++ b/internal/cuetxtar/inline.go @@ -30,6 +30,7 @@ package cuetxtar import ( + "bytes" "fmt" "io/fs" "maps" @@ -40,6 +41,7 @@ import ( "strings" "testing" + "github.com/google/go-cmp/cmp" "golang.org/x/tools/txtar" "cuelang.org/go/cue" @@ -824,6 +826,73 @@ func (r *inlineRunner) runArchive() { // AST-based write-backs re-parse the updated bytes. r.applyPosWritebacks() r.applyInlineFillWritebacks() + + // Update the optional out/errors.txt documentary section. + r.handleErrorsTxtSection(val) +} + +// handleErrorsTxtSection manages the out/errors.txt documentary section. +// The section is only processed if it already exists in the archive: +// - CUE_UPDATE=1: updates the section with current error output +// - CUE_UPDATE=diff: fails if the section is stale, showing a diff +// - otherwise: silently skips any difference +func (r *inlineRunner) handleErrorsTxtSection(val cue.Value) { + const sectionName = "out/errors.txt" + + // Find the section in the archive. + sectionIdx := -1 + for i, f := range r.archive.Files { + if f.Name == sectionName { + sectionIdx = i + break + } + } + // Never auto-create the section. + if sectionIdx < 0 { + return + } + + // Collect all errors (including incomplete) from the evaluated value. + // Do not pass Cwd: cueerrors.Print prepends "./" to relative paths for + // IDE compatibility, which we don't want in the golden section. Strip the + // directory prefix manually instead, consistent with how the rest of the + // inline runner normalizes paths. + var buf strings.Builder + core := val.Core() + if core.V != nil { + PrintErrors(&buf, core.V, &cueerrors.Config{ + Cwd: r.dir, + ToSlash: true, + }) + } + result := buf.String() + if result != "" && result[len(result)-1] != '\n' { + result += "\n" + } + resultBytes := []byte(result) + + existing := r.archive.Files[sectionIdx].Data + if bytes.Equal(existing, resultBytes) { + return + } + + if cuetest.UpdateGoldenFiles { + r.archive.Files[sectionIdx].Data = resultBytes + if r.filePath != "" { + out := txtar.Format(r.archive) + if err := os.WriteFile(r.filePath, out, 0o644); err != nil { + r.t.Errorf("inline: errors.txt write-back to %s: %v", r.filePath, err) + } + } + return + } + + if cuetest.DiffGoldenFiles { + r.t.Errorf("result for %s differs: (-want +got)\n%s", + sectionName, + cmp.Diff(string(existing), result), + ) + } } // subTestName returns the sub-test name for a root. diff --git a/internal/cuetxtar/txtar.go b/internal/cuetxtar/txtar.go index e0c4ab52e..34411cc06 100644 --- a/internal/cuetxtar/txtar.go +++ b/internal/cuetxtar/txtar.go @@ -150,7 +150,7 @@ var _ testing.TB = (*Test)(nil) func (t *Test) Write(b []byte) (n int, err error) { if t.buf == nil { t.buf = &bytes.Buffer{} - t.outFiles = append(t.outFiles, file{t.prefix, t.fallback, t.buf, false}) + t.outFiles = append(t.outFiles, file{t.prefix, t.fallback, t.buf, false, false}) } return t.buf.Write(b) } @@ -160,6 +160,7 @@ type file struct { fallback string buf *bytes.Buffer diff bool // true if this contains a diff between fallback and main + doc bool // true if this is a documentary section (never fails, only updated if already present) } // bytes returns the bytes in the file's buffer, and ensures that the @@ -275,7 +276,7 @@ func (t *Test) Writer(name string) io.Writer { } w := &bytes.Buffer{} - t.outFiles = append(t.outFiles, file{name, fallback, w, false}) + t.outFiles = append(t.outFiles, file{name, fallback, w, false, false}) if name == t.prefix { t.buf = w @@ -284,6 +285,25 @@ func (t *Test) Writer(name string) io.Writer { return w } +// WriterDoc returns a Writer with the given name that is marked as a +// documentary section. Documentary sections are never automatically created +// and never cause test failures. They are updated by CUE_UPDATE=1 only if +// the section already exists in the txtar archive. With CUE_UPDATE=diff, +// they are treated like normal golden sections and will fail if stale. +func (t *Test) WriterDoc(name string) io.Writer { + name = path.Join(t.prefix, name) + + for _, f := range t.outFiles { + if f.name == name { + return f.buf + } + } + + w := &bytes.Buffer{} + t.outFiles = append(t.outFiles, file{name, "", w, false, true}) + return w +} + func formatNode(t *testing.T, n ast.Node) []byte { t.Helper() @@ -596,6 +616,13 @@ func (x *TxTarTest) run(t *testing.T, m *cuetdtest.M, f func(tc *Test)) { for _, sub := range tc.outFiles { result := sub.bytes() + // Documentary sections are only included if they already exist + // in the archive; they are never auto-created. + _, inArchive := index[sub.name] + if sub.doc && !inArchive { + continue + } + files = append(files, txtar.File{Name: sub.name}) gold := &files[len(files)-1] @@ -631,6 +658,11 @@ func (x *TxTarTest) run(t *testing.T, m *cuetdtest.M, f func(tc *Test)) { continue } + // Doc sections never fail unless CUE_UPDATE=diff. + if sub.doc && !cuetest.DiffGoldenFiles { + continue + } + t.Errorf("result for %s differs: (-want +got)\n%s", sub.name, cmp.Diff(string(gold.Data), string(result)),