diff --git a/internal/cuetxtar/gc.go b/internal/cuetxtar/gc.go new file mode 100644 index 000000000..9ad82daf1 --- /dev/null +++ b/internal/cuetxtar/gc.go @@ -0,0 +1,141 @@ +//go:build ignore + +// This command implements "garbage collection" for cuetxtar output artifacts. +// It removes all out/* and diff/* entries from .txtar files used in the +// CUE tests that are not actually output targets of any test. +// +// It does that by running all the CUE tests to find out all the references +// (see the CUETXTAR_GC_URI logic in txtar.go), running +// a little HTTP server to gather the results. +// +// Run as: +// +// go run ./gc.go +package main + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "slices" + "strings" + "sync" + + "golang.org/x/tools/txtar" +) + +func main() { + h := &handler{ + references: make(map[string]map[string]bool), + } + srv := httptest.NewServer(h) + os.Setenv("CUETXTAR_GC_URI", srv.URL+"/ref") + + cmd := exec.Command("go", "test", "-count=1", "cuelang.org/go/...") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + if err := cmd.Run(); err != nil { + log.Fatal(err) + } + h.mu.Lock() + defer h.mu.Unlock() + for txtarFile, refs := range h.references { + hasDiff := false + for name := range refs { + if strings.HasPrefix(name, "diff/") && !isDiffAnnotationFile(name) { + hasDiff = true + break + } + } + a, err := txtar.ParseFile(txtarFile) + if err != nil { + log.Fatalf("error parsing txtar file: %v", err) + } + files := slices.DeleteFunc(a.Files, func(f txtar.File) bool { + if isOutputFile(f.Name) && !refs[f.Name] { + // Unreferenced output file. + return true + } + if !hasDiff && isDiffAnnotationFile(f.Name) { + // A diff-related annotation file when there are no diffs present. + return true + } + return false + }) + if len(files) == len(a.Files) { + continue + } + fmt.Printf("garbage collecting %d entries from %s\n", len(a.Files)-len(files), txtarFile) + a.Files = files + if err := os.WriteFile(txtarFile, txtar.Format(a), 0o644); err != nil { + log.Fatal(err) + } + } +} + +func isOutputFile(name string) bool { + switch { + case strings.HasPrefix(name, "out/"): + return true + case !strings.HasPrefix(name, "diff/"): + return false + case isDiffAnnotationFile(name): + return false + } + return true +} + +func isDiffAnnotationFile(name string) bool { + switch { + case !strings.HasPrefix(name, "diff/"): + return false + case strings.Contains(name, "/todo/"): + // e.g. + // /diff/todo/p3 + // /diff/dependencies-v3/todo/p2 + return true + case strings.HasSuffix(name, "/explanation"): + // e.g. + // /diff/explanation + // /diff/doc/explanation + return true + } + return false +} + +type handler struct { + mu sync.Mutex + references map[string]map[string]bool +} + +func (h *handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { + if req.Method != "PUT" || req.URL.Path != "/ref" { + http.Error(w, "bad request", http.StatusBadRequest) + return + } + var body struct { + TxtarFile string `json:"txtarfile"` + RetainFiles []string `json:"retainFiles"` + } + data, _ := io.ReadAll(req.Body) + if err := json.Unmarshal(data, &body); err != nil { + http.Error(w, "bad request body", http.StatusBadRequest) + return + } + h.mu.Lock() + defer h.mu.Unlock() + refs := h.references[body.TxtarFile] + if refs == nil { + refs = make(map[string]bool) + h.references[body.TxtarFile] = refs + } + for _, name := range body.RetainFiles { + refs[name] = true + } +} diff --git a/internal/cuetxtar/txtar.go b/internal/cuetxtar/txtar.go index 00e030af5..d0772d88b 100644 --- a/internal/cuetxtar/txtar.go +++ b/internal/cuetxtar/txtar.go @@ -17,10 +17,12 @@ package cuetxtar import ( "bufio" "bytes" + "encoding/json" "fmt" "io" "io/fs" "maps" + "net/http" "os" "path" "path/filepath" @@ -38,6 +40,7 @@ import ( "cuelang.org/go/internal/core/runtime" "cuelang.org/go/internal/cuetdtest" "cuelang.org/go/internal/cuetest" + "github.com/go-quicktest/qt" "github.com/google/go-cmp/cmp" "github.com/rogpeppe/go-internal/diff" "golang.org/x/tools/txtar" @@ -522,6 +525,7 @@ func (x *TxTarTest) run(t *testing.T, m *cuetdtest.M, f func(tc *Test)) { // later. ordering := maps.Clone(index) + usedFiles := make(map[string]bool) // Add diff files between fallback and main file. These are added // as regular output files so that they can be updated as well. for _, sub := range tc.outFiles { @@ -533,12 +537,14 @@ func (x *TxTarTest) run(t *testing.T, m *cuetdtest.M, f func(tc *Test)) { ordering[sub.name] = j } fallback := a.Files[j].Data + // The fallback file is "used" even though it's not actually + // an output file. + usedFiles[a.Files[j].Name] = true result := sub.bytes() if len(result) == 0 || len(fallback) == 0 { continue } - diffName := "diff/-" + sub.name + "<==>+" + sub.fallback if _, ok := ordering[diffName]; !ok { ordering[diffName] = j @@ -572,7 +578,6 @@ func (x *TxTarTest) run(t *testing.T, m *cuetdtest.M, f func(tc *Test)) { } files := make([]txtar.File, 0, len(a.Files)) - for _, sub := range tc.outFiles { result := sub.bytes() @@ -582,6 +587,7 @@ func (x *TxTarTest) run(t *testing.T, m *cuetdtest.M, f func(tc *Test)) { if i, ok := index[sub.name]; ok { gold.Data = a.Files[i].Data delete(index, sub.name) + usedFiles[sub.name] = true if bytes.Equal(gold.Data, result) { continue @@ -592,6 +598,7 @@ func (x *TxTarTest) run(t *testing.T, m *cuetdtest.M, f func(tc *Test)) { // Use the golden file of the fallback set if it matches. if bytes.Equal(gold.Data, result) { gold.Name = sub.fallback + usedFiles[gold.Name] = true delete(index, sub.fallback) continue } @@ -619,9 +626,34 @@ func (x *TxTarTest) run(t *testing.T, m *cuetdtest.M, f func(tc *Test)) { // Add remaining unrelated files, ignoring files that were already // added. for _, f := range a.Files { - if _, ok := index[f.Name]; ok { - files = append(files, f) + if _, ok := index[f.Name]; !ok { + continue } + files = append(files, f) + } + // log.Printf("usedFiles %v", slices.Sorted(maps.Keys(usedFiles))) + // for _, f := range a.Files { + // log.Printf("- %q", f.Name) + // } + if uri := os.Getenv("CUETXTAR_GC_URI"); uri != "" { + var retain []string + for _, f := range a.Files { + if isOutputFile(f.Name) && usedFiles[f.Name] { + retain = append(retain, f.Name) + } + } + absPath, err := filepath.Abs(fullpath) + qt.Assert(t, qt.IsNil(err)) + body, err := json.Marshal(struct { + TxtarFile string `json:"txtarfile"` + RetainFiles []string `json:"retainFiles"` + }{absPath, retain}) + qt.Assert(t, qt.IsNil(err)) + req, err := http.NewRequest("PUT", uri, bytes.NewReader(body)) + qt.Assert(t, qt.IsNil(err)) + resp, err := http.DefaultClient.Do(req) + qt.Assert(t, qt.IsNil(err)) + qt.Assert(t, qt.Equals(resp.StatusCode, http.StatusOK)) } a.Files = files @@ -652,3 +684,7 @@ func (x *TxTarTest) run(t *testing.T, m *cuetdtest.M, f func(tc *Test)) { t.Fatal(err) } } + +func isOutputFile(name string) bool { + return strings.HasPrefix(name, "out/") || strings.HasPrefix(name, "diff/") +}