From 6825394279bd8546fbdde42383fb82c4ad97a321 Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Mon, 2 Feb 2026 12:17:07 +0000 Subject: [PATCH] lsp/fscache: read json and yaml files as CUE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [module.ReadCUEFile] takes an arbitrary file path to read as a CUE AST: there's no requirement that the file path be to a cue file. However its one call site in modimports called it with only file paths with a ".cue" suffix. By relaxing this (giving the implementation of [module.ReadCUEFile] the option of converting non-.cue files to a CUE AST), and by making some minor changes to fscache, the fscache can read json and yaml files, and convert them to CUE ASTs. This means that the LSP, with suitable editor configuration, can work with json and yaml files too: for these files, they are treated as standalone packages. But the existing machinery managing their lifecycle, reloading and so forth, all works without further changes. There is one other part of the modpkgload package that tests for ".cue" suffix, and that's in isDirWithCUEFiles. We extend the dirfs.ReadCUEFS interface, adding a IsDirWithCUEFiles method, and so allowing the implementation to decide whether the directory in question contains any files which it would attempt to convert to CUE ASTs. Signed-off-by: Matthew Sackman Change-Id: Ie71b2b3d4f661f6e5ad82e0bb46bf82535e84b0f Reviewed-on: https://cue.gerrithub.io/c/cue-lang/cue/+/1230265 Unity-Result: CUE porcuepine TryBot-Result: CUEcueckoo Reviewed-by: Daniel Martí --- cue/load/fs.go | 9 +++ internal/lsp/fscache/fs_cache.go | 112 +++++++++++++++++++++++--- internal/lsp/fscache/fs_overlay.go | 61 ++++++++++++-- internal/lsp/fscache/fs_test.go | 63 +++++++++++++++ internal/mod/modimports/modimports.go | 17 ++-- internal/mod/modpkgload/import.go | 11 ++- mod/module/dirfs.go | 13 +++ 7 files changed, 259 insertions(+), 27 deletions(-) diff --git a/cue/load/fs.go b/cue/load/fs.go index 411a2f2db..08fc5c63e 100644 --- a/cue/load/fs.go +++ b/cue/load/fs.go @@ -17,6 +17,7 @@ package load import ( "bytes" "cmp" + stderrs "errors" "fmt" "io" iofs "io/fs" @@ -344,11 +345,19 @@ func (fs *ioFS) ReadFile(name string) ([]byte, error) { var _ module.ReadCUEFS = (*ioFS)(nil) +// IsDirWithCUEFiles implements [module.ReadCUEFS] +func (fs *ioFS) IsDirWithCUEFiles(path string) (bool, error) { + return false, stderrs.ErrUnsupported +} + // ReadCUEFile implements [module.ReadCUEFS] by // reading and updating the syntax file cache, which // is shared with the cache used by the [fileSystem.getCUESyntax] // method. func (fs *ioFS) ReadCUEFile(path string, cfg parser.Config) (*ast.File, error) { + if !strings.HasSuffix(path, ".cue") { + return nil, nil + } fpath, err := fs.absPathFromFSPath(path) if err != nil { return nil, err diff --git a/internal/lsp/fscache/fs_cache.go b/internal/lsp/fscache/fs_cache.go index b6517ee74..4671ba788 100644 --- a/internal/lsp/fscache/fs_cache.go +++ b/internal/lsp/fscache/fs_cache.go @@ -15,7 +15,9 @@ import ( "cuelang.org/go/cue/ast" "cuelang.org/go/cue/build" "cuelang.org/go/cue/parser" + "cuelang.org/go/encoding/yaml" "cuelang.org/go/internal" + "cuelang.org/go/internal/encoding/json" "cuelang.org/go/internal/filetypes" "cuelang.org/go/internal/golangorgx/gopls/protocol" "cuelang.org/go/internal/robustio" @@ -96,22 +98,42 @@ func (p *cueFileParser) ReadCUE(config parser.Config) (syntax *ast.File, cfg par } bf := p.buildFile - if !(bf != nil && bf.Encoding == build.CUE && bf.Form == "" && bf.Interpretation == "") { + if bf == nil { return nil, parser.Config{}, nil } + filename := bf.Filename content := p.content - parseComments := parser.NewConfig(config) - parseComments.Mode = parser.ParseComments - importsOnly := parser.NewConfig(config) - importsOnly.Mode = parser.ImportsOnly + switch bf.Encoding { + case build.CUE: + parseComments := parser.NewConfig(config) + parseComments.Mode = parser.ParseComments + importsOnly := parser.NewConfig(config) + importsOnly.Mode = parser.ImportsOnly - for _, cfg = range []parser.Config{parseComments, importsOnly} { - syntax, err = parser.ParseFile(bf.Filename, content, cfg) - if syntax != nil { - break + for _, cfg = range []parser.Config{parseComments, importsOnly} { + syntax, err = parser.ParseFile(filename, content, cfg) + if syntax != nil { + break + } } + + case build.JSON: + var expr ast.Expr + expr, err = parser.ParseExpr(filename, content) + syntax = internal.ToFile(expr) + if syntax == nil { + syntax = &ast.File{} + } + syntax.Filename = filename + json.PatchExpr(syntax, nil) + + case build.YAML: + syntax, err = yaml.Extract(filename, content) + + default: + return nil, parser.Config{}, nil } if syntax != nil { @@ -139,7 +161,7 @@ func (p *cueFileParser) ReadCUE(config parser.Config) (syntax *ast.File, cfg par } if pkg.Name == nil || pkg.Name.Name == "" || pkg.Name.Name == "_" { // Important that this ident has no position. - pkg.Name = ast.NewIdent(phantomPackageName(bf.Filename)) + pkg.Name = ast.NewIdent(phantomPackageName(filename)) } } @@ -329,12 +351,20 @@ func (fs *CUECacheFS) ReadFile(uri protocol.DocumentURI) (FileHandle, error) { func readFile(uri protocol.DocumentURI, mtime time.Time) (*diskFileEntry, error) { // NB filePath is GOOS-appropriate (uri.Path() calls [filepath.FromSlash]) filePath := uri.Path() - content, err := os.ReadFile(filePath) + + bf, err := filetypes.ParseFileAndType(filePath, "", filetypes.Input) if err != nil { - return nil, err + // Yes, throw the error away, and return an empty cueFileParser + // so that we record that this uri can never be converted to a + // CUE AST. + return &diskFileEntry{ + modTime: mtime, + uri: uri, + cueFileParser: &cueFileParser{}, + }, nil } - bf, err := filetypes.ParseFileAndType(filePath, "", filetypes.Input) + content, err := os.ReadFile(filePath) if err != nil { return nil, err } @@ -377,6 +407,7 @@ type CUEDirFS interface { DirFS module.OSRootFS module.ReadCUEFS + iofs.SubFS } // rootedCUECacheFS is a wrapper over [CUECacheFS] that implements @@ -396,6 +427,12 @@ func (fs *rootedCUECacheFS) OSRoot() string { return fs.root } +// Sub implements [iofs.Sub] +func (fs *rootedCUECacheFS) Sub(dir string) (iofs.FS, error) { + root := filepath.Join(fs.root, filepath.FromSlash(dir)) + return fs.cuecachefs.IoFS(root), nil +} + // Open implements [iofs.FS] func (fs *rootedCUECacheFS) Open(name string) (iofs.File, error) { return fs.delegatefs.Open(name) } @@ -418,6 +455,55 @@ func (fs *rootedCUECacheFS) ReadCUEFile(name string, config parser.Config) (*ast return ast, err } +// IsDirWithCUEFiles implements [module.ReadCUEFS] +func (fs *rootedCUECacheFS) IsDirWithCUEFiles(path string) (bool, error) { + if !iofs.ValidPath(path) { + return false, &iofs.PathError{Op: "IsDirWithCUEFiles", Path: path, Err: iofs.ErrInvalid} + } + path, err := filepath.Localize(path) + if err != nil { + return false, &iofs.PathError{Op: "IsDirWithCUEFiles", Path: path, Err: err} + } + path = filepath.Join(fs.root, path) + + entries, err := os.ReadDir(path) + if err != nil { + if errors.Is(err, iofs.ErrNotExist) { + return false, nil + } + return false, err + } + + for _, e := range entries { + if e.IsDir() { + continue + } + bf, err := filetypes.ParseFileAndType(e.Name(), "", filetypes.Input) + if err != nil { + continue + } + switch bf.Encoding { + case build.CUE, build.JSON, build.YAML: + default: + continue + } + + ftype := e.Type() + if ftype&iofs.ModeSymlink != 0 { + finfo, err := os.Stat(filepath.Join(path, e.Name())) + if err != nil { + continue // Ignore broken symlinks. + } + ftype = finfo.Mode() + } + if ftype.IsRegular() { + return true, nil + } + } + + return false, nil +} + // ReadDir implements [iofs.ReadDirFS] func (fs *rootedCUECacheFS) ReadDir(name string) ([]iofs.DirEntry, error) { return fs.delegatefs.ReadDir(name) diff --git a/internal/lsp/fscache/fs_overlay.go b/internal/lsp/fscache/fs_overlay.go index 2671ba04d..8f3a9adc6 100644 --- a/internal/lsp/fscache/fs_overlay.go +++ b/internal/lsp/fscache/fs_overlay.go @@ -15,6 +15,7 @@ import ( "time" "cuelang.org/go/cue/ast" + "cuelang.org/go/cue/build" "cuelang.org/go/cue/parser" "cuelang.org/go/internal/filetypes" "cuelang.org/go/internal/golangorgx/gopls/protocol" @@ -333,7 +334,7 @@ func (fs *OverlayFS) ReadFile(uri protocol.DocumentURI) (FileHandle, error) { file, isFile := entry.(*overlayFileEntry) if !isFile { - return nil, &iofs.PathError{Op: "open", Path: uri.Path(), Err: iofs.ErrInvalid} + return nil, &iofs.PathError{Op: "ReadFile", Path: uri.Path(), Err: iofs.ErrInvalid} } return file, nil @@ -558,6 +559,12 @@ func (fs *rootedOverlayFS) OSRoot() string { return fs.delegatefs.OSRoot() } +// Sub implements [iofs.Sub] +func (fs *rootedOverlayFS) Sub(dir string) (iofs.FS, error) { + root := filepath.Join(fs.delegatefs.root, filepath.FromSlash(dir)) + return fs.overlayfs.IoFS(root), nil +} + // pathComponents splits the name path into a slice of directory names // (which may be empty), and the final basename. The name must be // valid according to [iofs.ValidPath] @@ -576,14 +583,14 @@ func (fs *rootedOverlayFS) pathComponents(name string) ([]string, string) { // Open implements [iofs.FS] func (fs *rootedOverlayFS) Open(name string) (iofs.File, error) { if !iofs.ValidPath(name) { - return nil, &iofs.PathError{Op: "open", Path: name, Err: iofs.ErrInvalid} + return nil, &iofs.PathError{Op: "Open", Path: name, Err: iofs.ErrInvalid} } entry, err := fs.overlayfs.getEntry(fs.pathComponents(name)) if errors.Is(err, iofs.ErrNotExist) { return fs.delegatefs.Open(name) } else if err != nil { - return nil, &iofs.PathError{Op: "open", Path: name, Err: err} + return nil, &iofs.PathError{Op: "Open", Path: name, Err: err} } switch entry := entry.(type) { @@ -622,7 +629,7 @@ func (fs *rootedOverlayFS) Open(name string) (iofs.File, error) { // ReadCUEFile implements [module.ReadCUEFS] func (fs *rootedOverlayFS) ReadCUEFile(name string, config parser.Config) (*ast.File, error) { if !iofs.ValidPath(name) { - return nil, &iofs.PathError{Op: "readcuefile", Path: name, Err: iofs.ErrInvalid} + return nil, &iofs.PathError{Op: "ReadCUEFile", Path: name, Err: iofs.ErrInvalid} } entry, err := fs.overlayfs.getEntry(fs.pathComponents(name)) @@ -641,10 +648,50 @@ func (fs *rootedOverlayFS) ReadCUEFile(name string, config parser.Config) (*ast. return nil, iofs.ErrInvalid } +// IsDirWithCUEFiles implements [module.ReadCUEFS] +func (fs *rootedOverlayFS) IsDirWithCUEFiles(path string) (bool, error) { + if !iofs.ValidPath(path) { + return false, &iofs.PathError{Op: "IsDirWithCUEFiles", Path: path, Err: iofs.ErrInvalid} + } + + entry, err := fs.overlayfs.getEntry(fs.pathComponents(path)) + if errors.Is(err, iofs.ErrNotExist) { + return fs.delegatefs.IsDirWithCUEFiles(path) + } else if err != nil { + return false, err + } + + dir, isDir := entry.(*overlayDirEntry) + if !isDir { + return false, iofs.ErrInvalid + } + + overlayEntries, err := dir.open().ReadDir(0) + if err != nil { + return false, err + } + + for _, entry := range overlayEntries { + file, isFile := entry.(*overlayFileEntry) + if !isFile { + continue + } + + switch file.buildFile.Encoding { + case build.CUE, build.JSON, build.YAML: + return true, nil + default: + continue + } + } + + return fs.delegatefs.IsDirWithCUEFiles(path) +} + // ReadDir implements [iofs.ReadDirFS] func (fs *rootedOverlayFS) ReadDir(name string) ([]iofs.DirEntry, error) { if !iofs.ValidPath(name) { - return nil, &iofs.PathError{Op: "readdir", Path: name, Err: iofs.ErrInvalid} + return nil, &iofs.PathError{Op: "ReadDir", Path: name, Err: iofs.ErrInvalid} } entry, err := fs.overlayfs.getEntry(fs.pathComponents(name)) @@ -730,7 +777,7 @@ func mergeSort(as, bs []iofs.DirEntry) []iofs.DirEntry { // ReadFile implements [iofs.ReadFileFS] func (fs *rootedOverlayFS) ReadFile(name string) ([]byte, error) { if !iofs.ValidPath(name) { - return nil, &iofs.PathError{Op: "readfile", Path: name, Err: iofs.ErrInvalid} + return nil, &iofs.PathError{Op: "ReadFile", Path: name, Err: iofs.ErrInvalid} } entry, err := fs.overlayfs.getEntry(fs.pathComponents(name)) @@ -751,7 +798,7 @@ func (fs *rootedOverlayFS) ReadFile(name string) ([]byte, error) { // Stat implements [iofs.StatFS] func (fs *rootedOverlayFS) Stat(name string) (iofs.FileInfo, error) { if !iofs.ValidPath(name) { - return nil, &iofs.PathError{Op: "stat", Path: name, Err: iofs.ErrInvalid} + return nil, &iofs.PathError{Op: "Stat", Path: name, Err: iofs.ErrInvalid} } entry, err := fs.overlayfs.getEntry(fs.pathComponents(name)) diff --git a/internal/lsp/fscache/fs_test.go b/internal/lsp/fscache/fs_test.go index 74a3c106f..24895d757 100644 --- a/internal/lsp/fscache/fs_test.go +++ b/internal/lsp/fscache/fs_test.go @@ -3,6 +3,7 @@ package fscache_test import ( iofs "io/fs" "os" + "path" "path/filepath" "runtime" "slices" @@ -169,6 +170,68 @@ func TestOverlayFS(t *testing.T) { qt.Assert(t, qt.IsNil(err)) } +func TestConvertToCueFS(t *testing.T) { + now := time.Now() + dir := t.TempDir() + onDiskFiles := map[string]string{ + "foo/bar/a.json": `{"field": true}`, + "foo/bar/b.yaml": `field: false`, + "foo/c.txt": `no chance`, + } + for f, content := range onDiskFiles { + absPath := filepath.Join(dir, filepath.FromSlash(f)) + writeFile(t, absPath, content) + } + + fs := fscache.NewOverlayFS(fscache.NewCUECachedFS()) + iofs := fs.IoFS(dir) + + for f, content := range onDiskFiles { + dir := path.Dir(f) + isCueable, err := iofs.IsDirWithCUEFiles(dir) + qt.Assert(t, qt.IsNil(err)) + qt.Assert(t, qt.Equals(isCueable, path.Base(dir) == "bar")) + + syntax, err := iofs.ReadCUEFile(f, parser.NewConfig()) + qt.Assert(t, qt.IsNil(err)) + + qt.Assert(t, qt.Equals(syntax != nil, isCueable)) + if syntax != nil { + tokFile := syntax.Pos().File() + qt.Assert(t, qt.IsNotNil(tokFile)) + qt.Assert(t, qt.DeepEquals(tokFile.Content(), []byte(content))) + } + } + + // create extra file in the overlay only, that adds at .cue file to + // the foo directory. + extraPath := "foo/d.cue" + extraPathUri := protocol.URIFromPath(path.Join(dir, extraPath)) + err := fs.Update(func(txn *fscache.UpdateTxn) error { + _, err := txn.Set(extraPathUri, []byte(`field: yes`), now, 0) + return err + }) + qt.Assert(t, qt.IsNil(err)) + + dir = path.Dir(extraPath) + isCueable, err := iofs.IsDirWithCUEFiles(dir) + qt.Assert(t, qt.IsNil(err)) + qt.Assert(t, qt.IsTrue(isCueable)) + syntax, err := iofs.ReadCUEFile(extraPath, parser.NewConfig()) + qt.Assert(t, qt.IsNil(err)) + qt.Assert(t, qt.IsNotNil(syntax)) + + // delete that extra file + err = fs.Update(func(txn *fscache.UpdateTxn) error { + return txn.Delete(extraPathUri) + }) + qt.Assert(t, qt.IsNil(err)) + dir = path.Dir(extraPath) + isCueable, err = iofs.IsDirWithCUEFiles(dir) + qt.Assert(t, qt.IsNil(err)) + qt.Assert(t, qt.IsFalse(isCueable)) +} + func setup(t *testing.T) (dir string, onDiskFiles, onDiskFilesAbs []string) { t.Helper() dir = t.TempDir() diff --git a/internal/mod/modimports/modimports.go b/internal/mod/modimports/modimports.go index 774f2f752..402f071b1 100644 --- a/internal/mod/modimports/modimports.go +++ b/internal/mod/modimports/modimports.go @@ -220,9 +220,6 @@ func yieldAllModFiles(fsys fs.FS, fpath string, topDir bool, yield func(ModuleFi // It returns the yielded package name (if any) and reports whether // the iteration should continue. func yieldPackageFile(fsys fs.FS, fpath string, selectPackage func(pkgName string) bool, yield func(ModuleFile, error) bool) (pkgName string, cont bool) { - if !strings.HasSuffix(fpath, ".cue") { - return "", true - } pf := ModuleFile{ FilePath: fpath, } @@ -237,14 +234,24 @@ func yieldPackageFile(fsys fs.FS, fpath string, selectPackage func(pkgName strin // the default parser options used by cue/load for better // cache behavior. syntax, syntaxErr = cueFS.ReadCUEFile(fpath, parser.NewConfig(parser.ImportsOnly)) - if syntax == nil && !errors.Is(syntaxErr, errors.ErrUnsupported) { - return "", yield(pf, syntaxErr) + if syntax == nil { + if syntaxErr == nil { + // This file couldn't be read-and-converted to a CUE + // AST. Make no further attempts on this file. + return "", true + } else if !errors.Is(syntaxErr, errors.ErrUnsupported) { + return "", yield(pf, syntaxErr) + } } } if syntax == nil { // Either the FS doesn't implement [module.ReadCUEFS] // or the ReadCUEFile method returned ErrUnsupported, // so we need to acquire the syntax ourselves. + if !strings.HasSuffix(fpath, ".cue") { + // This fallback only supports reading `.cue` files. + return "", true + } f, err := fsys.Open(fpath) if err != nil { diff --git a/internal/mod/modpkgload/import.go b/internal/mod/modpkgload/import.go index 1b430ccff..d1123cad3 100644 --- a/internal/mod/modpkgload/import.go +++ b/internal/mod/modpkgload/import.go @@ -301,11 +301,18 @@ func (pkgs *Packages) findLocalPackage(pkgPath string) ([]module.SourceLoc, erro } func isDirWithCUEFiles(loc module.SourceLoc) (bool, error) { + fsys := loc.FS + if cueFS, ok := fsys.(module.ReadCUEFS); ok { + result, err := cueFS.IsDirWithCUEFiles(loc.Dir) + if !errors.Is(err, errors.ErrUnsupported) { + return result, err + } + } // It would be nice if we could inspect the error returned from ReadDir to see // if it's failing because it's not a directory, but unfortunately that doesn't // seem to be something defined by the Go fs interface. // For now, catching fs.ErrNotExist seems to be enough. - entries, err := fs.ReadDir(loc.FS, loc.Dir) + entries, err := fs.ReadDir(fsys, loc.Dir) if err != nil { if errors.Is(err, fs.ErrNotExist) { return false, nil @@ -320,7 +327,7 @@ func isDirWithCUEFiles(loc module.SourceLoc) (bool, error) { // If the directory entry is a symlink, stat it to obtain the info for the // link target instead of the link itself. if ftype&fs.ModeSymlink != 0 { - info, err := fs.Stat(loc.FS, filepath.Join(loc.Dir, e.Name())) + info, err := fs.Stat(fsys, filepath.Join(loc.Dir, e.Name())) if err != nil { continue // Ignore broken symlinks. } diff --git a/mod/module/dirfs.go b/mod/module/dirfs.go index f84469a62..fe66e830a 100644 --- a/mod/module/dirfs.go +++ b/mod/module/dirfs.go @@ -28,7 +28,20 @@ type ReadCUEFS interface { // If this method is implemented, but the implementation // does not support reading CUE files, // it should return [errors.ErrUnsupported]. + // + // This method may be called with paths which do not have a `.cue` + // suffix. If the implementation is unable to read-and-convert (as + // necessary) a path to a CUE AST, it should return (nil, nil). ReadCUEFile(path string, cfg parser.Config) (*ast.File, error) + + // IsDirWithCUEFiles reports whether the given path is a directory + // which contains files for which this implementation would attempt + // to read and parse, if its ReadCUEFile method were called. + // + // If this method is implemented, but the implementation does not + // support examining directories, it should return + // [errors.ErrUnsupported]. + IsDirWithCUEFiles(path string) (bool, error) } // OSRootFS can be implemented by an [fs.FS] -- 2.51.2