diff --git a/cue/load/fs.go b/cue/load/fs.go index 411a2f2db..08fc5c63e 100644 --- a/cue/load/fs.go +++ b/cue/load/fs.go @@ -17,6 +17,7 @@ package load import ( "bytes" "cmp" + stderrs "errors" "fmt" "io" iofs "io/fs" @@ -344,11 +345,19 @@ func (fs *ioFS) ReadFile(name string) ([]byte, error) { var _ module.ReadCUEFS = (*ioFS)(nil) +// IsDirWithCUEFiles implements [module.ReadCUEFS] +func (fs *ioFS) IsDirWithCUEFiles(path string) (bool, error) { + return false, stderrs.ErrUnsupported +} + // ReadCUEFile implements [module.ReadCUEFS] by // reading and updating the syntax file cache, which // is shared with the cache used by the [fileSystem.getCUESyntax] // method. func (fs *ioFS) ReadCUEFile(path string, cfg parser.Config) (*ast.File, error) { + if !strings.HasSuffix(path, ".cue") { + return nil, nil + } fpath, err := fs.absPathFromFSPath(path) if err != nil { return nil, err diff --git a/internal/lsp/fscache/fs_cache.go b/internal/lsp/fscache/fs_cache.go index b6517ee74..4671ba788 100644 --- a/internal/lsp/fscache/fs_cache.go +++ b/internal/lsp/fscache/fs_cache.go @@ -15,7 +15,9 @@ import ( "cuelang.org/go/cue/ast" "cuelang.org/go/cue/build" "cuelang.org/go/cue/parser" + "cuelang.org/go/encoding/yaml" "cuelang.org/go/internal" + "cuelang.org/go/internal/encoding/json" "cuelang.org/go/internal/filetypes" "cuelang.org/go/internal/golangorgx/gopls/protocol" "cuelang.org/go/internal/robustio" @@ -96,22 +98,42 @@ func (p *cueFileParser) ReadCUE(config parser.Config) (syntax *ast.File, cfg par } bf := p.buildFile - if !(bf != nil && bf.Encoding == build.CUE && bf.Form == "" && bf.Interpretation == "") { + if bf == nil { return nil, parser.Config{}, nil } + filename := bf.Filename content := p.content - parseComments := parser.NewConfig(config) - parseComments.Mode = parser.ParseComments - importsOnly := parser.NewConfig(config) - importsOnly.Mode = parser.ImportsOnly + switch bf.Encoding { + case build.CUE: + parseComments := parser.NewConfig(config) + parseComments.Mode = parser.ParseComments + importsOnly := parser.NewConfig(config) + importsOnly.Mode = parser.ImportsOnly - for _, cfg = range []parser.Config{parseComments, importsOnly} { - syntax, err = parser.ParseFile(bf.Filename, content, cfg) - if syntax != nil { - break + for _, cfg = range []parser.Config{parseComments, importsOnly} { + syntax, err = parser.ParseFile(filename, content, cfg) + if syntax != nil { + break + } } + + case build.JSON: + var expr ast.Expr + expr, err = parser.ParseExpr(filename, content) + syntax = internal.ToFile(expr) + if syntax == nil { + syntax = &ast.File{} + } + syntax.Filename = filename + json.PatchExpr(syntax, nil) + + case build.YAML: + syntax, err = yaml.Extract(filename, content) + + default: + return nil, parser.Config{}, nil } if syntax != nil { @@ -139,7 +161,7 @@ func (p *cueFileParser) ReadCUE(config parser.Config) (syntax *ast.File, cfg par } if pkg.Name == nil || pkg.Name.Name == "" || pkg.Name.Name == "_" { // Important that this ident has no position. - pkg.Name = ast.NewIdent(phantomPackageName(bf.Filename)) + pkg.Name = ast.NewIdent(phantomPackageName(filename)) } } @@ -329,12 +351,20 @@ func (fs *CUECacheFS) ReadFile(uri protocol.DocumentURI) (FileHandle, error) { func readFile(uri protocol.DocumentURI, mtime time.Time) (*diskFileEntry, error) { // NB filePath is GOOS-appropriate (uri.Path() calls [filepath.FromSlash]) filePath := uri.Path() - content, err := os.ReadFile(filePath) + + bf, err := filetypes.ParseFileAndType(filePath, "", filetypes.Input) if err != nil { - return nil, err + // Yes, throw the error away, and return an empty cueFileParser + // so that we record that this uri can never be converted to a + // CUE AST. + return &diskFileEntry{ + modTime: mtime, + uri: uri, + cueFileParser: &cueFileParser{}, + }, nil } - bf, err := filetypes.ParseFileAndType(filePath, "", filetypes.Input) + content, err := os.ReadFile(filePath) if err != nil { return nil, err } @@ -377,6 +407,7 @@ type CUEDirFS interface { DirFS module.OSRootFS module.ReadCUEFS + iofs.SubFS } // rootedCUECacheFS is a wrapper over [CUECacheFS] that implements @@ -396,6 +427,12 @@ func (fs *rootedCUECacheFS) OSRoot() string { return fs.root } +// Sub implements [iofs.Sub] +func (fs *rootedCUECacheFS) Sub(dir string) (iofs.FS, error) { + root := filepath.Join(fs.root, filepath.FromSlash(dir)) + return fs.cuecachefs.IoFS(root), nil +} + // Open implements [iofs.FS] func (fs *rootedCUECacheFS) Open(name string) (iofs.File, error) { return fs.delegatefs.Open(name) } @@ -418,6 +455,55 @@ func (fs *rootedCUECacheFS) ReadCUEFile(name string, config parser.Config) (*ast return ast, err } +// IsDirWithCUEFiles implements [module.ReadCUEFS] +func (fs *rootedCUECacheFS) IsDirWithCUEFiles(path string) (bool, error) { + if !iofs.ValidPath(path) { + return false, &iofs.PathError{Op: "IsDirWithCUEFiles", Path: path, Err: iofs.ErrInvalid} + } + path, err := filepath.Localize(path) + if err != nil { + return false, &iofs.PathError{Op: "IsDirWithCUEFiles", Path: path, Err: err} + } + path = filepath.Join(fs.root, path) + + entries, err := os.ReadDir(path) + if err != nil { + if errors.Is(err, iofs.ErrNotExist) { + return false, nil + } + return false, err + } + + for _, e := range entries { + if e.IsDir() { + continue + } + bf, err := filetypes.ParseFileAndType(e.Name(), "", filetypes.Input) + if err != nil { + continue + } + switch bf.Encoding { + case build.CUE, build.JSON, build.YAML: + default: + continue + } + + ftype := e.Type() + if ftype&iofs.ModeSymlink != 0 { + finfo, err := os.Stat(filepath.Join(path, e.Name())) + if err != nil { + continue // Ignore broken symlinks. + } + ftype = finfo.Mode() + } + if ftype.IsRegular() { + return true, nil + } + } + + return false, nil +} + // ReadDir implements [iofs.ReadDirFS] func (fs *rootedCUECacheFS) ReadDir(name string) ([]iofs.DirEntry, error) { return fs.delegatefs.ReadDir(name) diff --git a/internal/lsp/fscache/fs_overlay.go b/internal/lsp/fscache/fs_overlay.go index 2671ba04d..8f3a9adc6 100644 --- a/internal/lsp/fscache/fs_overlay.go +++ b/internal/lsp/fscache/fs_overlay.go @@ -15,6 +15,7 @@ import ( "time" "cuelang.org/go/cue/ast" + "cuelang.org/go/cue/build" "cuelang.org/go/cue/parser" "cuelang.org/go/internal/filetypes" "cuelang.org/go/internal/golangorgx/gopls/protocol" @@ -333,7 +334,7 @@ func (fs *OverlayFS) ReadFile(uri protocol.DocumentURI) (FileHandle, error) { file, isFile := entry.(*overlayFileEntry) if !isFile { - return nil, &iofs.PathError{Op: "open", Path: uri.Path(), Err: iofs.ErrInvalid} + return nil, &iofs.PathError{Op: "ReadFile", Path: uri.Path(), Err: iofs.ErrInvalid} } return file, nil @@ -558,6 +559,12 @@ func (fs *rootedOverlayFS) OSRoot() string { return fs.delegatefs.OSRoot() } +// Sub implements [iofs.Sub] +func (fs *rootedOverlayFS) Sub(dir string) (iofs.FS, error) { + root := filepath.Join(fs.delegatefs.root, filepath.FromSlash(dir)) + return fs.overlayfs.IoFS(root), nil +} + // pathComponents splits the name path into a slice of directory names // (which may be empty), and the final basename. The name must be // valid according to [iofs.ValidPath] @@ -576,14 +583,14 @@ func (fs *rootedOverlayFS) pathComponents(name string) ([]string, string) { // Open implements [iofs.FS] func (fs *rootedOverlayFS) Open(name string) (iofs.File, error) { if !iofs.ValidPath(name) { - return nil, &iofs.PathError{Op: "open", Path: name, Err: iofs.ErrInvalid} + return nil, &iofs.PathError{Op: "Open", Path: name, Err: iofs.ErrInvalid} } entry, err := fs.overlayfs.getEntry(fs.pathComponents(name)) if errors.Is(err, iofs.ErrNotExist) { return fs.delegatefs.Open(name) } else if err != nil { - return nil, &iofs.PathError{Op: "open", Path: name, Err: err} + return nil, &iofs.PathError{Op: "Open", Path: name, Err: err} } switch entry := entry.(type) { @@ -622,7 +629,7 @@ func (fs *rootedOverlayFS) Open(name string) (iofs.File, error) { // ReadCUEFile implements [module.ReadCUEFS] func (fs *rootedOverlayFS) ReadCUEFile(name string, config parser.Config) (*ast.File, error) { if !iofs.ValidPath(name) { - return nil, &iofs.PathError{Op: "readcuefile", Path: name, Err: iofs.ErrInvalid} + return nil, &iofs.PathError{Op: "ReadCUEFile", Path: name, Err: iofs.ErrInvalid} } entry, err := fs.overlayfs.getEntry(fs.pathComponents(name)) @@ -641,10 +648,50 @@ func (fs *rootedOverlayFS) ReadCUEFile(name string, config parser.Config) (*ast. return nil, iofs.ErrInvalid } +// IsDirWithCUEFiles implements [module.ReadCUEFS] +func (fs *rootedOverlayFS) IsDirWithCUEFiles(path string) (bool, error) { + if !iofs.ValidPath(path) { + return false, &iofs.PathError{Op: "IsDirWithCUEFiles", Path: path, Err: iofs.ErrInvalid} + } + + entry, err := fs.overlayfs.getEntry(fs.pathComponents(path)) + if errors.Is(err, iofs.ErrNotExist) { + return fs.delegatefs.IsDirWithCUEFiles(path) + } else if err != nil { + return false, err + } + + dir, isDir := entry.(*overlayDirEntry) + if !isDir { + return false, iofs.ErrInvalid + } + + overlayEntries, err := dir.open().ReadDir(0) + if err != nil { + return false, err + } + + for _, entry := range overlayEntries { + file, isFile := entry.(*overlayFileEntry) + if !isFile { + continue + } + + switch file.buildFile.Encoding { + case build.CUE, build.JSON, build.YAML: + return true, nil + default: + continue + } + } + + return fs.delegatefs.IsDirWithCUEFiles(path) +} + // ReadDir implements [iofs.ReadDirFS] func (fs *rootedOverlayFS) ReadDir(name string) ([]iofs.DirEntry, error) { if !iofs.ValidPath(name) { - return nil, &iofs.PathError{Op: "readdir", Path: name, Err: iofs.ErrInvalid} + return nil, &iofs.PathError{Op: "ReadDir", Path: name, Err: iofs.ErrInvalid} } entry, err := fs.overlayfs.getEntry(fs.pathComponents(name)) @@ -730,7 +777,7 @@ func mergeSort(as, bs []iofs.DirEntry) []iofs.DirEntry { // ReadFile implements [iofs.ReadFileFS] func (fs *rootedOverlayFS) ReadFile(name string) ([]byte, error) { if !iofs.ValidPath(name) { - return nil, &iofs.PathError{Op: "readfile", Path: name, Err: iofs.ErrInvalid} + return nil, &iofs.PathError{Op: "ReadFile", Path: name, Err: iofs.ErrInvalid} } entry, err := fs.overlayfs.getEntry(fs.pathComponents(name)) @@ -751,7 +798,7 @@ func (fs *rootedOverlayFS) ReadFile(name string) ([]byte, error) { // Stat implements [iofs.StatFS] func (fs *rootedOverlayFS) Stat(name string) (iofs.FileInfo, error) { if !iofs.ValidPath(name) { - return nil, &iofs.PathError{Op: "stat", Path: name, Err: iofs.ErrInvalid} + return nil, &iofs.PathError{Op: "Stat", Path: name, Err: iofs.ErrInvalid} } entry, err := fs.overlayfs.getEntry(fs.pathComponents(name)) diff --git a/internal/lsp/fscache/fs_test.go b/internal/lsp/fscache/fs_test.go index 74a3c106f..24895d757 100644 --- a/internal/lsp/fscache/fs_test.go +++ b/internal/lsp/fscache/fs_test.go @@ -3,6 +3,7 @@ package fscache_test import ( iofs "io/fs" "os" + "path" "path/filepath" "runtime" "slices" @@ -169,6 +170,68 @@ func TestOverlayFS(t *testing.T) { qt.Assert(t, qt.IsNil(err)) } +func TestConvertToCueFS(t *testing.T) { + now := time.Now() + dir := t.TempDir() + onDiskFiles := map[string]string{ + "foo/bar/a.json": `{"field": true}`, + "foo/bar/b.yaml": `field: false`, + "foo/c.txt": `no chance`, + } + for f, content := range onDiskFiles { + absPath := filepath.Join(dir, filepath.FromSlash(f)) + writeFile(t, absPath, content) + } + + fs := fscache.NewOverlayFS(fscache.NewCUECachedFS()) + iofs := fs.IoFS(dir) + + for f, content := range onDiskFiles { + dir := path.Dir(f) + isCueable, err := iofs.IsDirWithCUEFiles(dir) + qt.Assert(t, qt.IsNil(err)) + qt.Assert(t, qt.Equals(isCueable, path.Base(dir) == "bar")) + + syntax, err := iofs.ReadCUEFile(f, parser.NewConfig()) + qt.Assert(t, qt.IsNil(err)) + + qt.Assert(t, qt.Equals(syntax != nil, isCueable)) + if syntax != nil { + tokFile := syntax.Pos().File() + qt.Assert(t, qt.IsNotNil(tokFile)) + qt.Assert(t, qt.DeepEquals(tokFile.Content(), []byte(content))) + } + } + + // create extra file in the overlay only, that adds at .cue file to + // the foo directory. + extraPath := "foo/d.cue" + extraPathUri := protocol.URIFromPath(path.Join(dir, extraPath)) + err := fs.Update(func(txn *fscache.UpdateTxn) error { + _, err := txn.Set(extraPathUri, []byte(`field: yes`), now, 0) + return err + }) + qt.Assert(t, qt.IsNil(err)) + + dir = path.Dir(extraPath) + isCueable, err := iofs.IsDirWithCUEFiles(dir) + qt.Assert(t, qt.IsNil(err)) + qt.Assert(t, qt.IsTrue(isCueable)) + syntax, err := iofs.ReadCUEFile(extraPath, parser.NewConfig()) + qt.Assert(t, qt.IsNil(err)) + qt.Assert(t, qt.IsNotNil(syntax)) + + // delete that extra file + err = fs.Update(func(txn *fscache.UpdateTxn) error { + return txn.Delete(extraPathUri) + }) + qt.Assert(t, qt.IsNil(err)) + dir = path.Dir(extraPath) + isCueable, err = iofs.IsDirWithCUEFiles(dir) + qt.Assert(t, qt.IsNil(err)) + qt.Assert(t, qt.IsFalse(isCueable)) +} + func setup(t *testing.T) (dir string, onDiskFiles, onDiskFilesAbs []string) { t.Helper() dir = t.TempDir() diff --git a/internal/mod/modimports/modimports.go b/internal/mod/modimports/modimports.go index 774f2f752..402f071b1 100644 --- a/internal/mod/modimports/modimports.go +++ b/internal/mod/modimports/modimports.go @@ -220,9 +220,6 @@ func yieldAllModFiles(fsys fs.FS, fpath string, topDir bool, yield func(ModuleFi // It returns the yielded package name (if any) and reports whether // the iteration should continue. func yieldPackageFile(fsys fs.FS, fpath string, selectPackage func(pkgName string) bool, yield func(ModuleFile, error) bool) (pkgName string, cont bool) { - if !strings.HasSuffix(fpath, ".cue") { - return "", true - } pf := ModuleFile{ FilePath: fpath, } @@ -237,14 +234,24 @@ func yieldPackageFile(fsys fs.FS, fpath string, selectPackage func(pkgName strin // the default parser options used by cue/load for better // cache behavior. syntax, syntaxErr = cueFS.ReadCUEFile(fpath, parser.NewConfig(parser.ImportsOnly)) - if syntax == nil && !errors.Is(syntaxErr, errors.ErrUnsupported) { - return "", yield(pf, syntaxErr) + if syntax == nil { + if syntaxErr == nil { + // This file couldn't be read-and-converted to a CUE + // AST. Make no further attempts on this file. + return "", true + } else if !errors.Is(syntaxErr, errors.ErrUnsupported) { + return "", yield(pf, syntaxErr) + } } } if syntax == nil { // Either the FS doesn't implement [module.ReadCUEFS] // or the ReadCUEFile method returned ErrUnsupported, // so we need to acquire the syntax ourselves. + if !strings.HasSuffix(fpath, ".cue") { + // This fallback only supports reading `.cue` files. + return "", true + } f, err := fsys.Open(fpath) if err != nil { diff --git a/internal/mod/modpkgload/import.go b/internal/mod/modpkgload/import.go index 1b430ccff..d1123cad3 100644 --- a/internal/mod/modpkgload/import.go +++ b/internal/mod/modpkgload/import.go @@ -301,11 +301,18 @@ func (pkgs *Packages) findLocalPackage(pkgPath string) ([]module.SourceLoc, erro } func isDirWithCUEFiles(loc module.SourceLoc) (bool, error) { + fsys := loc.FS + if cueFS, ok := fsys.(module.ReadCUEFS); ok { + result, err := cueFS.IsDirWithCUEFiles(loc.Dir) + if !errors.Is(err, errors.ErrUnsupported) { + return result, err + } + } // It would be nice if we could inspect the error returned from ReadDir to see // if it's failing because it's not a directory, but unfortunately that doesn't // seem to be something defined by the Go fs interface. // For now, catching fs.ErrNotExist seems to be enough. - entries, err := fs.ReadDir(loc.FS, loc.Dir) + entries, err := fs.ReadDir(fsys, loc.Dir) if err != nil { if errors.Is(err, fs.ErrNotExist) { return false, nil @@ -320,7 +327,7 @@ func isDirWithCUEFiles(loc module.SourceLoc) (bool, error) { // If the directory entry is a symlink, stat it to obtain the info for the // link target instead of the link itself. if ftype&fs.ModeSymlink != 0 { - info, err := fs.Stat(loc.FS, filepath.Join(loc.Dir, e.Name())) + info, err := fs.Stat(fsys, filepath.Join(loc.Dir, e.Name())) if err != nil { continue // Ignore broken symlinks. } diff --git a/mod/module/dirfs.go b/mod/module/dirfs.go index f84469a62..fe66e830a 100644 --- a/mod/module/dirfs.go +++ b/mod/module/dirfs.go @@ -28,7 +28,20 @@ type ReadCUEFS interface { // If this method is implemented, but the implementation // does not support reading CUE files, // it should return [errors.ErrUnsupported]. + // + // This method may be called with paths which do not have a `.cue` + // suffix. If the implementation is unable to read-and-convert (as + // necessary) a path to a CUE AST, it should return (nil, nil). ReadCUEFile(path string, cfg parser.Config) (*ast.File, error) + + // IsDirWithCUEFiles reports whether the given path is a directory + // which contains files for which this implementation would attempt + // to read and parse, if its ReadCUEFile method were called. + // + // If this method is implemented, but the implementation does not + // support examining directories, it should return + // [errors.ErrUnsupported]. + IsDirWithCUEFiles(path string) (bool, error) } // OSRootFS can be implemented by an [fs.FS]