diff --git a/.github/workflows/trybot.yaml b/.github/workflows/trybot.yaml index 5079b7ae6..18306f3dd 100644 --- a/.github/workflows/trybot.yaml +++ b/.github/workflows/trybot.yaml @@ -117,7 +117,7 @@ jobs: GOARCH: "386" run: go test -short ./... - name: Test with -tags=cuewasm - run: go test -tags cuewasm ./cmd/cue/cmd ./cue/interpreter/wasm + run: go test -tags cuewasm ./cmd/cue/cmd ./cue/inject/wasm - id: auth if: |- github.repository == 'cue-lang/cue' && (((github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-branch.')) && (! (contains(github.event.head_commit.message, ' diff --git a/cmd/cue/cmd/common.go b/cmd/cue/cmd/common.go index 367011f98..4fff99567 100644 --- a/cmd/cue/cmd/common.go +++ b/cmd/cue/cmd/common.go @@ -734,7 +734,7 @@ func buildInstances(cmd *Command, binst []*build.Instance, ignoreErrors bool) ([ } func buildToolInstances(ctx *cue.Context, binst []*build.Instance) ([]*cue.Instance, error) { - // Reuse the same context, if there is one, so that the @embed interpreter can be used. + // Reuse the same context, if there is one, so that the @embed injection can be used. // Note that ctx may be nil when we do `cue help cmd`. r := new(cue.Runtime) if ctx != nil { diff --git a/cmd/cue/cmd/root.go b/cmd/cue/cmd/root.go index 0870b26eb..776bcdce9 100644 --- a/cmd/cue/cmd/root.go +++ b/cmd/cue/cmd/root.go @@ -54,8 +54,8 @@ import ( type runFunction func(cmd *Command, args []string) error -// wasmInterp is set when the cuewasm build tag is enbabled. -var wasmInterp cuecontext.ExternInterpreter +// wasmInterp is set when the cuewasm build tag is enabled. +var wasmInterp cuecontext.Injection func statsEncoder(cmd *Command) (*encoding.Encoder, error) { file := os.Getenv("CUE_STATS_FILE") @@ -181,7 +181,7 @@ func mkRunE(c *Command, f runFunction) func(*cobra.Command, []string) error { } var opts []cuecontext.Option if wasmInterp != nil { - opts = append(opts, cuecontext.Interpreter(wasmInterp)) + opts = append(opts, cuecontext.WithInjection(wasmInterp)) } c.ctx = cuecontext.New(opts...) // Some init work, such as in internal/filetypes, evaluates CUE by design. diff --git a/cmd/cue/cmd/root_cuewasm.go b/cmd/cue/cmd/root_cuewasm.go index 68baf838e..d74ccf8db 100644 --- a/cmd/cue/cmd/root_cuewasm.go +++ b/cmd/cue/cmd/root_cuewasm.go @@ -17,11 +17,11 @@ package cmd import ( - "cuelang.org/go/cue/interpreter/wasm" + "cuelang.org/go/cue/inject/wasm" ) func init() { - // The wasm interpreter can be enabled by default once we are ready to ship the feature. + // The wasm injection can be enabled by default once we are ready to ship the feature. // For now, it's not ready, and makes cue binaries heavier by over 2MiB. wasmInterp = wasm.New() } diff --git a/cue/cuecontext/cuecontext.go b/cue/cuecontext/cuecontext.go index 8970f5881..9ae775e72 100644 --- a/cue/cuecontext/cuecontext.go +++ b/cue/cuecontext/cuecontext.go @@ -21,7 +21,7 @@ import ( "fmt" "cuelang.org/go/cue" - "cuelang.org/go/cue/interpreter/embed" + "cuelang.org/go/cue/inject/embed" "cuelang.org/go/internal" "cuelang.org/go/internal/core/runtime" "cuelang.org/go/internal/cuedebug" @@ -45,22 +45,29 @@ type Option struct { func New(options ...Option) *cue.Context { r := runtime.New() // Embedding is always available. - r.SetInterpreter(embed.New()) + r.SetInjection(embed.New()) for _, o := range options { o.apply(r) } return (*cue.Context)(r) } -// An ExternInterpreter creates a compiler that can produce implementations of -// functions written in a language other than CUE. It is currently for internal -// use only. -type ExternInterpreter = runtime.Interpreter +// Deprecated: use [Injection] instead. +type ExternInterpreter = runtime.Injection -// Interpreter associates an interpreter for external code with this context. +// An Injection provides a way to inject runtime values +// into imported CUE code. +type Injection = runtime.Injection + +// Deprecated: use [WithInjection] instead. func Interpreter(i ExternInterpreter) Option { + return WithInjection(i) +} + +// WithInjection associates an injection for external code with this context. +func WithInjection(i Injection) Option { return Option{func(r *runtime.Runtime) { - r.SetInterpreter(i) + r.SetInjection(i) }} } diff --git a/cue/inject/embed/embed.go b/cue/inject/embed/embed.go new file mode 100644 index 000000000..b34d9906d --- /dev/null +++ b/cue/inject/embed/embed.go @@ -0,0 +1,592 @@ +// Copyright 2024 CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package embed provides capabilities to CUE to embed any file that resides +// within a CUE module into CUE either verbatim or decoded. +// +// This package is EXPERIMENTAL and subject to change. +// +// # Overview +// +// To enable file embedding, a file must include the file-level @extern(embed) +// attribute. This allows a quick glance to see if a file embeds any files at +// all. This allows the @embed attribute to be used to load a file within a CUE +// module into a field. +// +// References to files are always relative to the directory in which the +// referring file resides. Only files in the same module containing the CUE +// file can be embedded, and parent directory references are not allowed. +// +// # The @embed attribute +// +// There are two main ways to embed files which are distinguished by the file +// and glob arguments. The @embed attribute supports the following arguments: +// +// file=$filename +// +// The use of the file argument tells embed to load a single file into the +// field. This argument many not be used in conjunction with the glob argument. +// +// glob=$pattern +// +// The use of the glob argument tells embed to load multiple files into the +// field as a map of file paths to the decoded values. The paths are normalized +// to use forward slashes. This argument may not be used in conjunction with the +// file argument. +// +// type=$type +// +// By default, the file type is interpreted based on the file extension. This +// behavior can be overridden by the type argument. See cue help filetypes for +// the list of supported types. This field is required if a file extension is +// unknown, or if a wildcard is used for the file extension in the glob pattern. +// +// allowEmptyGlob +// +// By default, a glob pattern that matches no files results in an error. When +// allowEmptyGlob is present, a glob pattern with no matches will return an +// empty struct instead of an error. This option is only supported with glob patterns. +// +// # Limitations +// +// The embed injection currently does not support: +// - stream values, such as .ndjson or YAML streams. +// - schema-based decoding, such as needed for textproto +// +// # Example +// +// @extern(embed) +// +// package foo +// +// // interpreted as JSON +// a: _ @embed(file="file1.json") // the quotes are optional here +// +// // interpreted the same file as JSON schema +// #A: _ @embed(file=file1.json, type=jsonschema) +// +// // interpret a proprietary extension as OpenAPI represented as YAML +// b: _ @embed(file="file2.crd", type=openapi+yaml) +// +// // include all YAML files in the x directory interpreted as YAML +// // The result is a map of file paths to the decoded YAML values. +// files: _ @embed(glob=x/*.yaml) +// +// // include all files in the y directory as a map of file paths to binary +// // data. The entries are unified into the same map as above. +// files: _ @embed(glob=y/*.*, type=binary) +// +// // include all YAML files in the z directory, but allow empty result +// // if no files match (returns empty struct instead of error) +// optionalFiles: _ @embed(glob=z/*.yaml, allowEmptyGlob) +package embed + +import ( + iofs "io/fs" + "os" + "path" + "path/filepath" + "strings" + + "cuelang.org/go/cue" + "cuelang.org/go/cue/ast" + "cuelang.org/go/cue/build" + "cuelang.org/go/cue/errors" + "cuelang.org/go/cue/token" + "cuelang.org/go/internal" + "cuelang.org/go/internal/core/adt" + "cuelang.org/go/internal/core/runtime" + "cuelang.org/go/internal/encoding" + "cuelang.org/go/internal/filetypes" + "cuelang.org/go/internal/value" + pkgpath "cuelang.org/go/pkg/path" +) + +// TODO: record files in build.Instance + +// injection is a [runtime.Injection] for embedded files. +type injection struct{} + +// New returns a new injection for embedded files as a +// [runtime.Injection] suitable for passing to +// [cuelang.org/go/cue/cuecontext.WithInjection]. +func New() runtime.Injection { + return injection{} +} + +func (i injection) Kind() string { + return EmbedKind +} + +const EmbedKind = "embed" + +// InjectorForInstance returns an injector that can decode and embed files +// that exist within a CUE module. +func (i injection) InjectorForInstance(b *build.Instance, r *runtime.Runtime) (runtime.Injector, errors.Error) { + if b.Module == "" { + return nil, errors.Newf(token.Pos{}, "cannot embed files when not in a module") + } + if b.Root == "" { + return nil, errors.Newf(token.Pos{}, "cannot embed files: no module root found") + } + return &injector{ + b: b, + runtime: (*cue.Context)(r), + }, nil +} + +// An injector is a [runtime.Injector] that allows embedding files into CUE +// values. +type injector struct { + b *build.Instance + runtime *cue.Context + opCtx *adt.OpContext + + // file system cache + dir string + fs iofs.StatFS + pos token.Pos +} + +// validateAttr performs logical validation of the attr. It does not +// perform any checks against a filesystem. +func validateAttr(a *internal.Attr) (file, glob, typ string, allowEmptyGlob bool, errs errors.Error) { + if a.Err != nil { + return "", "", "", false, a.Err + } + + pos := a.Pos + + file, _, err := a.Lookup(0, "file") + if err != nil { + return "", "", "", false, errors.Promote(err, "invalid attribute") + } + + glob, _, err = a.Lookup(0, "glob") + if err != nil { + return "", "", "", false, errors.Promote(err, "invalid attribute") + } + + typ, _, err = a.Lookup(0, "type") + if err != nil { + return "", "", "", false, errors.Promote(err, "invalid type argument") + } + + allowEmptyGlob, err = a.Flag(0, "allowEmptyGlob") + if err != nil { + return "", "", "", false, errors.Promote(err, "invalid allowEmptyGlob argument") + } + + switch { + case file == "" && glob == "": + return "", "", "", false, errors.Newf(pos, "attribute must have file or glob field") + + case file != "" && glob != "": + return "", "", "", false, errors.Newf(pos, "attribute cannot have both file and glob field") + case allowEmptyGlob && glob == "": + return "", "", "", false, errors.Newf(pos, "allowEmptyGlob must be specified with a glob field") + + case file != "": + file, err := clean(pos, file) + if err != nil { + return "", "", "", false, err + } + return file, "", typ, allowEmptyGlob, nil + + default: + glob, err := clean(pos, glob) + if err != nil { + return "", "", "", false, err + } + + // Validate that the glob pattern is valid per [pkgpath.Match]. + // Note that we use Unix match semantics because all embed paths are Unix-like. + if _, err := pkgpath.Match(glob, "", pkgpath.Unix); err != nil { + return "", "", "", false, errors.Wrapf(err, pos, "invalid glob pattern %q", glob) + } + + // If we do not have a type, ensure the extension of the base is fully + // specified, i.e. does not contain any meta characters as specified by + // path.Match. + if typ == "" { + ext := path.Ext(path.Base(glob)) + if ext == "" || strings.ContainsAny(ext, "*?[\\") { + return "", "", "", false, errors.Newf(pos, "extension not fully specified; type argument required") + } + } + return "", glob, typ, allowEmptyGlob, nil + } +} + +// InjectedValue interprets an embed attribute to either load a file +// (@embed(file=...)) or a glob of files (@embed(glob=...)) +// and decodes the given files. +func (c *injector) InjectedValue(attr *runtime.ExternAttr, scope *adt.Vertex) (adt.Expr, errors.Error) { + a := attr.Attr + c.opCtx = adt.NewContext((*runtime.Runtime)(c.runtime), nil) + + pos := a.Pos + c.pos = pos + + // Jump through some hoops to get file operations to behave the same for + // Windows and Unix. + // TODO: obtain an iofs.FS from load or something similar. + dir := filepath.Dir(pos.File().Name()) + if c.dir != dir { + c.fs = os.DirFS(dir).(iofs.StatFS) // Documented as implementing iofs.StatFS + c.dir = dir + } + + file, glob, typ, allowEmptyGlob, err := validateAttr(a) + if err != nil { + return nil, err + } + + if file != "" { + for dir := path.Dir(file); dir != "."; dir = path.Dir(dir) { + if _, err := c.fs.Stat(path.Join(dir, "cue.mod")); err == nil { + return nil, errors.Newf(pos, "cannot embed file %q: in different module", file) + } + } + return c.decodeFile(file, typ) + } + return c.processGlob(glob, typ, allowEmptyGlob) +} + +func (c *injector) processGlob(glob, scope string, allowEmptyGlob bool) (adt.Expr, errors.Error) { + m := &adt.StructLit{} + + matches, err := fsGlob(c.fs, glob) + if err != nil { + return nil, errors.Promote(err, "failed to match glob") + } + if len(matches) == 0 && !allowEmptyGlob { + return nil, errors.Newf(c.pos, "no matches for glob pattern %q", glob) + } + + dirs := make(map[string]string) + for _, f := range matches { + // TODO: lots of stat calls happening in this MVP so another won't hurt. + // We don't support '**' initially, and '*' only matches files, so skip + // any directories. + if fi, err := c.fs.Stat(f); err != nil { + return nil, errors.Newf(c.pos, "failed to stat %s: %v", f, err) + } else if fi.IsDir() { + continue + } + // Add all parents of the embedded file that + // aren't the current directory (if there's a cue.mod + // in the current directory, that's the current module + // not nested). + for dir := path.Dir(f); dir != "."; dir = path.Dir(dir) { + dirs[dir] = f + } + + expr, err := c.decodeFile(f, scope) + if err != nil { + return nil, err + } + + m.Decls = append(m.Decls, &adt.Field{ + Label: c.opCtx.StringLabel(f), + Value: expr, + }) + } + // Check that none of the matches were in a nested module + // directory. + for dir, f := range dirs { + if _, err := c.fs.Stat(path.Join(dir, "cue.mod")); err == nil { + return nil, errors.Newf(c.pos, "cannot embed file %q: in different module", f) + } + } + return m, nil +} + +func clean(pos token.Pos, s string) (string, errors.Error) { + file := path.Clean(s) + if file != s { + return file, errors.Newf(pos, "path not normalized, use %q instead", file) + } + if path.IsAbs(file) { + return "", errors.Newf(pos, "only relative files are allowed") + } + if file == ".." || strings.HasPrefix(file, "../") { + return "", errors.Newf(pos, "cannot refer to parent directory") + } + return file, nil +} + +// fsGlob is like [iofs.Glob] but only includes dot-prefixed files +// when the dot is explictly present in an element. +// TODO: add option for including dot files? +func fsGlob(fsys iofs.FS, pattern string) ([]string, error) { + pattern = path.Clean(pattern) + matches, err := iofs.Glob(fsys, pattern) + if err != nil { + return nil, err + } + return filterFsGlobResults(pattern, matches...), nil +} + +// filterFsGlobResults applies additional filtering on the given +// matches to only include dot-prefixed files when the dot is +// explictly present in the corresponding pattern element. +func filterFsGlobResults(pattern string, matches ...string) []string { + patElems := strings.Split(pattern, "/") + included := func(m string) bool { + for i, elem := range strings.Split(m, "/") { + // Technically there should never be more elements in m than + // there are in patElems, but be defensive and check bounds just in case. + if strings.HasPrefix(elem, ".") && (i >= len(patElems) || !strings.HasPrefix(patElems[i], ".")) { + return false + } + } + return true + } + + i := 0 + for _, m := range matches { + if included(m) { + matches[i] = m + i++ + } + } + return matches[:i] +} + +func (c *injector) decodeFile(file, scope string) (adt.Expr, errors.Error) { + // Do not use the most obvious filetypes.Input in order to disable "auto" + // mode. + f, err := filetypes.ParseFileAndType(file, scope, filetypes.Def) + if err != nil { + return nil, errors.Promote(err, "invalid file type") + } + + // Open and pre-load the file system using iofs.FS. + r, err := c.fs.Open(file) + if err != nil { + return nil, errors.Newf(c.pos, "open %v: no such file or directory", file) + } + defer r.Close() + + info, err := r.Stat() + if err != nil { + return nil, errors.Promote(err, "failed to decode file") + } + if info.IsDir() { + return nil, errors.Newf(c.pos, "cannot embed directories") + } + f.Source = r + + // TODO: this really should be done at the start of the build process. + // c.b.ExternFiles = append(c.b.ExternFiles, f) + + config := &encoding.Config{ + // TODO: schema is currently the wrong schema, which is a bug in + // internal/core/runtime. There is also an outstanding design choice: + // do we imply the schema from the schema of the current field, or do + // we explicitly enable schema-based encoding with a "schema" argument. + // In the case of YAML it seems to be better to be explicit. In the case + // of textproto it seems to be more convenient to do it implicitly. + // Schema: value.Make(c.opCtx, schema), + } + + d := encoding.NewDecoder(c.runtime, f, config) + if err := d.Err(); err != nil { + return nil, errors.Promote(err, "failed to decode file") + } + + defer d.Close() + + n := d.File() + + if d.Next(); !d.Done() { + // TODO: support streaming values + return nil, errors.Newf(c.pos, "streaming not implemented: found more than one value in file") + } + + // TODO: each of these encodings should probably be supported in the future + switch f.Encoding { + case build.CUE: + return nil, errors.Newf(c.pos, "encoding %q not (yet) supported", f.Encoding) + case build.JSONL: + return nil, errors.Newf(c.pos, "encoding %q not (yet) supported: requires support for streaming", f.Encoding) + case build.BinaryProto, build.TextProto: + return nil, errors.Newf(c.pos, "encoding %q not (yet) supported: requires support for schema-guided decoding", f.Encoding) + } + + val := c.runtime.BuildFile(n) + if err := val.Err(); err != nil { + return nil, errors.Promote(err, "failed to build file") + } + + _, v := value.ToInternal(val) + return v, nil +} + +// EmbeddedPaths walks all the embed attributes in the given file, +// returning a slice of [Embed] structs for attributes +// that were successfully validated, and errors for those which were +// not. The filepath should be the filepath of the file from which +// these attributes were extracted, and relative to whatever root is +// going to be used in calls to [Embed.Matches] and [Embed.FindAll]. +func EmbeddedPaths(filepath string, syntax *ast.File) ([]*Embed, errors.Error) { + extAttrs, err := runtime.ExternAttrsForFile(syntax) + if err != nil { + return nil, err + } + if extAttrs.TopLevel[EmbedKind] == nil { + return nil, nil + } + var errs errors.Error + var embeds []*Embed + for attr := range extAttrs.Body { + if attr.Attr.Name != EmbedKind { + continue + } + if err := attr.Attr.Err; err != nil { + errs = errors.Append(errs, err) + continue + } + file, glob, typ, allowEmptyGlob, err := validateAttr(attr.Attr) + if err != nil { + errs = errors.Append(errs, err) + continue + } + embed := &Embed{ + Node: attr.Parent, + Attribute: attr.Attr, + FilePath: filepath, + Type: typ, + } + if file != "" { + embed.interpreter = &embeddedFile{ + filepath: file, + } + embeds = append(embeds, embed) + } else if glob != "" { + embed.interpreter = &embeddedGlob{ + glob: glob, + allowEmptyGlob: allowEmptyGlob, + } + embeds = append(embeds, embed) + } + } + return embeds, errs +} + +type Embed struct { + Node ast.Node + Attribute *internal.Attr + FilePath string + Type string + interpreter embedInterpreter +} + +// Matches reports whether the provided filepath is matched by this +// [Embed] attribute. The filepath should be relative to the same root +// as the filepath provided to [EmbeddedPaths]. E.g. if in +// `/wibble/foo/bar.cue` you have `@embed(filename=a/b.json)`, and +// `foo/bar.cue` is the filepath passed to [EmbeddedPaths], then +// [Embed.Matches] will return true if called with `foo/a/b.json`. +func (e *Embed) Matches(filepath string) bool { + return e.interpreter.matches(e, filepath) +} + +// FindAll uses the provided fs to report all the filepaths that +// match this [Embed] attribute. The fs must be relative to the same +// root as the filepath provided to [EmbeddedPaths]. I.e. for the +// filepath provided to [EmbeddedPaths], iofs.Stat(fs, filepath) +// should be accessing the same file which contained this [Embed] +// attribute. +func (e *Embed) FindAll(fs iofs.FS) ([]string, error) { + return e.interpreter.findAll(e, fs) +} + +// IsGlob reports whether this [Embed] attribute represents a glob +// embedding. +func (e *Embed) IsGlob() bool { + _, isGlob := e.interpreter.(*embeddedGlob) + return isGlob +} + +type embedInterpreter interface { + // NB: All filepaths (including any within the Embed) are + // considered relative to the same root. + + matches(e *Embed, filepath string) bool + findAll(e *Embed, fs iofs.FS) ([]string, error) +} + +type embeddedFile struct { + filepath string +} + +func (ef *embeddedFile) matches(e *Embed, filepath string) bool { + dir := path.Dir(e.FilePath) + return filepath == path.Join(dir, ef.filepath) +} + +func (ef *embeddedFile) findAll(e *Embed, fs iofs.FS) ([]string, error) { + dir := path.Dir(e.FilePath) + filepath := path.Join(dir, ef.filepath) + info, err := iofs.Stat(fs, filepath) + if err != nil { + return nil, errors.Wrapf(err, e.Attribute.Pos, "failed to stat %s: %v", filepath, err) + } + if info.IsDir() { + return nil, errors.Newf(e.Attribute.Pos, "%v is a directory", filepath) + } + return []string{filepath}, nil +} + +type embeddedGlob struct { + glob string + allowEmptyGlob bool +} + +func (eg *embeddedGlob) matches(e *Embed, filepath string) bool { + dir := path.Dir(e.FilePath) + if dir != "." { + wasCut := false + filepath, wasCut = strings.CutPrefix(filepath, dir+"/") + if !wasCut { + return false + } + } + result, err := pkgpath.Match(eg.glob, filepath, pkgpath.Unix) + if !result || err != nil { + return false + } + return len(filterFsGlobResults(eg.glob, filepath)) == 1 +} + +func (eg *embeddedGlob) findAll(e *Embed, fs iofs.FS) ([]string, error) { + dir := path.Dir(e.FilePath) + fs, err := iofs.Sub(fs, dir) + if err != nil { + return nil, errors.Wrapf(err, e.Attribute.Pos, "%v", err) + } + filepaths, err := fsGlob(fs, eg.glob) + if err != nil { + return nil, errors.Wrapf(err, e.Attribute.Pos, "%v", err) + } + if !eg.allowEmptyGlob && len(filepaths) == 0 { + return nil, errors.Newf(e.Attribute.Pos, "no matches for glob pattern %q", eg.glob) + } + for i, filepath := range filepaths { + filepaths[i] = path.Join(dir, filepath) + } + return filepaths, nil +} diff --git a/cue/interpreter/embed/embed_test.go b/cue/inject/embed/embed_test.go similarity index 100% rename from cue/interpreter/embed/embed_test.go rename to cue/inject/embed/embed_test.go diff --git a/cue/interpreter/wasm/builtin.go b/cue/inject/wasm/builtin.go similarity index 100% rename from cue/interpreter/wasm/builtin.go rename to cue/inject/wasm/builtin.go diff --git a/cue/interpreter/wasm/call.go b/cue/inject/wasm/call.go similarity index 100% rename from cue/interpreter/wasm/call.go rename to cue/inject/wasm/call.go diff --git a/cue/interpreter/wasm/doc.go b/cue/inject/wasm/doc.go similarity index 100% rename from cue/interpreter/wasm/doc.go rename to cue/inject/wasm/doc.go diff --git a/cue/interpreter/wasm/exe_test.go b/cue/inject/wasm/exe_test.go similarity index 99% rename from cue/interpreter/wasm/exe_test.go rename to cue/inject/wasm/exe_test.go index 311522205..8bd781b17 100644 --- a/cue/interpreter/wasm/exe_test.go +++ b/cue/inject/wasm/exe_test.go @@ -32,7 +32,7 @@ import ( "cuelang.org/go/cue/ast" "cuelang.org/go/cue/build" "cuelang.org/go/cue/cuecontext" - "cuelang.org/go/cue/interpreter/wasm" + "cuelang.org/go/cue/inject/wasm" "cuelang.org/go/cue/parser" "cuelang.org/go/internal" "cuelang.org/go/internal/core/runtime" diff --git a/cue/interpreter/wasm/extern.go b/cue/inject/wasm/extern.go similarity index 100% rename from cue/interpreter/wasm/extern.go rename to cue/inject/wasm/extern.go diff --git a/cue/interpreter/wasm/layout.go b/cue/inject/wasm/layout.go similarity index 100% rename from cue/interpreter/wasm/layout.go rename to cue/inject/wasm/layout.go diff --git a/cue/interpreter/wasm/runtime.go b/cue/inject/wasm/runtime.go similarity index 100% rename from cue/interpreter/wasm/runtime.go rename to cue/inject/wasm/runtime.go diff --git a/cue/interpreter/wasm/testdata/cue/basic.txtar b/cue/inject/wasm/testdata/cue/basic.txtar similarity index 100% rename from cue/interpreter/wasm/testdata/cue/basic.txtar rename to cue/inject/wasm/testdata/cue/basic.txtar diff --git a/cue/interpreter/wasm/testdata/cue/basic.wasm b/cue/inject/wasm/testdata/cue/basic.wasm similarity index 100% rename from cue/interpreter/wasm/testdata/cue/basic.wasm rename to cue/inject/wasm/testdata/cue/basic.wasm diff --git a/cue/interpreter/wasm/testdata/cue/basic1.wasm b/cue/inject/wasm/testdata/cue/basic1.wasm similarity index 100% rename from cue/interpreter/wasm/testdata/cue/basic1.wasm rename to cue/inject/wasm/testdata/cue/basic1.wasm diff --git a/cue/interpreter/wasm/testdata/cue/complex.txtar b/cue/inject/wasm/testdata/cue/complex.txtar similarity index 100% rename from cue/interpreter/wasm/testdata/cue/complex.txtar rename to cue/inject/wasm/testdata/cue/complex.txtar diff --git a/cue/interpreter/wasm/testdata/cue/def.txtar b/cue/inject/wasm/testdata/cue/def.txtar similarity index 100% rename from cue/interpreter/wasm/testdata/cue/def.txtar rename to cue/inject/wasm/testdata/cue/def.txtar diff --git a/cue/interpreter/wasm/testdata/cue/default.txtar b/cue/inject/wasm/testdata/cue/default.txtar similarity index 100% rename from cue/interpreter/wasm/testdata/cue/default.txtar rename to cue/inject/wasm/testdata/cue/default.txtar diff --git a/cue/interpreter/wasm/testdata/cue/empty.wasm b/cue/inject/wasm/testdata/cue/empty.wasm similarity index 100% rename from cue/interpreter/wasm/testdata/cue/empty.wasm rename to cue/inject/wasm/testdata/cue/empty.wasm diff --git a/cue/interpreter/wasm/testdata/cue/error.txtar b/cue/inject/wasm/testdata/cue/error.txtar similarity index 100% rename from cue/interpreter/wasm/testdata/cue/error.txtar rename to cue/inject/wasm/testdata/cue/error.txtar diff --git a/cue/interpreter/wasm/testdata/cue/missing.txtar b/cue/inject/wasm/testdata/cue/missing.txtar similarity index 100% rename from cue/interpreter/wasm/testdata/cue/missing.txtar rename to cue/inject/wasm/testdata/cue/missing.txtar diff --git a/cue/interpreter/wasm/testdata/cue/multiple.txtar b/cue/inject/wasm/testdata/cue/multiple.txtar similarity index 100% rename from cue/interpreter/wasm/testdata/cue/multiple.txtar rename to cue/inject/wasm/testdata/cue/multiple.txtar diff --git a/cue/interpreter/wasm/testdata/cue/nested.txtar b/cue/inject/wasm/testdata/cue/nested.txtar similarity index 100% rename from cue/interpreter/wasm/testdata/cue/nested.txtar rename to cue/inject/wasm/testdata/cue/nested.txtar diff --git a/cue/interpreter/wasm/testdata/cue/noload.txtar b/cue/inject/wasm/testdata/cue/noload.txtar similarity index 100% rename from cue/interpreter/wasm/testdata/cue/noload.txtar rename to cue/inject/wasm/testdata/cue/noload.txtar diff --git a/cue/interpreter/wasm/testdata/cue/nopackage.txtar b/cue/inject/wasm/testdata/cue/nopackage.txtar similarity index 100% rename from cue/interpreter/wasm/testdata/cue/nopackage.txtar rename to cue/inject/wasm/testdata/cue/nopackage.txtar diff --git a/cue/interpreter/wasm/testdata/cue/struct.txtar b/cue/inject/wasm/testdata/cue/struct.txtar similarity index 100% rename from cue/interpreter/wasm/testdata/cue/struct.txtar rename to cue/inject/wasm/testdata/cue/struct.txtar diff --git a/cue/interpreter/wasm/testdata/cue/struct.wasm b/cue/inject/wasm/testdata/cue/struct.wasm similarity index 100% rename from cue/interpreter/wasm/testdata/cue/struct.wasm rename to cue/inject/wasm/testdata/cue/struct.wasm diff --git a/cue/interpreter/wasm/testdata/cue/unused.txtar b/cue/inject/wasm/testdata/cue/unused.txtar similarity index 100% rename from cue/interpreter/wasm/testdata/cue/unused.txtar rename to cue/inject/wasm/testdata/cue/unused.txtar diff --git a/cue/interpreter/wasm/testdata/gen.go b/cue/inject/wasm/testdata/gen.go similarity index 100% rename from cue/interpreter/wasm/testdata/gen.go rename to cue/inject/wasm/testdata/gen.go diff --git a/cue/interpreter/wasm/testdata/rust/Cargo.lock b/cue/inject/wasm/testdata/rust/Cargo.lock similarity index 100% rename from cue/interpreter/wasm/testdata/rust/Cargo.lock rename to cue/inject/wasm/testdata/rust/Cargo.lock diff --git a/cue/interpreter/wasm/testdata/rust/Cargo.toml b/cue/inject/wasm/testdata/rust/Cargo.toml similarity index 100% rename from cue/interpreter/wasm/testdata/rust/Cargo.toml rename to cue/inject/wasm/testdata/rust/Cargo.toml diff --git a/cue/interpreter/wasm/testdata/rust/basic/Cargo.toml b/cue/inject/wasm/testdata/rust/basic/Cargo.toml similarity index 100% rename from cue/interpreter/wasm/testdata/rust/basic/Cargo.toml rename to cue/inject/wasm/testdata/rust/basic/Cargo.toml diff --git a/cue/interpreter/wasm/testdata/rust/basic/src/lib.rs b/cue/inject/wasm/testdata/rust/basic/src/lib.rs similarity index 100% rename from cue/interpreter/wasm/testdata/rust/basic/src/lib.rs rename to cue/inject/wasm/testdata/rust/basic/src/lib.rs diff --git a/cue/interpreter/wasm/testdata/rust/basic1/Cargo.toml b/cue/inject/wasm/testdata/rust/basic1/Cargo.toml similarity index 100% rename from cue/interpreter/wasm/testdata/rust/basic1/Cargo.toml rename to cue/inject/wasm/testdata/rust/basic1/Cargo.toml diff --git a/cue/interpreter/wasm/testdata/rust/basic1/src/lib.rs b/cue/inject/wasm/testdata/rust/basic1/src/lib.rs similarity index 100% rename from cue/interpreter/wasm/testdata/rust/basic1/src/lib.rs rename to cue/inject/wasm/testdata/rust/basic1/src/lib.rs diff --git a/cue/interpreter/wasm/testdata/rust/struct/Cargo.toml b/cue/inject/wasm/testdata/rust/struct/Cargo.toml similarity index 100% rename from cue/interpreter/wasm/testdata/rust/struct/Cargo.toml rename to cue/inject/wasm/testdata/rust/struct/Cargo.toml diff --git a/cue/interpreter/wasm/testdata/rust/struct/src/lib.rs b/cue/inject/wasm/testdata/rust/struct/src/lib.rs similarity index 100% rename from cue/interpreter/wasm/testdata/rust/struct/src/lib.rs rename to cue/inject/wasm/testdata/rust/struct/src/lib.rs diff --git a/cue/interpreter/wasm/testdata/rust/struct/src/mem.rs b/cue/inject/wasm/testdata/rust/struct/src/mem.rs similarity index 100% rename from cue/interpreter/wasm/testdata/rust/struct/src/mem.rs rename to cue/inject/wasm/testdata/rust/struct/src/mem.rs diff --git a/cue/interpreter/wasm/wasm.go b/cue/inject/wasm/wasm.go similarity index 72% rename from cue/interpreter/wasm/wasm.go rename to cue/inject/wasm/wasm.go index 080c8fb30..e35f0c471 100644 --- a/cue/interpreter/wasm/wasm.go +++ b/cue/inject/wasm/wasm.go @@ -19,6 +19,7 @@ import ( "strings" "sync" + "cuelang.org/go/cue/ast" "cuelang.org/go/cue/build" "cuelang.org/go/cue/cuecontext" "cuelang.org/go/cue/errors" @@ -28,23 +29,23 @@ import ( coreruntime "cuelang.org/go/internal/core/runtime" ) -// interpreter is a [cuecontext.ExternInterpreter] for Wasm files. -type interpreter struct{} +// injection is a [cuecontext.Injection] for Wasm files. +type injection struct{} -// New returns a new Wasm interpreter as a [cuecontext.ExternInterpreter] -// suitable for passing to [cuecontext.New]. +// New returns a new Wasm injection as a [cuecontext.Injection] +// suitable for passing to [cuecontext.WithInjection]. func New() cuecontext.ExternInterpreter { - return &interpreter{} + return &injection{} } -func (i *interpreter) Kind() string { +func (i *injection) Kind() string { return "wasm" } -// NewCompiler returns a Wasm compiler that services the specified +// InjectorForInstance returns a Wasm injector that services the specified // build.Instance. -func (i *interpreter) NewCompiler(b *build.Instance, r *coreruntime.Runtime) (coreruntime.Compiler, errors.Error) { - return &compiler{ +func (i *injection) InjectorForInstance(b *build.Instance, r *coreruntime.Runtime) (coreruntime.Injector, errors.Error) { + return &wasmInjector{ b: b, runtime: r, wasmRuntime: newRuntime(), @@ -52,9 +53,9 @@ func (i *interpreter) NewCompiler(b *build.Instance, r *coreruntime.Runtime) (co }, nil } -// A compiler is a [coreruntime.Compiler] +// A wasmInjector is a [coreruntime.Injector] // that provides Wasm functionality to the runtime. -type compiler struct { +type wasmInjector struct { b *build.Instance runtime *coreruntime.Runtime wasmRuntime runtime @@ -67,10 +68,21 @@ type compiler struct { instances map[string]*instance } -// Compile searches for a Wasm function described by the given `@extern` -// attribute and returns it as an [adt.Builtin] with the given function -// name. -func (c *compiler) Compile(funcName string, scope adt.Value, a *internal.Attr) (adt.Expr, errors.Error) { +// InjectedValue searches for a Wasm function described by the given +// extern attribute and returns it as an [adt.Builtin]. +func (c *wasmInjector) InjectedValue(attr *coreruntime.ExternAttr, scope *adt.Vertex) (adt.Expr, errors.Error) { + a := attr.Attr + + // Determine the function name from the parent field label, + // with an explicit name= attribute taking precedence. + var funcName string + if f, ok := attr.Parent.(*ast.Field); ok { + funcName, _, _ = ast.LabelName(f.Label) + } + if name, ok, _ := a.Lookup(1, "name"); ok { + funcName = name + } + baseFile, err := fileName(a) if err != nil { return nil, errors.Promote(err, "invalid attribute") @@ -104,7 +116,7 @@ func (c *compiler) Compile(funcName string, scope adt.Value, a *internal.Attr) ( // instance returns the instance corresponding to filename, compiling // and loading it if necessary. -func (c *compiler) instance(filename string) (inst *instance, err error) { +func (c *wasmInjector) instance(filename string) (inst *instance, err error) { c.mu.Lock() defer c.mu.Unlock() inst, ok := c.instances[filename] diff --git a/cue/interpreter/wasm/wasm_test.go b/cue/inject/wasm/wasm_test.go similarity index 98% rename from cue/interpreter/wasm/wasm_test.go rename to cue/inject/wasm/wasm_test.go index 92fc8df90..881029b02 100644 --- a/cue/interpreter/wasm/wasm_test.go +++ b/cue/inject/wasm/wasm_test.go @@ -24,7 +24,7 @@ import ( "cuelang.org/go/cue/cuecontext" "cuelang.org/go/cue/errors" "cuelang.org/go/cue/format" - "cuelang.org/go/cue/interpreter/wasm" + "cuelang.org/go/cue/inject/wasm" "cuelang.org/go/internal" "cuelang.org/go/internal/cuetxtar" ) diff --git a/cue/instance.go b/cue/instance.go index 0119798fc..f6bd2cec7 100644 --- a/cue/instance.go +++ b/cue/instance.go @@ -202,7 +202,7 @@ func (inst *hiddenInstance) Build(p *build.Instance) *Instance { cfg := &compile.Config{Scope: valueScope(Value{idx: r, v: inst.root})} v, err := compile.Instance(cfg, r, p) - // Just like [runtime.Runtime.Build], ensure that the @embed compiler is run as needed. + // Just like [runtime.Runtime.Build], ensure that the @embed injector is run as needed. err = errors.Append(err, r.InjectImplementations(p, v)) v.AddConjunct(adt.MakeRootConjunct(nil, inst.root)) diff --git a/cue/interpreter/embed/embed.go b/cue/interpreter/embed/embed.go index cdb7c74d0..e4d2579e0 100644 --- a/cue/interpreter/embed/embed.go +++ b/cue/interpreter/embed/embed.go @@ -1,4 +1,4 @@ -// Copyright 2024 CUE Authors +// Copyright 2026 CUE Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,584 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package embed provides capabilities to CUE to embed any file that resides -// within a CUE module into CUE either verbatim or decoded. -// -// This package is EXPERIMENTAL and subject to change. -// -// # Overview -// -// To enable file embedding, a file must include the file-level @extern(embed) -// attribute. This allows a quick glance to see if a file embeds any files at -// all. This allows the @embed attribute to be used to load a file within a CUE -// module into a field. -// -// References to files are always relative to the directory in which the -// referring file resides. Only files in the same module containing the CUE -// file can be embedded, and parent directory references are not allowed. -// -// # The @embed attribute -// -// There are two main ways to embed files which are distinguished by the file -// and glob arguments. The @embed attribute supports the following arguments: -// -// file=$filename -// -// The use of the file argument tells embed to load a single file into the -// field. This argument many not be used in conjunction with the glob argument. -// -// glob=$pattern -// -// The use of the glob argument tells embed to load multiple files into the -// field as a map of file paths to the decoded values. The paths are normalized -// to use forward slashes. This argument may not be used in conjunction with the -// file argument. -// -// type=$type -// -// By default, the file type is interpreted based on the file extension. This -// behavior can be overridden by the type argument. See cue help filetypes for -// the list of supported types. This field is required if a file extension is -// unknown, or if a wildcard is used for the file extension in the glob pattern. -// -// allowEmptyGlob -// -// By default, a glob pattern that matches no files results in an error. When -// allowEmptyGlob is present, a glob pattern with no matches will return an -// empty struct instead of an error. This option is only supported with glob patterns. -// -// # Limitations -// -// The embed interpreter currently does not support: -// - stream values, such as .ndjson or YAML streams. -// - schema-based decoding, such as needed for textproto -// -// # Example -// -// @extern(embed) -// -// package foo -// -// // interpreted as JSON -// a: _ @embed(file="file1.json") // the quotes are optional here -// -// // interpreted the same file as JSON schema -// #A: _ @embed(file=file1.json, type=jsonschema) -// -// // interpret a proprietary extension as OpenAPI represented as YAML -// b: _ @embed(file="file2.crd", type=openapi+yaml) -// -// // include all YAML files in the x directory interpreted as YAML -// // The result is a map of file paths to the decoded YAML values. -// files: _ @embed(glob=x/*.yaml) -// -// // include all files in the y directory as a map of file paths to binary -// // data. The entries are unified into the same map as above. -// files: _ @embed(glob=y/*.*, type=binary) -// -// // include all YAML files in the z directory, but allow empty result -// // if no files match (returns empty struct instead of error) -// optionalFiles: _ @embed(glob=z/*.yaml, allowEmptyGlob) package embed import ( - iofs "io/fs" - "os" - "path" - "path/filepath" - "strings" - - "cuelang.org/go/cue" - "cuelang.org/go/cue/ast" - "cuelang.org/go/cue/build" - "cuelang.org/go/cue/errors" - "cuelang.org/go/cue/token" - "cuelang.org/go/internal" - "cuelang.org/go/internal/core/adt" + "cuelang.org/go/cue/inject/embed" "cuelang.org/go/internal/core/runtime" - "cuelang.org/go/internal/encoding" - "cuelang.org/go/internal/filetypes" - "cuelang.org/go/internal/value" - pkgpath "cuelang.org/go/pkg/path" ) -// TODO: record files in build.Instance - -// interpreter is a [cuecontext.ExternInterpreter] for embedded files. -type interpreter struct{} - -// Note that [cuecontext.ExternInterpreter] is just an alias for [runtime.Interpreter] -// but because the [cuelang.org/go/cue/cuecontext] package depends on embedding -// we cannot refer to that type directly. - -// New returns a new interpreter for embedded files as a -// [cuelang.org/go/cue/cuecontext.ExternInterpreter] suitable for -// passing to [cuelang.org/go/cue/cuecontext.New]. -func New() runtime.Interpreter { - return interpreter{} -} - -func (i interpreter) Kind() string { - return EmbedKind -} - -const EmbedKind = "embed" - -// NewCompiler returns a compiler that can decode and embed files that exist -// within a CUE module. -func (i interpreter) NewCompiler(b *build.Instance, r *runtime.Runtime) (runtime.Compiler, errors.Error) { - if b.Module == "" { - return nil, errors.Newf(token.Pos{}, "cannot embed files when not in a module") - } - if b.Root == "" { - return nil, errors.Newf(token.Pos{}, "cannot embed files: no module root found") - } - return &compiler{ - b: b, - runtime: (*cue.Context)(r), - }, nil -} - -// A compiler is a [runtime.Compiler] that allows embedding files into CUE -// values. -type compiler struct { - b *build.Instance - runtime *cue.Context - opCtx *adt.OpContext - - // file system cache - dir string - fs iofs.StatFS - pos token.Pos -} - -// validateAttr performs logical validation of the attr. It does not -// perform any checks against a filesystem. -func validateAttr(a *internal.Attr) (file, glob, typ string, allowEmptyGlob bool, errs errors.Error) { - if a.Err != nil { - return "", "", "", false, a.Err - } - - pos := a.Pos - - file, _, err := a.Lookup(0, "file") - if err != nil { - return "", "", "", false, errors.Promote(err, "invalid attribute") - } - - glob, _, err = a.Lookup(0, "glob") - if err != nil { - return "", "", "", false, errors.Promote(err, "invalid attribute") - } - - typ, _, err = a.Lookup(0, "type") - if err != nil { - return "", "", "", false, errors.Promote(err, "invalid type argument") - } - - allowEmptyGlob, err = a.Flag(0, "allowEmptyGlob") - if err != nil { - return "", "", "", false, errors.Promote(err, "invalid allowEmptyGlob argument") - } - - switch { - case file == "" && glob == "": - return "", "", "", false, errors.Newf(pos, "attribute must have file or glob field") - - case file != "" && glob != "": - return "", "", "", false, errors.Newf(pos, "attribute cannot have both file and glob field") - case allowEmptyGlob && glob == "": - return "", "", "", false, errors.Newf(pos, "allowEmptyGlob must be specified with a glob field") - - case file != "": - file, err := clean(pos, file) - if err != nil { - return "", "", "", false, err - } - return file, "", typ, allowEmptyGlob, nil - - default: - glob, err := clean(pos, glob) - if err != nil { - return "", "", "", false, err - } - - // Validate that the glob pattern is valid per [pkgpath.Match]. - // Note that we use Unix match semantics because all embed paths are Unix-like. - if _, err := pkgpath.Match(glob, "", pkgpath.Unix); err != nil { - return "", "", "", false, errors.Wrapf(err, pos, "invalid glob pattern %q", glob) - } - - // If we do not have a type, ensure the extension of the base is fully - // specified, i.e. does not contain any meta characters as specified by - // path.Match. - if typ == "" { - ext := path.Ext(path.Base(glob)) - if ext == "" || strings.ContainsAny(ext, "*?[\\") { - return "", "", "", false, errors.Newf(pos, "extension not fully specified; type argument required") - } - } - return "", glob, typ, allowEmptyGlob, nil - } -} - -// Compile interprets an embed attribute to either load a file -// (@embed(file=...)) or a glob of files (@embed(glob=...)). -// and decodes the given files. -func (c *compiler) Compile(funcName string, scope adt.Value, a *internal.Attr) (adt.Expr, errors.Error) { - c.opCtx = adt.NewContext((*runtime.Runtime)(c.runtime), nil) - - pos := a.Pos - c.pos = pos - - // Jump through some hoops to get file operations to behave the same for - // Windows and Unix. - // TODO: obtain an iofs.FS from load or something similar. - dir := filepath.Dir(pos.File().Name()) - if c.dir != dir { - c.fs = os.DirFS(dir).(iofs.StatFS) // Documented as implementing iofs.StatFS - c.dir = dir - } - - file, glob, typ, allowEmptyGlob, err := validateAttr(a) - if err != nil { - return nil, err - } - - if file != "" { - for dir := path.Dir(file); dir != "."; dir = path.Dir(dir) { - if _, err := c.fs.Stat(path.Join(dir, "cue.mod")); err == nil { - return nil, errors.Newf(pos, "cannot embed file %q: in different module", file) - } - } - return c.decodeFile(file, typ) - } - return c.processGlob(glob, typ, allowEmptyGlob) -} - -func (c *compiler) processGlob(glob, scope string, allowEmptyGlob bool) (adt.Expr, errors.Error) { - m := &adt.StructLit{} - - matches, err := fsGlob(c.fs, glob) - if err != nil { - return nil, errors.Promote(err, "failed to match glob") - } - if len(matches) == 0 && !allowEmptyGlob { - return nil, errors.Newf(c.pos, "no matches for glob pattern %q", glob) - } - - dirs := make(map[string]string) - for _, f := range matches { - // TODO: lots of stat calls happening in this MVP so another won't hurt. - // We don't support '**' initially, and '*' only matches files, so skip - // any directories. - if fi, err := c.fs.Stat(f); err != nil { - return nil, errors.Newf(c.pos, "failed to stat %s: %v", f, err) - } else if fi.IsDir() { - continue - } - // Add all parents of the embedded file that - // aren't the current directory (if there's a cue.mod - // in the current directory, that's the current module - // not nested). - for dir := path.Dir(f); dir != "."; dir = path.Dir(dir) { - dirs[dir] = f - } - - expr, err := c.decodeFile(f, scope) - if err != nil { - return nil, err - } - - m.Decls = append(m.Decls, &adt.Field{ - Label: c.opCtx.StringLabel(f), - Value: expr, - }) - } - // Check that none of the matches were in a nested module - // directory. - for dir, f := range dirs { - if _, err := c.fs.Stat(path.Join(dir, "cue.mod")); err == nil { - return nil, errors.Newf(c.pos, "cannot embed file %q: in different module", f) - } - } - return m, nil -} - -func clean(pos token.Pos, s string) (string, errors.Error) { - file := path.Clean(s) - if file != s { - return file, errors.Newf(pos, "path not normalized, use %q instead", file) - } - if path.IsAbs(file) { - return "", errors.Newf(pos, "only relative files are allowed") - } - if file == ".." || strings.HasPrefix(file, "../") { - return "", errors.Newf(pos, "cannot refer to parent directory") - } - return file, nil -} - -// fsGlob is like [iofs.Glob] but only includes dot-prefixed files -// when the dot is explictly present in an element. -// TODO: add option for including dot files? -func fsGlob(fsys iofs.FS, pattern string) ([]string, error) { - pattern = path.Clean(pattern) - matches, err := iofs.Glob(fsys, pattern) - if err != nil { - return nil, err - } - return filterFsGlobResults(pattern, matches...), nil -} - -// filterFsGlobResults applies additional filtering on the given -// matches to only include dot-prefixed files when the dot is -// explictly present in the corresponding pattern element. -func filterFsGlobResults(pattern string, matches ...string) []string { - patElems := strings.Split(pattern, "/") - included := func(m string) bool { - for i, elem := range strings.Split(m, "/") { - // Technically there should never be more elements in m than - // there are in patElems, but be defensive and check bounds just in case. - if strings.HasPrefix(elem, ".") && (i >= len(patElems) || !strings.HasPrefix(patElems[i], ".")) { - return false - } - } - return true - } - - i := 0 - for _, m := range matches { - if included(m) { - matches[i] = m - i++ - } - } - return matches[:i] -} - -func (c *compiler) decodeFile(file, scope string) (adt.Expr, errors.Error) { - // Do not use the most obvious filetypes.Input in order to disable "auto" - // mode. - f, err := filetypes.ParseFileAndType(file, scope, filetypes.Def) - if err != nil { - return nil, errors.Promote(err, "invalid file type") - } - - // Open and pre-load the file system using iofs.FS. - r, err := c.fs.Open(file) - if err != nil { - return nil, errors.Newf(c.pos, "open %v: no such file or directory", file) - } - defer r.Close() - - info, err := r.Stat() - if err != nil { - return nil, errors.Promote(err, "failed to decode file") - } - if info.IsDir() { - return nil, errors.Newf(c.pos, "cannot embed directories") - } - f.Source = r - - // TODO: this really should be done at the start of the build process. - // c.b.ExternFiles = append(c.b.ExternFiles, f) - - config := &encoding.Config{ - // TODO: schema is currently the wrong schema, which is a bug in - // internal/core/runtime. There is also an outstanding design choice: - // do we imply the schema from the schema of the current field, or do - // we explicitly enable schema-based encoding with a "schema" argument. - // In the case of YAML it seems to be better to be explicit. In the case - // of textproto it seems to be more convenient to do it implicitly. - // Schema: value.Make(c.opCtx, schema), - } - - d := encoding.NewDecoder(c.runtime, f, config) - if err := d.Err(); err != nil { - return nil, errors.Promote(err, "failed to decode file") - } - - defer d.Close() - - n := d.File() - - if d.Next(); !d.Done() { - // TODO: support streaming values - return nil, errors.Newf(c.pos, "streaming not implemented: found more than one value in file") - } - - // TODO: each of these encodings should probably be supported in the future - switch f.Encoding { - case build.CUE: - return nil, errors.Newf(c.pos, "encoding %q not (yet) supported", f.Encoding) - case build.JSONL: - return nil, errors.Newf(c.pos, "encoding %q not (yet) supported: requires support for streaming", f.Encoding) - case build.BinaryProto, build.TextProto: - return nil, errors.Newf(c.pos, "encoding %q not (yet) supported: requires support for schema-guided decoding", f.Encoding) - } - - val := c.runtime.BuildFile(n) - if err := val.Err(); err != nil { - return nil, errors.Promote(err, "failed to build file") - } - - _, v := value.ToInternal(val) - return v, nil -} - -// EmbeddedPaths walks all the embed attributes in the given file, -// returning a slice of [Embed] structs for attributes -// that were successfully validated, and errors for those which were -// not. The filepath should be the filepath of the file from which -// these attributes were extracted, and relative to whatever root is -// going to be used in calls to [Embed.Matches] and [Embed.FindAll]. -func EmbeddedPaths(filepath string, syntax *ast.File) ([]*Embed, errors.Error) { - extAttrs, err := runtime.ExternAttrsForFile(syntax) - if err != nil { - return nil, err - } - if extAttrs.TopLevel[EmbedKind] == nil { - return nil, nil - } - var errs errors.Error - var embeds []*Embed - for attr := range extAttrs.Body { - if attr.Attr.Name != EmbedKind { - continue - } - if err := attr.Attr.Err; err != nil { - errs = errors.Append(errs, err) - continue - } - file, glob, typ, allowEmptyGlob, err := validateAttr(attr.Attr) - if err != nil { - errs = errors.Append(errs, err) - continue - } - embed := &Embed{ - Node: attr.Parent, - Attribute: attr.Attr, - FilePath: filepath, - Type: typ, - } - if file != "" { - embed.interpreter = &embeddedFile{ - filepath: file, - } - embeds = append(embeds, embed) - } else if glob != "" { - embed.interpreter = &embeddedGlob{ - glob: glob, - allowEmptyGlob: allowEmptyGlob, - } - embeds = append(embeds, embed) - } - } - return embeds, errs -} - -type Embed struct { - Node ast.Node - Attribute *internal.Attr - FilePath string - Type string - interpreter embedInterpreter -} - -// Matches reports whether the provided filepath is matched by this -// [Embed] attribute. The filepath should be relative to the same root -// as the filepath provided to [EmbeddedPaths]. E.g. if in -// `/wibble/foo/bar.cue` you have `@embed(filename=a/b.json)`, and -// `foo/bar.cue` is the filepath passed to [EmbeddedPaths], then -// [Embed.Matches] will return true if called with `foo/a/b.json`. -func (e *Embed) Matches(filepath string) bool { - return e.interpreter.matches(e, filepath) -} - -// FindAll uses the provided fs to report all the filepaths that -// match this [Embed] attribute. The fs must be relative to the same -// root as the filepath provided to [EmbeddedPaths]. I.e. for the -// filepath provided to [EmbeddedPaths], iofs.Stat(fs, filepath) -// should be accessing the same file which contained this [Embed] -// attribute. -func (e *Embed) FindAll(fs iofs.FS) ([]string, error) { - return e.interpreter.findAll(e, fs) -} - -// IsGlob reports whether this [Embed] attribute represents a glob -// embedding. -func (e *Embed) IsGlob() bool { - _, isGlob := e.interpreter.(*embeddedGlob) - return isGlob -} - -type embedInterpreter interface { - // NB: All filepaths (including any within the Embed) are - // considered relative to the same root. - - matches(e *Embed, filepath string) bool - findAll(e *Embed, fs iofs.FS) ([]string, error) -} - -type embeddedFile struct { - filepath string -} - -func (ef *embeddedFile) matches(e *Embed, filepath string) bool { - dir := path.Dir(e.FilePath) - return filepath == path.Join(dir, ef.filepath) -} - -func (ef *embeddedFile) findAll(e *Embed, fs iofs.FS) ([]string, error) { - dir := path.Dir(e.FilePath) - filepath := path.Join(dir, ef.filepath) - info, err := iofs.Stat(fs, filepath) - if err != nil { - return nil, errors.Wrapf(err, e.Attribute.Pos, "failed to stat %s: %v", filepath, err) - } - if info.IsDir() { - return nil, errors.Newf(e.Attribute.Pos, "%v is a directory", filepath) - } - return []string{filepath}, nil -} - -type embeddedGlob struct { - glob string - allowEmptyGlob bool -} - -func (eg *embeddedGlob) matches(e *Embed, filepath string) bool { - dir := path.Dir(e.FilePath) - if dir != "." { - wasCut := false - filepath, wasCut = strings.CutPrefix(filepath, dir+"/") - if !wasCut { - return false - } - } - result, err := pkgpath.Match(eg.glob, filepath, pkgpath.Unix) - if !result || err != nil { - return false - } - return len(filterFsGlobResults(eg.glob, filepath)) == 1 -} - -func (eg *embeddedGlob) findAll(e *Embed, fs iofs.FS) ([]string, error) { - dir := path.Dir(e.FilePath) - fs, err := iofs.Sub(fs, dir) - if err != nil { - return nil, errors.Wrapf(err, e.Attribute.Pos, "%v", err) - } - filepaths, err := fsGlob(fs, eg.glob) - if err != nil { - return nil, errors.Wrapf(err, e.Attribute.Pos, "%v", err) - } - if !eg.allowEmptyGlob && len(filepaths) == 0 { - return nil, errors.Newf(e.Attribute.Pos, "no matches for glob pattern %q", eg.glob) - } - for i, filepath := range filepaths { - filepaths[i] = path.Join(dir, filepath) - } - return filepaths, nil +// Deprecated: use [embed.New]. +// +//go:fix inline +func New() runtime.Injection { + return embed.New() } diff --git a/internal/ci/github/trybot.cue b/internal/ci/github/trybot.cue index 91ad8f9bc..8a875e2b3 100644 --- a/internal/ci/github/trybot.cue +++ b/internal/ci/github/trybot.cue @@ -215,6 +215,6 @@ workflows: trybot: _repo.bashWorkflow & { name: "Test with -tags=cuewasm" // The wasm interpreter is only bundled into cmd/cue with the cuewasm build tag. // Test the related packages with the build tag enabled as well. - run: "go test -tags cuewasm ./cmd/cue/cmd ./cue/interpreter/wasm" + run: "go test -tags cuewasm ./cmd/cue/cmd ./cue/inject/wasm" } } diff --git a/internal/core/compile/compile.go b/internal/core/compile/compile.go index f7526c315..8dd4928b4 100644 --- a/internal/core/compile/compile.go +++ b/internal/core/compile/compile.go @@ -1121,9 +1121,9 @@ func (c *compiler) expr(expr ast.Expr) adt.Expr { case *ast.Func: // We don't yet support function types natively in // CUE. ast.Func exists only to support external - // interpreters. Function values (really, adt.Builtin) + // injections. Function values (really, adt.Builtin) // are only created by the runtime, or injected by - // external interpreters. + // external injections. // // TODO: revise this when we add function types. return c.resolve(ast.NewIdent("_")) diff --git a/internal/core/runtime/extern.go b/internal/core/runtime/extern.go index 0af5010cf..f88c6ba3a 100644 --- a/internal/core/runtime/extern.go +++ b/internal/core/runtime/extern.go @@ -26,37 +26,35 @@ import ( "cuelang.org/go/internal/core/walk" ) -// SetInterpreter sets the interpreter for interpretation of files marked with -// @extern(kind). -func (r *Runtime) SetInterpreter(i Interpreter) { - if r.interpreters == nil { - r.interpreters = map[string]Interpreter{} +// SetInjection sets the injection value to be used for injection +// of values with an @extern(kind) attribute where kind is i.Kind(). +func (r *Runtime) SetInjection(i Injection) { + if r.injections == nil { + r.injections = map[string]Injection{} } - r.interpreters[i.Kind()] = i + r.injections[i.Kind()] = i } -// TODO: consider also passing the top-level attribute to NewCompiler to allow -// passing default values. +// Injection defines an entrypoint for creating per-instance injectors. +type Injection interface { + // InjectorForInstance returns a new injector for the + // given build instance. + InjectorForInstance(b *build.Instance, r *Runtime) (Injector, errors.Error) -// Interpreter defines an entrypoint for creating per-package interpreters. -type Interpreter interface { - // NewCompiler creates a compiler for b and reports any errors. - NewCompiler(b *build.Instance, r *Runtime) (Compiler, errors.Error) - - // Kind returns the string to be used in the file-level @extern attribute. + // Kind returns the @extern kind for this injection, + // for example "embed" for @extern(embed). + // A given Injection instance should always return + // the same value. Kind() string } -// A Compiler fills in an adt.Expr for fields marked with `@extern(kind)`. -type Compiler interface { - // Compile creates an adt.Expr (usually a builtin) for the - // given external named resource (usually a function). name - // is the name of the resource to compile, taken from altName - // in `@extern(name=altName)`, or from the field name if that's - // not defined. Scope is the struct that contains the field. - // Other than "name", the fields in a are implementation - // specific. - Compile(name string, scope adt.Value, a *internal.Attr) (adt.Expr, errors.Error) +// An Injector fills in an adt.Expr for fields marked with `@extern(kind)`. +type Injector interface { + // InjectedValue returns a value to be unified at the position of + // the given external attribute. The scope argument + // holds the value of the instance before any injections + // have been unified into it. + InjectedValue(attr *ExternAttr, scope *adt.Vertex) (adt.Expr, errors.Error) } // InjectImplementations modifies v to include implementations of functions @@ -82,22 +80,23 @@ func (r *Runtime) InjectImplementations(b *build.Instance, v *adt.Vertex) (errs return d.errs } -// externDecorator locates extern attributes and calls the relevant interpreters -// to inject builtins. +// externDecorator locates extern attributes and calls the relevant injectors +// to inject values. type externDecorator struct { runtime *Runtime pkg *build.Instance - compilers map[string]Compiler + injectors map[string]Injector - // fileKinds maps each AST file to the set of extern kinds declared in it. - fileKinds map[*token.File]map[string]bool + // fileKinds maps each AST file to the extern kinds declared in it, + // along with their file-level @extern attribute. + fileKinds map[*token.File]map[string]*internal.Attr errs errors.Error } // addFile finds injection points in the given ast.File for external -// implementations of Builtins. +// implementations. func (d *externDecorator) addFile(f *ast.File) (errs errors.Error) { kinds, _, err := findExternFileAttrs(f) if err != nil { @@ -108,16 +107,12 @@ func (d *externDecorator) addFile(f *ast.File) (errs errors.Error) { } if d.fileKinds == nil { - d.fileKinds = map[*token.File]map[string]bool{} - } - km := make(map[string]bool) - for kind := range kinds { - km[kind] = true + d.fileKinds = map[*token.File]map[string]*internal.Attr{} } - d.fileKinds[f.Pos().File()] = km + d.fileKinds[f.Pos().File()] = kinds for kind, attr := range kinds { - if err := d.initCompiler(kind, attr.Pos); err != nil { + if err := d.initInjector(kind, attr.Pos); err != nil { errs = errors.Append(errs, err) } } @@ -163,7 +158,7 @@ loop: if k == "" { err = errors.Append(err, errors.Newf(attr.Pos, - "interpreter name must be non-empty")) + "injection name must be non-empty")) continue } @@ -208,25 +203,24 @@ loop: return kinds, f.Decls[p:], err } -// initCompiler initializes the runtime for kind, if applicable. The pos +// initInjector initializes the injector for kind, if applicable. The pos // argument represents the position of the file-level @extern attribute. -func (d *externDecorator) initCompiler(kind string, pos token.Pos) errors.Error { - if _, ok := d.compilers[kind]; ok { +func (d *externDecorator) initInjector(kind string, pos token.Pos) errors.Error { + if _, ok := d.injectors[kind]; ok { return nil } - // initialize the compiler. - if d.compilers == nil { - d.compilers = map[string]Compiler{} + if d.injectors == nil { + d.injectors = map[string]Injector{} } - x := d.runtime.interpreters[kind] + x := d.runtime.injections[kind] if x == nil { - return errors.Newf(pos, "no interpreter defined for %q", kind) + return errors.Newf(pos, "no injection defined for %q", kind) } - c, err := x.NewCompiler(d.pkg, d.runtime) + inj, err := x.InjectorForInstance(d.pkg, d.runtime) if err != nil { return err } - d.compilers[kind] = c + d.injectors[kind] = inj return nil } @@ -351,9 +345,8 @@ func (d *externDecorator) processNode(n adt.Node, scope *adt.Vertex) { continue } srcField := decl.Source().(*ast.Field) // We know all the above types come from ast.Field. - name, _, _ := ast.LabelName(srcField.Label) for _, attr := range srcField.Attrs { - if expr := d.externValue(attr, name, kinds, scope); expr != nil { + if expr := d.injectedValue(attr, srcField, kinds, scope); expr != nil { *valuePtr = &adt.BinaryExpr{ Op: adt.AndOp, X: *valuePtr, @@ -364,26 +357,30 @@ func (d *externDecorator) processNode(n adt.Node, scope *adt.Vertex) { } // Process embedded attributes. + var srcParent ast.Node var srcDecls []ast.Decl switch src := s.Src.(type) { case *ast.File: + srcParent = src srcDecls = src.Decls case *ast.StructLit: + srcParent = src srcDecls = src.Elts default: panic("unexpected type in adt.StructLit.Src") } for _, decl := range srcDecls { if attr, ok := decl.(*ast.Attribute); ok { - if expr := d.externValue(attr, "", kinds, scope); expr != nil { + if expr := d.injectedValue(attr, srcParent, kinds, scope); expr != nil { s.Decls = append(s.Decls, expr) } } } } -func (d *externDecorator) externValue(astAttr *ast.Attribute, name string, kinds map[string]bool, scope *adt.Vertex) adt.Expr { - if !kinds[astAttr.Name()] { +func (d *externDecorator) injectedValue(astAttr *ast.Attribute, parent ast.Node, kinds map[string]*internal.Attr, scope *adt.Vertex) adt.Expr { + topLevel := kinds[astAttr.Name()] + if topLevel == nil { return nil } attr := internal.ParseAttr(astAttr) @@ -391,14 +388,16 @@ func (d *externDecorator) externValue(astAttr *ast.Attribute, name string, kinds d.errs = errors.Append(d.errs, attr.Err) return nil } - c := d.compilers[attr.Name] - if c == nil { + inj := d.injectors[attr.Name] + if inj == nil { return nil } - if a, ok, _ := attr.Lookup(1, "name"); ok { - name = a + ea := &ExternAttr{ + TopLevel: topLevel, + Parent: parent, + Attr: attr, } - b, err := c.Compile(name, scope, attr) + b, err := inj.InjectedValue(ea, scope) if err != nil { d.errs = errors.Append(d.errs, errors.Wrap(errors.Newf(attr.Pos, "@%s", attr.Name), err)) return nil diff --git a/internal/core/runtime/extern_test.go b/internal/core/runtime/extern_test.go index c8bb9d32e..1ad0227fd 100644 --- a/internal/core/runtime/extern_test.go +++ b/internal/core/runtime/extern_test.go @@ -20,11 +20,11 @@ import ( "testing" "cuelang.org/go/cue" + "cuelang.org/go/cue/ast" "cuelang.org/go/cue/build" "cuelang.org/go/cue/cuecontext" "cuelang.org/go/cue/errors" "cuelang.org/go/cue/token" - "cuelang.org/go/internal" "cuelang.org/go/internal/core/adt" "cuelang.org/go/internal/core/runtime" "cuelang.org/go/internal/cuetxtar" @@ -38,8 +38,8 @@ func Test(t *testing.T) { } test.Run(t, func(t *cuetxtar.Test) { - interpreter := &interpreterFake{files: map[string]int{}} - ctx := cuecontext.New(cuecontext.Interpreter(interpreter)) + inj := &injectionFake{files: map[string]int{}} + ctx := cuecontext.New(cuecontext.WithInjection(inj)) b := t.Instance() v := ctx.BuildInstance(b) @@ -52,25 +52,26 @@ func Test(t *testing.T) { }) } -type interpreterFake struct { +type injectionFake struct { files map[string]int } -func (i *interpreterFake) Kind() string { return "testfn" } +func (i *injectionFake) Kind() string { return "testfn" } -func (i *interpreterFake) NewCompiler(b *build.Instance, r *runtime.Runtime) (runtime.Compiler, errors.Error) { +func (i *injectionFake) InjectorForInstance(b *build.Instance, r *runtime.Runtime) (runtime.Injector, errors.Error) { switch b.PkgName { case "failinit": return nil, errors.Newf(token.NoPos, "TEST: fail initialization") case "nullinit": return nil, nil case "scopetest": - return newCompilerFake(b, r) + return newInjectorFake(b, r) } return i, nil } -func (i *interpreterFake) Compile(funcName string, _ adt.Value, a *internal.Attr) (adt.Expr, errors.Error) { +func (i *injectionFake) InjectedValue(attr *runtime.ExternAttr, _ *adt.Vertex) (adt.Expr, errors.Error) { + a := attr.Attr if ok, _ := a.Flag(1, "fail"); ok { return nil, errors.Newf(token.NoPos, "TEST: fail compilation") } @@ -92,6 +93,16 @@ func (i *interpreterFake) Compile(funcName string, _ adt.Value, a *internal.Attr i.files[str] = len(i.files) + 1 } + // Derive the function name from the parent field label, + // with an explicit name= attribute taking precedence. + var funcName string + if f, ok := attr.Parent.(*ast.Field); ok { + funcName, _, _ = ast.LabelName(f.Label) + } + if name, ok, _ := a.Lookup(1, "name"); ok { + funcName = name + } + return &adt.Builtin{ Name: "impl" + funcName + strconv.Itoa(i.files[str]), Params: []adt.Param{{Value: &adt.BasicType{K: adt.IntKind}}}, @@ -99,19 +110,20 @@ func (i *interpreterFake) Compile(funcName string, _ adt.Value, a *internal.Attr }, nil } -type compilerFake struct { +type injectorFake struct { runtime *runtime.Runtime b *build.Instance } -func newCompilerFake(b *build.Instance, r *runtime.Runtime) (runtime.Compiler, errors.Error) { - return &compilerFake{ +func newInjectorFake(b *build.Instance, r *runtime.Runtime) (runtime.Injector, errors.Error) { + return &injectorFake{ runtime: r, b: b, }, nil } -func (c *compilerFake) Compile(name string, scope adt.Value, a *internal.Attr) (adt.Expr, errors.Error) { +func (c *injectorFake) InjectedValue(attr *runtime.ExternAttr, scope *adt.Vertex) (adt.Expr, errors.Error) { + a := attr.Attr typStr, err := a.String(0) if err != nil { return nil, errors.Promote(err, "test") diff --git a/internal/core/runtime/runtime.go b/internal/core/runtime/runtime.go index 0f18c7360..b961c4851 100644 --- a/internal/core/runtime/runtime.go +++ b/internal/core/runtime/runtime.go @@ -28,9 +28,9 @@ type Runtime struct { loaded map[*build.Instance]interface{} - // interpreters implement extern functionality. The map key corresponds to + // injections implement extern functionality. The map key corresponds to // the kind in a file-level @extern(kind) attribute. - interpreters map[string]Interpreter + injections map[string]Injection version internal.EvaluatorVersion diff --git a/internal/core/runtime/testdata/errors.txtar b/internal/core/runtime/testdata/errors.txtar index 755b3eb8b..6ea38bfeb 100644 --- a/internal/core/runtime/testdata/errors.txtar +++ b/internal/core/runtime/testdata/errors.txtar @@ -93,11 +93,11 @@ duplicate @extern attribute for kind "testfn": ./double_extern_b.cue:2:1 duplicate @extern attribute for kind "testfn": ./double_extern_b.cue:3:1 -interpreter name must be non-empty: +injection name must be non-empty: ./empty_extern.cue:1:1 -no interpreter defined for "\"testfn\" foo": +no injection defined for "\"testfn\" foo": ./invalid_file_attr.cue:1:1 extern attribute must appear before package clause: ./late_extern.cue:3:1 -no interpreter defined for "wazem": +no injection defined for "wazem": ./unknown_interpreter.cue:1:1 diff --git a/internal/lsp/cache/package.go b/internal/lsp/cache/package.go index 297af0f26..62227d562 100644 --- a/internal/lsp/cache/package.go +++ b/internal/lsp/cache/package.go @@ -21,7 +21,7 @@ import ( "strings" "cuelang.org/go/cue/ast" - "cuelang.org/go/cue/interpreter/embed" + "cuelang.org/go/cue/inject/embed" "cuelang.org/go/cue/token" "cuelang.org/go/internal/golangorgx/gopls/protocol" "cuelang.org/go/internal/lsp/eval" diff --git a/internal/lsp/eval/eval.go b/internal/lsp/eval/eval.go index 7a12d0956..be5c9758b 100644 --- a/internal/lsp/eval/eval.go +++ b/internal/lsp/eval/eval.go @@ -339,7 +339,7 @@ import ( "strings" "cuelang.org/go/cue/ast" - "cuelang.org/go/cue/interpreter/embed" + "cuelang.org/go/cue/inject/embed" "cuelang.org/go/cue/token" "cuelang.org/go/internal/golangorgx/gopls/protocol" "cuelang.org/go/internal/lsp/fscache"