diff --git a/encoding/jsonschema/constraints.go b/encoding/jsonschema/constraints.go new file mode 100644 index 000000000..c4c5add2d --- /dev/null +++ b/encoding/jsonschema/constraints.go @@ -0,0 +1,535 @@ +// Copyright 2019 CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package jsonschema + +import ( + "math/big" + "net/url" + "path" + "strings" + + "cuelang.org/go/cue" + "cuelang.org/go/cue/ast" + "cuelang.org/go/cue/token" +) + +type constraint struct { + key string + + // phase indicates on which pass c constraint should be added. This ensures + // that constraints are applied in the correct order. For instance, the + // "required" constraint validates that a listed field is contained in + // "properties". For this to work, "properties" must be processed before + // "required" and thus must have a lower phase number than the latter. + phase int + + // Indicates the draft number in which this constraint is defined. + draft int + fn constraintFunc +} + +// A constraintFunc converts a given JSON Schema constraint (specified in n) +// to a CUE constraint recorded in state. +type constraintFunc func(n cue.Value, s *state) + +func p0(name string, f constraintFunc) *constraint { + return &constraint{key: name, fn: f} +} + +func p0d(name string, draft int, f constraintFunc) *constraint { + return &constraint{key: name, draft: draft, fn: f} +} + +func p1(name string, f constraintFunc) *constraint { + return &constraint{key: name, phase: 1, fn: f} +} + +func p2(name string, f constraintFunc) *constraint { + return &constraint{key: name, phase: 2, fn: f} +} + +func combineSequence(name string, n cue.Value, s *state, op token.Token, f func(n cue.Value) ast.Expr) { + if n.Kind() != cue.ListKind { + s.errf(n, `value of %q must be an array, found %v`, name, n.Kind()) + } + var a ast.Expr + for _, n := range list(n) { + if a == nil { + a = f(n) + continue + } + a = ast.NewBinExpr(token.OR, a, f(n)) + } + if a == nil { + s.errf(n, `empty array for %q`, name) + return + } + s.add(a) +} + +// TODO: +// writeOnly, readOnly + +var constraintMap = map[string]*constraint{} + +func init() { + for _, c := range constraints { + constraintMap[c.key] = c + } +} + +func addDefinitions(n cue.Value, s *state) { + s.kind |= cue.StructKind + if n.Kind() != cue.StructKind { + s.errf(n, `"definitions" expected an object, found %v`, n.Kind) + } + + if len(s.path) != 1 { + s.errf(n, `"definitions" expected an object, found %v`, n.Kind) + } + + s.processMap(n, func(key string, n cue.Value) { + f := &ast.Field{ + Label: ast.NewString(s.path[len(s.path)-1]), + Token: token.ISA, + Value: s.schema(n), + } + f = &ast.Field{ + Label: ast.NewIdent(rootDefs), + Value: &ast.StructLit{Elts: []ast.Decl{f}}, + } + ast.SetRelPos(f, token.NewSection) + s.definitions = append(s.definitions, f) + }) +} + +var constraints = []*constraint{ + // Meta data. + + p0("$schema", func(n cue.Value, s *state) { + // Identifies this as a JSON schema and specifies its version. + // TODO: extract version. + s.jsonschema, _ = s.strValue(n) + }), + + p0("$id", func(n cue.Value, s *state) { + // URL: https://domain.com/schemas/foo.json + // Use Title(foo) as CUE identifier. + // anchors: #identifier + // + // TODO: mark identifiers. + s.id, _ = s.strValue(n) + }), + + // Generic constraint + + p0("type", func(n cue.Value, s *state) { + switch n.Kind() { + case cue.StringKind: + s.types = append(s.types, n) + case cue.ListKind: + for i, _ := n.List(); i.Next(); { + s.types = append(s.types, i.Value()) + } + default: + s.errf(n, `value of "type" must be a string or list of strings`) + } + }), + + p0("enum", func(n cue.Value, s *state) { + combineSequence("enum", n, s, token.OR, s.value) + s.typeOptional = true + }), + + p0d("const", 6, func(n cue.Value, s *state) { + s.add(s.value(n)) + }), + + p0("default", func(n cue.Value, s *state) { + s.default_ = s.value(n) + // must validate that the default is subsumed by the normal value, + // as CUE will otherwise broaden the accepted values with the default. + s.examples = append(s.examples, s.default_) + }), + + p0("deprecated", func(n cue.Value, s *state) { + if s.boolValue(n) { + s.deprecated = true + } + }), + + p0("examples", func(n cue.Value, s *state) { + if n.Kind() != cue.ListKind { + s.errf(n, `value of "examples" must be an array, found %v`, n.Kind) + } + for _, n := range list(n) { + s.examples = append(s.examples, s.schema(n)) + } + }), + + p0("description", func(n cue.Value, s *state) { + s.description, _ = s.strValue(n) + }), + + p0("title", func(n cue.Value, s *state) { + s.title, _ = s.strValue(n) + }), + + p0d("$comment", 7, func(n cue.Value, s *state) { + }), + + p0("$def", addDefinitions), + p0("definitions", addDefinitions), + p0("$ref", func(n cue.Value, s *state) { + if str, ok := s.strValue(n); ok { + u, err := url.Parse(str) + if err != nil { + s.add(s.errf(n, "invalid JSON reference: %s", err)) + return + } + + if u.Host != "" || u.Path != "" { + s.add(s.errf(n, "external references (%s) not supported", str)) + // TODO: handle + // host: + // If the host corresponds to a package known to cue, + // load it from there. It would prefer schema converted to + // CUE, although we could consider loading raw JSON schema + // if present. + // If not present, advise the user to run cue get. + // path: + // Look up on file system or relatively to authority location. + return + } + + if !path.IsAbs(u.Fragment) { + s.add(s.errf(n, "anchors (%s) not supported", u.Fragment)) + // TODO: support anchors + return + } + + // NOTE: Go bug?: url.URL has no raw representation of the fragment. + // This means that %2F gets translated to `/` before it can be + // split. This, in turn, means that field names cannot have a `/` + // as name. + a := strings.Split(u.Fragment[1:], "/") + if a[0] != "definitions" && a[0] != "$def" { + s.add(s.errf(n, "reference %q must resolve to definition", u.Fragment)) + return + } + s.add(ast.NewSel(ast.NewIdent(rootDefs), a[1:]...)) + + // TODO: technically, a references could reference a non-definition. + // In that case this will not resolve. We should detect cases that + // are not definitions and then resolve those as literal values. + } + }), + + // Combinators + + // TODO: work this out in more detail: oneOf and anyOf below have the same + // implementation in CUE. The distinction is that for anyOf a result is + // allowed to be ambiguous at the end, whereas for oneOf a disjunction must + // be fully resolved. There is currently no easy way to set this distinction + // in CUE. + // + // One could correctly write oneOf like this once 'not' is implemented: + // + // oneOf(a, b, c) :- + // anyOf( + // allOf(a, not(b), not(c)), + // allOf(not(a), b, not(c)), + // allOf(not(a), not(b), c), + // )) + // + // This is not necessary if the values are mutually exclusive/ have a + // discriminator. + + p0("allOf", func(n cue.Value, s *state) { + combineSequence("allOf", n, s, token.AND, s.schema) + }), + + p0("anyOf", func(n cue.Value, s *state) { + combineSequence("anyOf", n, s, token.OR, s.schema) + }), + + p0("oneOf", func(n cue.Value, s *state) { + combineSequence("allOf", n, s, token.OR, s.schema) + }), + + // String constraints + + p0("pattern", func(n cue.Value, s *state) { + s.kind |= cue.StringKind + s.add(&ast.UnaryExpr{Op: token.MAT, X: s.string(n)}) + }), + + p0d("contentMediaType", 7, func(n cue.Value, s *state) { + s.kind |= cue.StringKind + }), + + p0d("contentEncoding", 7, func(n cue.Value, s *state) { + s.kind |= cue.StringKind + // 7bit, 8bit, binary, quoted-printable and base64. + // RFC 2054, part 6.1. + // https://tools.ietf.org/html/rfc2045 + // TODO: at least handle bytes. + }), + + // Number constraints + + p0("minimum", func(n cue.Value, s *state) { + s.kind |= cue.NumberKind + s.add(&ast.UnaryExpr{Op: token.GEQ, X: s.number(n)}) + }), + + p0("exclusiveMinimum", func(n cue.Value, s *state) { + // TODO: should we support Draft 4 booleans? + s.kind |= cue.NumberKind + s.add(&ast.UnaryExpr{Op: token.GTR, X: s.number(n)}) + }), + + p0("maximum", func(n cue.Value, s *state) { + s.kind |= cue.NumberKind + s.add(&ast.UnaryExpr{Op: token.LEQ, X: s.number(n)}) + }), + + p0("exclusiveMaximum", func(n cue.Value, s *state) { + // TODO: should we support Draft 4 booleans? + s.kind |= cue.NumberKind + s.add(&ast.UnaryExpr{Op: token.LSS, X: s.number(n)}) + }), + + p0("multipleOf", func(n cue.Value, s *state) { + s.kind |= cue.NumberKind + multiple := s.number(n) + var x big.Int + _, _ = n.MantExp(&x) + if x.Cmp(big.NewInt(0)) != 1 { + s.errf(n, `"multipleOf" value must be < 0; found %s`, n) + } + math := s.addImport("math") + s.add(ast.NewCall(ast.NewSel(math, "MultipleOf"), multiple)) + }), + + // Object constraints + + p0("properties", func(n cue.Value, s *state) { + s.kind |= cue.StructKind + if s.obj == nil { + s.obj = &ast.StructLit{} + } + if n.Kind() != cue.StructKind { + s.errf(n, `"properties" expected an object, found %v`, n.Kind) + } + + s.processMap(n, func(key string, n cue.Value) { + // property?: value + expr, state := s.schemaState(n) + f := &ast.Field{Label: ast.NewString(key), Value: expr} + state.doc(f) + f.Optional = token.Blank.Pos() + if len(s.obj.Elts) > 0 && len(f.Comments()) > 0 { + // TODO: change formatter such that either a a NewSection on the + // field or doc comment will cause a new section. + ast.SetRelPos(f.Comments()[0], token.NewSection) + } + s.obj.Elts = append(s.obj.Elts, f) + }) + }), + + p1("required", func(n cue.Value, s *state) { + s.kind |= cue.StructKind + if s.obj == nil { + s.errf(n, `"required" without a "properties" field`) + } + if n.Kind() != cue.ListKind { + s.errf(n, `value of "required" must be list of strings, found %v`, n.Kind) + } + + // Create field map + fields := map[string]*ast.Field{} + for _, d := range s.obj.Elts { + f := d.(*ast.Field) + str, _, err := ast.LabelName(f.Label) + if err == nil { + fields[str] = f + } + } + + for _, n := range list(n) { + str, ok := s.strValue(n) + f := fields[str] + if f == nil && ok { + s.errf(n, "required field %q not in properties", str) + continue + } + if f.Optional == token.NoPos { + s.errf(n, "duplicate required field %q", str) + } + f.Optional = token.NoPos + } + }), + + p0d("propertyNames", 6, func(n cue.Value, s *state) { + s.kind |= cue.StructKind + // [=~pattern]: _ + s.add(&ast.StructLit{Elts: []ast.Decl{&ast.Field{ + Label: ast.NewList(s.schema(n)), + Value: ast.NewIdent("_"), + }}}) + }), + + p0("minProperties", func(n cue.Value, s *state) { + s.kind |= cue.StructKind + pkg := s.addImport("struct") + s.add(ast.NewCall(ast.NewSel(pkg, "MinFields"), s.uint(n))) + }), + + p0("maxProperties", func(n cue.Value, s *state) { + s.kind |= cue.StructKind + pkg := s.addImport("struct") + s.add(ast.NewCall(ast.NewSel(pkg, "MaxFields"), s.uint(n))) + }), + + p0("dependencies", func(n cue.Value, s *state) { + s.kind |= cue.StructKind + // Schema and property dependencies. + // TODO: the easiest implementation is with comprehensions. + // The nicer implementation is with disjunctions. This has to be done + // at the very end, replacing properties. + /* + *{ property?: _|_ } | { + property: _ + schema + } + */ + }), + + p1("patternProperties", func(n cue.Value, s *state) { + s.kind |= cue.StructKind + if n.Kind() != cue.StructKind { + s.errf(n, `value of "patternProperties" must be an an object, found %v`, n.Kind) + } + if s.obj == nil { + s.obj = &ast.StructLit{} + } + existing := excludeFields(s.obj.Elts) + s.processMap(n, func(key string, n cue.Value) { + // [!~(properties) & pattern]: schema + s.patterns = append(s.patterns, + &ast.UnaryExpr{Op: token.NMAT, X: ast.NewString(key)}) + f := &ast.Field{ + Label: ast.NewList(ast.NewBinExpr(token.AND, + &ast.UnaryExpr{Op: token.MAT, X: ast.NewString(key)}, + existing)), + Value: s.schema(n), + } + ast.SetRelPos(f, token.NewSection) + s.obj.Elts = append(s.obj.Elts, f) + }) + }), + + p2("additionalProperties", func(n cue.Value, s *state) { + s.kind |= cue.StructKind + switch n.Kind() { + case cue.BoolKind: + s.closeStruct = !s.boolValue(n) + + case cue.StructKind: + s.closeStruct = true + if s.obj == nil { + s.obj = &ast.StructLit{} + } + if len(s.obj.Elts) == 0 { + s.obj.Elts = append(s.obj.Elts, &ast.Field{ + Label: ast.NewList(ast.NewIdent("string")), + Value: s.schema(n), + }) + return + } + // [!~(properties|patternProperties)]: schema + existing := append(s.patterns, excludeFields(s.obj.Elts)) + f := &ast.Field{ + Label: ast.NewList(ast.NewBinExpr(token.AND, existing...)), + Value: s.schema(n), + } + ast.SetRelPos(f, token.NewSection) + s.obj.Elts = append(s.obj.Elts, f) + + default: + s.errf(n, `value of "additionalProperties" must be an object or boolean`) + } + }), + + // Array constraints. + + p0("items", func(n cue.Value, s *state) { + s.kind |= cue.ListKind + if s.list != nil { + s.errf(n, `"items" declared more than once, previous declaration at %s`, s.list.Pos()) + } + switch n.Kind() { + case cue.StructKind: + elem := s.schema(n) + ast.SetRelPos(elem, token.NoRelPos) + s.add(ast.NewList(&ast.Ellipsis{Type: elem})) + + case cue.ListKind: + var a []ast.Expr + for _, n := range list(n) { + v := s.schema(n) + ast.SetRelPos(v, token.NoRelPos) + a = append(a, v) + } + s.add(ast.NewList(a...)) + + default: + s.errf(n, `value of "items" must be an object or array`) + } + }), + + p0("contains", func(n cue.Value, s *state) { + s.kind |= cue.ListKind + list := s.addImport("list") + // TODO: Passing non-concrete values is not yet supported in CUE. + s.add(ast.NewCall(ast.NewSel(list, "Contains"), clearPos(s.schema(n)))) + }), + + p0("minItems", func(n cue.Value, s *state) { + s.kind |= cue.ListKind + list := s.addImport("list") + s.add(ast.NewCall(ast.NewSel(list, "MinItems"), clearPos(s.uint(n)))) + }), + + p0("maxItems", func(n cue.Value, s *state) { + s.kind |= cue.ListKind + list := s.addImport("list") + s.add(ast.NewCall(ast.NewSel(list, "MaxItems"), clearPos(s.uint(n)))) + }), + + p0("uniqueItems", func(n cue.Value, s *state) { + s.kind |= cue.ListKind + if s.boolValue(n) { + list := s.addImport("list") + s.add(ast.NewCall(ast.NewSel(list, "UniqueItems"))) + } + }), +} + +func clearPos(e ast.Expr) ast.Expr { + ast.SetRelPos(e, token.NoRelPos) + return e +} diff --git a/encoding/jsonschema/decode.go b/encoding/jsonschema/decode.go new file mode 100644 index 000000000..dff0cc743 --- /dev/null +++ b/encoding/jsonschema/decode.go @@ -0,0 +1,427 @@ +// Copyright 2019 CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package jsonschema + +// TODO: +// - replace converter from YAML to CUE to CUE (schema) to CUE. +// - define OpenAPI definitions als CUE. + +import ( + "fmt" + "path" + "sort" + "strings" + + "cuelang.org/go/cue" + "cuelang.org/go/cue/ast" + "cuelang.org/go/cue/errors" + "cuelang.org/go/cue/token" + "cuelang.org/go/internal" +) + +const rootDefs = "def" + +func decode(filename string, inst *cue.Instance) (*ast.File, errors.Error) { + d := &decoder{ + imports: map[string]*ast.Ident{}, + } + e := d.decode(filename, inst) + if d.errs != nil { + return nil, d.errs + } + return e, nil +} + +// A decoder converts JSON schema to CUE. +type decoder struct { + errs errors.Error + imports map[string]*ast.Ident + + definitions []ast.Decl +} + +// addImport registers +func (d *decoder) addImport(pkg string) *ast.Ident { + ident, ok := d.imports[pkg] + if !ok { + ident = ast.NewIdent(pkg) + d.imports[pkg] = ident + } + return ident +} + +func (d *decoder) decode(filename string, inst *cue.Instance) *ast.File { + root := state{decoder: d} + expr, state := root.schemaState(inst.Value()) + + var a []ast.Decl + + name := filename + if state.id != "" { + name = strings.Trim(name, "#") + } + pkgName := path.Base(name) + pkgName = pkgName[:len(pkgName)-len(path.Ext(pkgName))] + pkg := &ast.Package{Name: ast.NewIdent(pkgName)} + state.doc(pkg) + + a = append(a, pkg) + + var imports []string + for k := range d.imports { + imports = append(imports, k) + } + sort.Strings(imports) + + if len(imports) > 0 { + x := &ast.ImportDecl{} + for _, p := range imports { + x.Specs = append(x.Specs, ast.NewImport(nil, p)) + } + a = append(a, x) + } + + tags := []string{} + if state.jsonschema != "" { + tags = append(tags, fmt.Sprintf("schema=%q", state.jsonschema)) + } + if state.id != "" { + tags = append(tags, fmt.Sprintf("id=%q", state.id)) + } + if len(tags) > 0 { + a = append(a, addTag("Schema", "jsonschema", strings.Join(tags, ","))) + } + + if state.deprecated { + a = append(a, addTag("Schema", "deprecated", "")) + } + + f := &ast.Field{ + Label: ast.NewIdent("Schema"), + Value: expr, + } + + f.Token = token.ISA + a = append(a, f) + a = append(a, d.definitions...) + + return &ast.File{Decls: a} +} + +func (d *decoder) errf(n cue.Value, format string, args ...interface{}) ast.Expr { + d.warnf(n, format, args...) + return &ast.BadExpr{From: n.Pos()} +} + +func (d *decoder) warnf(n cue.Value, format string, args ...interface{}) { + d.errs = errors.Append(d.errs, errors.Newf(n.Pos(), format, args...)) +} + +func (d *decoder) number(n cue.Value) ast.Expr { + return n.Syntax().(ast.Expr) +} + +func (d *decoder) uint(n cue.Value) ast.Expr { + _, err := n.Uint64() + if err != nil { + d.errf(n, "invalid uint") + } + return n.Syntax().(ast.Expr) +} + +func (d *decoder) bool(n cue.Value) ast.Expr { + return n.Syntax().(ast.Expr) +} + +func (d *decoder) boolValue(n cue.Value) bool { + x, err := n.Bool() + if err != nil { + d.errf(n, "invalid bool") + } + return x +} + +func (d *decoder) string(n cue.Value) ast.Expr { + return n.Syntax().(ast.Expr) +} + +func (d *decoder) strValue(n cue.Value) (s string, ok bool) { + s, err := n.String() + if err != nil { + d.errf(n, "invalid string") + return "", false + } + return s, true +} + +// const draftCutoff = 5 + +type state struct { + *decoder + + path []string + + pos cue.Value + + types []cue.Value + typeOptional bool + kind cue.Kind + + default_ ast.Expr + examples []ast.Expr + title string + description string + deprecated bool + jsonschema string + id string + + conjuncts []ast.Expr + + obj *ast.StructLit + closeStruct bool + patterns []ast.Expr + + list *ast.ListLit +} + +// finalize constructs a CUE type from the collected constraints. +func (s *state) finalize() (e ast.Expr) { + if s.typeOptional || s.kind != 0 { + if len(s.types) > 1 { + s.errf(s.pos, "constraints require specific type") + } + s.types = nil + } + + conjuncts := []ast.Expr{} + disjuncts := []ast.Expr{} + for _, n := range s.types { + add := func(e ast.Expr) { + disjuncts = append(disjuncts, setPos(e, n)) + } + str, ok := s.strValue(n) + if !ok { + s.errf(n, "type value should be a string") + return + } + switch str { + case "null": + // TODO: handle OpenAPI restrictions. + add(ast.NewIdent("null")) + case "boolean": + add(ast.NewIdent("bool")) + case "string": + add(ast.NewIdent("string")) + case "number": + add(ast.NewIdent("number")) + case "integer": + add(ast.NewIdent("int")) + case "array": + if s.kind&cue.ListKind == 0 { + add(ast.NewList(&ast.Ellipsis{})) + } + case "object": + elps := &ast.Ellipsis{} + st := &ast.StructLit{Elts: []ast.Decl{elps}} + add(st) + default: + s.errf(n, "unknown type %q", n) + } + } + if len(disjuncts) > 0 { + conjuncts = append(conjuncts, ast.NewBinExpr(token.OR, disjuncts...)) + } + + conjuncts = append(conjuncts, s.conjuncts...) + + if s.obj != nil { + if !s.closeStruct { + s.obj.Elts = append(s.obj.Elts, &ast.Ellipsis{}) + } + conjuncts = append(conjuncts, s.obj) + } + + if len(conjuncts) == 0 { + return ast.NewString(fmt.Sprint(s.pos)) + } + + e = ast.NewBinExpr(token.AND, conjuncts...) + + if s.default_ != nil { + // check conditions where default can be skipped. + switch x := s.default_.(type) { + case *ast.ListLit: + if s.kind == cue.ListKind && len(x.Elts) == 0 { + return e + } + } + e = ast.NewBinExpr(token.OR, e, &ast.UnaryExpr{Op: token.MUL, X: s.default_}) + } + return e +} + +func (s *state) doc(n ast.Node) { + // Create documentation. + doc := strings.TrimSpace(s.title) + if s.description != "" { + if doc != "" { + doc += "\n\n" + } + doc += s.description + doc = strings.TrimSpace(doc) + } + if doc != "" { + ast.SetComments(n, []*ast.CommentGroup{ + internal.NewComment(true, doc), + }) + } + + // TODO: add examples as well? +} + +func (s *state) add(e ast.Expr) { + s.conjuncts = append(s.conjuncts, e) +} + +func (s *state) schema(n cue.Value) ast.Expr { + expr, _ := s.schemaState(n) + // TODO: report unused doc. + return expr +} + +func (s *state) schemaState(n cue.Value) (ast.Expr, *state) { + state := &state{path: s.path, pos: n, decoder: s.decoder} + + if n.Kind() != cue.StructKind { + return s.errf(n, "schema expects mapping node, found %s", n.Kind()), state + } + + // do multiple passes over the constraints to ensure they are done in order. + for pass := 0; pass < 3; pass++ { + state.processMap(n, func(key string, value cue.Value) { + // Convert each constraint into a either a value or a functor. + c := constraintMap[key] + if c == nil { + if pass == 0 { + s.warnf(n, "unsupported constraint %q", key) + } + return + } + if c.phase == pass { + c.fn(value, state) + } + }) + } + + return state.finalize(), state +} + +func (s *state) value(n cue.Value) ast.Expr { + switch n.Kind() { + case cue.ListKind: + a := []ast.Expr{} + for i, _ := n.List(); i.Next(); { + a = append(a, s.value(i.Value())) + } + return setPos(ast.NewList(a...), n) + + case cue.StructKind: + a := []ast.Decl{} + s.processMap(n, func(key string, n cue.Value) { + a = append(a, &ast.Field{ + Label: ast.NewString(key), + Value: s.value(n), + }) + }) + a = append(a, &ast.Ellipsis{}) + return setPos(&ast.StructLit{Elts: a}, n) + + default: + if !n.IsConcrete() { + s.errf(n, "invalid non-concerte value") + } + return n.Syntax().(ast.Expr) + } +} + +// processMap processes a yaml node, expanding merges. +// +// TODO: in some cases we can translate merges into CUE embeddings. +// This may also prevent exponential blow-up (as may happen when +// converting YAML to JSON). +func (s *state) processMap(n cue.Value, f func(key string, n cue.Value)) { + saved := s.path + defer func() { s.path = saved }() + + // TODO: intercept references to allow for optimized performance. + for i, _ := n.Fields(); i.Next(); { + key := i.Label() + s.path = append(saved, key) + f(key, i.Value()) + } +} + +func list(n cue.Value) (a []cue.Value) { + for i, _ := n.List(); i.Next(); { + a = append(a, i.Value()) + } + return a +} + +// excludeFields returns a CUE expression that can be used to exclude the +// fields of the given declaration in a label expression. For instance, for +// +// { foo: 1, bar: int } +// +// it creates +// +// "^(foo|bar)$" +// +// which can be used in a label expression to define types for all fields but +// those existing: +// +// [!~"^(foo|bar)$"]: string +// +func excludeFields(decls []ast.Decl) ast.Expr { + var a []string + for _, d := range decls { + f, ok := d.(*ast.Field) + if !ok { + continue + } + str, _, _ := ast.LabelName(f.Label) + if str != "" { + a = append(a, str) + } + } + re := fmt.Sprintf("^(%s)$", strings.Join(a, "|")) + return &ast.UnaryExpr{Op: token.NMAT, X: ast.NewString(re)} +} + +func addTag(field, tag, value string) *ast.Field { + return &ast.Field{ + Label: ast.NewIdent(field), + Token: token.ISA, + Value: ast.NewIdent("_"), + Attrs: []*ast.Attribute{ + &ast.Attribute{Text: fmt.Sprintf("@%s(%s)", tag, value)}, + }, + } +} + +func setPos(e ast.Expr, v cue.Value) ast.Expr { + ast.SetPos(e, v.Pos()) + return e +} diff --git a/encoding/jsonschema/decode_test.go b/encoding/jsonschema/decode_test.go new file mode 100644 index 000000000..da64a24ef --- /dev/null +++ b/encoding/jsonschema/decode_test.go @@ -0,0 +1,122 @@ +// Copyright 2019 CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package jsonschema + +import ( + "bytes" + "flag" + "io/ioutil" + "os" + "path" + "path/filepath" + "strings" + "testing" + + "cuelang.org/go/cue" + "cuelang.org/go/cue/errors" + "cuelang.org/go/cue/format" + "cuelang.org/go/encoding/json" + "cuelang.org/go/encoding/yaml" + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + "golang.org/x/tools/txtar" +) + +var update = flag.Bool("update", false, "update the test files") + +// TestDecode reads the testdata/*.txtar files, converts the contained +// JSON schema to CUE and compares it against the output. +// +// Use the --update flag to update test files with the corresponding output. +func TestDecode(t *testing.T) { + err := filepath.Walk("testdata", func(fullpath string, info os.FileInfo, err error) error { + _ = err + if !strings.HasSuffix(fullpath, ".txtar") { + return nil + } + + t.Run(fullpath, func(t *testing.T) { + a, err := txtar.ParseFile(fullpath) + if err != nil { + t.Fatal(err) + } + + r := &cue.Runtime{} + var in *cue.Instance + var out, errout []byte + outIndex := -1 + + for i, f := range a.Files { + switch path.Ext(f.Name) { + case ".json": + in, err = json.Decode(r, f.Name, f.Data) + case ".yaml": + in, err = yaml.Decode(r, f.Name, f.Data) + case ".cue": + out = f.Data + outIndex = i + case ".err": + errout = f.Data + } + } + if err != nil { + t.Fatal(err) + } + + // TODO: use actual API. + expr, err := decode(fullpath, in) + if err != nil && errout == nil { + t.Fatal(errors.Details(err, nil)) + } + got := []byte(nil) + if err != nil { + got = []byte(err.Error()) + } + if !cmp.Equal(errout, got) { + t.Error(cmp.Diff(string(got), string(errout))) + } + + if expr != nil { + b, err := format.Node(expr, format.Simplify()) + if err != nil { + t.Fatal(err) + } + + // verify the generated CUE. + if _, err = r.Compile(fullpath, b); err != nil { + t.Fatal(errors.Details(err, nil)) + } + + b = bytes.TrimSpace(b) + out = bytes.TrimSpace(out) + + if !cmp.Equal(b, out) { + if *update { + a.Files[outIndex].Data = b + b = txtar.Format(a) + err = ioutil.WriteFile(fullpath, b, 0644) + if err != nil { + t.Fatal(err) + } + return + } + t.Error(cmp.Diff(string(out), string(b))) + } + } + }) + return nil + }) + assert.NoError(t, err) +} diff --git a/encoding/jsonschema/doc.go b/encoding/jsonschema/doc.go new file mode 100644 index 000000000..c0b813ce2 --- /dev/null +++ b/encoding/jsonschema/doc.go @@ -0,0 +1,19 @@ +// Copyright 2020 CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package jsonschema converts JSON Schema to CUE +// +// JSON Schema data is presented in CUE, so any of the supported encodings +// that can represent JSON Schema data can be used as a source. +package jsonschema diff --git a/encoding/jsonschema/testdata/basic.txtar b/encoding/jsonschema/testdata/basic.txtar new file mode 100644 index 000000000..0c1b255a2 --- /dev/null +++ b/encoding/jsonschema/testdata/basic.txtar @@ -0,0 +1,53 @@ +-- basic.json -- +{ + "$schema": "http://json-schema.org/draft-07/schema#", + + "type": "object", + "title": "Main schema", + "description": "Specify who you are and all.", + + "properties": { + "person": { + "description": "A person is a human being.", + "type": "object", + "required": [ "name" ], + "properties": { + "name": { "type": "string" }, + "address": { + "description": "where does this person live?", + "type": "string" + }, + "children": { + "type": "array", + "items": { "type": "string" }, + "default": [] + }, + "home phone": { + "type": "string", + "deprecated": true + } + } + } + } +} + +-- out.cue -- +// Main schema +// +// Specify who you are and all. +package basic + +Schema :: _ @jsonschema(schema="http://json-schema.org/draft-07/schema#") +Schema :: { + // A person is a human being. + person?: { + name: string + + // where does this person live? + address?: string + children?: [...string] + "home phone"?: string + ... + } + ... +} diff --git a/encoding/jsonschema/testdata/def.txtar b/encoding/jsonschema/testdata/def.txtar new file mode 100644 index 000000000..d5d92e904 --- /dev/null +++ b/encoding/jsonschema/testdata/def.txtar @@ -0,0 +1,63 @@ +// This test tests the conversion and ordering of definitions. + +-- definition.json -- +{ + "$schema": "http://json-schema.org/draft-07/schema#", + + "$id": "http://cuelang.org/go/encoding/openapi/testdata/order.json", + + "definitions": { + "address": { + "type": "object", + "properties": { + "street_address": { "type": "string" }, + "city": { "type": "string" }, + "state": { "type": "string" } + }, + "required": ["street_address", "city", "state"] + }, + "person": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "children": { + "type": "array", + "items": { "$ref": "#/definitions/person" }, + "default": [] + } + } + } + }, + + "type": "object", + + "properties": { + "person": { "$ref": "#/definitions/person" }, + "billing_address": { "$ref": "#/definitions/address" }, + "shipping_address": { "$ref": "#/definitions/address" } + } +} + +-- out.cue -- +package def + +Schema :: _ @jsonschema(schema="http://json-schema.org/draft-07/schema#",id="http://cuelang.org/go/encoding/openapi/testdata/order.json") +Schema :: { + person?: def.person + billing_address?: def.address + shipping_address?: def.address + ... +} + +def: address :: { + street_address: string + city: string + state: string + ... +} + +def: person :: { + name?: string + children?: [...def.person] + ... +} diff --git a/encoding/jsonschema/testdata/list.txtar b/encoding/jsonschema/testdata/list.txtar new file mode 100644 index 000000000..d3046dd8c --- /dev/null +++ b/encoding/jsonschema/testdata/list.txtar @@ -0,0 +1,36 @@ +-- list.yaml -- +type: object + +properties: + foo: + items: + type: string + + tuple: + items: + - type: string + - type: integer + - const: 2 + + has: + contains: + const: 3 + + size: + minItems: 3 + maxItems: 9 + uniqueItems: true + +additionalProperties: false + +-- out.cue -- +package list + +import "list" + +Schema :: { + foo?: [...string] + tuple?: [string, int, 2] + has?: list.Contains(3) + size?: list.MinItems(3) & list.MaxItems(9) & list.UniqueItems() +} diff --git a/encoding/jsonschema/testdata/num.txtar b/encoding/jsonschema/testdata/num.txtar new file mode 100644 index 000000000..5652628fd --- /dev/null +++ b/encoding/jsonschema/testdata/num.txtar @@ -0,0 +1,37 @@ +-- type.json -- +{ + "type": "object", + + "properties": { + "constant": { "const": 2 }, + "several": { + "enum": [ 1, 2, 3, 4 ] + }, + "inclusive": { + "type": "number", + "minimum": 2, + "maximum": 3 + }, + "exclusive": { + "exclusiveMinimum": 2, + "exclusiveMaximum": 3 + }, + "cents": { + "multipleOf": 0.05 + } + }, + "additionalProperties": false +} + +-- out.cue -- +package num + +import "math" + +Schema :: { + constant?: 2 + several?: 1 | 2 | 3 | 4 + inclusive?: >=2 & <=3 + exclusive?: >2 & <3 + cents?: math.MultipleOf(0.05) +} diff --git a/encoding/jsonschema/testdata/object.txtar b/encoding/jsonschema/testdata/object.txtar new file mode 100644 index 000000000..59bbbe359 --- /dev/null +++ b/encoding/jsonschema/testdata/object.txtar @@ -0,0 +1,103 @@ +-- type.json -- +{ + "type": "object", + "title": "Main schema", + + "properties": { + "fields" : { + "type": "object", + "minProperties": 3, + "maxProperties": 10, + "propertyNames": { + "pattern": "^\\P{Lu}" + } + }, + "additional": { + "type": "object", + "properties": { + "foo": { "type": "number" }, + "bar": { "type": "number" } + }, + "additionalProperties": { "type": "string" } + }, + "map": { + "type": "object", + "additionalProperties": { "type": "string" } + }, + "patterns": { + "type": "object", + "properties": { + "foo": { "type": "number" }, + "bar": { "type": "number" } + }, + "patternProperties": { + "^\\P{Lu}": { "type": "string" }, + "^\\P{Lo}": { "type": "integer" } + } + }, + "patternsNoProps": { + "type": "object", + "patternProperties": { + "^\\P{Lu}": { "type": "string" }, + "^\\P{Lo}": { "type": "integer" } + } + }, + "complex": { + "type": "object", + "properties": { + "foo": { "type": "number" }, + "bar": { "type": "number" } + }, + "patternProperties": { + "^\\P{Lu}": { "type": "string" }, + "^\\P{Lo}": { "type": "integer" } + }, + "additionalProperties": { "type": "string" } + } + }, + "additionalProperties": false +} + +-- out.cue -- +// Main schema +package object + +import "struct" + +Schema :: { + fields?: struct.MinFields(3) & struct.MaxFields(10) & { + [=~"^\\P{Lu}"]: _ + } + additional?: { + foo?: number + bar?: number + + [!~"^(foo|bar)$"]: string + } + map?: [string]: string + patterns?: { + foo?: number + bar?: number + + [=~"^\\P{Lu}" & !~"^(foo|bar)$"]: string + + [=~"^\\P{Lo}" & !~"^(foo|bar)$"]: int + ... + } + patternsNoProps?: { + [=~"^\\P{Lu}" & !~"^()$"]: string + + [=~"^\\P{Lo}" & !~"^()$"]: int + ... + } + complex?: { + foo?: number + bar?: number + + [=~"^\\P{Lu}" & !~"^(foo|bar)$"]: string + + [=~"^\\P{Lo}" & !~"^(foo|bar)$"]: int + + [!~"^\\P{Lu}" & !~"^\\P{Lo}" & !~"^(foo|bar)$"]: string + } +} diff --git a/encoding/jsonschema/testdata/type.txtar b/encoding/jsonschema/testdata/type.txtar new file mode 100644 index 000000000..2c13d3f83 --- /dev/null +++ b/encoding/jsonschema/testdata/type.txtar @@ -0,0 +1,44 @@ +-- type.json -- +{ + "type": "object", + "title": "Main schema", + + "properties": { + "intString": { + "description": "an integer or string.", + "type": [ "string", "integer", "boolean", "array", "null" ] + }, + "object": { + "type": "object", + "default": { + "foo": "bar", + "baz": 1.3 + } + }, + "numOrList": { + "oneOf": [ + { "type": "number" }, + { "items": { "type": "number" } } + ], + "default": [ 1, 2, 3 ] + } + }, + "additionalProperties": false +} + +-- out.cue -- +// Main schema +package type + +Schema :: { + // an integer or string. + intString?: string | int | bool | [...] | null + object?: { + ... + } | *{ + foo: "bar" + baz: 1.3 + ... + } + numOrList?: number | [...number] | *[1, 2, 3] +} diff --git a/go.mod b/go.mod index dafaba4bd..7b07b3c64 100644 --- a/go.mod +++ b/go.mod @@ -18,11 +18,11 @@ require ( github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.3 github.com/stretchr/testify v1.2.0 - golang.org/x/net v0.0.0-20190311183353-d8887717615a + golang.org/x/net v0.0.0-20190620200207-3b0461eec859 golang.org/x/sync v0.0.0-20190423024810-112230192c58 golang.org/x/text v0.3.2 - golang.org/x/tools v0.0.0-20190328211700-ab21143f2384 - golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 + golang.org/x/tools v0.0.0-20200113154838-30cae5f2fb06 + golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 gopkg.in/yaml.v2 v2.2.2 // indirect ) diff --git a/go.sum b/go.sum index 644c89173..8329368e9 100644 --- a/go.sum +++ b/go.sum @@ -38,19 +38,29 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/stretchr/testify v1.2.0 h1:LThGCOvhuJic9Gyd1VBCkhyUXmO8vKaBFvBsJ2k03rg= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384 h1:TFlARGu6Czu1z7q93HTxcP1P+/ZFC/IKythI5RzrnRg= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20200113154838-30cae5f2fb06 h1:2CO16rKD6U0HaguVX/TdalEJwNYfoUrHC+KShDytyqc= +golang.org/x/tools v0.0.0-20200113154838-30cae5f2fb06/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=