From 6cb0878a1ef86cd2e63dc3bd31f32811ecf69730 Mon Sep 17 00:00:00 2001 From: Marcel van Lohuizen Date: Mon, 13 Jan 2020 18:03:27 +0100 Subject: [PATCH] encoding/jsonschema: initial JSON Schema to CUE conversion This does not yet introduce the API, but merely the conversion of JSON Schema to CUE. Also, several things, like formats, are not yet supported. Note that the input to the conversion is CUE itself. This allows any of the encodings (like YAML or JSON) to be used as a base. The conversion will preserve line information from the original file. That is, if a YAML file was converted to CUE, which is then converted to OpenAPI, the line information in the resulting AST corresponds to the original YAML. Change-Id: I512502ce3d98443a530e75f606ca3c533b0c4299 Reviewed-on: https://cue-review.googlesource.com/c/cue/+/4604 Reviewed-by: Marcel van Lohuizen --- encoding/jsonschema/constraints.go | 535 ++++++++++++++++++++++ encoding/jsonschema/decode.go | 427 +++++++++++++++++ encoding/jsonschema/decode_test.go | 122 +++++ encoding/jsonschema/doc.go | 19 + encoding/jsonschema/testdata/basic.txtar | 53 +++ encoding/jsonschema/testdata/def.txtar | 63 +++ encoding/jsonschema/testdata/list.txtar | 36 ++ encoding/jsonschema/testdata/num.txtar | 37 ++ encoding/jsonschema/testdata/object.txtar | 103 +++++ encoding/jsonschema/testdata/type.txtar | 44 ++ go.mod | 6 +- go.sum | 10 + 12 files changed, 1452 insertions(+), 3 deletions(-) create mode 100644 encoding/jsonschema/constraints.go create mode 100644 encoding/jsonschema/decode.go create mode 100644 encoding/jsonschema/decode_test.go create mode 100644 encoding/jsonschema/doc.go create mode 100644 encoding/jsonschema/testdata/basic.txtar create mode 100644 encoding/jsonschema/testdata/def.txtar create mode 100644 encoding/jsonschema/testdata/list.txtar create mode 100644 encoding/jsonschema/testdata/num.txtar create mode 100644 encoding/jsonschema/testdata/object.txtar create mode 100644 encoding/jsonschema/testdata/type.txtar diff --git a/encoding/jsonschema/constraints.go b/encoding/jsonschema/constraints.go new file mode 100644 index 000000000..c4c5add2d --- /dev/null +++ b/encoding/jsonschema/constraints.go @@ -0,0 +1,535 @@ +// Copyright 2019 CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package jsonschema + +import ( + "math/big" + "net/url" + "path" + "strings" + + "cuelang.org/go/cue" + "cuelang.org/go/cue/ast" + "cuelang.org/go/cue/token" +) + +type constraint struct { + key string + + // phase indicates on which pass c constraint should be added. This ensures + // that constraints are applied in the correct order. For instance, the + // "required" constraint validates that a listed field is contained in + // "properties". For this to work, "properties" must be processed before + // "required" and thus must have a lower phase number than the latter. + phase int + + // Indicates the draft number in which this constraint is defined. + draft int + fn constraintFunc +} + +// A constraintFunc converts a given JSON Schema constraint (specified in n) +// to a CUE constraint recorded in state. +type constraintFunc func(n cue.Value, s *state) + +func p0(name string, f constraintFunc) *constraint { + return &constraint{key: name, fn: f} +} + +func p0d(name string, draft int, f constraintFunc) *constraint { + return &constraint{key: name, draft: draft, fn: f} +} + +func p1(name string, f constraintFunc) *constraint { + return &constraint{key: name, phase: 1, fn: f} +} + +func p2(name string, f constraintFunc) *constraint { + return &constraint{key: name, phase: 2, fn: f} +} + +func combineSequence(name string, n cue.Value, s *state, op token.Token, f func(n cue.Value) ast.Expr) { + if n.Kind() != cue.ListKind { + s.errf(n, `value of %q must be an array, found %v`, name, n.Kind()) + } + var a ast.Expr + for _, n := range list(n) { + if a == nil { + a = f(n) + continue + } + a = ast.NewBinExpr(token.OR, a, f(n)) + } + if a == nil { + s.errf(n, `empty array for %q`, name) + return + } + s.add(a) +} + +// TODO: +// writeOnly, readOnly + +var constraintMap = map[string]*constraint{} + +func init() { + for _, c := range constraints { + constraintMap[c.key] = c + } +} + +func addDefinitions(n cue.Value, s *state) { + s.kind |= cue.StructKind + if n.Kind() != cue.StructKind { + s.errf(n, `"definitions" expected an object, found %v`, n.Kind) + } + + if len(s.path) != 1 { + s.errf(n, `"definitions" expected an object, found %v`, n.Kind) + } + + s.processMap(n, func(key string, n cue.Value) { + f := &ast.Field{ + Label: ast.NewString(s.path[len(s.path)-1]), + Token: token.ISA, + Value: s.schema(n), + } + f = &ast.Field{ + Label: ast.NewIdent(rootDefs), + Value: &ast.StructLit{Elts: []ast.Decl{f}}, + } + ast.SetRelPos(f, token.NewSection) + s.definitions = append(s.definitions, f) + }) +} + +var constraints = []*constraint{ + // Meta data. + + p0("$schema", func(n cue.Value, s *state) { + // Identifies this as a JSON schema and specifies its version. + // TODO: extract version. + s.jsonschema, _ = s.strValue(n) + }), + + p0("$id", func(n cue.Value, s *state) { + // URL: https://domain.com/schemas/foo.json + // Use Title(foo) as CUE identifier. + // anchors: #identifier + // + // TODO: mark identifiers. + s.id, _ = s.strValue(n) + }), + + // Generic constraint + + p0("type", func(n cue.Value, s *state) { + switch n.Kind() { + case cue.StringKind: + s.types = append(s.types, n) + case cue.ListKind: + for i, _ := n.List(); i.Next(); { + s.types = append(s.types, i.Value()) + } + default: + s.errf(n, `value of "type" must be a string or list of strings`) + } + }), + + p0("enum", func(n cue.Value, s *state) { + combineSequence("enum", n, s, token.OR, s.value) + s.typeOptional = true + }), + + p0d("const", 6, func(n cue.Value, s *state) { + s.add(s.value(n)) + }), + + p0("default", func(n cue.Value, s *state) { + s.default_ = s.value(n) + // must validate that the default is subsumed by the normal value, + // as CUE will otherwise broaden the accepted values with the default. + s.examples = append(s.examples, s.default_) + }), + + p0("deprecated", func(n cue.Value, s *state) { + if s.boolValue(n) { + s.deprecated = true + } + }), + + p0("examples", func(n cue.Value, s *state) { + if n.Kind() != cue.ListKind { + s.errf(n, `value of "examples" must be an array, found %v`, n.Kind) + } + for _, n := range list(n) { + s.examples = append(s.examples, s.schema(n)) + } + }), + + p0("description", func(n cue.Value, s *state) { + s.description, _ = s.strValue(n) + }), + + p0("title", func(n cue.Value, s *state) { + s.title, _ = s.strValue(n) + }), + + p0d("$comment", 7, func(n cue.Value, s *state) { + }), + + p0("$def", addDefinitions), + p0("definitions", addDefinitions), + p0("$ref", func(n cue.Value, s *state) { + if str, ok := s.strValue(n); ok { + u, err := url.Parse(str) + if err != nil { + s.add(s.errf(n, "invalid JSON reference: %s", err)) + return + } + + if u.Host != "" || u.Path != "" { + s.add(s.errf(n, "external references (%s) not supported", str)) + // TODO: handle + // host: + // If the host corresponds to a package known to cue, + // load it from there. It would prefer schema converted to + // CUE, although we could consider loading raw JSON schema + // if present. + // If not present, advise the user to run cue get. + // path: + // Look up on file system or relatively to authority location. + return + } + + if !path.IsAbs(u.Fragment) { + s.add(s.errf(n, "anchors (%s) not supported", u.Fragment)) + // TODO: support anchors + return + } + + // NOTE: Go bug?: url.URL has no raw representation of the fragment. + // This means that %2F gets translated to `/` before it can be + // split. This, in turn, means that field names cannot have a `/` + // as name. + a := strings.Split(u.Fragment[1:], "/") + if a[0] != "definitions" && a[0] != "$def" { + s.add(s.errf(n, "reference %q must resolve to definition", u.Fragment)) + return + } + s.add(ast.NewSel(ast.NewIdent(rootDefs), a[1:]...)) + + // TODO: technically, a references could reference a non-definition. + // In that case this will not resolve. We should detect cases that + // are not definitions and then resolve those as literal values. + } + }), + + // Combinators + + // TODO: work this out in more detail: oneOf and anyOf below have the same + // implementation in CUE. The distinction is that for anyOf a result is + // allowed to be ambiguous at the end, whereas for oneOf a disjunction must + // be fully resolved. There is currently no easy way to set this distinction + // in CUE. + // + // One could correctly write oneOf like this once 'not' is implemented: + // + // oneOf(a, b, c) :- + // anyOf( + // allOf(a, not(b), not(c)), + // allOf(not(a), b, not(c)), + // allOf(not(a), not(b), c), + // )) + // + // This is not necessary if the values are mutually exclusive/ have a + // discriminator. + + p0("allOf", func(n cue.Value, s *state) { + combineSequence("allOf", n, s, token.AND, s.schema) + }), + + p0("anyOf", func(n cue.Value, s *state) { + combineSequence("anyOf", n, s, token.OR, s.schema) + }), + + p0("oneOf", func(n cue.Value, s *state) { + combineSequence("allOf", n, s, token.OR, s.schema) + }), + + // String constraints + + p0("pattern", func(n cue.Value, s *state) { + s.kind |= cue.StringKind + s.add(&ast.UnaryExpr{Op: token.MAT, X: s.string(n)}) + }), + + p0d("contentMediaType", 7, func(n cue.Value, s *state) { + s.kind |= cue.StringKind + }), + + p0d("contentEncoding", 7, func(n cue.Value, s *state) { + s.kind |= cue.StringKind + // 7bit, 8bit, binary, quoted-printable and base64. + // RFC 2054, part 6.1. + // https://tools.ietf.org/html/rfc2045 + // TODO: at least handle bytes. + }), + + // Number constraints + + p0("minimum", func(n cue.Value, s *state) { + s.kind |= cue.NumberKind + s.add(&ast.UnaryExpr{Op: token.GEQ, X: s.number(n)}) + }), + + p0("exclusiveMinimum", func(n cue.Value, s *state) { + // TODO: should we support Draft 4 booleans? + s.kind |= cue.NumberKind + s.add(&ast.UnaryExpr{Op: token.GTR, X: s.number(n)}) + }), + + p0("maximum", func(n cue.Value, s *state) { + s.kind |= cue.NumberKind + s.add(&ast.UnaryExpr{Op: token.LEQ, X: s.number(n)}) + }), + + p0("exclusiveMaximum", func(n cue.Value, s *state) { + // TODO: should we support Draft 4 booleans? + s.kind |= cue.NumberKind + s.add(&ast.UnaryExpr{Op: token.LSS, X: s.number(n)}) + }), + + p0("multipleOf", func(n cue.Value, s *state) { + s.kind |= cue.NumberKind + multiple := s.number(n) + var x big.Int + _, _ = n.MantExp(&x) + if x.Cmp(big.NewInt(0)) != 1 { + s.errf(n, `"multipleOf" value must be < 0; found %s`, n) + } + math := s.addImport("math") + s.add(ast.NewCall(ast.NewSel(math, "MultipleOf"), multiple)) + }), + + // Object constraints + + p0("properties", func(n cue.Value, s *state) { + s.kind |= cue.StructKind + if s.obj == nil { + s.obj = &ast.StructLit{} + } + if n.Kind() != cue.StructKind { + s.errf(n, `"properties" expected an object, found %v`, n.Kind) + } + + s.processMap(n, func(key string, n cue.Value) { + // property?: value + expr, state := s.schemaState(n) + f := &ast.Field{Label: ast.NewString(key), Value: expr} + state.doc(f) + f.Optional = token.Blank.Pos() + if len(s.obj.Elts) > 0 && len(f.Comments()) > 0 { + // TODO: change formatter such that either a a NewSection on the + // field or doc comment will cause a new section. + ast.SetRelPos(f.Comments()[0], token.NewSection) + } + s.obj.Elts = append(s.obj.Elts, f) + }) + }), + + p1("required", func(n cue.Value, s *state) { + s.kind |= cue.StructKind + if s.obj == nil { + s.errf(n, `"required" without a "properties" field`) + } + if n.Kind() != cue.ListKind { + s.errf(n, `value of "required" must be list of strings, found %v`, n.Kind) + } + + // Create field map + fields := map[string]*ast.Field{} + for _, d := range s.obj.Elts { + f := d.(*ast.Field) + str, _, err := ast.LabelName(f.Label) + if err == nil { + fields[str] = f + } + } + + for _, n := range list(n) { + str, ok := s.strValue(n) + f := fields[str] + if f == nil && ok { + s.errf(n, "required field %q not in properties", str) + continue + } + if f.Optional == token.NoPos { + s.errf(n, "duplicate required field %q", str) + } + f.Optional = token.NoPos + } + }), + + p0d("propertyNames", 6, func(n cue.Value, s *state) { + s.kind |= cue.StructKind + // [=~pattern]: _ + s.add(&ast.StructLit{Elts: []ast.Decl{&ast.Field{ + Label: ast.NewList(s.schema(n)), + Value: ast.NewIdent("_"), + }}}) + }), + + p0("minProperties", func(n cue.Value, s *state) { + s.kind |= cue.StructKind + pkg := s.addImport("struct") + s.add(ast.NewCall(ast.NewSel(pkg, "MinFields"), s.uint(n))) + }), + + p0("maxProperties", func(n cue.Value, s *state) { + s.kind |= cue.StructKind + pkg := s.addImport("struct") + s.add(ast.NewCall(ast.NewSel(pkg, "MaxFields"), s.uint(n))) + }), + + p0("dependencies", func(n cue.Value, s *state) { + s.kind |= cue.StructKind + // Schema and property dependencies. + // TODO: the easiest implementation is with comprehensions. + // The nicer implementation is with disjunctions. This has to be done + // at the very end, replacing properties. + /* + *{ property?: _|_ } | { + property: _ + schema + } + */ + }), + + p1("patternProperties", func(n cue.Value, s *state) { + s.kind |= cue.StructKind + if n.Kind() != cue.StructKind { + s.errf(n, `value of "patternProperties" must be an an object, found %v`, n.Kind) + } + if s.obj == nil { + s.obj = &ast.StructLit{} + } + existing := excludeFields(s.obj.Elts) + s.processMap(n, func(key string, n cue.Value) { + // [!~(properties) & pattern]: schema + s.patterns = append(s.patterns, + &ast.UnaryExpr{Op: token.NMAT, X: ast.NewString(key)}) + f := &ast.Field{ + Label: ast.NewList(ast.NewBinExpr(token.AND, + &ast.UnaryExpr{Op: token.MAT, X: ast.NewString(key)}, + existing)), + Value: s.schema(n), + } + ast.SetRelPos(f, token.NewSection) + s.obj.Elts = append(s.obj.Elts, f) + }) + }), + + p2("additionalProperties", func(n cue.Value, s *state) { + s.kind |= cue.StructKind + switch n.Kind() { + case cue.BoolKind: + s.closeStruct = !s.boolValue(n) + + case cue.StructKind: + s.closeStruct = true + if s.obj == nil { + s.obj = &ast.StructLit{} + } + if len(s.obj.Elts) == 0 { + s.obj.Elts = append(s.obj.Elts, &ast.Field{ + Label: ast.NewList(ast.NewIdent("string")), + Value: s.schema(n), + }) + return + } + // [!~(properties|patternProperties)]: schema + existing := append(s.patterns, excludeFields(s.obj.Elts)) + f := &ast.Field{ + Label: ast.NewList(ast.NewBinExpr(token.AND, existing...)), + Value: s.schema(n), + } + ast.SetRelPos(f, token.NewSection) + s.obj.Elts = append(s.obj.Elts, f) + + default: + s.errf(n, `value of "additionalProperties" must be an object or boolean`) + } + }), + + // Array constraints. + + p0("items", func(n cue.Value, s *state) { + s.kind |= cue.ListKind + if s.list != nil { + s.errf(n, `"items" declared more than once, previous declaration at %s`, s.list.Pos()) + } + switch n.Kind() { + case cue.StructKind: + elem := s.schema(n) + ast.SetRelPos(elem, token.NoRelPos) + s.add(ast.NewList(&ast.Ellipsis{Type: elem})) + + case cue.ListKind: + var a []ast.Expr + for _, n := range list(n) { + v := s.schema(n) + ast.SetRelPos(v, token.NoRelPos) + a = append(a, v) + } + s.add(ast.NewList(a...)) + + default: + s.errf(n, `value of "items" must be an object or array`) + } + }), + + p0("contains", func(n cue.Value, s *state) { + s.kind |= cue.ListKind + list := s.addImport("list") + // TODO: Passing non-concrete values is not yet supported in CUE. + s.add(ast.NewCall(ast.NewSel(list, "Contains"), clearPos(s.schema(n)))) + }), + + p0("minItems", func(n cue.Value, s *state) { + s.kind |= cue.ListKind + list := s.addImport("list") + s.add(ast.NewCall(ast.NewSel(list, "MinItems"), clearPos(s.uint(n)))) + }), + + p0("maxItems", func(n cue.Value, s *state) { + s.kind |= cue.ListKind + list := s.addImport("list") + s.add(ast.NewCall(ast.NewSel(list, "MaxItems"), clearPos(s.uint(n)))) + }), + + p0("uniqueItems", func(n cue.Value, s *state) { + s.kind |= cue.ListKind + if s.boolValue(n) { + list := s.addImport("list") + s.add(ast.NewCall(ast.NewSel(list, "UniqueItems"))) + } + }), +} + +func clearPos(e ast.Expr) ast.Expr { + ast.SetRelPos(e, token.NoRelPos) + return e +} diff --git a/encoding/jsonschema/decode.go b/encoding/jsonschema/decode.go new file mode 100644 index 000000000..dff0cc743 --- /dev/null +++ b/encoding/jsonschema/decode.go @@ -0,0 +1,427 @@ +// Copyright 2019 CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package jsonschema + +// TODO: +// - replace converter from YAML to CUE to CUE (schema) to CUE. +// - define OpenAPI definitions als CUE. + +import ( + "fmt" + "path" + "sort" + "strings" + + "cuelang.org/go/cue" + "cuelang.org/go/cue/ast" + "cuelang.org/go/cue/errors" + "cuelang.org/go/cue/token" + "cuelang.org/go/internal" +) + +const rootDefs = "def" + +func decode(filename string, inst *cue.Instance) (*ast.File, errors.Error) { + d := &decoder{ + imports: map[string]*ast.Ident{}, + } + e := d.decode(filename, inst) + if d.errs != nil { + return nil, d.errs + } + return e, nil +} + +// A decoder converts JSON schema to CUE. +type decoder struct { + errs errors.Error + imports map[string]*ast.Ident + + definitions []ast.Decl +} + +// addImport registers +func (d *decoder) addImport(pkg string) *ast.Ident { + ident, ok := d.imports[pkg] + if !ok { + ident = ast.NewIdent(pkg) + d.imports[pkg] = ident + } + return ident +} + +func (d *decoder) decode(filename string, inst *cue.Instance) *ast.File { + root := state{decoder: d} + expr, state := root.schemaState(inst.Value()) + + var a []ast.Decl + + name := filename + if state.id != "" { + name = strings.Trim(name, "#") + } + pkgName := path.Base(name) + pkgName = pkgName[:len(pkgName)-len(path.Ext(pkgName))] + pkg := &ast.Package{Name: ast.NewIdent(pkgName)} + state.doc(pkg) + + a = append(a, pkg) + + var imports []string + for k := range d.imports { + imports = append(imports, k) + } + sort.Strings(imports) + + if len(imports) > 0 { + x := &ast.ImportDecl{} + for _, p := range imports { + x.Specs = append(x.Specs, ast.NewImport(nil, p)) + } + a = append(a, x) + } + + tags := []string{} + if state.jsonschema != "" { + tags = append(tags, fmt.Sprintf("schema=%q", state.jsonschema)) + } + if state.id != "" { + tags = append(tags, fmt.Sprintf("id=%q", state.id)) + } + if len(tags) > 0 { + a = append(a, addTag("Schema", "jsonschema", strings.Join(tags, ","))) + } + + if state.deprecated { + a = append(a, addTag("Schema", "deprecated", "")) + } + + f := &ast.Field{ + Label: ast.NewIdent("Schema"), + Value: expr, + } + + f.Token = token.ISA + a = append(a, f) + a = append(a, d.definitions...) + + return &ast.File{Decls: a} +} + +func (d *decoder) errf(n cue.Value, format string, args ...interface{}) ast.Expr { + d.warnf(n, format, args...) + return &ast.BadExpr{From: n.Pos()} +} + +func (d *decoder) warnf(n cue.Value, format string, args ...interface{}) { + d.errs = errors.Append(d.errs, errors.Newf(n.Pos(), format, args...)) +} + +func (d *decoder) number(n cue.Value) ast.Expr { + return n.Syntax().(ast.Expr) +} + +func (d *decoder) uint(n cue.Value) ast.Expr { + _, err := n.Uint64() + if err != nil { + d.errf(n, "invalid uint") + } + return n.Syntax().(ast.Expr) +} + +func (d *decoder) bool(n cue.Value) ast.Expr { + return n.Syntax().(ast.Expr) +} + +func (d *decoder) boolValue(n cue.Value) bool { + x, err := n.Bool() + if err != nil { + d.errf(n, "invalid bool") + } + return x +} + +func (d *decoder) string(n cue.Value) ast.Expr { + return n.Syntax().(ast.Expr) +} + +func (d *decoder) strValue(n cue.Value) (s string, ok bool) { + s, err := n.String() + if err != nil { + d.errf(n, "invalid string") + return "", false + } + return s, true +} + +// const draftCutoff = 5 + +type state struct { + *decoder + + path []string + + pos cue.Value + + types []cue.Value + typeOptional bool + kind cue.Kind + + default_ ast.Expr + examples []ast.Expr + title string + description string + deprecated bool + jsonschema string + id string + + conjuncts []ast.Expr + + obj *ast.StructLit + closeStruct bool + patterns []ast.Expr + + list *ast.ListLit +} + +// finalize constructs a CUE type from the collected constraints. +func (s *state) finalize() (e ast.Expr) { + if s.typeOptional || s.kind != 0 { + if len(s.types) > 1 { + s.errf(s.pos, "constraints require specific type") + } + s.types = nil + } + + conjuncts := []ast.Expr{} + disjuncts := []ast.Expr{} + for _, n := range s.types { + add := func(e ast.Expr) { + disjuncts = append(disjuncts, setPos(e, n)) + } + str, ok := s.strValue(n) + if !ok { + s.errf(n, "type value should be a string") + return + } + switch str { + case "null": + // TODO: handle OpenAPI restrictions. + add(ast.NewIdent("null")) + case "boolean": + add(ast.NewIdent("bool")) + case "string": + add(ast.NewIdent("string")) + case "number": + add(ast.NewIdent("number")) + case "integer": + add(ast.NewIdent("int")) + case "array": + if s.kind&cue.ListKind == 0 { + add(ast.NewList(&ast.Ellipsis{})) + } + case "object": + elps := &ast.Ellipsis{} + st := &ast.StructLit{Elts: []ast.Decl{elps}} + add(st) + default: + s.errf(n, "unknown type %q", n) + } + } + if len(disjuncts) > 0 { + conjuncts = append(conjuncts, ast.NewBinExpr(token.OR, disjuncts...)) + } + + conjuncts = append(conjuncts, s.conjuncts...) + + if s.obj != nil { + if !s.closeStruct { + s.obj.Elts = append(s.obj.Elts, &ast.Ellipsis{}) + } + conjuncts = append(conjuncts, s.obj) + } + + if len(conjuncts) == 0 { + return ast.NewString(fmt.Sprint(s.pos)) + } + + e = ast.NewBinExpr(token.AND, conjuncts...) + + if s.default_ != nil { + // check conditions where default can be skipped. + switch x := s.default_.(type) { + case *ast.ListLit: + if s.kind == cue.ListKind && len(x.Elts) == 0 { + return e + } + } + e = ast.NewBinExpr(token.OR, e, &ast.UnaryExpr{Op: token.MUL, X: s.default_}) + } + return e +} + +func (s *state) doc(n ast.Node) { + // Create documentation. + doc := strings.TrimSpace(s.title) + if s.description != "" { + if doc != "" { + doc += "\n\n" + } + doc += s.description + doc = strings.TrimSpace(doc) + } + if doc != "" { + ast.SetComments(n, []*ast.CommentGroup{ + internal.NewComment(true, doc), + }) + } + + // TODO: add examples as well? +} + +func (s *state) add(e ast.Expr) { + s.conjuncts = append(s.conjuncts, e) +} + +func (s *state) schema(n cue.Value) ast.Expr { + expr, _ := s.schemaState(n) + // TODO: report unused doc. + return expr +} + +func (s *state) schemaState(n cue.Value) (ast.Expr, *state) { + state := &state{path: s.path, pos: n, decoder: s.decoder} + + if n.Kind() != cue.StructKind { + return s.errf(n, "schema expects mapping node, found %s", n.Kind()), state + } + + // do multiple passes over the constraints to ensure they are done in order. + for pass := 0; pass < 3; pass++ { + state.processMap(n, func(key string, value cue.Value) { + // Convert each constraint into a either a value or a functor. + c := constraintMap[key] + if c == nil { + if pass == 0 { + s.warnf(n, "unsupported constraint %q", key) + } + return + } + if c.phase == pass { + c.fn(value, state) + } + }) + } + + return state.finalize(), state +} + +func (s *state) value(n cue.Value) ast.Expr { + switch n.Kind() { + case cue.ListKind: + a := []ast.Expr{} + for i, _ := n.List(); i.Next(); { + a = append(a, s.value(i.Value())) + } + return setPos(ast.NewList(a...), n) + + case cue.StructKind: + a := []ast.Decl{} + s.processMap(n, func(key string, n cue.Value) { + a = append(a, &ast.Field{ + Label: ast.NewString(key), + Value: s.value(n), + }) + }) + a = append(a, &ast.Ellipsis{}) + return setPos(&ast.StructLit{Elts: a}, n) + + default: + if !n.IsConcrete() { + s.errf(n, "invalid non-concerte value") + } + return n.Syntax().(ast.Expr) + } +} + +// processMap processes a yaml node, expanding merges. +// +// TODO: in some cases we can translate merges into CUE embeddings. +// This may also prevent exponential blow-up (as may happen when +// converting YAML to JSON). +func (s *state) processMap(n cue.Value, f func(key string, n cue.Value)) { + saved := s.path + defer func() { s.path = saved }() + + // TODO: intercept references to allow for optimized performance. + for i, _ := n.Fields(); i.Next(); { + key := i.Label() + s.path = append(saved, key) + f(key, i.Value()) + } +} + +func list(n cue.Value) (a []cue.Value) { + for i, _ := n.List(); i.Next(); { + a = append(a, i.Value()) + } + return a +} + +// excludeFields returns a CUE expression that can be used to exclude the +// fields of the given declaration in a label expression. For instance, for +// +// { foo: 1, bar: int } +// +// it creates +// +// "^(foo|bar)$" +// +// which can be used in a label expression to define types for all fields but +// those existing: +// +// [!~"^(foo|bar)$"]: string +// +func excludeFields(decls []ast.Decl) ast.Expr { + var a []string + for _, d := range decls { + f, ok := d.(*ast.Field) + if !ok { + continue + } + str, _, _ := ast.LabelName(f.Label) + if str != "" { + a = append(a, str) + } + } + re := fmt.Sprintf("^(%s)$", strings.Join(a, "|")) + return &ast.UnaryExpr{Op: token.NMAT, X: ast.NewString(re)} +} + +func addTag(field, tag, value string) *ast.Field { + return &ast.Field{ + Label: ast.NewIdent(field), + Token: token.ISA, + Value: ast.NewIdent("_"), + Attrs: []*ast.Attribute{ + &ast.Attribute{Text: fmt.Sprintf("@%s(%s)", tag, value)}, + }, + } +} + +func setPos(e ast.Expr, v cue.Value) ast.Expr { + ast.SetPos(e, v.Pos()) + return e +} diff --git a/encoding/jsonschema/decode_test.go b/encoding/jsonschema/decode_test.go new file mode 100644 index 000000000..da64a24ef --- /dev/null +++ b/encoding/jsonschema/decode_test.go @@ -0,0 +1,122 @@ +// Copyright 2019 CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package jsonschema + +import ( + "bytes" + "flag" + "io/ioutil" + "os" + "path" + "path/filepath" + "strings" + "testing" + + "cuelang.org/go/cue" + "cuelang.org/go/cue/errors" + "cuelang.org/go/cue/format" + "cuelang.org/go/encoding/json" + "cuelang.org/go/encoding/yaml" + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + "golang.org/x/tools/txtar" +) + +var update = flag.Bool("update", false, "update the test files") + +// TestDecode reads the testdata/*.txtar files, converts the contained +// JSON schema to CUE and compares it against the output. +// +// Use the --update flag to update test files with the corresponding output. +func TestDecode(t *testing.T) { + err := filepath.Walk("testdata", func(fullpath string, info os.FileInfo, err error) error { + _ = err + if !strings.HasSuffix(fullpath, ".txtar") { + return nil + } + + t.Run(fullpath, func(t *testing.T) { + a, err := txtar.ParseFile(fullpath) + if err != nil { + t.Fatal(err) + } + + r := &cue.Runtime{} + var in *cue.Instance + var out, errout []byte + outIndex := -1 + + for i, f := range a.Files { + switch path.Ext(f.Name) { + case ".json": + in, err = json.Decode(r, f.Name, f.Data) + case ".yaml": + in, err = yaml.Decode(r, f.Name, f.Data) + case ".cue": + out = f.Data + outIndex = i + case ".err": + errout = f.Data + } + } + if err != nil { + t.Fatal(err) + } + + // TODO: use actual API. + expr, err := decode(fullpath, in) + if err != nil && errout == nil { + t.Fatal(errors.Details(err, nil)) + } + got := []byte(nil) + if err != nil { + got = []byte(err.Error()) + } + if !cmp.Equal(errout, got) { + t.Error(cmp.Diff(string(got), string(errout))) + } + + if expr != nil { + b, err := format.Node(expr, format.Simplify()) + if err != nil { + t.Fatal(err) + } + + // verify the generated CUE. + if _, err = r.Compile(fullpath, b); err != nil { + t.Fatal(errors.Details(err, nil)) + } + + b = bytes.TrimSpace(b) + out = bytes.TrimSpace(out) + + if !cmp.Equal(b, out) { + if *update { + a.Files[outIndex].Data = b + b = txtar.Format(a) + err = ioutil.WriteFile(fullpath, b, 0644) + if err != nil { + t.Fatal(err) + } + return + } + t.Error(cmp.Diff(string(out), string(b))) + } + } + }) + return nil + }) + assert.NoError(t, err) +} diff --git a/encoding/jsonschema/doc.go b/encoding/jsonschema/doc.go new file mode 100644 index 000000000..c0b813ce2 --- /dev/null +++ b/encoding/jsonschema/doc.go @@ -0,0 +1,19 @@ +// Copyright 2020 CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package jsonschema converts JSON Schema to CUE +// +// JSON Schema data is presented in CUE, so any of the supported encodings +// that can represent JSON Schema data can be used as a source. +package jsonschema diff --git a/encoding/jsonschema/testdata/basic.txtar b/encoding/jsonschema/testdata/basic.txtar new file mode 100644 index 000000000..0c1b255a2 --- /dev/null +++ b/encoding/jsonschema/testdata/basic.txtar @@ -0,0 +1,53 @@ +-- basic.json -- +{ + "$schema": "http://json-schema.org/draft-07/schema#", + + "type": "object", + "title": "Main schema", + "description": "Specify who you are and all.", + + "properties": { + "person": { + "description": "A person is a human being.", + "type": "object", + "required": [ "name" ], + "properties": { + "name": { "type": "string" }, + "address": { + "description": "where does this person live?", + "type": "string" + }, + "children": { + "type": "array", + "items": { "type": "string" }, + "default": [] + }, + "home phone": { + "type": "string", + "deprecated": true + } + } + } + } +} + +-- out.cue -- +// Main schema +// +// Specify who you are and all. +package basic + +Schema :: _ @jsonschema(schema="http://json-schema.org/draft-07/schema#") +Schema :: { + // A person is a human being. + person?: { + name: string + + // where does this person live? + address?: string + children?: [...string] + "home phone"?: string + ... + } + ... +} diff --git a/encoding/jsonschema/testdata/def.txtar b/encoding/jsonschema/testdata/def.txtar new file mode 100644 index 000000000..d5d92e904 --- /dev/null +++ b/encoding/jsonschema/testdata/def.txtar @@ -0,0 +1,63 @@ +// This test tests the conversion and ordering of definitions. + +-- definition.json -- +{ + "$schema": "http://json-schema.org/draft-07/schema#", + + "$id": "http://cuelang.org/go/encoding/openapi/testdata/order.json", + + "definitions": { + "address": { + "type": "object", + "properties": { + "street_address": { "type": "string" }, + "city": { "type": "string" }, + "state": { "type": "string" } + }, + "required": ["street_address", "city", "state"] + }, + "person": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "children": { + "type": "array", + "items": { "$ref": "#/definitions/person" }, + "default": [] + } + } + } + }, + + "type": "object", + + "properties": { + "person": { "$ref": "#/definitions/person" }, + "billing_address": { "$ref": "#/definitions/address" }, + "shipping_address": { "$ref": "#/definitions/address" } + } +} + +-- out.cue -- +package def + +Schema :: _ @jsonschema(schema="http://json-schema.org/draft-07/schema#",id="http://cuelang.org/go/encoding/openapi/testdata/order.json") +Schema :: { + person?: def.person + billing_address?: def.address + shipping_address?: def.address + ... +} + +def: address :: { + street_address: string + city: string + state: string + ... +} + +def: person :: { + name?: string + children?: [...def.person] + ... +} diff --git a/encoding/jsonschema/testdata/list.txtar b/encoding/jsonschema/testdata/list.txtar new file mode 100644 index 000000000..d3046dd8c --- /dev/null +++ b/encoding/jsonschema/testdata/list.txtar @@ -0,0 +1,36 @@ +-- list.yaml -- +type: object + +properties: + foo: + items: + type: string + + tuple: + items: + - type: string + - type: integer + - const: 2 + + has: + contains: + const: 3 + + size: + minItems: 3 + maxItems: 9 + uniqueItems: true + +additionalProperties: false + +-- out.cue -- +package list + +import "list" + +Schema :: { + foo?: [...string] + tuple?: [string, int, 2] + has?: list.Contains(3) + size?: list.MinItems(3) & list.MaxItems(9) & list.UniqueItems() +} diff --git a/encoding/jsonschema/testdata/num.txtar b/encoding/jsonschema/testdata/num.txtar new file mode 100644 index 000000000..5652628fd --- /dev/null +++ b/encoding/jsonschema/testdata/num.txtar @@ -0,0 +1,37 @@ +-- type.json -- +{ + "type": "object", + + "properties": { + "constant": { "const": 2 }, + "several": { + "enum": [ 1, 2, 3, 4 ] + }, + "inclusive": { + "type": "number", + "minimum": 2, + "maximum": 3 + }, + "exclusive": { + "exclusiveMinimum": 2, + "exclusiveMaximum": 3 + }, + "cents": { + "multipleOf": 0.05 + } + }, + "additionalProperties": false +} + +-- out.cue -- +package num + +import "math" + +Schema :: { + constant?: 2 + several?: 1 | 2 | 3 | 4 + inclusive?: >=2 & <=3 + exclusive?: >2 & <3 + cents?: math.MultipleOf(0.05) +} diff --git a/encoding/jsonschema/testdata/object.txtar b/encoding/jsonschema/testdata/object.txtar new file mode 100644 index 000000000..59bbbe359 --- /dev/null +++ b/encoding/jsonschema/testdata/object.txtar @@ -0,0 +1,103 @@ +-- type.json -- +{ + "type": "object", + "title": "Main schema", + + "properties": { + "fields" : { + "type": "object", + "minProperties": 3, + "maxProperties": 10, + "propertyNames": { + "pattern": "^\\P{Lu}" + } + }, + "additional": { + "type": "object", + "properties": { + "foo": { "type": "number" }, + "bar": { "type": "number" } + }, + "additionalProperties": { "type": "string" } + }, + "map": { + "type": "object", + "additionalProperties": { "type": "string" } + }, + "patterns": { + "type": "object", + "properties": { + "foo": { "type": "number" }, + "bar": { "type": "number" } + }, + "patternProperties": { + "^\\P{Lu}": { "type": "string" }, + "^\\P{Lo}": { "type": "integer" } + } + }, + "patternsNoProps": { + "type": "object", + "patternProperties": { + "^\\P{Lu}": { "type": "string" }, + "^\\P{Lo}": { "type": "integer" } + } + }, + "complex": { + "type": "object", + "properties": { + "foo": { "type": "number" }, + "bar": { "type": "number" } + }, + "patternProperties": { + "^\\P{Lu}": { "type": "string" }, + "^\\P{Lo}": { "type": "integer" } + }, + "additionalProperties": { "type": "string" } + } + }, + "additionalProperties": false +} + +-- out.cue -- +// Main schema +package object + +import "struct" + +Schema :: { + fields?: struct.MinFields(3) & struct.MaxFields(10) & { + [=~"^\\P{Lu}"]: _ + } + additional?: { + foo?: number + bar?: number + + [!~"^(foo|bar)$"]: string + } + map?: [string]: string + patterns?: { + foo?: number + bar?: number + + [=~"^\\P{Lu}" & !~"^(foo|bar)$"]: string + + [=~"^\\P{Lo}" & !~"^(foo|bar)$"]: int + ... + } + patternsNoProps?: { + [=~"^\\P{Lu}" & !~"^()$"]: string + + [=~"^\\P{Lo}" & !~"^()$"]: int + ... + } + complex?: { + foo?: number + bar?: number + + [=~"^\\P{Lu}" & !~"^(foo|bar)$"]: string + + [=~"^\\P{Lo}" & !~"^(foo|bar)$"]: int + + [!~"^\\P{Lu}" & !~"^\\P{Lo}" & !~"^(foo|bar)$"]: string + } +} diff --git a/encoding/jsonschema/testdata/type.txtar b/encoding/jsonschema/testdata/type.txtar new file mode 100644 index 000000000..2c13d3f83 --- /dev/null +++ b/encoding/jsonschema/testdata/type.txtar @@ -0,0 +1,44 @@ +-- type.json -- +{ + "type": "object", + "title": "Main schema", + + "properties": { + "intString": { + "description": "an integer or string.", + "type": [ "string", "integer", "boolean", "array", "null" ] + }, + "object": { + "type": "object", + "default": { + "foo": "bar", + "baz": 1.3 + } + }, + "numOrList": { + "oneOf": [ + { "type": "number" }, + { "items": { "type": "number" } } + ], + "default": [ 1, 2, 3 ] + } + }, + "additionalProperties": false +} + +-- out.cue -- +// Main schema +package type + +Schema :: { + // an integer or string. + intString?: string | int | bool | [...] | null + object?: { + ... + } | *{ + foo: "bar" + baz: 1.3 + ... + } + numOrList?: number | [...number] | *[1, 2, 3] +} diff --git a/go.mod b/go.mod index dafaba4bd..7b07b3c64 100644 --- a/go.mod +++ b/go.mod @@ -18,11 +18,11 @@ require ( github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.3 github.com/stretchr/testify v1.2.0 - golang.org/x/net v0.0.0-20190311183353-d8887717615a + golang.org/x/net v0.0.0-20190620200207-3b0461eec859 golang.org/x/sync v0.0.0-20190423024810-112230192c58 golang.org/x/text v0.3.2 - golang.org/x/tools v0.0.0-20190328211700-ab21143f2384 - golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 + golang.org/x/tools v0.0.0-20200113154838-30cae5f2fb06 + golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 gopkg.in/yaml.v2 v2.2.2 // indirect ) diff --git a/go.sum b/go.sum index 644c89173..8329368e9 100644 --- a/go.sum +++ b/go.sum @@ -38,19 +38,29 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/stretchr/testify v1.2.0 h1:LThGCOvhuJic9Gyd1VBCkhyUXmO8vKaBFvBsJ2k03rg= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384 h1:TFlARGu6Czu1z7q93HTxcP1P+/ZFC/IKythI5RzrnRg= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20200113154838-30cae5f2fb06 h1:2CO16rKD6U0HaguVX/TdalEJwNYfoUrHC+KShDytyqc= +golang.org/x/tools v0.0.0-20200113154838-30cae5f2fb06/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= -- 2.51.2