diff --git a/lex/lexgen/codegen.go b/lex/lexgen/codegen.go new file mode 100644 index 00000000..1d7820a8 --- /dev/null +++ b/lex/lexgen/codegen.go @@ -0,0 +1,867 @@ +package lexgen + +import ( + "fmt" + "io" + "log/slog" + "sort" + "strings" + + "github.com/bluesky-social/indigo/atproto/lexicon" + "github.com/bluesky-social/indigo/atproto/syntax" +) + +// Configuration for [CodeGenerator] output +type GenConfig struct { + RegisterLexiconTypeID bool + PackageMappings map[string]string + // one of: "type-decoder", "map-string-any", "json-raw-message" + UnknownType string + WarningText string + LegacyMode bool +} + +func NewGenConfig() *GenConfig { + return &GenConfig{ + UnknownType: "map-string-any", + WarningText: "Code generated by indigo lexgen tool. DO NOT EDIT MANUALLY.", + } +} + +func LegacyConfig() *GenConfig { + return &GenConfig{ + RegisterLexiconTypeID: true, + UnknownType: "type-decoder", + WarningText: "Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT.", + LegacyMode: true, + } +} + +// Core implementation of Go code generation for a single Lexicon schema file (multiple definitions), building on pre-parsed [FlatLexicon] +type CodeGenerator struct { + Config *GenConfig + Lex *FlatLexicon + Cat lexicon.Catalog + Out io.Writer +} + +// Outputs Go source code to the "Out" [io.Writer]. +func (gen *CodeGenerator) WriteLexicon() error { + + if gen.Config.WarningText != "" { + fmt.Fprintf(gen.Out, "// %s\n\n", gen.Config.WarningText) + } + fmt.Fprintf(gen.Out, "// Lexicon schema: %s\n\n", gen.Lex.NSID) + fmt.Fprintf(gen.Out, "package %s\n\n", gen.PkgName()) + fmt.Fprintln(gen.Out, "import (") + for dep, _ := range gen.deps() { + fmt.Fprintf(gen.Out, " %s\n", dep) + } + fmt.Fprint(gen.Out, ")\n\n") + + for _, ft := range gen.Lex.Types { + slog.Info("generating type", "nsid", gen.Lex.NSID, "def", ft.DefName, "path", ft.Path, "type", ft.Type) + if err := gen.WriteType(&ft); err != nil { + return err + } + } + return nil +} + +func (gen *CodeGenerator) PkgName() string { + n := nsidPkgName(gen.Lex.NSID) + if gen.Config.LegacyMode { + switch n { + case "appbsky": + return "bsky" + case "comatproto": + return "atproto" + case "toolsozone": + return "ozone" + case "chatbsky": + return "chat" + } + } + return n +} + +func (gen *CodeGenerator) baseName() string { + // TODO: memoize this value? this method gets called a lot + return nsidBaseName(gen.Lex.NSID) +} + +func (gen *CodeGenerator) FileName() string { + return nsidFileName(gen.Lex.NSID) + ".go" +} + +func (gen *CodeGenerator) deps() map[string]bool { + d := map[string]bool{ + "\"context\"": true, + "\"fmt\"": true, + "\"io\"": true, + "\"bytes\"": true, + "\"encoding/json\"": true, + "lexutil \"github.com/bluesky-social/indigo/lex/util\"": true, + "cbg \"github.com/whyrusleeping/cbor-gen\"": true, + } + + for ext, _ := range gen.Lex.ExternalRefs { + // TODO: replace this with configurable/extensible mappings + if strings.HasPrefix(ext, "com.atproto.") { + d["comatproto \"github.com/bluesky-social/indigo/api/atproto\""] = true + } else if strings.HasPrefix(ext, "app.bsky.") { + d["appbsky \"github.com/bluesky-social/indigo/api/bsky\""] = true + } else if strings.HasPrefix(ext, "tools.ozone.") { + d["toolsozone \"github.com/bluesky-social/indigo/api/ozone\""] = true + } else if strings.HasPrefix(ext, "chat.bsky.") { + d["chatbsky \"github.com/bluesky-social/indigo/api/chat\""] = true + } else { + // TODO: configurable mappings; and return error if none found + slog.Error("unhandled external reference", "ref", ext) + } + } + return d +} + +func (gen *CodeGenerator) WriteType(ft *FlatType) error { + + switch v := ft.Schema.Inner.(type) { + case lexicon.SchemaRecord: + if gen.Config.RegisterLexiconTypeID { + fmt.Fprintf(gen.Out, "func init() {\n") + fmt.Fprintf(gen.Out, " lexutil.RegisterType(\"%s\", &%s{})", gen.Lex.NSID, gen.baseName()) + fmt.Fprintf(gen.Out, "}\n\n") + } + // HACK: insert record-level description in to object if nil + if v.Description != nil && v.Record.Description == nil { + v.Record.Description = v.Description + } + if err := gen.writeStruct(ft, &v.Record); err != nil { + return err + } + case lexicon.SchemaQuery: + return gen.writeEndpoint(ft, defDescription(ft.Schema), v.Parameters, v.Output, nil, false) + case lexicon.SchemaProcedure: + return gen.writeEndpoint(ft, defDescription(ft.Schema), v.Parameters, v.Output, v.Input, true) + case lexicon.SchemaSubscription: + // pass; we only generate message types, not overall subscription + case lexicon.SchemaPermissionSet, lexicon.SchemaPermission: + // pass for Go codegen + case lexicon.SchemaToken: + // TODO: pass for now; could be a var/const? + case lexicon.SchemaString, lexicon.SchemaInteger, lexicon.SchemaBoolean, lexicon.SchemaUnknown: + // skip + case lexicon.SchemaObject: + if gen.Config.RegisterLexiconTypeID && ft.DefName == "main" && len(ft.Path) == 0 { + fmt.Fprintf(gen.Out, "func init() {\n") + fmt.Fprintf(gen.Out, " lexutil.RegisterType(\"%s#main\", &%s{})", gen.Lex.NSID, gen.baseName()) + fmt.Fprintf(gen.Out, "}\n\n") + } + if err := gen.writeStruct(ft, &v); err != nil { + return err + } + case lexicon.SchemaUnion: + return gen.writeUnion(ft, &v) + case lexicon.SchemaRef: + // skip for now. could be an alias type? + default: + return fmt.Errorf("unhandled schema type for codegen: %T", ft.Schema.Inner) + } + + return nil +} + +func isRequired(required []string, fname string) bool { + for _, k := range required { + if k == fname { + return true + } + } + return false +} + +func (gen *CodeGenerator) fieldType(fname string, def *lexicon.SchemaDef, optional bool) (string, error) { + // NOTE: SchemaObject and SchemaUnion should be handled outside this function; as well as arrays of those types also count + // TODO: another pass to check for type completeness + switch v := def.Inner.(type) { + case lexicon.SchemaNull: + // NOTE: using "any" as a generic 'nil' type + return "any", nil + case lexicon.SchemaBoolean: + if optional { + return "*bool", nil + } else { + return "bool", nil + } + case lexicon.SchemaInteger: + if optional { + return "*int64", nil + } else { + return "int64", nil + } + case lexicon.SchemaString: + if optional { + return "*string", nil + } else { + return "string", nil + } + case lexicon.SchemaBytes: + // NOTE: not using a pointer for optional + return "lexutil.LexBytes", nil + case lexicon.SchemaCIDLink: + if optional { + return "*lexutil.LexLink", nil + } else { + return "lexutil.LexLink", nil + } + case lexicon.SchemaBlob: + if optional || gen.Config.LegacyMode { + return "*lexutil.LexBlob", nil + } else { + return "lexutil.LexBlob", nil + } + case lexicon.SchemaArray: + t, err := gen.fieldType(fname, &v.Items, false) + if err != nil { + return "", err + } + // NOTE: not using a pointer for optional + return "[]" + t, nil + case lexicon.SchemaUnknown: + switch gen.Config.UnknownType { + case "type-decoder": + if gen.Config.LegacyMode && (fname == "didDoc" || fname == "plcOp" || fname == "meta" || fname == "debug") { + if optional { + return "*interface{}", nil + } else { + return "interface{}", nil + } + } + return "*lexutil.LexiconTypeDecoder", nil + case "json-raw-message": + if optional { + return "*json.RawMessage", nil + } else { + return "json.RawMessage", nil + } + case "map-string-any": + return "map[string]any", nil + default: + return "map[string]any", nil + } + case lexicon.SchemaRef: + ptr := "" + if optional { + ptr = "*" + } + + // check for local references to concrete types first + if strings.HasPrefix(v.Ref, "#") { + dt, ok := gen.Lex.Defs[v.Ref[1:]] + if !ok { + return "", fmt.Errorf("broken self-reference: %s", v.Ref) + } + switch dt.Type { + case "string": + if gen.Config.LegacyMode { + ptr = "*" + } + return ptr + "string", nil + case "integer": + return ptr + "int64", nil + case "boolean": + return ptr + "bool", nil + // TODO: "unknown", "ref", "token", etc + case "array": + // TODO: more completeness here (eg, non-object types) + structPtr := "" + if gen.Config.LegacyMode { + structPtr = "*" + } + return fmt.Sprintf("[]%s%s_%s_Elem", structPtr, gen.baseName(), strings.Title(v.Ref[1:])), nil + default: // presumed "object", "union" + if gen.Config.LegacyMode { + ptr = "*" + } + if v.Ref == "#main" { + return ptr + gen.baseName(), nil + } + return fmt.Sprintf("%s%s_%s", ptr, gen.baseName(), strings.Title(v.Ref[1:])), nil + } + } + + // external reference + t, err := gen.externalRefType(v.Ref) + if err != nil { + return "", err + } + if gen.Config.LegacyMode { + ptr = "*" + } + return ptr + t, nil + default: + return "", fmt.Errorf("unhandled schema type in struct field: %T", def.Inner) + } +} + +func (gen *CodeGenerator) externalRefType(ref string) (string, error) { + s, err := gen.Cat.Resolve(ref) + if err != nil { + return "", fmt.Errorf("could not resolve lexicon reference (%s): %w", ref, err) + } + + switch s.Def.(type) { + case lexicon.SchemaString: + return "string", nil + // TODO: other concrete types and special-cases types, like arrays + } + + parts := strings.SplitN(ref, "#", 3) + if len(parts) > 2 { + return "", fmt.Errorf("failed to parse external ref: %s", ref) + } + nsid, err := syntax.ParseNSID(parts[0]) + if err != nil { + return "", fmt.Errorf("failed to parse external ref NSID (%s): %w", ref, err) + } + + // check if this is actually in the same package (which might not mean the same NSID authority) + if nsidPkgName(nsid) == nsidPkgName(gen.Lex.NSID) { + if len(parts) == 1 || parts[1] == "main" { + return nsidBaseName(nsid), nil + } else { + return fmt.Sprintf("%s_%s", nsidBaseName(nsid), strings.Title(parts[1])), nil + } + } + + if len(parts) == 1 || parts[1] == "main" { + return fmt.Sprintf("%s.%s", nsidPkgName(nsid), nsidBaseName(nsid)), nil + } else { + return fmt.Sprintf("%s.%s_%s", nsidPkgName(nsid), nsidBaseName(nsid), strings.Title(parts[1])), nil + } +} + +func (gen *CodeGenerator) writeStruct(ft *FlatType, obj *lexicon.SchemaObject) error { + + name := gen.baseName() + if ft.DefName != "main" { + name += "_" + strings.Title(ft.DefName) + } + for _, sub := range ft.Path { + name += "_" + strings.Title(sub) + } + + if ft.DefName != "main" && len(ft.Path) == 0 { + fmt.Fprintf(gen.Out, "// %s is a \"%s\" in the %s schema.\n", name, ft.DefName, gen.Lex.NSID) + if obj.Description != nil { + fmt.Fprintln(gen.Out, "//") + } + } + if gen.Lex.Defs[ft.DefName].Type == "procedure" && len(ft.Path) == 1 && ft.Path[0] == "input" { + // TODO: "request body" + fmt.Fprintf(gen.Out, "// %s is the input argument to a %s call.\n", name, gen.Lex.NSID) + } + if (gen.Lex.Defs[ft.DefName].Type == "query" || gen.Lex.Defs[ft.DefName].Type == "procedure") && len(ft.Path) == 1 && ft.Path[0] == "output" { + // TODO: "response body" + fmt.Fprintf(gen.Out, "// %s is the output of a %s call.\n", name, gen.Lex.NSID) + } + skipDesc := false + if gen.Config.LegacyMode && ft.Type == "record" { + skipDesc = true + } + if obj.Description != nil && !skipDesc { + for _, l := range strings.Split(*obj.Description, "\n") { + fmt.Fprintf(gen.Out, "// %s\n", l) + } + } + fmt.Fprintf(gen.Out, "type %s struct {\n", name) + + // iterate field in sorted order + fieldNames := []string{} + for fname := range obj.Properties { + fieldNames = append(fieldNames, fname) + } + sort.Strings(fieldNames) + + // if this is a def-level struct, write out type decoder + skipType := false + if gen.Config.LegacyMode { + // TODO: skip $type for all defs in subscription. this isn't robust! + switch gen.Lex.MainType() { + case "subscription": + skipType = true + } + } + if len(ft.Path) == 0 && !skipType { + // TODO: can skip in some more situations? + fullName := gen.Lex.NSID.String() + if ft.DefName != "main" { + fullName += "#" + ft.DefName + } + omitempty := ",omitempty" + if ft.Type == "record" { + omitempty = "" + } + fmt.Fprintf(gen.Out, " LexiconTypeID string `json:\"$type%s\" cborgen:\"$type,const=%s%s\"`\n", omitempty, fullName, omitempty) + } + + for _, fname := range fieldNames { + field := obj.Properties[fname] + optional := false + omitempty := "" + if !isRequired(obj.Required, fname) { + optional = true + omitempty = ",omitempty" + } + if obj.IsNullable(fname) { + optional = true + omitempty = "" + } + + var t string + var err error + + switch v := field.Inner.(type) { + case lexicon.SchemaObject, lexicon.SchemaUnion: + t = name + "_" + strings.Title(fname) + if optional || gen.Config.LegacyMode { + t = "*" + t + } + case lexicon.SchemaArray: + switch v.Items.Inner.(type) { + case lexicon.SchemaObject, lexicon.SchemaUnion: + elemPtr := "" + if gen.Config.LegacyMode { + elemPtr = "*" + } + // NOTE: not using ptr for optional + t = fmt.Sprintf("[]%s%s_%s_Elem", elemPtr, name, strings.Title(fname)) + default: + t, err = gen.fieldType(fname, &field, optional) + if err != nil { + return err + } + } + default: + t, err = gen.fieldType(fname, &field, optional) + if err != nil { + return err + } + } + + if gen.Config.LegacyMode { + switch field.Inner.(type) { + case lexicon.SchemaBytes: + omitempty = ",omitempty" + } + } + + cborExtra := "" + // HACK: copied from legacy code for now + if gen.Lex.NSID.String() == "com.atproto.label.defs" && name == "LabelDefs_SelfLabels" && fname == "values" { + cborExtra = ",preservenil" + } + + desc := defDescription(&field) + if desc != "" { + fmt.Fprintf(gen.Out, " // %s: %s\n", fname, desc) + } + fmt.Fprintf(gen.Out, " %s %s", strings.ReplaceAll(strings.Title(fname), "-", ""), t) + fmt.Fprintf(gen.Out, " `json:\"%s%s\" cborgen:\"%s%s%s\"`\n", fname, omitempty, fname, omitempty, cborExtra) + } + fmt.Fprintf(gen.Out, "}\n\n") + + return nil +} + +type unionRef struct { + FieldName string + TypeName string + LexName string +} + +func (gen *CodeGenerator) writeUnion(ft *FlatType, union *lexicon.SchemaUnion) error { + + name := gen.baseName() + if ft.DefName != "main" { + name += "_" + strings.Title(ft.DefName) + } + for _, sub := range ft.Path { + name += "_" + strings.Title(sub) + } + + unionRefs := map[string]unionRef{} + refNames := []string{} + for _, ref := range union.Refs { + r := unionRef{ + LexName: ref, + } + + if strings.HasPrefix(ref, "#") { + r.LexName = gen.Lex.NSID.String() + ref + n := gen.baseName() + if ref != "#main" { + n += "_" + strings.Title(ref[1:]) + } + r.FieldName = n + r.TypeName = n + } else { + n, err := gen.externalRefType(ref) + if err != nil { + return err + } + r.FieldName = n + r.TypeName = n + if strings.Contains(n, ".") { + parts := strings.SplitN(n, ".", 2) + r.FieldName = parts[1] + } + } + refNames = append(refNames, r.FieldName) + unionRefs[r.FieldName] = r + } + if !gen.Config.LegacyMode { + sort.Strings(refNames) + } + + // first print out the union struct type + if union.Description != nil { + for _, l := range strings.Split(*union.Description, "\n") { + fmt.Fprintf(gen.Out, "// %s\n", l) + } + } + fmt.Fprintf(gen.Out, "type %s struct {\n", name) + for _, rname := range refNames { + ref := unionRefs[rname] + fmt.Fprintf(gen.Out, " %s *%s\n", ref.FieldName, ref.TypeName) + } + fmt.Fprintf(gen.Out, "}\n\n") + + // ... then MarshalJSON + fmt.Fprintf(gen.Out, "func (t *%s) MarshalJSON() ([]byte, error) {\n", name) + for _, rname := range refNames { + ref := unionRefs[rname] + fmt.Fprintf(gen.Out, " if t.%s != nil {\n", ref.FieldName) + fmt.Fprintf(gen.Out, " t.%s.LexiconTypeID = \"%s\"\n", ref.FieldName, ref.LexName) + fmt.Fprintf(gen.Out, " return json.Marshal(t.%s)\n", ref.FieldName) + fmt.Fprintf(gen.Out, " }\n") + } + fmt.Fprintf(gen.Out, " return nil, fmt.Errorf(\"can not marshal empty union as JSON\")") + fmt.Fprintf(gen.Out, "}\n\n") + + // ... then UnmarshalJSON + fmt.Fprintf(gen.Out, "func (t *%s) UnmarshalJSON(b []byte) error {\n", name) + fmt.Fprintf(gen.Out, " typ, err := lexutil.TypeExtract(b)\n") + fmt.Fprintf(gen.Out, " if err != nil {\n") + fmt.Fprintf(gen.Out, " return err\n") + fmt.Fprintf(gen.Out, " }\n\n") + fmt.Fprintf(gen.Out, " switch typ {\n") + for _, rname := range refNames { + ref := unionRefs[rname] + fmt.Fprintf(gen.Out, " case \"%s\":\n", ref.LexName) + fmt.Fprintf(gen.Out, " t.%s = new(%s)\n", ref.FieldName, ref.TypeName) + fmt.Fprintf(gen.Out, " return json.Unmarshal(b, t.%s)\n", ref.FieldName) + } + fmt.Fprintf(gen.Out, " default:\n") + if union.Closed != nil && *union.Closed { + // TODO: better error message + fmt.Fprintf(gen.Out, " return fmt.Errorf(\"closed unions must match a listed schema\")\n") + } else { + fmt.Fprintf(gen.Out, " return nil\n") + } + fmt.Fprintf(gen.Out, " }\n") + fmt.Fprintf(gen.Out, "}\n\n") + + // only import CBOR marshalling of unions in legacy mode + if !gen.Config.LegacyMode { + return nil + } + + switch gen.Lex.MainType() { + case "record", "subscription": + // no-op + case "object": + // hacks for legacy serialization + nsid := gen.Lex.NSID.String() + if !(nsid == "app.bsky.richtext.facet" || (nsid == "app.bsky.embed.recordWithMedia" && ft.DefName == "main")) { + return nil + } + default: + return nil + } + + // ... then MarshalCBOR + fmt.Fprintf(gen.Out, "func (t *%s) MarshalCBOR(w io.Writer) error {\n\n", name) + fmt.Fprintf(gen.Out, " if t == nil {\n") + fmt.Fprintf(gen.Out, " _, err := w.Write(cbg.CborNull)\n") + fmt.Fprintf(gen.Out, " return err") + fmt.Fprintf(gen.Out, " }\n") + for _, rname := range refNames { + ref := unionRefs[rname] + fmt.Fprintf(gen.Out, " if t.%s != nil {\n", ref.FieldName) + fmt.Fprintf(gen.Out, " return t.%s.MarshalCBOR(w)\n", ref.FieldName) + fmt.Fprintf(gen.Out, " }\n") + } + fmt.Fprintf(gen.Out, " return fmt.Errorf(\"can not marshal empty union as CBOR\")") + fmt.Fprintf(gen.Out, "}\n\n") + + // ... then UnmarshalCBOR + fmt.Fprintf(gen.Out, "func (t *%s) UnmarshalCBOR(r io.Reader) error {\n", name) + fmt.Fprintf(gen.Out, " typ, b, err := lexutil.CborTypeExtractReader(r)\n") + fmt.Fprintf(gen.Out, " if err != nil {\n") + fmt.Fprintf(gen.Out, " return err\n") + fmt.Fprintf(gen.Out, " }\n\n") + fmt.Fprintf(gen.Out, " switch typ {\n") + for _, rname := range refNames { + ref := unionRefs[rname] + fmt.Fprintf(gen.Out, " case \"%s\":\n", ref.LexName) + fmt.Fprintf(gen.Out, " t.%s = new(%s)\n", ref.FieldName, ref.TypeName) + fmt.Fprintf(gen.Out, " return t.%s.UnmarshalCBOR(bytes.NewReader(b))\n", ref.FieldName) + } + fmt.Fprintf(gen.Out, " default:\n") + fmt.Fprintf(gen.Out, " return nil\n") + fmt.Fprintf(gen.Out, " }\n") + fmt.Fprintf(gen.Out, "}\n\n") + + return nil +} + +func (gen *CodeGenerator) writeEndpoint(ft *FlatType, desc string, params *lexicon.SchemaParams, output, input *lexicon.SchemaBody, isProcedure bool) error { + name := gen.baseName() + + fmt.Fprintf(gen.Out, "// %s calls the XRPC method \"%s\".\n", name, gen.Lex.NSID) + if desc != "" && !gen.Config.LegacyMode { + fmt.Fprintln(gen.Out, "//") + for _, l := range strings.Split(desc, "\n") { + fmt.Fprintf(gen.Out, "// %s\n", l) + } + } + + outputBytes := false + outputStruct := "" + if output != nil && output.Schema != nil { + switch v := output.Schema.Inner.(type) { + case lexicon.SchemaObject, lexicon.SchemaUnion: + outputStruct = name + "_Output" + case lexicon.SchemaRef: + if strings.HasPrefix(v.Ref, "#") { + // local reference + outputStruct = fmt.Sprintf("%s_%s", gen.baseName(), strings.Title(v.Ref[1:])) + } else { + // external reference + t, err := gen.externalRefType(v.Ref) + if err != nil { + return err + } + outputStruct = t + } + default: + return fmt.Errorf("unsupported endpoint output schema def type: %T", output.Schema.Inner) + } + } else if output != nil && output.Encoding != "" { + outputBytes = true + } + + paramNames := []string{} + if params != nil { + for name := range params.Properties { + paramNames = append(paramNames, name) + } + } + sort.Strings(paramNames) + + args := []string{"ctx context.Context", "c lexutil.LexClient"} + reqParams := []string{} + optParams := []string{} + if len(paramNames) > 0 { + fmt.Fprintln(gen.Out, "//") + for _, name := range paramNames { + param := params.Properties[name] + ptr := "*" + if isRequired(params.Required, name) { + ptr = "" + reqParams = append(reqParams, name) + } else { + optParams = append(optParams, name) + } + switch v := param.Inner.(type) { + case lexicon.SchemaBoolean: + if v.Description != nil && *v.Description != "" { + fmt.Fprintf(gen.Out, "// %s: %s\n", name, *v.Description) + } + if gen.Config.LegacyMode { + ptr = "" + } + args = append(args, fmt.Sprintf("%s %sbool", name, ptr)) + case lexicon.SchemaInteger: + if v.Description != nil && *v.Description != "" { + fmt.Fprintf(gen.Out, "// %s: %s\n", name, *v.Description) + } + if gen.Config.LegacyMode { + ptr = "" + } + args = append(args, fmt.Sprintf("%s %sint64", name, ptr)) + case lexicon.SchemaString: + if v.Description != nil && *v.Description != "" { + fmt.Fprintf(gen.Out, "// %s: %s\n", name, *v.Description) + } + args = append(args, fmt.Sprintf("%s string", name)) + case lexicon.SchemaUnknown: + if v.Description != nil && *v.Description != "" { + fmt.Fprintf(gen.Out, "// %s: %s\n", name, *v.Description) + } + args = append(args, fmt.Sprintf("%s any", name)) + case lexicon.SchemaArray: + if v.Description != nil && *v.Description != "" { + suffix := "[]" + if gen.Config.LegacyMode { + suffix = "" + } + fmt.Fprintf(gen.Out, "// %s%s: %s\n", name, suffix, *v.Description) + } + switch v.Items.Inner.(type) { + case lexicon.SchemaBoolean: + args = append(args, fmt.Sprintf("%s []bool", name)) + case lexicon.SchemaInteger: + args = append(args, fmt.Sprintf("%s []int64", name)) + case lexicon.SchemaString: + args = append(args, fmt.Sprintf("%s []string", name)) + default: + return fmt.Errorf("unsupported parameter array type: %T", param.Inner) + } + default: + return fmt.Errorf("unsupported parameter type: %T", param.Inner) + } + } + } + + inputArg := "nil" + inputEncoding := "" + inputStruct := "" + if isProcedure && input != nil && input.Schema != nil { + inputArg = "input" + inputEncoding = input.Encoding + switch v := input.Schema.Inner.(type) { + case lexicon.SchemaObject, lexicon.SchemaUnion: + inputStruct = name + "_Input" + case lexicon.SchemaRef: + if strings.HasPrefix(v.Ref, "#") { + // local reference + inputStruct = fmt.Sprintf("%s_%s", gen.baseName(), strings.Title(v.Ref[1:])) + } else { + // external reference + t, err := gen.externalRefType(v.Ref) + if err != nil { + return err + } + inputStruct = t + } + } + args = append(args, fmt.Sprintf("input *%s", inputStruct)) + } else if isProcedure && input != nil && input.Encoding != "" { + inputArg = "input" + inputEncoding = input.Encoding + args = append(args, "input io.Reader") + } + + doOutParam := "" + returnType := "" + fmt.Fprintf(gen.Out, "func %s(%s) ", name, strings.Join(args, ", ")) + if outputStruct != "" { + fmt.Fprintf(gen.Out, "(*%s, error) {\n", outputStruct) + fmt.Fprintf(gen.Out, " var out %s\n", outputStruct) + if !gen.Config.LegacyMode { + fmt.Fprintln(gen.Out, "") + } + doOutParam = "&out" + returnType = "&out" + } else if outputBytes { + fmt.Fprintf(gen.Out, "([]byte, error) {\n") + fmt.Fprintf(gen.Out, " buf := new(bytes.Buffer)\n\n") + doOutParam = "buf" + returnType = "buf.Bytes()" + } else { + fmt.Fprintf(gen.Out, "error {\n") + doOutParam = "nil" + } + paramsArg := "nil" + if params != nil && len(params.Properties) > 0 { + paramsArg = "params" + if gen.Config.LegacyMode { + fmt.Fprintln(gen.Out, "") + } + // TODO: switch to map[string]any + fmt.Fprintf(gen.Out, " params := map[string]interface{}{}\n") + } + for _, name := range optParams { + param := params.Properties[name] + switch param.Inner.(type) { + case lexicon.SchemaString: + fmt.Fprintf(gen.Out, " if %s != \"\" {\n", name) + fmt.Fprintf(gen.Out, " params[\"%s\"] = %s\n", name, name) + fmt.Fprintf(gen.Out, " }\n") + case lexicon.SchemaArray: + fmt.Fprintf(gen.Out, " if len(%s) != 0 {\n", name) + fmt.Fprintf(gen.Out, " params[\"%s\"] = %s\n", name, name) + fmt.Fprintf(gen.Out, " }\n") + case lexicon.SchemaUnknown: + fmt.Fprintf(gen.Out, " if %s != nil {\n", name) + fmt.Fprintf(gen.Out, " params[\"%s\"] = %s\n", name, name) + fmt.Fprintf(gen.Out, " }\n") + case lexicon.SchemaInteger: + if gen.Config.LegacyMode { + fmt.Fprintf(gen.Out, " if %s != 0 {\n", name) + fmt.Fprintf(gen.Out, " params[\"%s\"] = %s\n", name, name) + fmt.Fprintf(gen.Out, " }\n") + } else { + fmt.Fprintf(gen.Out, " if %s != nil {\n", name) + fmt.Fprintf(gen.Out, " params[\"%s\"] = *%s\n", name, name) + fmt.Fprintf(gen.Out, " }\n") + } + case lexicon.SchemaBoolean: + if gen.Config.LegacyMode { + fmt.Fprintf(gen.Out, " if %s {\n", name) + fmt.Fprintf(gen.Out, " params[\"%s\"] = %s\n", name, name) + fmt.Fprintf(gen.Out, " }\n") + } else { + fmt.Fprintf(gen.Out, " if %s != nil {\n", name) + fmt.Fprintf(gen.Out, " params[\"%s\"] = *%s\n", name, name) + fmt.Fprintf(gen.Out, " }\n") + } + default: + fmt.Fprintf(gen.Out, " if %s != nil {\n", name) + fmt.Fprintf(gen.Out, " params[\"%s\"] = *%s\n", name, name) + fmt.Fprintf(gen.Out, " }\n") + } + } + for _, name := range reqParams { + fmt.Fprintf(gen.Out, " params[\"%s\"] = %s\n", name, name) + } + if !gen.Config.LegacyMode { + fmt.Fprintln(gen.Out, "") + } + + method := "lexutil.Query" + if isProcedure { + method = "lexutil.Procedure" + } + + fmt.Fprintf(gen.Out, " if err := c.LexDo(ctx, %s, \"%s\", \"%s\", %s, %s, %s); err != nil {\n", method, inputEncoding, gen.Lex.NSID, paramsArg, inputArg, doOutParam) + if returnType != "" { + fmt.Fprintf(gen.Out, " return nil, err\n") + } else { + fmt.Fprintf(gen.Out, " return err\n") + } + fmt.Fprintf(gen.Out, " }\n") + if gen.Config.LegacyMode { + fmt.Fprintln(gen.Out, "") + } + if returnType != "" { + fmt.Fprintf(gen.Out, " return %s, nil\n", returnType) + } else { + fmt.Fprintf(gen.Out, " return nil\n") + } + fmt.Fprintf(gen.Out, "}\n\n") + + return nil +} diff --git a/lex/lexgen/doc.go b/lex/lexgen/doc.go new file mode 100644 index 00000000..2133f81e --- /dev/null +++ b/lex/lexgen/doc.go @@ -0,0 +1,17 @@ +/* +Package implementing Go code generation for lexicon schemas. + +Used by the 'lexgen' CLI tool to output Go structs and client API helpers based on Lexicon schemas. This package currently includes a "legacy" mode to stay as close as possible to the previous code generation output. + +WARNING: this package is still a work in progress. Both the package API and the generated code are likely to change, possibly in backwards-incompatible ways. + +# Package Structure + +The package works in two steps: + +- "flattening" parses a full lexicon schema file and copies nested type definitions in to a top-level array +- code generation outputs a single Go source code file corresponding to a flattened lexicon schema file + +Wrapping code is expected to handle code formatting and fixing imports (which mostly means removing unused imports). +*/ +package lexgen diff --git a/lex/lexgen/flatten.go b/lex/lexgen/flatten.go new file mode 100644 index 00000000..a8388c0d --- /dev/null +++ b/lex/lexgen/flatten.go @@ -0,0 +1,247 @@ +package lexgen + +import ( + "fmt" + "slices" + "sort" + "strings" + + "github.com/bluesky-social/indigo/atproto/lexicon" + "github.com/bluesky-social/indigo/atproto/syntax" +) + +// Intermediate representation of a complete lexicon schema file, containing one or more definitions. +type FlatLexicon struct { + NSID syntax.NSID + Description *string + ExternalRefs map[string]bool // NSID with optional ref + Defs map[string]FlatDef + Types []FlatType +} + +// Minimal context about an individual top-level schema definition: just the short name and schema type. +type FlatDef struct { + Name string + Type string +} + +// An individual "type definition", which is a small unit of schema definition that corresponds to a named unit of generated code. For example, a struct or API endpoint. +type FlatType struct { + // the short name of the schema def that this type is under + DefName string + Path []string + Type string + Schema *lexicon.SchemaDef +} + +func FlattenSchemaFile(sf *lexicon.SchemaFile) (*FlatLexicon, error) { + nsid, err := syntax.ParseNSID(sf.ID) + if err != nil { + return nil, err + } + + fl := FlatLexicon{ + NSID: nsid, + Description: sf.Description, + ExternalRefs: map[string]bool{}, + Defs: map[string]FlatDef{}, + Types: []FlatType{}, + } + + // iterate defs in sorted order; except "main" is always first if present + defNames := []string{} + hasMain := false + for name := range sf.Defs { + if name == "main" { + hasMain = true + continue + } + defNames = append(defNames, name) + } + sort.Strings(defNames) + if hasMain { + defNames = append([]string{"main"}, defNames...) + } + + for _, name := range defNames { + def := sf.Defs[name] + if err := fl.flattenDef(name, &def); err != nil { + return nil, err + } + } + + return &fl, nil +} + +func (fl *FlatLexicon) flattenDef(name string, def *lexicon.SchemaDef) error { + + t, err := defType(def) + if err != nil { + return err + } + + fd := FlatDef{ + Name: name, + Type: t, + } + fl.Defs[name] = fd + + return fl.flattenType(&fd, []string{}, def) +} + +func (fl *FlatLexicon) flattenType(fd *FlatDef, tpath []string, def *lexicon.SchemaDef) error { + + t, err := defType(def) + if err != nil { + return err + } + + ft := FlatType{ + DefName: fd.Name, + Path: slices.Clone(tpath), + Type: t, + Schema: def, + } + + switch v := def.Inner.(type) { + case lexicon.SchemaRecord: + fl.Types = append(fl.Types, ft) + if err := fl.flattenObject(fd, tpath, &v.Record); err != nil { + return err + } + case lexicon.SchemaQuery: + // v.Properties: only boolean, integer, string, or unknown are allowed, so recursion not really needed? + if v.Output != nil && v.Output.Schema != nil { + tp := slices.Clone(tpath) + tp = append(tp, "output") + if err := fl.flattenType(fd, tp, v.Output.Schema); err != nil { + return err + } + } + fl.Types = append(fl.Types, ft) + case lexicon.SchemaProcedure: + // v.Properties: same as above + if v.Input != nil && v.Input.Schema != nil { + tp := slices.Clone(tpath) + tp = append(tp, "input") + if err := fl.flattenType(fd, tp, v.Input.Schema); err != nil { + return err + } + } + if v.Output != nil && v.Output.Schema != nil { + tp := slices.Clone(tpath) + tp = append(tp, "output") + if err := fl.flattenType(fd, tp, v.Output.Schema); err != nil { + return err + } + } + fl.Types = append(fl.Types, ft) + case lexicon.SchemaSubscription: + // v.Properties: same as above + if v.Message != nil { + switch vv := v.Message.Schema.Inner.(type) { + case lexicon.SchemaUnion: + for _, ref := range vv.Refs { + if !strings.HasPrefix(ref, "#") { + fl.ExternalRefs[strings.TrimSuffix(ref, "#main")] = true + } + } + default: + return fmt.Errorf("subscription with non-union message schema: %T", v.Message.Schema.Inner) + } + } + fl.Types = append(fl.Types, ft) + case lexicon.SchemaObject: + fl.Types = append(fl.Types, ft) + if err := fl.flattenObject(fd, tpath, &v); err != nil { + return err + } + case lexicon.SchemaRef: + if !strings.HasPrefix(v.Ref, "#") { + fl.ExternalRefs[strings.TrimSuffix(v.Ref, "#main")] = true + } + fl.Types = append(fl.Types, ft) + case lexicon.SchemaUnion: + for _, ref := range v.Refs { + if !strings.HasPrefix(ref, "#") { + fl.ExternalRefs[strings.TrimSuffix(ref, "#main")] = true + } + } + fl.Types = append(fl.Types, ft) + case lexicon.SchemaArray: + // flatten the inner item + tp := slices.Clone(tpath) + tp = append(tp, "elem") + if err := fl.flattenType(fd, tpath, &v.Items); err != nil { + return err + } + // don't emit the array itself + return nil + case lexicon.SchemaString, lexicon.SchemaNull, lexicon.SchemaInteger, lexicon.SchemaBoolean, lexicon.SchemaUnknown, lexicon.SchemaBytes: + // don't emit + // NOTE: might want to emit some string "knownValue" lists in the future? + case lexicon.SchemaCIDLink, lexicon.SchemaBlob: + // don't emit + case lexicon.SchemaToken: + // pass-through (emit) + fl.Types = append(fl.Types, ft) + case lexicon.SchemaPermissionSet, lexicon.SchemaPermission: + // pass-through (emit) + fl.Types = append(fl.Types, ft) + default: + return fmt.Errorf("unsupported def type for flattening (%s): %T", fd.Name, def.Inner) + } + + return nil +} + +func (fl *FlatLexicon) flattenObject(fd *FlatDef, tpath []string, obj *lexicon.SchemaObject) error { + + keys := []string{} + for n := range obj.Properties { + keys = append(keys, n) + } + sort.Strings(keys) + + for _, fname := range keys { + field := obj.Properties[fname] + tp := slices.Clone(tpath) + tp = append(tp, fname) + switch v := field.Inner.(type) { + case lexicon.SchemaNull, lexicon.SchemaBoolean, lexicon.SchemaInteger, lexicon.SchemaString, lexicon.SchemaBytes: + // no-op + case lexicon.SchemaCIDLink, lexicon.SchemaBlob, lexicon.SchemaUnknown: + // no-op, but maybe set a flag on def? + case lexicon.SchemaArray: + tp = append(tp, "elem") + if err := fl.flattenType(fd, tp, &v.Items); err != nil { + return err + } + case lexicon.SchemaObject: + if err := fl.flattenType(fd, tp, &field); err != nil { + return err + } + case lexicon.SchemaRef: + if !strings.HasPrefix(v.Ref, "#") { + // remove any #main suffix + fl.ExternalRefs[strings.TrimSuffix(v.Ref, "#main")] = true + } + case lexicon.SchemaUnion: + if err := fl.flattenType(fd, tp, &field); err != nil { + return err + } + default: + return fmt.Errorf("unsupported field type for object flattening: %T", field.Inner) + } + } + return nil +} + +// Returns the type of any "#main" definition in this file (or else an empty string) +func (fl *FlatLexicon) MainType() string { + main, ok := fl.Defs["main"] + if !ok { + return "" + } + return main.Type +} diff --git a/lex/lexgen/util.go b/lex/lexgen/util.go new file mode 100644 index 00000000..0a8e110d --- /dev/null +++ b/lex/lexgen/util.go @@ -0,0 +1,160 @@ +package lexgen + +import ( + "fmt" + "slices" + "strings" + + "github.com/bluesky-social/indigo/atproto/lexicon" + "github.com/bluesky-social/indigo/atproto/syntax" + + "golang.org/x/net/publicsuffix" +) + +func defType(sd *lexicon.SchemaDef) (string, error) { + switch sd.Inner.(type) { + case lexicon.SchemaRecord: + return "record", nil + case lexicon.SchemaQuery: + return "query", nil + case lexicon.SchemaProcedure: + return "procedure", nil + case lexicon.SchemaSubscription: + return "subscription", nil + case lexicon.SchemaPermissionSet: + return "permission-set", nil + case lexicon.SchemaPermission: + return "permission", nil + case lexicon.SchemaNull: + return "null", nil + case lexicon.SchemaBoolean: + return "boolean", nil + case lexicon.SchemaInteger: + return "integer", nil + case lexicon.SchemaString: + return "string", nil + case lexicon.SchemaBytes: + return "bytes", nil + case lexicon.SchemaCIDLink: + return "cid-link", nil + case lexicon.SchemaArray: + return "array", nil + case lexicon.SchemaObject: + return "object", nil + case lexicon.SchemaBlob: + return "blob", nil + case lexicon.SchemaParams: + return "params", nil + case lexicon.SchemaToken: + return "token", nil + case lexicon.SchemaRef: + return "ref", nil + case lexicon.SchemaUnion: + return "union", nil + case lexicon.SchemaUnknown: + return "unknown", nil + default: + return "", fmt.Errorf("unhandled schema type: %T", sd.Inner) + } +} + +func defDescription(sd *lexicon.SchemaDef) string { + var desc *string + + switch v := sd.Inner.(type) { + case lexicon.SchemaRecord: + desc = v.Description + case lexicon.SchemaQuery: + desc = v.Description + case lexicon.SchemaProcedure: + desc = v.Description + case lexicon.SchemaSubscription: + desc = v.Description + case lexicon.SchemaPermissionSet: + // TODO: extract *some* description? + case lexicon.SchemaPermission: + desc = v.Description + case lexicon.SchemaNull: + desc = v.Description + case lexicon.SchemaBoolean: + desc = v.Description + case lexicon.SchemaInteger: + desc = v.Description + case lexicon.SchemaString: + desc = v.Description + case lexicon.SchemaBytes: + desc = v.Description + case lexicon.SchemaCIDLink: + desc = v.Description + case lexicon.SchemaArray: + desc = v.Description + case lexicon.SchemaObject: + desc = v.Description + case lexicon.SchemaBlob: + desc = v.Description + case lexicon.SchemaParams: + desc = v.Description + case lexicon.SchemaToken: + desc = v.Description + case lexicon.SchemaRef: + desc = v.Description + case lexicon.SchemaUnion: + desc = v.Description + case lexicon.SchemaUnknown: + desc = v.Description + } + if desc != nil && *desc != "" { + return *desc + } + return "" +} + +func isCompoundDef(sd *lexicon.SchemaDef) bool { + switch sd.Inner.(type) { + case lexicon.SchemaRecord, lexicon.SchemaQuery, lexicon.SchemaProcedure, lexicon.SchemaSubscription, lexicon.SchemaArray, lexicon.SchemaObject, lexicon.SchemaUnion: + return true + default: + return false + } +} + +func nsidPkgName(nsid syntax.NSID) string { + domain := strings.ToLower(nsid.Authority()) + reg, err := publicsuffix.EffectiveTLDPlusOne(domain) + if err != nil { + return "FAIL" + } + parts := strings.Split(reg, ".") + slices.Reverse(parts) + + return strings.Join(parts, "") +} + +func nsidBaseName(nsid syntax.NSID) string { + domain := strings.ToLower(nsid.Authority()) + reg, err := publicsuffix.EffectiveTLDPlusOne(domain) + if err != nil { + return "FAIL" + } + rem := domain[0 : len(domain)-len(reg)] + parts := strings.Split(rem, ".") + slices.Reverse(parts) + parts = append(parts, nsid.Name()) + for i := range parts { + parts[i] = strings.Title(parts[i]) + } + return strings.Join(parts, "") +} + +func nsidFileName(nsid syntax.NSID) string { + domain := strings.ToLower(nsid.Authority()) + reg, err := publicsuffix.EffectiveTLDPlusOne(domain) + if err != nil { + return "FAIL" + } + rem := domain[0 : len(domain)-len(reg)] + parts := strings.Split(rem, ".") + slices.Reverse(parts) + parts = append(parts, nsid.Name()) + return strings.Join(parts, "") +} diff --git a/lex/lexgen/util_test.go b/lex/lexgen/util_test.go new file mode 100644 index 00000000..fba7f4ae --- /dev/null +++ b/lex/lexgen/util_test.go @@ -0,0 +1,25 @@ +package lexgen + +import ( + "testing" + + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/stretchr/testify/assert" +) + +func TestNSIDNames(t *testing.T) { + assert := assert.New(t) + + testVectors := [][]string{ + {"app.bsky.feed.post", "appbsky", "FeedPost"}, + {"com.atproto.admin.deleteAccount", "comatproto", "AdminDeleteAccount"}, + {"uk.ac.school.lab.COOL.project", "ukacschool", "LabCoolProject"}, + } + + for _, vec := range testVectors { + nsid := syntax.NSID(vec[0]) + assert.Equal(vec[1], nsidPkgName(nsid)) + assert.Equal(vec[2], nsidBaseName(nsid)) + } +}