From 8a404c31dfb3c13d02c4c11cbc7a2d582637508d Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Wed, 29 Mar 2023 15:32:04 -0700 Subject: [PATCH] lex: refactor $blob, $link, and $bytes serialization --- gen/main.go | 2 +- lex/gen.go | 8 +- lex/util/cbor_gen.go | 188 +++++++++++++++++++++++++++--- lex/util/lex_types.go | 262 ++++++++++++++++++++++++++++++++++++++++++ lex/util/util.go | 70 ----------- 5 files changed, 441 insertions(+), 89 deletions(-) create mode 100644 lex/util/lex_types.go diff --git a/gen/main.go b/gen/main.go index 86b27d65..02022384 100644 --- a/gen/main.go +++ b/gen/main.go @@ -32,7 +32,7 @@ func main() { panic(err) } - if err := cbg.WriteMapEncodersToFile("lex/util/cbor_gen.go", "util", lexutil.CborChecker{}, lexutil.Blob{}); err != nil { + if err := cbg.WriteMapEncodersToFile("lex/util/cbor_gen.go", "util", lexutil.CborChecker{}, lexutil.LegacyBlob{}, lexutil.BlobSchema{}); err != nil { panic(err) } diff --git a/lex/gen.go b/lex/gen.go index 4eb053f5..ef4d72ce 100644 --- a/lex/gen.go +++ b/lex/gen.go @@ -1018,10 +1018,8 @@ func (s *TypeSchema) typeNameForField(name, k string, v TypeSchema) (string, err return "util.LexiconTypeDecoder", nil case "union": return "*" + name + "_" + strings.Title(k), nil - case "image": - return "*util.Blob", nil case "blob": - return "*util.Blob", nil + return "*util.LexBlob", nil case "array": subt, err := s.typeNameForField(name+"_"+strings.Title(k), "Elem", *v.Items) if err != nil { @@ -1030,7 +1028,9 @@ func (s *TypeSchema) typeNameForField(name, k string, v TypeSchema) (string, err return "[]" + subt, nil case "cid-link": - return "cid.Cid", nil + return "util.LexLink", nil + case "bytes": + return "util.LexBytes", nil default: return "", fmt.Errorf("field %q in %s has unsupported type name (%s)", k, name, v.Type) } diff --git a/lex/util/cbor_gen.go b/lex/util/cbor_gen.go index 4814538e..b93a560a 100644 --- a/lex/util/cbor_gen.go +++ b/lex/util/cbor_gen.go @@ -113,7 +113,7 @@ func (t *CborChecker) UnmarshalCBOR(r io.Reader) (err error) { return nil } -func (t *Blob) MarshalCBOR(w io.Writer) error { +func (t *LegacyBlob) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err @@ -121,11 +121,140 @@ func (t *Blob) MarshalCBOR(w io.Writer) error { cw := cbg.NewCborWriter(w) - if _, err := cw.Write([]byte{163}); err != nil { + if _, err := cw.Write([]byte{162}); err != nil { return err } - // t.Ref (cid.Cid) (struct) + // t.Cid (string) (string) + if len("cid") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"cid\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("cid"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("cid")); err != nil { + return err + } + + if len(t.Cid) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.Cid was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Cid))); err != nil { + return err + } + if _, err := io.WriteString(w, string(t.Cid)); err != nil { + return err + } + + // t.MimeType (string) (string) + if len("mimeType") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"mimeType\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("mimeType"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("mimeType")); err != nil { + return err + } + + if len(t.MimeType) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.MimeType was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.MimeType))); err != nil { + return err + } + if _, err := io.WriteString(w, string(t.MimeType)); err != nil { + return err + } + return nil +} + +func (t *LegacyBlob) UnmarshalCBOR(r io.Reader) (err error) { + *t = LegacyBlob{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("LegacyBlob: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Cid (string) (string) + case "cid": + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + t.Cid = string(sval) + } + // t.MimeType (string) (string) + case "mimeType": + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + t.MimeType = string(sval) + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *BlobSchema) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.Ref (util.LexLink) (struct) if len("ref") > cbg.MaxLength { return xerrors.Errorf("Value in field \"ref\" was too long") } @@ -137,8 +266,8 @@ func (t *Blob) MarshalCBOR(w io.Writer) error { return err } - if err := cbg.WriteCid(cw, t.Ref); err != nil { - return xerrors.Errorf("failed to write cid field t.Ref: %w", err) + if err := t.Ref.MarshalCBOR(cw); err != nil { + return err } // t.Size (int64) (int64) @@ -185,11 +314,34 @@ func (t *Blob) MarshalCBOR(w io.Writer) error { if _, err := io.WriteString(w, string(t.MimeType)); err != nil { return err } + + // t.LexiconTypeID (string) (string) + if len("LexiconTypeID") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"LexiconTypeID\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("LexiconTypeID"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("LexiconTypeID")); err != nil { + return err + } + + if len(t.LexiconTypeID) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.LexiconTypeID was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.LexiconTypeID))); err != nil { + return err + } + if _, err := io.WriteString(w, string(t.LexiconTypeID)); err != nil { + return err + } return nil } -func (t *Blob) UnmarshalCBOR(r io.Reader) (err error) { - *t = Blob{} +func (t *BlobSchema) UnmarshalCBOR(r io.Reader) (err error) { + *t = BlobSchema{} cr := cbg.NewCborReader(r) @@ -208,7 +360,7 @@ func (t *Blob) UnmarshalCBOR(r io.Reader) (err error) { } if extra > cbg.MaxLength { - return fmt.Errorf("Blob: map struct too large (%d)", extra) + return fmt.Errorf("BlobSchema: map struct too large (%d)", extra) } var name string @@ -226,18 +378,15 @@ func (t *Blob) UnmarshalCBOR(r io.Reader) (err error) { } switch name { - // t.Ref (cid.Cid) (struct) + // t.Ref (util.LexLink) (struct) case "ref": { - c, err := cbg.ReadCid(cr) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Ref: %w", err) + if err := t.Ref.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Ref: %w", err) } - t.Ref = c - } // t.Size (int64) (int64) case "size": @@ -276,6 +425,17 @@ func (t *Blob) UnmarshalCBOR(r io.Reader) (err error) { t.MimeType = string(sval) } + // t.LexiconTypeID (string) (string) + case "LexiconTypeID": + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + t.LexiconTypeID = string(sval) + } default: // Field doesn't exist on this type, so ignore it diff --git a/lex/util/lex_types.go b/lex/util/lex_types.go new file mode 100644 index 00000000..e2a4d8cc --- /dev/null +++ b/lex/util/lex_types.go @@ -0,0 +1,262 @@ +package util + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "io" + + "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +const ( + // TODO: this is an arbitrary size. lexicons can set more realistic limits, + // and we should pass those limits through and only fall back to this when + // undefined. + MAX_BYTE_ARRAY_SIZE = 128 * 1024 * 1024 +) + +type LexLink cid.Cid + +type jsonLink struct { + Link string `json:"$type"` +} + +// convenience helper +func (ll LexLink) String() string { + return cid.Cid(ll).String() +} + +// convenience helper +func (ll LexLink) Defined() bool { + return cid.Cid(ll).Defined() +} + +func (ll *LexLink) MarshalJSON() ([]byte, error) { + if ll == nil || !ll.Defined() { + return nil, xerrors.Errorf("tried to marshal nil or undefined cid-link") + } + jl := jsonLink{ + Link: (*cid.Cid)(ll).String(), + } + return json.Marshal(jl) +} + +func (ll *LexLink) UnmarshalJSON(raw []byte) error { + var jl jsonLink + err := json.Unmarshal(raw, &jl) + if err != nil { + return xerrors.Errorf("parsing cid-link JSON: %v", err) + } + c, err := cid.Decode(jl.Link) + if err != nil { + return xerrors.Errorf("parsing cid-link CID: %v", err) + } + *ll = LexLink(c) + return nil +} + +func (ll *LexLink) MarshalCBOR(w io.Writer) error { + if ll == nil || !ll.Defined() { + return xerrors.Errorf("tried to marshal nil or undefined cid-link") + } + cw := cbg.NewCborWriter(w) + if err := cbg.WriteCid(cw, cid.Cid(*ll)); err != nil { + return xerrors.Errorf("failed to write cid-link as CBOR: %w", err) + } + return nil +} + +func (ll *LexLink) UnmarshalCBOR(r io.Reader) error { + cr := cbg.NewCborReader(r) + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid-link from CBOR: %w", err) + } + *ll = LexLink(c) + return nil +} + +type LexBytes []byte + +type jsonBytes struct { + Bytes string `json:"$bytes"` +} + +func (lb *LexBytes) MarshalJSON() ([]byte, error) { + if lb == nil { + return nil, xerrors.Errorf("tried to marshal nil $bytes") + } + jb := jsonBytes{ + Bytes: base64.StdEncoding.EncodeToString([]byte(*lb)), + } + return json.Marshal(jb) +} + +func (lb *LexBytes) UnmarshalJSON(raw []byte) error { + var jb jsonBytes + err := json.Unmarshal(raw, &jb) + if err != nil { + return xerrors.Errorf("parsing $bytes JSON: %v", err) + } + out, err := base64.StdEncoding.DecodeString(jb.Bytes) + if err != nil { + return xerrors.Errorf("parsing $bytes base64: %v", err) + } + *lb = LexBytes(out) + return nil +} + +func (lb *LexBytes) MarshalCBOR(w io.Writer) error { + if lb == nil { + return xerrors.Errorf("tried to marshal nil or undefined $bytes") + } + cw := cbg.NewCborWriter(w) + if err := cbg.WriteByteArray(cw, ([]byte)(*lb)); err != nil { + return xerrors.Errorf("failed to write $bytes as CBOR: %w", err) + } + return nil +} + +func (lb *LexBytes) UnmarshalCBOR(r io.Reader) error { + cr := cbg.NewCborReader(r) + b, err := cbg.ReadByteArray(cr, MAX_BYTE_ARRAY_SIZE) + if err != nil { + return xerrors.Errorf("failed to read $bytes from CBOR: %w", err) + } + *lb = LexBytes(b) + return nil +} + +// used in schemas, and can represent either a legacy blob or a "new" (lex +// refactor) blob. size=-1 indicates that this is (and should be serialized as) +// a legacy blob (string CID, no size, etc). +type LexBlob struct { + Ref LexLink `json:"ref" cborgen:"ref"` + MimeType string `json:"mimeType" cborgen:"mimeType"` + Size int64 `json:"size" cborgen:"size"` +} + +type LegacyBlob struct { + Cid string `json:"cid" cborgen:"cid"` + MimeType string `json:"mimeType" cborgen:"mimeType"` +} + +type BlobSchema struct { + LexiconTypeID string `json:"$type,omitempty"` + Ref LexLink `json:"ref" cborgen:"ref"` + MimeType string `json:"mimeType" cborgen:"mimeType"` + Size int64 `json:"size" cborgen:"size"` +} + +func (b *LexBlob) MarshalJSON() ([]byte, error) { + if b.Size < 0 { + lb := LegacyBlob{ + Cid: b.Ref.String(), + MimeType: b.MimeType, + } + return json.Marshal(lb) + } else { + nb := BlobSchema{ + LexiconTypeID: "blob", + Ref: b.Ref, + MimeType: b.MimeType, + Size: b.Size, + } + return json.Marshal(nb) + } +} + +func (b *LexBlob) UnmarshalJSON(raw []byte) error { + typ, err := TypeExtract(raw) + if err != nil { + return xerrors.Errorf("parsing blob: %v", err) + } + + if typ == "blob" { + var bs BlobSchema + err := json.Unmarshal(raw, &bs) + if err != nil { + return xerrors.Errorf("parsing blob JSON: %v", err) + } + b.Ref = bs.Ref + b.MimeType = bs.MimeType + b.Size = bs.Size + if bs.Size < 0 { + return xerrors.Errorf("parsing blob: negative size: %d", bs.Size) + } + } else { + var legacy *LegacyBlob + err := json.Unmarshal(raw, legacy) + if err != nil { + return xerrors.Errorf("parsing legacy blob: %v", err) + } + refCid, err := cid.Decode(legacy.Cid) + if err != nil { + return xerrors.Errorf("parsing CID in legacy blob: %v", err) + } + b.Ref = LexLink(refCid) + b.MimeType = legacy.MimeType + b.Size = -1 + } + return nil +} + +func (b *LexBlob) MarshalCBOR(w io.Writer) error { + if b == nil { + return nil + } + if b.Size < 0 { + lb := LegacyBlob{ + Cid: b.Ref.String(), + MimeType: b.MimeType, + } + return lb.MarshalCBOR(w) + } else { + bs := BlobSchema{ + LexiconTypeID: "blob", + Ref: b.Ref, + MimeType: b.MimeType, + Size: b.Size, + } + return bs.MarshalCBOR(w) + } +} + +func (lb *LexBlob) UnmarshalCBOR(r io.Reader) error { + typ, b, err := CborTypeExtractReader(r) + if err != nil { + return xerrors.Errorf("parsing $blob CBOR type: %w", err) + } + *lb = LexBlob{} + if typ == "blob" { + var bs BlobSchema + err := bs.UnmarshalCBOR(bytes.NewReader(b)) + if err != nil { + return xerrors.Errorf("parsing $blob CBOR: %v", err) + } + lb.Ref = bs.Ref + lb.MimeType = bs.MimeType + lb.Size = bs.Size + if bs.Size < 0 { + return xerrors.Errorf("parsing $blob CBOR: negative size: %d", bs.Size) + } + } else { + legacy := LegacyBlob{} + err := legacy.UnmarshalCBOR(bytes.NewReader(b)) + if err != nil { + return xerrors.Errorf("parsing legacy blob CBOR: %v", err) + } + refCid, err := cid.Decode(legacy.Cid) + if err != nil { + return xerrors.Errorf("parsing CID in legacy blob CBOR: %v", err) + } + lb.Ref = LexLink(refCid) + lb.MimeType = legacy.MimeType + lb.Size = -1 + } + + return nil +} diff --git a/lex/util/util.go b/lex/util/util.go index 57a6f79c..b7396f38 100644 --- a/lex/util/util.go +++ b/lex/util/util.go @@ -5,8 +5,6 @@ import ( "encoding/json" "fmt" "io" - - "github.com/ipfs/go-cid" ) type typeExtractor struct { @@ -22,74 +20,6 @@ func TypeExtract(b []byte) (string, error) { return te.Type, nil } -type LegacyBlob struct { - Cid string `json:"cid" cborgen:"cid"` - MimeType string `json:"mimeType" cborgen:"mimeType"` -} - -type CidLink struct { - Cid string `json:"$link"` -} - -type NewBlob struct { - LexiconTypeID string `json:"$type,omitempty"` - Ref CidLink `json:"ref" cborgen:"ref"` - MimeType string `json:"mimeType" cborgen:"mimeType"` - Size int64 `json:"size" cborgen:"size"` -} - -type Blob struct { - Ref cid.Cid `json:"ref" cborgen:"ref"` - MimeType string `json:"mimeType" cborgen:"mimeType"` - Size int64 `json:"size" cborgen:"size"` -} - -func (b *Blob) MarshalJSON() ([]byte, error) { - nb := NewBlob{ - LexiconTypeID: "blob", - Ref: CidLink{b.Ref.String()}, - MimeType: b.MimeType, - Size: b.Size, - } - return json.Marshal(nb) -} - -func (b *Blob) UnmarshalJSON(raw []byte) error { - typ, err := TypeExtract(raw) - if err != nil { - return fmt.Errorf("parsing blob: %v", err) - } - - if typ == "blob" { - var nb NewBlob - err := json.Unmarshal(raw, &nb) - if err != nil { - return fmt.Errorf("parsing blob JSON: %v", err) - } - b.Ref, err = cid.Decode(nb.Ref.Cid) - if err != nil { - return fmt.Errorf("parsing blob CID: %v", err) - } - b.MimeType = nb.MimeType - b.Size = nb.Size - } else { - var legacy *LegacyBlob - err := json.Unmarshal(raw, legacy) - if err != nil { - return fmt.Errorf("parsing legacy blob: %v", err) - } - b.Ref, err = cid.Decode(legacy.Cid) - if err != nil { - return fmt.Errorf("parsing CID in legacy blob: %v", err) - } - b.MimeType = legacy.MimeType - // TODO: copying the -1 here from atproto behavior. should verify if it - // should be *size instead - b.Size = -1 - } - return nil -} - type CborChecker struct { Type string `json:"$type" cborgen:"$type"` } -- 2.51.2