From a90b8b15f655a29aee875eeb79986a517bd40e01 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Fri, 12 Jan 2024 15:16:58 -0800 Subject: [PATCH] initial pass at blob fetching and rules in automod --- automod/engine/blobs.go | 115 ++++++++++++++++++++++++++++++++++++ automod/engine/ruleset.go | 26 ++++++++ automod/engine/ruletypes.go | 2 + automod/pkg.go | 1 + automod/rules/all.go | 3 + automod/rules/blobs.go | 24 ++++++++ 6 files changed, 171 insertions(+) create mode 100644 automod/engine/blobs.go create mode 100644 automod/rules/blobs.go diff --git a/automod/engine/blobs.go b/automod/engine/blobs.go new file mode 100644 index 00000000..be975522 --- /dev/null +++ b/automod/engine/blobs.go @@ -0,0 +1,115 @@ +package engine + +import ( + "fmt" + "io" + "net/http" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// Parses out any blobs from the enclosed record. +// +// TODO: currently this function uses schema-specific logic, and won't work with generic lexicon records. A future version could use the indigo/atproto/data package and the raw record CBOR to extract blobs from arbitrary records +// +// NOTE: for consistency with other RecordContext methods, which don't usually return errors, maybe the error-returning version of this function should be a helper function, or definted on RecordOp, and the RecordContext version should return an empty array on error? +func (c *RecordContext) Blobs() ([]lexutil.LexBlob, error) { + + if c.RecordOp.Action != CreateOp { + // TODO: should this really error, or return empty array? + return nil, fmt.Errorf("expected record creation, got: %s", c.RecordOp.Action) + } + + var blobs []lexutil.LexBlob + + switch c.RecordOp.Collection.String() { + case "app.bsky.feed.post": + post, ok := c.RecordOp.Value.(*appbsky.FeedPost) + if !ok { + return nil, fmt.Errorf("mismatch between collection (%s) and type", c.RecordOp.Collection) + } + if post.Embed != nil && post.Embed.EmbedImages != nil { + for _, eii := range post.Embed.EmbedImages.Images { + if eii.Image != nil { + blobs = append(blobs, *eii.Image) + } + } + } + if post.Embed != nil && post.Embed.EmbedExternal != nil { + ext := post.Embed.EmbedExternal.External + if ext != nil && ext.Thumb != nil { + blobs = append(blobs, *ext.Thumb) + } + } + if post.Embed != nil && post.Embed.EmbedRecordWithMedia != nil { + media := post.Embed.EmbedRecordWithMedia.Media + if media != nil && media.EmbedImages != nil { + for _, eii := range media.EmbedImages.Images { + if eii.Image != nil { + blobs = append(blobs, *eii.Image) + } + } + } + if media != nil && media.EmbedExternal != nil { + ext := media.EmbedExternal.External + if ext != nil && ext.Thumb != nil { + blobs = append(blobs, *ext.Thumb) + } + } + } + case "app.bsky.actor.profile": + profile, ok := c.RecordOp.Value.(*appbsky.ActorProfile) + if !ok { + return nil, fmt.Errorf("mismatch between collection (%s) and type", c.RecordOp.Collection) + } + if profile.Avatar != nil { + blobs = append(blobs, *profile.Avatar) + } + if profile.Banner != nil { + blobs = append(blobs, *profile.Banner) + } + case "app.bsky.graph.list": + list, ok := c.RecordOp.Value.(*appbsky.GraphList) + if !ok { + return nil, fmt.Errorf("mismatch between collection (%s) and type", c.RecordOp.Collection) + } + if list.Avatar != nil { + blobs = append(blobs, *list.Avatar) + } + case "app.bsky.feed.generator": + generator, ok := c.RecordOp.Value.(*appbsky.FeedGenerator) + if !ok { + return nil, fmt.Errorf("mismatch between collection (%s) and type", c.RecordOp.Collection) + } + if generator.Avatar != nil { + blobs = append(blobs, *generator.Avatar) + } + } + return blobs, nil +} + +func fetchBlob(c *RecordContext, blob lexutil.LexBlob) ([]byte, error) { + + var blobBytes []byte + + // TODO: more robust way to write this? + xrpcURL := fmt.Sprintf("%s/xrpc/com.atproto.sync.getBlob?did=%s&cid=%s", c.Account.Identity.PDSEndpoint(), c.Account.Identity.DID, blob.Ref) + + resp, err := http.Get(xrpcURL) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return nil, fmt.Errorf("failed to fetch blob from PDS. did=%s cid=%s statusCode=%d", c.Account.Identity.DID, blob.Ref, resp.StatusCode) + } + + blobBytes, err = io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return blobBytes, nil +} diff --git a/automod/engine/ruleset.go b/automod/engine/ruleset.go index 6e63437c..a5d27165 100644 --- a/automod/engine/ruleset.go +++ b/automod/engine/ruleset.go @@ -12,6 +12,7 @@ type RuleSet struct { RecordRules []RecordRuleFunc RecordDeleteRules []RecordRuleFunc IdentityRules []IdentityRuleFunc + BlobRules []BlobRuleFunc } func (r *RuleSet) CallRecordRules(c *RecordContext) error { @@ -47,6 +48,31 @@ func (r *RuleSet) CallRecordRules(c *RecordContext) error { } } } + // then blob rules, if any + if len(r.BlobRules) == 0 || c.RecordOp.Action != CreateOp { + return nil + } + blobs, err := c.Blobs() + if err != nil { + // TODO: should this really return error, or just log? + return err + } + if len(blobs) == 0 { + return nil + } + // TODO: concurrency + for _, blob := range blobs { + data, err := fetchBlob(c, blob) + if err != nil { + return err + } + for _, f := range r.BlobRules { + err := f(c, blob, data) + if err != nil { + return err + } + } + } return nil } diff --git a/automod/engine/ruletypes.go b/automod/engine/ruletypes.go index 5cdb23c9..5579e29c 100644 --- a/automod/engine/ruletypes.go +++ b/automod/engine/ruletypes.go @@ -2,9 +2,11 @@ package engine import ( appbsky "github.com/bluesky-social/indigo/api/bsky" + lexutil "github.com/bluesky-social/indigo/lex/util" ) type IdentityRuleFunc = func(c *AccountContext) error type RecordRuleFunc = func(c *RecordContext) error type PostRuleFunc = func(c *RecordContext, post *appbsky.FeedPost) error type ProfileRuleFunc = func(c *RecordContext, profile *appbsky.ActorProfile) error +type BlobRuleFunc = func(c *RecordContext, blob lexutil.LexBlob, data []byte) error diff --git a/automod/pkg.go b/automod/pkg.go index e3bad4ba..86b6ffe6 100644 --- a/automod/pkg.go +++ b/automod/pkg.go @@ -17,6 +17,7 @@ type IdentityRuleFunc = engine.IdentityRuleFunc type RecordRuleFunc = engine.RecordRuleFunc type PostRuleFunc = engine.PostRuleFunc type ProfileRuleFunc = engine.ProfileRuleFunc +type BlobRuleFunc = engine.BlobRuleFunc var ( ReportReasonSpam = engine.ReportReasonSpam diff --git a/automod/rules/all.go b/automod/rules/all.go index 815db436..11d92198 100644 --- a/automod/rules/all.go +++ b/automod/rules/all.go @@ -35,6 +35,9 @@ func DefaultRules() automod.RuleSet { IdentityRules: []automod.IdentityRuleFunc{ NewAccountRule, }, + BlobRules: []automod.BlobRuleFunc{ + //BlobVerifyRule, + }, } return rules } diff --git a/automod/rules/blobs.go b/automod/rules/blobs.go new file mode 100644 index 00000000..9ebbeadc --- /dev/null +++ b/automod/rules/blobs.go @@ -0,0 +1,24 @@ +package rules + +import ( + "github.com/bluesky-social/indigo/automod" + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +var _ automod.BlobRuleFunc = BlobVerifyRule + +func BlobVerifyRule(c *automod.RecordContext, blob lexutil.LexBlob, data []byte) error { + + if len(data) == 0 { + c.AddRecordFlag("empty-blob") + } + + // check size + if blob.Size >= 0 && int64(len(data)) != blob.Size { + c.AddRecordFlag("invalid-blob") + } else { + c.Logger.Info("blob checks out", "cid", blob.Ref, "size", blob.Size, "mimetype", blob.MimeType) + } + + return nil +} -- 2.51.2