From e1b57bf8a281f5c68293a23b533eae854f71de7a Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Tue, 29 Oct 2024 17:51:36 -0700 Subject: [PATCH] refactor rules to use new helpers package --- automod/rules/harassment.go | 11 ++++++----- automod/rules/hashtags.go | 7 ++++--- automod/rules/identity.go | 3 ++- automod/rules/keyword.go | 9 +++++---- automod/rules/mentions.go | 3 ++- automod/rules/misleading.go | 7 ++++--- automod/rules/misleading_test.go | 21 +++++++++++---------- automod/rules/nostr.go | 3 ++- automod/rules/promo.go | 9 +++++---- automod/rules/quick.go | 5 +++-- automod/rules/replies.go | 25 +++++++++++++------------ automod/rules/reposts.go | 3 ++- automod/visual/hiveai_rule.go | 4 ++-- 13 files changed, 61 insertions(+), 49 deletions(-) diff --git a/automod/rules/harassment.go b/automod/rules/harassment.go index 5212b69e..2cf7ce19 100644 --- a/automod/rules/harassment.go +++ b/automod/rules/harassment.go @@ -8,18 +8,19 @@ import ( "github.com/bluesky-social/indigo/atproto/syntax" "github.com/bluesky-social/indigo/automod" "github.com/bluesky-social/indigo/automod/countstore" + "github.com/bluesky-social/indigo/automod/helpers" ) var _ automod.PostRuleFunc = HarassmentTargetInteractionPostRule // looks for new accounts, which interact with frequently-harassed accounts, and report them for review func HarassmentTargetInteractionPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { - if c.Account.Identity == nil || !AccountIsYoungerThan(&c.AccountContext, 24*time.Hour) { + if c.Account.Identity == nil || !helpers.AccountIsYoungerThan(&c.AccountContext, 24*time.Hour) { return nil } var interactionDIDs []string - facets, err := ExtractFacets(post) + facets, err := helpers.ExtractFacets(post) if err != nil { return err } @@ -28,7 +29,7 @@ func HarassmentTargetInteractionPostRule(c *automod.RecordContext, post *appbsky interactionDIDs = append(interactionDIDs, *pf.DID) } } - if post.Reply != nil && !IsSelfThread(c, post) { + if post.Reply != nil && !helpers.IsSelfThread(c, post) { parentURI, err := syntax.ParseATURI(post.Reply.Parent.Uri) if err != nil { return err @@ -57,7 +58,7 @@ func HarassmentTargetInteractionPostRule(c *automod.RecordContext, post *appbsky return nil } - interactionDIDs = dedupeStrings(interactionDIDs) + interactionDIDs = helpers.DedupeStrings(interactionDIDs) for _, d := range interactionDIDs { did, err := syntax.ParseDID(d) if err != nil { @@ -114,7 +115,7 @@ var _ automod.PostRuleFunc = HarassmentTrivialPostRule // looks for new accounts, which frequently post the same type of content func HarassmentTrivialPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { - if c.Account.Identity == nil || !AccountIsYoungerThan(&c.AccountContext, 7*24*time.Hour) { + if c.Account.Identity == nil || !helpers.AccountIsYoungerThan(&c.AccountContext, 7*24*time.Hour) { return nil } diff --git a/automod/rules/hashtags.go b/automod/rules/hashtags.go index c6d73480..682ce746 100644 --- a/automod/rules/hashtags.go +++ b/automod/rules/hashtags.go @@ -5,13 +5,14 @@ import ( appbsky "github.com/bluesky-social/indigo/api/bsky" "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/automod/helpers" "github.com/bluesky-social/indigo/automod/keyword" ) // looks for specific hashtags from known lists func BadHashtagsPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { - for _, tag := range ExtractHashtagsPost(post) { - tag = NormalizeHashtag(tag) + for _, tag := range helpers.ExtractHashtagsPost(post) { + tag = helpers.NormalizeHashtag(tag) // skip some bad-word hashtags which frequently false-positive if tag == "nazi" || tag == "hitler" { continue @@ -35,7 +36,7 @@ var _ automod.PostRuleFunc = BadHashtagsPostRule // if a post is "almost all" hashtags, it might be a form of search spam func TooManyHashtagsPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { - tags := ExtractHashtagsPost(post) + tags := helpers.ExtractHashtagsPost(post) tagChars := 0 for _, tag := range tags { tagChars += len(tag) diff --git a/automod/rules/identity.go b/automod/rules/identity.go index 365d63f9..e7499123 100644 --- a/automod/rules/identity.go +++ b/automod/rules/identity.go @@ -7,11 +7,12 @@ import ( "github.com/bluesky-social/indigo/automod" "github.com/bluesky-social/indigo/automod/countstore" + "github.com/bluesky-social/indigo/automod/helpers" ) // triggers on first identity event for an account (DID) func NewAccountRule(c *automod.AccountContext) error { - if c.Account.Identity == nil || !AccountIsYoungerThan(c, 4*time.Hour) { + if c.Account.Identity == nil || !helpers.AccountIsYoungerThan(c, 4*time.Hour) { return nil } diff --git a/automod/rules/keyword.go b/automod/rules/keyword.go index abb20260..8d5caa39 100644 --- a/automod/rules/keyword.go +++ b/automod/rules/keyword.go @@ -7,6 +7,7 @@ import ( appbsky "github.com/bluesky-social/indigo/api/bsky" "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/automod/helpers" "github.com/bluesky-social/indigo/automod/keyword" ) @@ -17,7 +18,7 @@ func BadWordPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { isJapanese = true } } - for _, tok := range ExtractTextTokensPost(post) { + for _, tok := range helpers.ExtractTextTokensPost(post) { word := keyword.SlugIsExplicitSlur(tok) // used very frequently in a reclaimed context if word != "" && word != "faggot" && word != "tranny" && word != "coon" && !(word == "kike" && isJapanese) { @@ -54,7 +55,7 @@ func BadWordProfileRule(c *automod.RecordContext, profile *appbsky.ActorProfile) //c.Notify("slack") } } - for _, tok := range ExtractTextTokensProfile(profile) { + for _, tok := range helpers.ExtractTextTokensProfile(profile) { // de-pluralize tok = strings.TrimSuffix(tok, "s") if c.InSet("worst-words", tok) { @@ -71,8 +72,8 @@ var _ automod.ProfileRuleFunc = BadWordProfileRule // looks for the specific harassment situation of a replay to another user with only a single word func ReplySingleBadWordPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { - if post.Reply != nil && !IsSelfThread(c, post) { - tokens := ExtractTextTokensPost(post) + if post.Reply != nil && !helpers.IsSelfThread(c, post) { + tokens := helpers.ExtractTextTokensPost(post) if len(tokens) != 1 { return nil } diff --git a/automod/rules/mentions.go b/automod/rules/mentions.go index 8155b4a4..98d419d0 100644 --- a/automod/rules/mentions.go +++ b/automod/rules/mentions.go @@ -8,6 +8,7 @@ import ( "github.com/bluesky-social/indigo/atproto/syntax" "github.com/bluesky-social/indigo/automod" "github.com/bluesky-social/indigo/automod/countstore" + "github.com/bluesky-social/indigo/automod/helpers" ) var _ automod.PostRuleFunc = DistinctMentionsRule @@ -47,7 +48,7 @@ var youngMentionAccountLimit = 12 var _ automod.PostRuleFunc = YoungAccountDistinctMentionsRule func YoungAccountDistinctMentionsRule(c *automod.RecordContext, post *appbsky.FeedPost) error { - if c.Account.Identity == nil || !AccountIsYoungerThan(&c.AccountContext, 14*24*time.Hour) { + if c.Account.Identity == nil || !helpers.AccountIsYoungerThan(&c.AccountContext, 14*24*time.Hour) { return nil } diff --git a/automod/rules/misleading.go b/automod/rules/misleading.go index df4525cf..31822ccc 100644 --- a/automod/rules/misleading.go +++ b/automod/rules/misleading.go @@ -9,9 +9,10 @@ import ( appbsky "github.com/bluesky-social/indigo/api/bsky" "github.com/bluesky-social/indigo/atproto/syntax" "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/automod/helpers" ) -func isMisleadingURLFacet(facet PostFacet, logger *slog.Logger) bool { +func isMisleadingURLFacet(facet helpers.PostFacet, logger *slog.Logger) bool { linkURL, err := url.Parse(*facet.URL) if err != nil { logger.Warn("invalid link metadata URL", "url", facet.URL) @@ -84,7 +85,7 @@ func MisleadingURLPostRule(c *automod.RecordContext, post *appbsky.FeedPost) err if c.Account.Identity.Handle == "nowbreezing.ntw.app" { return nil } - facets, err := ExtractFacets(post) + facets, err := helpers.ExtractFacets(post) if err != nil { c.Logger.Warn("invalid facets", "err", err) // TODO: or some other "this record is corrupt" indicator? @@ -105,7 +106,7 @@ func MisleadingURLPostRule(c *automod.RecordContext, post *appbsky.FeedPost) err var _ automod.PostRuleFunc = MisleadingMentionPostRule func MisleadingMentionPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { - facets, err := ExtractFacets(post) + facets, err := helpers.ExtractFacets(post) if err != nil { c.Logger.Warn("invalid facets", "err", err) // TODO: or some other "this record is corrupt" indicator? diff --git a/automod/rules/misleading_test.go b/automod/rules/misleading_test.go index cf8e814a..2e47883a 100644 --- a/automod/rules/misleading_test.go +++ b/automod/rules/misleading_test.go @@ -11,6 +11,7 @@ import ( "github.com/bluesky-social/indigo/atproto/syntax" "github.com/bluesky-social/indigo/automod" "github.com/bluesky-social/indigo/automod/engine" + "github.com/bluesky-social/indigo/automod/helpers" "github.com/stretchr/testify/assert" ) @@ -118,67 +119,67 @@ func TestIsMisleadingURL(t *testing.T) { logger := slog.Default() fixtures := []struct { - facet PostFacet + facet helpers.PostFacet out bool }{ { - facet: PostFacet{ + facet: helpers.PostFacet{ Text: "https://atproto.com", URL: pstr("https://atproto.com"), }, out: false, }, { - facet: PostFacet{ + facet: helpers.PostFacet{ Text: "https://atproto.com", URL: pstr("https://evil.com"), }, out: true, }, { - facet: PostFacet{ + facet: helpers.PostFacet{ Text: "https://www.atproto.com", URL: pstr("https://atproto.com"), }, out: false, }, { - facet: PostFacet{ + facet: helpers.PostFacet{ Text: "https://atproto.com", URL: pstr("https://www.atproto.com"), }, out: false, }, { - facet: PostFacet{ + facet: helpers.PostFacet{ Text: "[example.com]", URL: pstr("https://www.example.com"), }, out: false, }, { - facet: PostFacet{ + facet: helpers.PostFacet{ Text: "example.com...", URL: pstr("https://example.com.evil.com"), }, out: true, }, { - facet: PostFacet{ + facet: helpers.PostFacet{ Text: "ATPROTO.com...", URL: pstr("https://atproto.com"), }, out: false, }, { - facet: PostFacet{ + facet: helpers.PostFacet{ Text: "1234.5678", URL: pstr("https://arxiv.org/abs/1234.5678"), }, out: false, }, { - facet: PostFacet{ + facet: helpers.PostFacet{ Text: "www.techdirt.com…", URL: pstr("https://www.techdirt.com/"), }, diff --git a/automod/rules/nostr.go b/automod/rules/nostr.go index 0291d066..5f91e7ee 100644 --- a/automod/rules/nostr.go +++ b/automod/rules/nostr.go @@ -7,13 +7,14 @@ import ( appbsky "github.com/bluesky-social/indigo/api/bsky" "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/automod/helpers" ) var _ automod.PostRuleFunc = NostrSpamPostRule // looks for new accounts, which frequently post the same type of content func NostrSpamPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { - if c.Account.Identity == nil || !AccountIsYoungerThan(&c.AccountContext, 2*24*time.Hour) { + if c.Account.Identity == nil || !helpers.AccountIsYoungerThan(&c.AccountContext, 2*24*time.Hour) { return nil } diff --git a/automod/rules/promo.go b/automod/rules/promo.go index 0dad7aaf..f6fe23a2 100644 --- a/automod/rules/promo.go +++ b/automod/rules/promo.go @@ -9,6 +9,7 @@ import ( appbsky "github.com/bluesky-social/indigo/api/bsky" "github.com/bluesky-social/indigo/automod" "github.com/bluesky-social/indigo/automod/countstore" + "github.com/bluesky-social/indigo/automod/helpers" ) var _ automod.PostRuleFunc = AggressivePromotionRule @@ -17,16 +18,16 @@ var _ automod.PostRuleFunc = AggressivePromotionRule // // this rule depends on ReplyCountPostRule() to set counts func AggressivePromotionRule(c *automod.RecordContext, post *appbsky.FeedPost) error { - if c.Account.Identity == nil || !AccountIsYoungerThan(&c.AccountContext, 7*24*time.Hour) { + if c.Account.Identity == nil || !helpers.AccountIsYoungerThan(&c.AccountContext, 7*24*time.Hour) { return nil } - if post.Reply == nil || IsSelfThread(c, post) { + if post.Reply == nil || helpers.IsSelfThread(c, post) { return nil } - allURLs := ExtractTextURLs(post.Text) + allURLs := helpers.ExtractTextURLs(post.Text) if c.Account.Profile.Description != nil { - profileURLs := ExtractTextURLs(*c.Account.Profile.Description) + profileURLs := helpers.ExtractTextURLs(*c.Account.Profile.Description) allURLs = append(allURLs, profileURLs...) } hasPromo := false diff --git a/automod/rules/quick.go b/automod/rules/quick.go index 77075d94..ea6a69e3 100644 --- a/automod/rules/quick.go +++ b/automod/rules/quick.go @@ -7,6 +7,7 @@ import ( appbsky "github.com/bluesky-social/indigo/api/bsky" "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/automod/helpers" ) var botLinkStrings = []string{"ainna13762491", "LINK押して", "→ https://tiny", "⇒ http://tiny"} @@ -54,7 +55,7 @@ func SimpleBotPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { var _ automod.IdentityRuleFunc = NewAccountBotEmailRule func NewAccountBotEmailRule(c *automod.AccountContext) error { - if c.Account.Identity == nil || !AccountIsYoungerThan(c, 1*time.Hour) { + if c.Account.Identity == nil || !helpers.AccountIsYoungerThan(c, 1*time.Hour) { return nil } @@ -73,7 +74,7 @@ var _ automod.PostRuleFunc = TrivialSpamPostRule // looks for new accounts, which frequently post the same type of content func TrivialSpamPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { - if c.Account.Identity == nil || !AccountIsYoungerThan(&c.AccountContext, 8*24*time.Hour) { + if c.Account.Identity == nil || !helpers.AccountIsYoungerThan(&c.AccountContext, 8*24*time.Hour) { return nil } diff --git a/automod/rules/replies.go b/automod/rules/replies.go index aed98673..e03e9de5 100644 --- a/automod/rules/replies.go +++ b/automod/rules/replies.go @@ -9,13 +9,14 @@ import ( "github.com/bluesky-social/indigo/atproto/syntax" "github.com/bluesky-social/indigo/automod" "github.com/bluesky-social/indigo/automod/countstore" + "github.com/bluesky-social/indigo/automod/helpers" ) var _ automod.PostRuleFunc = ReplyCountPostRule // does not count "self-replies" (direct to self, or in own post thread) func ReplyCountPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { - if post.Reply == nil || IsSelfThread(c, post) { + if post.Reply == nil || helpers.IsSelfThread(c, post) { return nil } @@ -47,7 +48,7 @@ var _ automod.PostRuleFunc = IdenticalReplyPostRule // // There can be legitimate situations that trigger this rule, so in most situations should be a "report" not "label" action. func IdenticalReplyPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { - if post.Reply == nil || IsSelfThread(c, post) { + if post.Reply == nil || helpers.IsSelfThread(c, post) { return nil } @@ -55,18 +56,18 @@ func IdenticalReplyPostRule(c *automod.RecordContext, post *appbsky.FeedPost) er if utf8.RuneCountInString(post.Text) <= 10 { return nil } - if AccountIsOlderThan(&c.AccountContext, 14*24*time.Hour) { + if helpers.AccountIsOlderThan(&c.AccountContext, 14*24*time.Hour) { return nil } // don't count if there is a follow-back relationship - if ParentOrRootIsFollower(c, post) { + if helpers.ParentOrRootIsFollower(c, post) { return nil } // increment before read. use a specific period (IncrementPeriod()) to reduce the number of counters (one per unique post text) period := countstore.PeriodDay - bucket := c.Account.Identity.DID.String() + "/" + HashOfString(post.Text) + bucket := c.Account.Identity.DID.String() + "/" + helpers.HashOfString(post.Text) c.IncrementPeriod("reply-text", bucket, period) count := c.GetCount("reply-text", bucket, period) @@ -91,21 +92,21 @@ var identicalReplySameParentMaxPosts int64 = 50 var _ automod.PostRuleFunc = IdenticalReplyPostSameParentRule func IdenticalReplyPostSameParentRule(c *automod.RecordContext, post *appbsky.FeedPost) error { - if post.Reply == nil || IsSelfThread(c, post) { + if post.Reply == nil || helpers.IsSelfThread(c, post) { return nil } - if ParentOrRootIsFollower(c, post) { + if helpers.ParentOrRootIsFollower(c, post) { return nil } postCount := c.Account.PostsCount - if AccountIsOlderThan(&c.AccountContext, identicalReplySameParentMaxAge) || postCount >= identicalReplySameParentMaxPosts { + if helpers.AccountIsOlderThan(&c.AccountContext, identicalReplySameParentMaxAge) || postCount >= identicalReplySameParentMaxPosts { return nil } period := countstore.PeriodHour - bucket := c.Account.Identity.DID.String() + "/" + post.Reply.Parent.Uri + "/" + HashOfString(post.Text) + bucket := c.Account.Identity.DID.String() + "/" + post.Reply.Parent.Uri + "/" + helpers.HashOfString(post.Text) c.IncrementPeriod("reply-text-same-post", bucket, period) count := c.GetCount("reply-text-same-post", bucket, period) @@ -126,7 +127,7 @@ var _ automod.PostRuleFunc = YoungAccountDistinctRepliesRule func YoungAccountDistinctRepliesRule(c *automod.RecordContext, post *appbsky.FeedPost) error { // only replies, and skip self-replies (eg, threads) - if post.Reply == nil || IsSelfThread(c, post) { + if post.Reply == nil || helpers.IsSelfThread(c, post) { return nil } @@ -134,12 +135,12 @@ func YoungAccountDistinctRepliesRule(c *automod.RecordContext, post *appbsky.Fee if utf8.RuneCountInString(post.Text) <= 10 { return nil } - if AccountIsOlderThan(&c.AccountContext, 14*24*time.Hour) { + if helpers.AccountIsOlderThan(&c.AccountContext, 14*24*time.Hour) { return nil } // don't count if there is a follow-back relationship - if ParentOrRootIsFollower(c, post) { + if helpers.ParentOrRootIsFollower(c, post) { return nil } diff --git a/automod/rules/reposts.go b/automod/rules/reposts.go index 75b24846..57314655 100644 --- a/automod/rules/reposts.go +++ b/automod/rules/reposts.go @@ -7,6 +7,7 @@ import ( "github.com/bluesky-social/indigo/automod" "github.com/bluesky-social/indigo/automod/countstore" + "github.com/bluesky-social/indigo/automod/helpers" ) var dailyRepostThresholdWithoutPost = 30 @@ -18,7 +19,7 @@ var _ automod.RecordRuleFunc = TooManyRepostRule // looks for accounts which do frequent reposts func TooManyRepostRule(c *automod.RecordContext) error { // Don't bother checking reposts from accounts older than 30 days - if c.Account.Identity == nil || !AccountIsYoungerThan(&c.AccountContext, 30*24*time.Hour) { + if c.Account.Identity == nil || !helpers.AccountIsYoungerThan(&c.AccountContext, 30*24*time.Hour) { return nil } diff --git a/automod/visual/hiveai_rule.go b/automod/visual/hiveai_rule.go index 32bcf6a9..850ee83b 100644 --- a/automod/visual/hiveai_rule.go +++ b/automod/visual/hiveai_rule.go @@ -5,7 +5,7 @@ import ( "time" "github.com/bluesky-social/indigo/automod" - "github.com/bluesky-social/indigo/automod/rules" + "github.com/bluesky-social/indigo/automod/helpers" lexutil "github.com/bluesky-social/indigo/lex/util" ) @@ -43,7 +43,7 @@ func (hal *HiveAIClient) HiveLabelBlobRule(c *automod.RecordContext, blob lexuti for _, l := range labels { // NOTE: experimenting with profile reporting for new accounts - if l == "sexual" && c.RecordOp.Collection.String() == "app.bsky.actor.profile" && rules.AccountIsYoungerThan(&c.AccountContext, 2*24*time.Hour) { + if l == "sexual" && c.RecordOp.Collection.String() == "app.bsky.actor.profile" && helpers.AccountIsYoungerThan(&c.AccountContext, 2*24*time.Hour) { c.ReportRecord(automod.ReportReasonSexual, "possible sexual profile (not labeled yet)") c.Logger.Info("skipping record label", "label", l, "reason", "sexual-profile-experiment") } else { -- 2.51.2