diff --git a/automod/visual/doc.go b/automod/visual/doc.go index 2798b2ec..ad30c999 100644 --- a/automod/visual/doc.go +++ b/automod/visual/doc.go @@ -1,3 +1,2 @@ // automod helpers for visual content (image blobs) package visual - diff --git a/automod/visual/hiveai.go b/automod/visual/hiveai.go index 21594f60..cec618f0 100644 --- a/automod/visual/hiveai.go +++ b/automod/visual/hiveai.go @@ -1,12 +1,12 @@ package visual import ( - "log/slog" "bytes" "context" "encoding/json" "fmt" "io" + "log/slog" "mime/multipart" "net/http" @@ -51,39 +51,105 @@ func NewHiveAILabeler(token string) HiveAILabeler { } } +// Simple direct mappings from individual classes to to labels +// +// hive gore and violence: https://docs.thehive.ai/docs/class-descriptions-violence-gore +func summarizeSimpleLabels(cl []HiveAIResp_Class) []string { + var labels []string + + for _, cls := range cl { + if cls.Class == "very_bloody" && cls.Score >= 0.90 { + labels = append(labels, "gore") + } + if cls.Class == "human_corpse" && cls.Score >= 0.90 { + labels = append(labels, "corpse") + } + if cls.Class == "hanging" && cls.Score >= 0.90 { + labels = append(labels, "corpse") + } + if cls.Class == "yes_self_harm" && cls.Score >= 0.96 { + labels = append(labels, "self-harm") + } + } + return labels +} + +// Matches only one (or none) of: porn, sexual, nudity +// +// porn: sexual and nudity. including both explicit activity or full-frontal and suggestive/intent +// sexual: sexually suggestive, not explicit; may include some forms of nudity +// nudity: non-sexual nudity (eg, artistic, possibly some photographic) +// +// hive docs/definitions: https://docs.thehive.ai/docs/sexual-content +func summarizeSexualLabels(cl []HiveAIResp_Class) string { + + scores := make(map[string]float64) + for _, cls := range cl { + scores[cls.Class] = cls.Score + } + + // first check if porn... + for _, pornClass := range []string{"yes_sexual_activity", "animal_genitalia_and_human", "yes_realistic_nsfw"} { + if scores[pornClass] >= 0.9 { + return "porn" + } + } + if scores["general_nsfw"] >= 0.9 { + // special case for some anime examples + if scores["animated_animal_genitalia"] >= 0.5 { + return "porn" + } + + // special case for some pornographic/explicit classic drawings + if scores["yes_undressed"] >= 0.9 && scores["yes_sexual_activity"] >= 0.9 { + return "porn" + } + } + + // then check for sexual suggestive (which may include nudity)... + for _, sexualClass := range []string{"yes_sexual_intent", "yes_sex_toy"} { + if scores[sexualClass] >= 0.9 { + return "sexual" + } + } + if scores["yes_undressed"] >= 0.9 { + // special case for bondage examples + if scores["yes_sex_toy"] > 0.75 { + return "sexual" + } + } + + // then non-sexual nudity... + for _, nudityClass := range []string{"yes_male_nudity", "yes_female_nudity", "yes_undressed"} { + if scores[nudityClass] >= 0.9 { + return "nudity" + } + } + + // then finally flag remaining "underwear" images in to sexually suggestive + // (after non-sexual content already labeled above) + for _, underwearClass := range []string{"yes_male_underwear", "yes_female_underwear"} { + if scores[underwearClass] >= 0.9 { + return "sexual" + } + } + + return "" +} + func (resp *HiveAIResp) SummarizeLabels() []string { var labels []string for _, status := range resp.Status { for _, out := range status.Response.Output { - for _, cls := range out.Classes { - // TODO(bnewbold): lots more upstream tags could be included here. - // for example, "sexy" for not nude but still explicit/suggestive, - // or lolicon (animated, not nude, "sugggestive" - - // sexual: https://docs.thehive.ai/docs/sexual-content - // note: won't apply "nude" if "porn" already applied - if cls.Class == "yes_sexual_activity" && cls.Score >= 0.90 { - // NOTE: will include "hentai" - labels = append(labels, "porn") - } else if cls.Class == "animal_genitalia_and_human" && cls.Score >= 0.90 { - labels = append(labels, "porn") - } else if cls.Class == "yes_male_nudity" && cls.Score >= 0.90 { - labels = append(labels, "nude") - } else if cls.Class == "yes_female_nudity" && cls.Score >= 0.90 { - labels = append(labels, "nude") - } - - // gore and violence: https://docs.thehive.ai/docs/class-descriptions-violence-gore - if cls.Class == "very_bloody" && cls.Score >= 0.90 { - labels = append(labels, "gore") - } - if cls.Class == "human_corpse" && cls.Score >= 0.90 { - labels = append(labels, "corpse") - } - if cls.Class == "yes_self_harm" && cls.Score >= 0.90 { - labels = append(labels, "self-harm") - } + simple := summarizeSimpleLabels(out.Classes) + if len(simple) > 0 { + labels = append(labels, simple...) + } + + sexual := summarizeSexualLabels(out.Classes) + if sexual != "" { + labels = append(labels, sexual) } } } diff --git a/automod/visual/testdata/hiveai_resp_example.json b/automod/visual/testdata/hiveai_resp_example.json index 2a378604..2315fa9d 100644 --- a/automod/visual/testdata/hiveai_resp_example.json +++ b/automod/visual/testdata/hiveai_resp_example.json @@ -44,7 +44,7 @@ }, { "class": "general_nsfw", - "score": 8.857344804177162e-05 + "score": 8.857344804177162e-5 }, { "class": "general_suggestive", @@ -56,7 +56,7 @@ }, { "class": "yes_female_underwear", - "score": 7.692095961599136e-06 + "score": 7.692095961599136e-6 }, { "class": "no_male_underwear", @@ -64,7 +64,7 @@ }, { "class": "yes_male_underwear", - "score": 1.5095132367094679e-06 + "score": 1.5095132367094679e-6 }, { "class": "no_sex_toy", @@ -72,7 +72,7 @@ }, { "class": "yes_sex_toy", - "score": 2.9029237450490604e-06 + "score": 2.9029237450490604e-6 }, { "class": "no_female_nudity", @@ -80,7 +80,7 @@ }, { "class": "yes_female_nudity", - "score": 2.60971090699536e-05 + "score": 2.60971090699536e-5 }, { "class": "no_male_nudity", @@ -88,7 +88,7 @@ }, { "class": "yes_male_nudity", - "score": 2.8862691625255323e-05 + "score": 2.8862691625255323e-5 }, { "class": "no_female_swimwear", @@ -96,7 +96,7 @@ }, { "class": "yes_female_swimwear", - "score": 8.239010034025379e-06 + "score": 8.239010034025379e-6 }, { "class": "no_male_shirtless", @@ -104,7 +104,7 @@ }, { "class": "yes_male_shirtless", - "score": 4.166492556688088e-05 + "score": 4.166492556688088e-5 }, { "class": "no_text", @@ -128,15 +128,15 @@ }, { "class": "animated_gun", - "score": 2.0713000782979496e-05 + "score": 2.0713000782979496e-5 }, { "class": "gun_in_hand", - "score": 1.5844730446534659e-06 + "score": 1.5844730446534659e-6 }, { "class": "gun_not_in_hand", - "score": 1.0338973818006654e-06 + "score": 1.0338973818006654e-6 }, { "class": "no_gun", @@ -144,19 +144,19 @@ }, { "class": "culinary_knife_in_hand", - "score": 3.8063500083369785e-06 + "score": 3.8063500083369785e-6 }, { "class": "culinary_knife_not_in_hand", - "score": 7.94057948996249e-07 + "score": 7.94057948996249e-7 }, { "class": "knife_in_hand", - "score": 4.5578955723278505e-07 + "score": 4.5578955723278505e-7 }, { "class": "knife_not_in_hand", - "score": 3.842124714748908e-07 + "score": 3.842124714748908e-7 }, { "class": "no_knife", @@ -164,7 +164,7 @@ }, { "class": "a_little_bloody", - "score": 2.1317745626539786e-07 + "score": 2.1317745626539786e-7 }, { "class": "no_blood", @@ -172,11 +172,11 @@ }, { "class": "other_blood", - "score": 2.0322054269591763e-05 + "score": 2.0322054269591763e-5 }, { "class": "very_bloody", - "score": 1.306446309561673e-07 + "score": 1.306446309561673e-7 }, { "class": "no_pills", @@ -184,7 +184,7 @@ }, { "class": "yes_pills", - "score": 1.0407623044588633e-06 + "score": 1.0407623044588633e-6 }, { "class": "no_smoking", @@ -192,15 +192,15 @@ }, { "class": "yes_smoking", - "score": 6.089803082758281e-06 + "score": 6.089803082758281e-6 }, { "class": "illicit_injectables", - "score": 6.925695592003094e-07 + "score": 6.925695592003094e-7 }, { "class": "medical_injectables", - "score": 8.587808234452378e-07 + "score": 8.587808234452378e-7 }, { "class": "no_injectables", @@ -212,7 +212,7 @@ }, { "class": "yes_nazi", - "score": 1.2550371902234279e-06 + "score": 1.2550371902234279e-6 }, { "class": "no_kkk", @@ -220,7 +220,7 @@ }, { "class": "yes_kkk", - "score": 2.3758245111050425e-07 + "score": 2.3758245111050425e-7 }, { "class": "no_middle_finger", @@ -228,7 +228,7 @@ }, { "class": "yes_middle_finger", - "score": 1.184847681536747e-05 + "score": 1.184847681536747e-5 }, { "class": "no_terrorist", @@ -236,7 +236,7 @@ }, { "class": "yes_terrorist", - "score": 1.1292067715380635e-07 + "score": 1.1292067715380635e-7 }, { "class": "no_overlay_text", @@ -253,11 +253,11 @@ { "class": "yes_sexual_activity", "score": 0.99, - "realScore": 4.364196252012032e-05 + "realScore": 4.364196252012032e-5 }, { "class": "hanging", - "score": 3.6435135762510905e-07 + "score": 3.6435135762510905e-7 }, { "class": "no_hanging_no_noose", @@ -265,7 +265,7 @@ }, { "class": "noose", - "score": 1.5577290007796094e-06 + "score": 1.5577290007796094e-6 }, { "class": "no_realistic_nsfw", @@ -273,15 +273,15 @@ }, { "class": "yes_realistic_nsfw", - "score": 5.565899219571182e-06 + "score": 5.565899219571182e-6 }, { "class": "animated_corpse", - "score": 5.276802046755426e-07 + "score": 5.276802046755426e-7 }, { "class": "human_corpse", - "score": 2.5449360984211012e-08 + "score": 2.5449360984211012e-8 }, { "class": "no_corpse", @@ -293,7 +293,7 @@ }, { "class": "yes_self_harm", - "score": 5.484374493605692e-07 + "score": 5.484374493605692e-7 }, { "class": "no_drawing", @@ -309,7 +309,7 @@ }, { "class": "yes_emaciated_body", - "score": 1.853499568724518e-07 + "score": 1.853499568724518e-7 }, { "class": "no_child_present", @@ -317,7 +317,7 @@ }, { "class": "yes_child_present", - "score": 2.950148455380443e-06 + "score": 2.950148455380443e-6 }, { "class": "no_sexual_intent", @@ -325,19 +325,19 @@ }, { "class": "yes_sexual_intent", - "score": 3.613845370766111e-06 + "score": 3.613845370766111e-6 }, { "class": "animal_genitalia_and_human", - "score": 2.255472023465222e-08 + "score": 2.255472023465222e-8 }, { "class": "animal_genitalia_only", - "score": 4.6783185199931176e-07 + "score": 4.6783185199931176e-7 }, { "class": "animated_animal_genitalia", - "score": 6.707857419436447e-07 + "score": 6.707857419436447e-7 }, { "class": "no_animal_genitalia", @@ -349,7 +349,7 @@ }, { "class": "yes_gambling", - "score": 3.906031285604864e-06 + "score": 3.906031285604864e-6 }, { "class": "no_undressed", @@ -357,7 +357,7 @@ }, { "class": "yes_undressed", - "score": 7.664378199789045e-07 + "score": 7.664378199789045e-7 }, { "class": "no_confederate", @@ -365,11 +365,11 @@ }, { "class": "yes_confederate", - "score": 7.454309962453175e-06 + "score": 7.454309962453175e-6 }, { "class": "animated_alcohol", - "score": 1.8109949948066074e-06 + "score": 1.8109949948066074e-6 }, { "class": "no_alcohol", @@ -377,11 +377,11 @@ }, { "class": "yes_alcohol", - "score": 5.88781463445443e-06 + "score": 5.88781463445443e-6 }, { "class": "yes_drinking_alcohol", - "score": 6.390945746578106e-07 + "score": 6.390945746578106e-7 }, { "class": "no_religious_icon", @@ -389,7 +389,7 @@ }, { "class": "yes_religious_icon", - "score": 1.3784141931119298e-05 + "score": 1.3784141931119298e-5 } ] }