diff --git a/client/src/graphql/generated.ts b/client/src/graphql/generated.ts index 6c530df..58217be 100644 --- a/client/src/graphql/generated.ts +++ b/client/src/graphql/generated.ts @@ -4450,6 +4450,12 @@ export const GQLSignalType = { OpenAiHateTextModel: 'OPEN_AI_HATE_TEXT_MODEL', OpenAiHateThreateningTextModel: 'OPEN_AI_HATE_THREATENING_TEXT_MODEL', OpenAiSelfHarmImageModel: 'OPEN_AI_SELF_HARM_IMAGE_MODEL', + OpenAiSelfHarmInstructionsImageModel: + 'OPEN_AI_SELF_HARM_INSTRUCTIONS_IMAGE_MODEL', + OpenAiSelfHarmInstructionsTextModel: + 'OPEN_AI_SELF_HARM_INSTRUCTIONS_TEXT_MODEL', + OpenAiSelfHarmIntentImageModel: 'OPEN_AI_SELF_HARM_INTENT_IMAGE_MODEL', + OpenAiSelfHarmIntentTextModel: 'OPEN_AI_SELF_HARM_INTENT_TEXT_MODEL', OpenAiSelfHarmTextModel: 'OPEN_AI_SELF_HARM_TEXT_MODEL', OpenAiSexualImageModel: 'OPEN_AI_SEXUAL_IMAGE_MODEL', OpenAiSexualMinorsTextModel: 'OPEN_AI_SEXUAL_MINORS_TEXT_MODEL', diff --git a/client/src/models/signal.ts b/client/src/models/signal.ts index 24f06f6..5c74378 100644 --- a/client/src/models/signal.ts +++ b/client/src/models/signal.ts @@ -29,6 +29,10 @@ export function integrationForSignalType(type: string) { case 'OPEN_AI_HATE_TEXT_MODEL': case 'OPEN_AI_HATE_THREATENING_TEXT_MODEL': case 'OPEN_AI_SELF_HARM_IMAGE_MODEL': + case 'OPEN_AI_SELF_HARM_INSTRUCTIONS_IMAGE_MODEL': + case 'OPEN_AI_SELF_HARM_INSTRUCTIONS_TEXT_MODEL': + case 'OPEN_AI_SELF_HARM_INTENT_IMAGE_MODEL': + case 'OPEN_AI_SELF_HARM_INTENT_TEXT_MODEL': case 'OPEN_AI_SELF_HARM_TEXT_MODEL': case 'OPEN_AI_SEXUAL_IMAGE_MODEL': case 'OPEN_AI_SEXUAL_MINORS_TEXT_MODEL': diff --git a/server/graphql/generated.ts b/server/graphql/generated.ts index 178d0f8..12d6be1 100644 --- a/server/graphql/generated.ts +++ b/server/graphql/generated.ts @@ -4518,6 +4518,12 @@ export const GQLSignalType = { OpenAiHateTextModel: 'OPEN_AI_HATE_TEXT_MODEL', OpenAiHateThreateningTextModel: 'OPEN_AI_HATE_THREATENING_TEXT_MODEL', OpenAiSelfHarmImageModel: 'OPEN_AI_SELF_HARM_IMAGE_MODEL', + OpenAiSelfHarmInstructionsImageModel: + 'OPEN_AI_SELF_HARM_INSTRUCTIONS_IMAGE_MODEL', + OpenAiSelfHarmInstructionsTextModel: + 'OPEN_AI_SELF_HARM_INSTRUCTIONS_TEXT_MODEL', + OpenAiSelfHarmIntentImageModel: 'OPEN_AI_SELF_HARM_INTENT_IMAGE_MODEL', + OpenAiSelfHarmIntentTextModel: 'OPEN_AI_SELF_HARM_INTENT_TEXT_MODEL', OpenAiSelfHarmTextModel: 'OPEN_AI_SELF_HARM_TEXT_MODEL', OpenAiSexualImageModel: 'OPEN_AI_SEXUAL_IMAGE_MODEL', OpenAiSexualMinorsTextModel: 'OPEN_AI_SEXUAL_MINORS_TEXT_MODEL', diff --git a/server/graphql/modules/signal.ts b/server/graphql/modules/signal.ts index 060d0c3..a6aae1c 100644 --- a/server/graphql/modules/signal.ts +++ b/server/graphql/modules/signal.ts @@ -104,6 +104,10 @@ const typeDefs = /* GraphQL */ ` OPEN_AI_HATE_TEXT_MODEL OPEN_AI_HATE_THREATENING_TEXT_MODEL OPEN_AI_SELF_HARM_IMAGE_MODEL + OPEN_AI_SELF_HARM_INSTRUCTIONS_IMAGE_MODEL + OPEN_AI_SELF_HARM_INSTRUCTIONS_TEXT_MODEL + OPEN_AI_SELF_HARM_INTENT_IMAGE_MODEL + OPEN_AI_SELF_HARM_INTENT_TEXT_MODEL OPEN_AI_SELF_HARM_TEXT_MODEL OPEN_AI_SEXUAL_IMAGE_MODEL OPEN_AI_SEXUAL_MINORS_TEXT_MODEL diff --git a/server/services/signalsService/helpers/instantiateBuiltInSignals.ts b/server/services/signalsService/helpers/instantiateBuiltInSignals.ts index d7a6728..ec236dd 100644 --- a/server/services/signalsService/helpers/instantiateBuiltInSignals.ts +++ b/server/services/signalsService/helpers/instantiateBuiltInSignals.ts @@ -28,6 +28,10 @@ import OpenAiGraphicViolenceTextSignal from '../signals/third_party_signals/open import OpenAiHateTextSignal from '../signals/third_party_signals/open_ai/moderation/OpenAiHateTextSignal.js'; import OpenAiHateThreateningTextSignal from '../signals/third_party_signals/open_ai/moderation/OpenAiHateThreateningTextSignal.js'; import OpenAiSelfHarmImageSignal from '../signals/third_party_signals/open_ai/moderation/OpenAiSelfHarmImageSignal.js'; +import OpenAiSelfHarmInstructionsImageSignal from '../signals/third_party_signals/open_ai/moderation/OpenAiSelfHarmInstructionsImageSignal.js'; +import OpenAiSelfHarmInstructionsTextSignal from '../signals/third_party_signals/open_ai/moderation/OpenAiSelfHarmInstructionsTextSignal.js'; +import OpenAiSelfHarmIntentImageSignal from '../signals/third_party_signals/open_ai/moderation/OpenAiSelfHarmIntentImageSignal.js'; +import OpenAiSelfHarmIntentTextSignal from '../signals/third_party_signals/open_ai/moderation/OpenAiSelfHarmIntentTextSignal.js'; import OpenAiSelfHarmTextSignal from '../signals/third_party_signals/open_ai/moderation/OpenAiSelfHarmTextSignal.js'; import OpenAiSexualImageSignal from '../signals/third_party_signals/open_ai/moderation/OpenAiSexualImageSignal.js'; import OpenAiSexualMinorsTextSignal from '../signals/third_party_signals/open_ai/moderation/OpenAiSexualMinorsTextSignal.js'; @@ -108,6 +112,26 @@ export function instantiateBuiltInSignals( credentialGetters.OPEN_AI, getOpenAiScores, ), + [SignalType.OPEN_AI_SELF_HARM_INSTRUCTIONS_IMAGE_MODEL]: + new OpenAiSelfHarmInstructionsImageSignal( + credentialGetters.OPEN_AI, + getOpenAiScores, + ), + [SignalType.OPEN_AI_SELF_HARM_INSTRUCTIONS_TEXT_MODEL]: + new OpenAiSelfHarmInstructionsTextSignal( + credentialGetters.OPEN_AI, + getOpenAiScores, + ), + [SignalType.OPEN_AI_SELF_HARM_INTENT_IMAGE_MODEL]: + new OpenAiSelfHarmIntentImageSignal( + credentialGetters.OPEN_AI, + getOpenAiScores, + ), + [SignalType.OPEN_AI_SELF_HARM_INTENT_TEXT_MODEL]: + new OpenAiSelfHarmIntentTextSignal( + credentialGetters.OPEN_AI, + getOpenAiScores, + ), [SignalType.OPEN_AI_SELF_HARM_TEXT_MODEL]: new OpenAiSelfHarmTextSignal( credentialGetters.OPEN_AI, getOpenAiScores, diff --git a/server/services/signalsService/signals/third_party_signals/open_ai/moderation/OpenAiGraphicViolenceImageSignal.ts b/server/services/signalsService/signals/third_party_signals/open_ai/moderation/OpenAiGraphicViolenceImageSignal.ts index 0aa1f5f..168061e 100644 --- a/server/services/signalsService/signals/third_party_signals/open_ai/moderation/OpenAiGraphicViolenceImageSignal.ts +++ b/server/services/signalsService/signals/third_party_signals/open_ai/moderation/OpenAiGraphicViolenceImageSignal.ts @@ -1,5 +1,5 @@ import { SignalType } from '../../../../types/SignalType.js'; -import { makeOpenAiImageModerationSignal } from './openAIModerationUtils.js'; +import { makeOpenAiImageModerationSignal } from './openAiModerationSignalFactory.js'; /** * OpenAI image-moderation signal scoring whether an image depicts death, diff --git a/server/services/signalsService/signals/third_party_signals/open_ai/moderation/OpenAiSelfHarmImageSignal.ts b/server/services/signalsService/signals/third_party_signals/open_ai/moderation/OpenAiSelfHarmImageSignal.ts index 14d3e7e..b7cb56d 100644 --- a/server/services/signalsService/signals/third_party_signals/open_ai/moderation/OpenAiSelfHarmImageSignal.ts +++ b/server/services/signalsService/signals/third_party_signals/open_ai/moderation/OpenAiSelfHarmImageSignal.ts @@ -1,5 +1,5 @@ import { SignalType } from '../../../../types/SignalType.js'; -import { makeOpenAiImageModerationSignal } from './openAIModerationUtils.js'; +import { makeOpenAiImageModerationSignal } from './openAiModerationSignalFactory.js'; /** * OpenAI image-moderation signal scoring whether an image promotes, diff --git a/server/services/signalsService/signals/third_party_signals/open_ai/moderation/OpenAiSelfHarmInstructionsImageSignal.ts b/server/services/signalsService/signals/third_party_signals/open_ai/moderation/OpenAiSelfHarmInstructionsImageSignal.ts new file mode 100644 index 0000000..5bc0cc9 --- /dev/null +++ b/server/services/signalsService/signals/third_party_signals/open_ai/moderation/OpenAiSelfHarmInstructionsImageSignal.ts @@ -0,0 +1,19 @@ +import { SignalType } from '../../../../types/SignalType.js'; +import { makeOpenAiImageModerationSignal } from './openAiModerationSignalFactory.js'; + +/** + * OpenAI image-moderation signal scoring whether an image encourages or + * provides instructions for self-harm. Routes through omni-moderation-latest's + * multimodal endpoint and returns the `self-harm/instructions` category score + * (0..1). + */ +const OpenAiSelfHarmInstructionsImageSignal = makeOpenAiImageModerationSignal({ + type: SignalType.OPEN_AI_SELF_HARM_INSTRUCTIONS_IMAGE_MODEL, + displayName: 'OpenAI Self-Harm Instructions Image score', + description: `OpenAI's model that detects content that encourages performing acts of self-harm, such as suicide, cutting, and eating disorders, or that gives instructions or advice on how to commit such acts. Scored against the image. + + This model produces a confidence score between 0 and 1, indicating the model's confidence that the image contains self-harm instructions.`, + modelName: 'self-harm/instructions', +}); + +export default OpenAiSelfHarmInstructionsImageSignal; diff --git a/server/services/signalsService/signals/third_party_signals/open_ai/moderation/OpenAiSelfHarmInstructionsTextSignal.ts b/server/services/signalsService/signals/third_party_signals/open_ai/moderation/OpenAiSelfHarmInstructionsTextSignal.ts new file mode 100644 index 0000000..0ff2fe1 --- /dev/null +++ b/server/services/signalsService/signals/third_party_signals/open_ai/moderation/OpenAiSelfHarmInstructionsTextSignal.ts @@ -0,0 +1,19 @@ +import { SignalType } from '../../../../types/SignalType.js'; +import { makeOpenAiTextModerationSignal } from './openAiModerationSignalFactory.js'; + +/** + * OpenAI text-moderation signal scoring whether text encourages or provides + * instructions for self-harm (suicide, cutting, eating disorders, etc.). + * Routes through omni-moderation-latest and returns the + * `self-harm/instructions` category score (0..1). + */ +const OpenAiSelfHarmInstructionsTextSignal = makeOpenAiTextModerationSignal({ + type: SignalType.OPEN_AI_SELF_HARM_INSTRUCTIONS_TEXT_MODEL, + displayName: 'OpenAI Self-Harm Instructions Text score', + description: `OpenAI's model that detects content that encourages performing acts of self-harm, such as suicide, cutting, and eating disorders, or that gives instructions or advice on how to commit such acts. + + This model produces a confidence score between 0 and 1, indicating the model's confidence that the content contains self-harm instructions.`, + modelName: 'self-harm/instructions', +}); + +export default OpenAiSelfHarmInstructionsTextSignal; diff --git a/server/services/signalsService/signals/third_party_signals/open_ai/moderation/OpenAiSelfHarmIntentImageSignal.ts b/server/services/signalsService/signals/third_party_signals/open_ai/moderation/OpenAiSelfHarmIntentImageSignal.ts new file mode 100644 index 0000000..3ec08f3 --- /dev/null +++ b/server/services/signalsService/signals/third_party_signals/open_ai/moderation/OpenAiSelfHarmIntentImageSignal.ts @@ -0,0 +1,19 @@ +import { SignalType } from '../../../../types/SignalType.js'; +import { makeOpenAiImageModerationSignal } from './openAiModerationSignalFactory.js'; + +/** + * OpenAI image-moderation signal scoring whether an image expresses the + * speaker's intent to engage in acts of self-harm. Routes through + * omni-moderation-latest's multimodal endpoint and returns the + * `self-harm/intent` category score (0..1). + */ +const OpenAiSelfHarmIntentImageSignal = makeOpenAiImageModerationSignal({ + type: SignalType.OPEN_AI_SELF_HARM_INTENT_IMAGE_MODEL, + displayName: 'OpenAI Self-Harm Intent Image score', + description: `OpenAI's model that detects self-harm intent, which is defined as content where the speaker expresses that they are engaging or intend to engage in acts of self-harm, such as suicide, cutting, and eating disorders. Scored against the image. + + This model produces a confidence score between 0 and 1, indicating the model's confidence that the image expresses self-harm intent.`, + modelName: 'self-harm/intent', +}); + +export default OpenAiSelfHarmIntentImageSignal; diff --git a/server/services/signalsService/signals/third_party_signals/open_ai/moderation/OpenAiSelfHarmIntentTextSignal.ts b/server/services/signalsService/signals/third_party_signals/open_ai/moderation/OpenAiSelfHarmIntentTextSignal.ts new file mode 100644 index 0000000..62acec3 --- /dev/null +++ b/server/services/signalsService/signals/third_party_signals/open_ai/moderation/OpenAiSelfHarmIntentTextSignal.ts @@ -0,0 +1,19 @@ +import { SignalType } from '../../../../types/SignalType.js'; +import { makeOpenAiTextModerationSignal } from './openAiModerationSignalFactory.js'; + +/** + * OpenAI text-moderation signal scoring whether text expresses the speaker's + * intent to engage in acts of self-harm (suicide, cutting, eating disorders, + * etc.). Routes through omni-moderation-latest and returns the + * `self-harm/intent` category score (0..1). + */ +const OpenAiSelfHarmIntentTextSignal = makeOpenAiTextModerationSignal({ + type: SignalType.OPEN_AI_SELF_HARM_INTENT_TEXT_MODEL, + displayName: 'OpenAI Self-Harm Intent Text score', + description: `OpenAI's model that detects self-harm intent, which is defined as content where the speaker expresses that they are engaging or intend to engage in acts of self-harm, such as suicide, cutting, and eating disorders. + + This model produces a confidence score between 0 and 1, indicating the model's confidence that the content expresses self-harm intent.`, + modelName: 'self-harm/intent', +}); + +export default OpenAiSelfHarmIntentTextSignal; diff --git a/server/services/signalsService/signals/third_party_signals/open_ai/moderation/OpenAiSexualImageSignal.ts b/server/services/signalsService/signals/third_party_signals/open_ai/moderation/OpenAiSexualImageSignal.ts index f2bdee7..22067d4 100644 --- a/server/services/signalsService/signals/third_party_signals/open_ai/moderation/OpenAiSexualImageSignal.ts +++ b/server/services/signalsService/signals/third_party_signals/open_ai/moderation/OpenAiSexualImageSignal.ts @@ -1,5 +1,5 @@ import { SignalType } from '../../../../types/SignalType.js'; -import { makeOpenAiImageModerationSignal } from './openAIModerationUtils.js'; +import { makeOpenAiImageModerationSignal } from './openAiModerationSignalFactory.js'; /** * OpenAI image-moderation signal scoring whether an image is meant to arouse diff --git a/server/services/signalsService/signals/third_party_signals/open_ai/moderation/OpenAiViolenceImageSignal.ts b/server/services/signalsService/signals/third_party_signals/open_ai/moderation/OpenAiViolenceImageSignal.ts index f251fc3..1946d37 100644 --- a/server/services/signalsService/signals/third_party_signals/open_ai/moderation/OpenAiViolenceImageSignal.ts +++ b/server/services/signalsService/signals/third_party_signals/open_ai/moderation/OpenAiViolenceImageSignal.ts @@ -1,5 +1,5 @@ import { SignalType } from '../../../../types/SignalType.js'; -import { makeOpenAiImageModerationSignal } from './openAIModerationUtils.js'; +import { makeOpenAiImageModerationSignal } from './openAiModerationSignalFactory.js'; /** * OpenAI image-moderation signal scoring whether an image promotes, diff --git a/server/services/signalsService/signals/third_party_signals/open_ai/moderation/openAIModerationUtils.ts b/server/services/signalsService/signals/third_party_signals/open_ai/moderation/openAIModerationUtils.ts index 0ea1e8c..d8b4d8a 100644 --- a/server/services/signalsService/signals/third_party_signals/open_ai/moderation/openAIModerationUtils.ts +++ b/server/services/signalsService/signals/third_party_signals/open_ai/moderation/openAIModerationUtils.ts @@ -12,8 +12,7 @@ import { type CachedGetCredentials } from '../../../../../signalAuthService/sign import { Integration } from '../../../../types/Integration.js'; import { type RecommendedThresholds } from '../../../../types/RecommendedThresholds.js'; import { SignalPricingStructure } from '../../../../types/SignalPricingStructure.js'; -import { type SignalType } from '../../../../types/SignalType.js'; -import SignalBase, { +import { type SignalDisabledInfo, type SignalInput, } from '../../../SignalBase.js'; @@ -24,6 +23,8 @@ export type OpenAiModelName = | 'hate' | 'hate/threatening' | 'self-harm' + | 'self-harm/instructions' + | 'self-harm/intent' | 'sexual' | 'sexual/minors' | 'violence' @@ -38,7 +39,12 @@ export type OpenAiModelName = */ export type OpenAiImageModelName = Extract< OpenAiModelName, - 'self-harm' | 'sexual' | 'violence' | 'violence/graphic' + | 'self-harm' + | 'self-harm/instructions' + | 'self-harm/intent' + | 'sexual' + | 'violence' + | 'violence/graphic' >; const OPEN_AI_MODERATION_MODEL = 'omni-moderation-latest'; @@ -285,115 +291,3 @@ export async function getOpenAiModerationScores( throw e; } } - -/** - * Factory for OpenAI image-moderation signals. All four image signals - * (`violence`, `violence/graphic`, `self-harm`, `sexual`) share identical - * boilerplate: same integration, pricing, language coverage, eligible inputs, - * etc. — the only per-signal config is the SignalType id, display name, - * description, and the model category to read. - * - * Returns a `SignalBase` subclass that the IoC container instantiates with - * `(credentials, scores)` like any other signal class, preserving the - * existing registration pattern in `instantiateBuiltInSignals.ts`. - */ -export function makeOpenAiImageModerationSignal(config: { - type: SignalType; - displayName: string; - description: string; - modelName: OpenAiImageModelName; -}) { - return class OpenAiImageModerationSignal extends SignalBase< - ScalarTypes['IMAGE'], - { scalarType: ScalarTypes['NUMBER'] } - > { - constructor( - protected readonly getOpenAiCredentials: CachedGetCredentials<'OPEN_AI'>, - protected readonly getOpenAiScores: FetchOpenAiModerationScores, - ) { - super(); - } - - override get id() { - return { type: config.type }; - } - - override get displayName() { - return config.displayName; - } - - override get description() { - return config.description; - } - - override get docsUrl() { - return openAiModerationDocsUrl(); - } - - override get integration() { - return openAiModerationIntegration(); - } - - override get pricingStructure() { - return openAiModerationPricingStructure(); - } - - override get recommendedThresholds() { - return openAiModerationRecommendedThresholds(); - } - - override get supportedLanguages() { - return openAiModerationSupportedLanguages(); - } - - override get eligibleSubcategories() { - return openAiModerationEligibleSubcategories(); - } - - override get needsActionPenalties() { - return openAiModerationNeedsActionPenalties(); - } - - override get needsMatchingValues() { - return openAiModerationNeedsMatchingValues(); - } - - override async getDisabledInfo(orgId: string) { - return openAiModerationGetDisabledInfo(orgId, this.getOpenAiCredentials); - } - - override get eligibleInputs() { - return [ScalarTypes.IMAGE]; - } - - override get outputType() { - return { scalarType: ScalarTypes.NUMBER }; - } - - // Inherits the placeholder cost convention from the existing OpenAI text - // signals (see OpenAiViolenceTextSignal etc.). Cost units are unitless - // ordering hints used by the engine to prefer cheaper signals; the - // current ~20 baseline reflects that OpenAI moderation is a paid - // remote-API call (vs. local heuristics) — not a calibrated value. - override getCost() { - return 20; - } - - override get allowedInAutomatedRules() { - return true; - } - - /** - * Fetches the omni-moderation `${config.modelName}` score for the image - * and returns it as a number between 0 and 1. - */ - async run(input: SignalInput) { - return runOpenAiModerationImageImpl( - this.getOpenAiCredentials, - input, - this.getOpenAiScores, - config.modelName, - ); - } - }; -} diff --git a/server/services/signalsService/signals/third_party_signals/open_ai/moderation/openAiModerationSignalFactory.ts b/server/services/signalsService/signals/third_party_signals/open_ai/moderation/openAiModerationSignalFactory.ts new file mode 100644 index 0000000..b6c527f --- /dev/null +++ b/server/services/signalsService/signals/third_party_signals/open_ai/moderation/openAiModerationSignalFactory.ts @@ -0,0 +1,191 @@ +/** + * Factories for OpenAI moderation signals. Collapses the boilerplate that + * was duplicated across every per-category signal class — id, displayName, + * description, integration, pricing, languages, cost, etc. The two public + * exports (`makeOpenAiImageModerationSignal`, + * `makeOpenAiTextModerationSignal`) are thin wrappers around the private + * `makeOpenAiModerationSignal` so the class body lives in one place. + * + * The IoC container instantiates the returned class with + * `(credentials, scores)` like any other signal, preserving the existing + * registration pattern in `instantiateBuiltInSignals.ts`. + */ +import { ScalarTypes } from '@roostorg/coop-types'; + +import { type CachedGetCredentials } from '../../../../../signalAuthService/signalAuthService.js'; +import { type SignalType } from '../../../../types/SignalType.js'; +import SignalBase, { + type SignalInput, + type SignalInputType, +} from '../../../SignalBase.js'; +import { + openAiModerationDocsUrl, + openAiModerationEligibleSubcategories, + openAiModerationGetDisabledInfo, + openAiModerationIntegration, + openAiModerationNeedsActionPenalties, + openAiModerationNeedsMatchingValues, + openAiModerationPricingStructure, + openAiModerationRecommendedThresholds, + openAiModerationSupportedLanguages, + runOpenAiModerationImageImpl, + runOpenAiModerationImpl, + type FetchOpenAiModerationScores, + type OpenAiImageModelName, + type OpenAiModelName, +} from './openAIModerationUtils.js'; + +type SignalRunResult = { + score: number; + outputType: { scalarType: ScalarTypes['NUMBER'] }; +}; + +type ModerationMode = { + /** ScalarType key returned by `eligibleInputs` (e.g. `ScalarTypes.IMAGE`). */ + inputScalar: InputScalar; + /** Routes to either `runOpenAiModerationImpl` or `runOpenAiModerationImageImpl`. */ + runImpl: ( + getOpenAiCredentials: CachedGetCredentials<'OPEN_AI'>, + input: SignalInput, + getOpenAiScores: FetchOpenAiModerationScores, + modelName: ModelName, + ) => Promise; +}; + +type SignalConfig = { + type: SignalType; + displayName: string; + description: string; + modelName: ModelName; +}; + +function makeOpenAiModerationSignal< + InputScalar extends SignalInputType, + ModelName, +>(config: SignalConfig & ModerationMode) { + return class OpenAiModerationSignal extends SignalBase< + InputScalar, + { scalarType: ScalarTypes['NUMBER'] } + > { + constructor( + protected readonly getOpenAiCredentials: CachedGetCredentials<'OPEN_AI'>, + protected readonly getOpenAiScores: FetchOpenAiModerationScores, + ) { + super(); + } + + override get id() { + return { type: config.type }; + } + + override get displayName() { + return config.displayName; + } + + override get description() { + return config.description; + } + + override get docsUrl() { + return openAiModerationDocsUrl(); + } + + override get integration() { + return openAiModerationIntegration(); + } + + override get pricingStructure() { + return openAiModerationPricingStructure(); + } + + override get recommendedThresholds() { + return openAiModerationRecommendedThresholds(); + } + + override get supportedLanguages() { + return openAiModerationSupportedLanguages(); + } + + override get eligibleSubcategories() { + return openAiModerationEligibleSubcategories(); + } + + override get needsActionPenalties() { + return openAiModerationNeedsActionPenalties(); + } + + override get needsMatchingValues() { + return openAiModerationNeedsMatchingValues(); + } + + override async getDisabledInfo(orgId: string) { + return openAiModerationGetDisabledInfo(orgId, this.getOpenAiCredentials); + } + + override get eligibleInputs() { + return [config.inputScalar]; + } + + override get outputType() { + return { scalarType: ScalarTypes.NUMBER }; + } + + // Matches the legacy OpenAI text-signal baseline (`getCost: 20`). Values + // here are unitless ordering hints used by the rule engine to prefer + // cheaper signals — they're not calibrated against latency or $ across + // the codebase. Re-calibrating is tracked separately; perpetuating the + // existing baseline keeps new signals consistent with their text peers. + override getCost() { + return 20; + } + + override get allowedInAutomatedRules() { + return true; + } + + /** + * Fetches the omni-moderation `${config.modelName}` score for the input + * and returns it as a number between 0 and 1. + */ + async run(input: SignalInput) { + return config.runImpl( + this.getOpenAiCredentials, + input, + this.getOpenAiScores, + config.modelName, + ); + } + }; +} + +/** + * Factory for image-input OpenAI moderation signals. The `modelName` + * parameter is constrained to {@link OpenAiImageModelName} so callers can't + * request a category OpenAI only scores against text. + */ +export function makeOpenAiImageModerationSignal( + config: SignalConfig, +) { + return makeOpenAiModerationSignal( + { + ...config, + inputScalar: ScalarTypes.IMAGE, + runImpl: runOpenAiModerationImageImpl, + }, + ); +} + +/** + * Factory for text-input OpenAI moderation signals. Accepts the full + * {@link OpenAiModelName} since omni-moderation scores every category on + * text. + */ +export function makeOpenAiTextModerationSignal( + config: SignalConfig, +) { + return makeOpenAiModerationSignal({ + ...config, + inputScalar: ScalarTypes.STRING, + runImpl: runOpenAiModerationImpl, + }); +} diff --git a/server/services/signalsService/types/SignalArgsByType.ts b/server/services/signalsService/types/SignalArgsByType.ts index 44c0b0d..83ac4ce 100644 --- a/server/services/signalsService/types/SignalArgsByType.ts +++ b/server/services/signalsService/types/SignalArgsByType.ts @@ -29,6 +29,10 @@ export type SignalArgsByType = Satisfies< [SignalType.OPEN_AI_HATE_TEXT_MODEL]: undefined; [SignalType.OPEN_AI_HATE_THREATENING_TEXT_MODEL]: undefined; [SignalType.OPEN_AI_SELF_HARM_IMAGE_MODEL]: undefined; + [SignalType.OPEN_AI_SELF_HARM_INSTRUCTIONS_IMAGE_MODEL]: undefined; + [SignalType.OPEN_AI_SELF_HARM_INSTRUCTIONS_TEXT_MODEL]: undefined; + [SignalType.OPEN_AI_SELF_HARM_INTENT_IMAGE_MODEL]: undefined; + [SignalType.OPEN_AI_SELF_HARM_INTENT_TEXT_MODEL]: undefined; [SignalType.OPEN_AI_SELF_HARM_TEXT_MODEL]: undefined; [SignalType.OPEN_AI_SEXUAL_IMAGE_MODEL]: undefined; [SignalType.OPEN_AI_SEXUAL_MINORS_TEXT_MODEL]: undefined; @@ -67,6 +71,10 @@ export type RuntimeSignalArgsByType = Satisfies< [SignalType.OPEN_AI_HATE_TEXT_MODEL]: undefined; [SignalType.OPEN_AI_HATE_THREATENING_TEXT_MODEL]: undefined; [SignalType.OPEN_AI_SELF_HARM_IMAGE_MODEL]: undefined; + [SignalType.OPEN_AI_SELF_HARM_INSTRUCTIONS_IMAGE_MODEL]: undefined; + [SignalType.OPEN_AI_SELF_HARM_INSTRUCTIONS_TEXT_MODEL]: undefined; + [SignalType.OPEN_AI_SELF_HARM_INTENT_IMAGE_MODEL]: undefined; + [SignalType.OPEN_AI_SELF_HARM_INTENT_TEXT_MODEL]: undefined; [SignalType.OPEN_AI_SELF_HARM_TEXT_MODEL]: undefined; [SignalType.OPEN_AI_SEXUAL_IMAGE_MODEL]: undefined; [SignalType.OPEN_AI_SEXUAL_MINORS_TEXT_MODEL]: undefined; diff --git a/server/services/signalsService/types/SignalType.ts b/server/services/signalsService/types/SignalType.ts index 2c3c807..f6570ec 100644 --- a/server/services/signalsService/types/SignalType.ts +++ b/server/services/signalsService/types/SignalType.ts @@ -44,6 +44,10 @@ export const BuiltInThirdPartySignalType = makeEnumLike([ 'OPEN_AI_HATE_TEXT_MODEL', 'OPEN_AI_HATE_THREATENING_TEXT_MODEL', 'OPEN_AI_SELF_HARM_IMAGE_MODEL', + 'OPEN_AI_SELF_HARM_INSTRUCTIONS_IMAGE_MODEL', + 'OPEN_AI_SELF_HARM_INSTRUCTIONS_TEXT_MODEL', + 'OPEN_AI_SELF_HARM_INTENT_IMAGE_MODEL', + 'OPEN_AI_SELF_HARM_INTENT_TEXT_MODEL', 'OPEN_AI_SELF_HARM_TEXT_MODEL', 'OPEN_AI_SEXUAL_IMAGE_MODEL', 'OPEN_AI_SEXUAL_MINORS_TEXT_MODEL', @@ -98,6 +102,10 @@ export function integrationForSignalType(type: SignalType | string) { case 'OPEN_AI_HATE_TEXT_MODEL': case 'OPEN_AI_HATE_THREATENING_TEXT_MODEL': case 'OPEN_AI_SELF_HARM_IMAGE_MODEL': + case 'OPEN_AI_SELF_HARM_INSTRUCTIONS_IMAGE_MODEL': + case 'OPEN_AI_SELF_HARM_INSTRUCTIONS_TEXT_MODEL': + case 'OPEN_AI_SELF_HARM_INTENT_IMAGE_MODEL': + case 'OPEN_AI_SELF_HARM_INTENT_TEXT_MODEL': case 'OPEN_AI_SELF_HARM_TEXT_MODEL': case 'OPEN_AI_SEXUAL_IMAGE_MODEL': case 'OPEN_AI_SEXUAL_MINORS_TEXT_MODEL':