diff --git a/.env.githubci b/.env.githubci index 0c43148..3d4c697 100644 --- a/.env.githubci +++ b/.env.githubci @@ -50,7 +50,7 @@ SCYLLA_HAS_ENTERPRISE_FEATURES='false' GRAPHQL_MAX_DEPTH=10 EXPOSE_SENSITIVE_IMPLEMENTATION_DETAILS_IN_ERRORS=true ALLOW_USER_INPUT_LOCALHOST_URIS=true -SEQUELIZE_PRINT_LOGS=true +DATABASE_PRINT_LOGS=true GROQ_SECRET_KEY= diff --git a/codegen.yaml b/codegen.yaml index d13ec88..cbc640a 100644 --- a/codegen.yaml +++ b/codegen.yaml @@ -95,10 +95,10 @@ generates: EnqueueToMrtAction: ../services/moderationConfigService/types/actions.js#EnqueueToMrtAction EnqueueToNcmecAction: ../services/moderationConfigService/types/actions.js#EnqueueToNcmecAction EnqueueAuthorToMrtAction: ../services/moderationConfigService/types/actions.js#EnqueueAuthorToMrtAction - Backtest: ../models/rules/BacktestModel.js#Backtest - ContentType: ../models/rules/ItemTypeModel.js#ItemType + Backtest: ../graphql/datasources/ruleKyselyPersistence.js#GraphQLBacktestParent + ContentType: ../services/moderationConfigService/types/itemTypes.js#ItemType DerivedFieldSource: ../services/derivedFieldsService/helpers.js#DerivedFieldSpecSource - HashBank: ../models/HashBankModel.js#HashBank + HashBank: ../services/hmaService/index.js#HashBank Org: ../graphql/datasources/orgKyselyPersistence.js#GraphQLOrgParent # NB: `MatchingBanks` is currently resolved by returning the `Org` # parent, and its sub-resolvers only read `org.id`. A bit unusual, as @@ -106,22 +106,16 @@ generates: MatchingBanks: ../graphql/datasources/orgKyselyPersistence.js#GraphQLOrgParent User: ../graphql/datasources/userKyselyPersistence.js#GraphQLUserParent LocationBank: ./datasources/LocationBankApi.js#LocationBankWithoutFullPlacesAPIResponse - # TODO(Kysely migration): these parents will flip to - # `../models/rules/ruleTypes.js#Rule` once the Action/Policy resolvers - # and the remaining Sequelize-backed callers of `org.getRules()` / - # `user.getFavoriteRules()` are migrated off Sequelize. Until then the - # GraphQL Rule parent is the Sequelize instance type, and - # `buildGraphqlRuleParent` casts to it at the boundary. - Rule: ../models/rules/RuleModel.js#Rule + Rule: ../graphql/datasources/ruleKyselyPersistence.js#GraphQLRuleParent Condition: ../services/moderationConfigService/index.js#Condition ConditionWithResult: ../services/moderationConfigService/index.js#ConditionWithResult ConditionSet: ../services/moderationConfigService/index.js#ConditionSet ConditionSetWithResult: ../services/moderationConfigService/index.js#ConditionSetWithResult LeafCondition: ../services/moderationConfigService/index.js#LeafCondition LeafConditionWithResult: ../services/moderationConfigService/index.js#LeafConditionWithResult - UserRule: ../models/rules/RuleModel.js#Rule - ContentRule: ../models/rules/RuleModel.js#Rule - RuleInsights: ../models/rules/RuleModel.js#Rule + UserRule: ../graphql/datasources/ruleKyselyPersistence.js#GraphQLRuleParent + ContentRule: ../graphql/datasources/ruleKyselyPersistence.js#GraphQLRuleParent + RuleInsights: ../graphql/datasources/ruleKyselyPersistence.js#GraphQLRuleParent Notification: ../services/notificationsService/notificationsService.js#Notification Signal: ../services/signalsService/index.js#Signal ManualReviewJobPayload: ../services/manualReviewToolService/index.js#ManualReviewJobPayload diff --git a/server/.env.example b/server/.env.example index 30d832f..261225f 100644 --- a/server/.env.example +++ b/server/.env.example @@ -1,15 +1,28 @@ NODE_ENV=development # Db connection info. -# Providing a single connection url is more common, but Sequelize requires the -# individual component values, and AWS RDS secrets expose each component -# separately by default, so we might as well have a separate variable for each. +# AWS RDS secrets expose each component separately by default, so we keep one +# variable per component rather than a single connection url. DATABASE_HOST=127.0.0.1 DATABASE_READ_ONLY_HOST=127.0.0.1 DATABASE_PORT=5432 DATABASE_NAME=postgres DATABASE_USER=postgres DATABASE_PASSWORD=postgres123 +# Uncomment if your local Postgres is configured for TLS. +# DATABASE_SSL=true + +# Postgres pool / client tuning (applies to both the writer and read-replica +# Kysely pools). The pool sizes below are sized for a local dev Postgres +# (max_connections=100 by default); production overrides these via env. +DATABASE_POOL_MAX=5 +DATABASE_READ_POOL_MAX=10 +DATABASE_POOL_IDLE_TIMEOUT_MS=300000 +DATABASE_POOL_CONNECTION_TIMEOUT_MS=15000 +DATABASE_QUERY_TIMEOUT_MS=1000000 +DATABASE_IDLE_IN_TRANSACTION_TIMEOUT_MS=300000 +# Log every Kysely query (SQL, params, duration). Errors always log regardless. +DATABASE_PRINT_LOGS=true # Postgres connection pool sizes. Defaults shown. # DATABASE_POOL_MAX=30 @@ -43,7 +56,6 @@ SCYLLA_LOCAL_DATACENTER='datacenter1' NODE_ENV=development EXPOSE_SENSITIVE_IMPLEMENTATION_DETAILS_IN_ERRORS=true ALLOW_USER_INPUT_LOCALHOST_URIS=true -SEQUELIZE_PRINT_LOGS=true # Redis Credentials REDIS_USE_CLUSTER=false diff --git a/server/api.ts b/server/api.ts index 4e78a7c..0de75c0 100644 --- a/server/api.ts +++ b/server/api.ts @@ -14,11 +14,11 @@ import { SEMATTRS_EXCEPTION_STACKTRACE, SEMATTRS_EXCEPTION_TYPE, } from '@opentelemetry/semantic-conventions'; -import { GraphQLError, type GraphQLFormattedError } from 'graphql'; import connectPgSimple from 'connect-pg-simple'; import cors from 'cors'; import express, { type ErrorRequestHandler } from 'express'; import session from 'express-session'; +import { GraphQLError, type GraphQLFormattedError } from 'graphql'; import { buildContext, GraphQLLocalStrategy } from 'graphql-passport'; import helmet from 'helmet'; import passport from 'passport'; @@ -33,13 +33,13 @@ import { kyselyUserFindById, } from './graphql/datasources/userKyselyPersistence.js'; import resolvers, { type Context } from './graphql/resolvers.js'; -import { passwordMatchesHash } from './services/userManagementService/index.js'; import typeDefs from './graphql/schema.js'; import { authSchemaWrapper } from './graphql/utils/authorization.js'; import { safeDepthLimit } from './graphql/utils/safeDepthLimit.js'; import { type Dependencies } from './iocContainer/index.js'; import { isEnvTrue, safeGetEnvInt } from './iocContainer/utils.js'; import controllers from './routes/index.js'; +import { passwordMatchesHash } from './services/userManagementService/index.js'; import { createBodySchemaValidator } from './utils/bodySchemaValidation.js'; import { jsonStringify } from './utils/encoding.js'; import { @@ -187,9 +187,8 @@ export default async function makeApiServer(deps: Dependencies) { ); } - const samlSettings = await deps.OrgSettingsService.getSamlSettings( - orgId, - ); + const samlSettings = + await deps.OrgSettingsService.getSamlSettings(orgId); if (!samlSettings) return done( @@ -230,7 +229,7 @@ export default async function makeApiServer(deps: Dependencies) { ); } - return done(null, user as any); + return done(null, user); } catch (e) { return done( makeInternalServerError('Unknown error during login attempt', { @@ -253,7 +252,7 @@ export default async function makeApiServer(deps: Dependencies) { ); } - return done(null, user as any); + return done(null, user); } catch (e) { return done( makeInternalServerError('Unknown error during login attempt', { @@ -405,7 +404,7 @@ export default async function makeApiServer(deps: Dependencies) { // For all other errors (CoopError, unexpected errors, context errors), // sanitize to remove sensitive details and reformat for the client. const sanitizedError = sanitizeError( - rawError instanceof Error ? rawError : (error as Error), + rawError instanceof Error ? rawError : error, ); const { title: sanitizedErrorTitle, ...extensions } = sanitizedError; @@ -430,10 +429,10 @@ export default async function makeApiServer(deps: Dependencies) { code: extensions.type.includes(ErrorType.Unauthenticated) ? 'UNAUTHENTICATED' : extensions.type.includes(ErrorType.Unauthorized) - ? 'FORBIDDEN' - : extensions.type.includes(ErrorType.InvalidUserInput) - ? 'BAD_USER_INPUT' - : 'INTERNAL_SERVER_ERROR', + ? 'FORBIDDEN' + : extensions.type.includes(ErrorType.InvalidUserInput) + ? 'BAD_USER_INPUT' + : 'INTERNAL_SERVER_ERROR', }, message: sanitizedErrorTitle, }; @@ -447,11 +446,12 @@ export default async function makeApiServer(deps: Dependencies) { '/graphql', express.json(), expressMiddleware(apolloServer, { - context: async ({ req, res }) => ({ - ...buildContext({ req, res }), - services: makeGqlServices(deps), - dataSources: deps.DataSources, - } as unknown as Context), + context: async ({ req, res }) => + ({ + ...buildContext({ req, res }), + services: makeGqlServices(deps), + dataSources: deps.DataSources, + }) as unknown as Context, }), ); @@ -465,10 +465,7 @@ export default async function makeApiServer(deps: Dependencies) { const middlewares = it.bodySchema ? [createBodySchemaValidator(it.bodySchema), ...handlers] : handlers; - app[it.method]( - path.join(controller.pathPrefix, it.path), - ...middlewares, - ); + app[it.method](path.join(controller.pathPrefix, it.path), ...middlewares); }); }); diff --git a/server/bin/create-org-and-user.ts b/server/bin/create-org-and-user.ts index 5c6daa9..e5a969b 100644 --- a/server/bin/create-org-and-user.ts +++ b/server/bin/create-org-and-user.ts @@ -2,7 +2,7 @@ /* eslint-disable no-console */ /** * Script to create a new organization and admin user - * + * * Usage: * npm run create-org -- \ * --name "My Org" \ @@ -12,14 +12,17 @@ * --lastName "Doe" \ * --password "securePassword123" */ - import { uid } from 'uid'; import yargs from 'yargs'; import { hideBin } from 'yargs/helpers'; import { kyselyOrgInsert } from '../graphql/datasources/orgKyselyPersistence.js'; +import { kyselyUserInsert } from '../graphql/datasources/userKyselyPersistence.js'; import getBottle from '../iocContainer/index.js'; -import { hashPassword } from '../services/userManagementService/index.js'; +import { + hashPassword, + UserRole, +} from '../services/userManagementService/index.js'; const argv = await yargs(hideBin(process.argv)) .options({ @@ -83,13 +86,12 @@ async function createOrgAndUser() { // Create signing keys and API key (both reference org via FK) await container.SigningKeyPairService.createAndStoreSigningKeys(orgId); - const { apiKey: rawApiKey } = - await container.ApiKeyService.createApiKey( - orgId, - 'Main API Key', - 'Primary API key for organization', - null, - ); + const { apiKey: rawApiKey } = await container.ApiKeyService.createApiKey( + orgId, + 'Main API Key', + 'Primary API key for organization', + null, + ); // Initialize org settings await Promise.all([ @@ -110,15 +112,16 @@ async function createOrgAndUser() { // Hash the password and create the admin user const hashedPassword = await hashPassword(argv.password); - const user = await container.Sequelize.User.create({ + const user = await kyselyUserInsert({ + db: container.KyselyPg, id: userId, + orgId, email: argv.email, password: hashedPassword, firstName: argv.firstName, lastName: argv.lastName, - role: 'ADMIN', + role: UserRole.ADMIN, approvedByAdmin: true, - orgId, loginMethods: ['password'], }); @@ -141,7 +144,9 @@ async function createOrgAndUser() { console.log('\n' + '═'.repeat(60)); console.log('šŸ”‘ API KEY (STORE THIS SECURELY!)'); console.log('═'.repeat(60)); - console.log('\nNew API key generated successfully! Please copy and store it securely.\n'); + console.log( + '\nNew API key generated successfully! Please copy and store it securely.\n', + ); console.log(`API Key: ${rawApiKey}\n`); console.log('āš ļø This API key will not be shown again. Save it now!'); console.log('═'.repeat(60) + '\n'); @@ -152,14 +157,14 @@ async function createOrgAndUser() { } catch (error: unknown) { console.error('\nāŒ Error creating organization and user:\n'); console.error(error); - + // Try to close resources even on error try { await container.closeSharedResourcesForShutdown(); } catch (shutdownError) { console.error('Error during shutdown:', shutdownError); } - + process.exit(1); } } @@ -168,4 +173,3 @@ createOrgAndUser().catch((error) => { console.error('Unhandled error:', error); process.exit(1); }); - diff --git a/server/condition_evaluator/conditionSet.test.ts b/server/condition_evaluator/conditionSet.test.ts index d462a82..77b710c 100644 --- a/server/condition_evaluator/conditionSet.test.ts +++ b/server/condition_evaluator/conditionSet.test.ts @@ -1,5 +1,6 @@ import fc from 'fast-check'; import _ from 'lodash'; +import type { ReadonlyDeep } from 'type-fest'; import { ConditionCompletionOutcome, @@ -9,7 +10,6 @@ import { import { getSignalIdString, type ExternalSignalId, - type SignalId, } from '../services/signalsService/index.js'; import { ConditionOutcomeArbitrary as anyOutcomeArbitrary, @@ -27,7 +27,6 @@ import { getConditionSetResults, tryGetOutcomeFromPartialOutcomes, } from './conditionSet.js'; -import type { ReadonlyDeep } from 'type-fest'; const { sampleSize, shuffle, groupBy } = _; const { AND, OR, XOR } = ConditionConjunction; @@ -35,9 +34,11 @@ const { AND, OR, XOR } = ConditionConjunction; describe('Condition Evaluation', () => { describe('getConditionSetResults', () => { test('should run conditions in cost order, skipping unnecessary ones', async () => { - const stubRunLeafCondition = jest.fn(async (_it: ReadonlyDeep) => ({ - outcome: ConditionCompletionOutcome.PASSED, - })); + const stubRunLeafCondition = jest.fn( + async (_it: ReadonlyDeep) => ({ + outcome: ConditionCompletionOutcome.PASSED, + }), + ); await fc.assert( fc @@ -70,9 +71,7 @@ describe('Condition Evaluation', () => { return async (id: ExternalSignalId) => costsBySignalId.get(getSignalIdString(id))!; - })() satisfies (id: ExternalSignalId) => Promise as ( - id: SignalId, - ) => Promise; + })() satisfies (id: ExternalSignalId) => Promise; const conditions = leafConditionsAndCostsWithUniqueSignalIds.map( (it) => it[0], @@ -242,11 +241,11 @@ describe('Condition Evaluation', () => { outcomesByType['true']?.length === 1 && !outcomesByType['null'] ? true : // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - outcomesByType['null']?.length > 0 && - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - (outcomesByType['true']?.length ?? 0) < 2 - ? null - : false, + outcomesByType['null']?.length > 0 && + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + (outcomesByType['true']?.length ?? 0) < 2 + ? null + : false, ); }), ); diff --git a/server/condition_evaluator/leafCondition.ts b/server/condition_evaluator/leafCondition.ts index d5fe0d3..f7bbc1d 100644 --- a/server/condition_evaluator/leafCondition.ts +++ b/server/condition_evaluator/leafCondition.ts @@ -7,10 +7,6 @@ import { } from '@roostorg/types'; import { type ReadonlyDeep } from 'type-fest'; -import { - isTaggedItemData, - type TaggedItemData, -} from '../models/rules/item-type-fields.js'; import { getUserFromRuleInput, isFullSubmission, @@ -25,14 +21,16 @@ import { type ItemSubmission, } from '../services/itemProcessingService/index.js'; import { - CoopInput, ConditionCompletionOutcome, ConditionFailureOutcome, + CoopInput, + isTaggedItemData, + ValueComparator, type ConditionInput, type ConditionResult, type ConditionSignalInfo, type LeafCondition, - ValueComparator, + type TaggedItemData, } from '../services/moderationConfigService/index.js'; import { isSignalErrorResult, @@ -119,14 +117,12 @@ export async function runLeafCondition( const { input: conditionInput, signal } = condition; const { input: ruleInput } = evaluationContext; - // Figure out what data we're going to extract to pass to the signal. const selectedValueOrValues = await (conditionInput.type !== 'CONTENT_DERIVED_FIELD' ? getSignalInputValueOrValues(conditionInput, ruleInput) : evaluationContext.getDerivedFieldValue(conditionInput.spec)); - // We got an error computing the value for a derived field. if (isCoopError(selectedValueOrValues)) { return condition.comparator === ValueComparator.IS_UNAVAILABLE @@ -162,7 +158,6 @@ export async function runLeafCondition( ? selectedValueOrValues : [selectedValueOrValues]; - // Now, transform the extracted content values by running them through the // signal. If the signal is null, we just treat it as the identity function // and leave the extracted values as-is. (A null/identity signal often happens @@ -188,15 +183,15 @@ export async function runLeafCondition( ? ruleInput.data['spectrum-context-id'] ? String(ruleInput.data['spectrum-context-id']) : ruleInput.data['spectrum_context_id'] - ? String(ruleInput.data['spectrum_context_id']) - : ruleInput.itemType.kind === 'CONTENT' - ? getFieldValueForRole( - ruleInput.itemType.schema, - ruleInput.itemType.schemaFieldRoles, - 'threadId', - ruleInput.data, - )?.id - : undefined + ? String(ruleInput.data['spectrum_context_id']) + : ruleInput.itemType.kind === 'CONTENT' + ? getFieldValueForRole( + ruleInput.itemType.schema, + ruleInput.itemType.schemaFieldRoles, + 'threadId', + ruleInput.data, + )?.id + : undefined : undefined, contentType: isFullSubmission(ruleInput) ? ruleInput.itemType.name @@ -459,7 +454,7 @@ export function extractContentValueOrValues( schemaFields.filter( (it) => getScalarType(it) === ScalarTypes.IMAGE, ), - ) as TaggedScalar[]; + ); case CoopInput.ANY_GEOHASH: return getValuesFromFields( @@ -467,7 +462,7 @@ export function extractContentValueOrValues( schemaFields.filter( (it) => getScalarType(it) === ScalarTypes.GEOHASH, ), - ) as TaggedScalar[]; + ); case CoopInput.ANY_VIDEO: return getValuesFromFields( @@ -475,7 +470,7 @@ export function extractContentValueOrValues( schemaFields.filter( (it) => getScalarType(it) === ScalarTypes.VIDEO, ), - ) as TaggedScalar[]; + ); case CoopInput.POLICY_ID: case CoopInput.SOURCE: diff --git a/server/decs.d.ts b/server/decs.d.ts index e00b395..4780f9e 100644 --- a/server/decs.d.ts +++ b/server/decs.d.ts @@ -230,6 +230,14 @@ namespace NodeJS { DATABASE_NAME?: string; DATABASE_USER?: string; DATABASE_PASSWORD?: string; + DATABASE_SSL?: string; + DATABASE_POOL_MAX?: string; + DATABASE_READ_POOL_MAX?: string; + DATABASE_POOL_IDLE_TIMEOUT_MS?: string; + DATABASE_POOL_CONNECTION_TIMEOUT_MS?: string; + DATABASE_QUERY_TIMEOUT_MS?: string; + DATABASE_IDLE_IN_TRANSACTION_TIMEOUT_MS?: string; + DATABASE_PRINT_LOGS?: string; SESSION_SECRET?: string; WAREHOUSE_ADAPTER?: string; ANALYTICS_ADAPTER?: string; @@ -237,7 +245,6 @@ namespace NodeJS { NODE_ENV?: string; EXPOSE_SENSITIVE_IMPLEMENTATION_DETAILS_IN_ERRORS?: string; ALLOW_USER_INPUT_LOCALHOST_URIS?: string; - SEQUELIZE_PRINT_LOGS?: string; REDIS_USE_CLUSTER?: string; REDIS_HOST?: string; REDIS_PORT?: string; diff --git a/server/graphql/datasources/ActionApi.ts b/server/graphql/datasources/ActionApi.ts index 783bd64..6b0670a 100644 --- a/server/graphql/datasources/ActionApi.ts +++ b/server/graphql/datasources/ActionApi.ts @@ -100,8 +100,7 @@ class ActionAPI { callbackUrlHeaders, callbackUrlBody, applyUserStrikes: applyUserStrikes ?? undefined, - parameters: - parameters === undefined ? undefined : (parameters ?? []), + parameters: parameters === undefined ? undefined : (parameters ?? []), }, itemTypeIds: itemTypeIds ?? undefined, }); @@ -209,10 +208,12 @@ class ActionAPI { const triggered = actions.map((action) => ({ action, - matchingRules: undefined as undefined, - ruleEnvironment: undefined as undefined, + matchingRules: undefined, + ruleEnvironment: undefined, policies, - customMrtApiParamDecisionPayload: validatedParameters.get(action.id), + customMrtApiParamDecisionPayload: validatedParameters.get( + action.id, + ), })); // If the item isn't found, pass it along to the action publisher @@ -221,7 +222,11 @@ class ActionAPI { // never been submitted to us at all. const targetItem = itemSubmission ?? { itemId, - itemType: { id: itemType.id, kind: itemType.kind, name: itemType.name }, + itemType: { + id: itemType.id, + kind: itemType.kind, + name: itemType.name, + }, }; return this.actionPublisher.publishActions(triggered, { orgId, @@ -250,10 +255,15 @@ class ActionAPI { const out = new Map | undefined>(); for (const action of actions) { const spec = parseStoredParameters( - action.actionType === 'CUSTOM_ACTION' ? action.customMrtApiParams : null, + action.actionType === 'CUSTOM_ACTION' + ? action.customMrtApiParams + : null, ); const supplied = rawByActionId?.[action.id]; - if (spec.length === 0 && (supplied === undefined || Object.keys(supplied).length === 0)) { + if ( + spec.length === 0 && + (supplied === undefined || Object.keys(supplied).length === 0) + ) { // No spec, no values — nothing to do for this action. continue; } @@ -261,7 +271,10 @@ class ActionAPI { // keys, etc. Validation runs even when no values are supplied so that // missing-required-with-no-default is caught. const validated = validateActionParameterValues(spec, supplied ?? null); - out.set(action.id, Object.keys(validated).length > 0 ? validated : undefined); + out.set( + action.id, + Object.keys(validated).length > 0 ? validated : undefined, + ); } return out; } diff --git a/server/graphql/datasources/LocationBankApi.ts b/server/graphql/datasources/LocationBankApi.ts index 8922458..ce217c8 100644 --- a/server/graphql/datasources/LocationBankApi.ts +++ b/server/graphql/datasources/LocationBankApi.ts @@ -4,13 +4,15 @@ import { uid } from 'uid'; import { v1 as uuidV1 } from 'uuid'; import { inject, type Dependencies } from '../../iocContainer/index.js'; -import { type LocationArea } from '../../models/types/locationArea.js'; import { type CombinedPg } from '../../services/combinedDbTypes.js'; -import { makeLocationBankNameExistsError } from '../../services/moderationConfigService/index.js'; +import { + makeLocationBankNameExistsError, + type LocationArea, +} from '../../services/moderationConfigService/index.js'; import { type PlacesApiService } from '../../services/placesApiService/index.js'; +import { makeNotFoundError } from '../../utils/errors.js'; import { isUniqueViolationError } from '../../utils/kysely.js'; import { makeKyselyTransactionWithRetry } from '../../utils/kyselyTransactionWithRetry.js'; -import { makeNotFoundError } from '../../utils/errors.js'; import { safePick } from '../../utils/misc.js'; import { type GQLCreateLocationBankInput, @@ -78,12 +80,12 @@ class LocationBankAPI { async getGraphQLLocationBankFromId(opts: { id: string; orgId: string }) { const { id, orgId } = opts; - const row = (await this.db + const row = await this.db .selectFrom('public.location_banks') .select(['id', 'name', 'description', 'org_id', 'owner_id']) .where('id', '=', id) .where('org_id', '=', orgId) - .executeTakeFirst()) as LocationBankRow | undefined; + .executeTakeFirst(); if (row == null) { throw makeNotFoundError('Location bank not found', { @@ -165,12 +167,12 @@ class LocationBankAPI { ? await this.expandLocationAreaInputs(id, locationsToAdd) : undefined; - const row = (await this.db + const row = await this.db .selectFrom('public.location_banks') .select(['id', 'name', 'description', 'org_id', 'owner_id']) .where('id', '=', id) .where('org_id', '=', orgId) - .executeTakeFirst()) as LocationBankRow | undefined; + .executeTakeFirst(); if (row == null) { throw makeNotFoundError('Location bank not found', { @@ -184,7 +186,7 @@ class LocationBankAPI { const nextName = name ?? row.name; const nextDescription = - description !== undefined ? description ?? null : row.description; + description !== undefined ? (description ?? null) : row.description; try { return await this.transactionWithRetry(async (trx) => { diff --git a/server/graphql/datasources/OrgApi.test.ts b/server/graphql/datasources/OrgApi.test.ts index e92bef2..367f93b 100644 --- a/server/graphql/datasources/OrgApi.test.ts +++ b/server/graphql/datasources/OrgApi.test.ts @@ -1,9 +1,16 @@ +import { faker } from '@faker-js/faker'; import { uid } from 'uid'; +import { UserRole } from '../../services/userManagementService/index.js'; +import createContentItemTypes from '../../test/fixtureHelpers/createContentItemTypes.js'; import createOrg from '../../test/fixtureHelpers/createOrg.js'; import { makeMockedServer } from '../../test/setupMockedServer.js'; import { makeTestWithFixture } from '../../test/utils.js'; import { CoopError } from '../../utils/errors.js'; +import { + kyselyUserDeleteById, + kyselyUserInsert, +} from './userKyselyPersistence.js'; describe('OrgAPI', () => { const testWithFixture = makeTestWithFixture(async () => { @@ -27,17 +34,17 @@ describe('OrgAPI', () => { }); describe('getGraphQLOrgFromId', () => { - testWithFixture('returns the org parent for an existing id', async ({ - deps, - org, - }) => { - const result = await deps.OrgAPIDataSource.getGraphQLOrgFromId(org.id); - expect(result).toMatchObject({ - id: org.id, - name: org.name, - email: org.email, - }); - }); + testWithFixture( + 'returns the org parent for an existing id', + async ({ deps, org }) => { + const result = await deps.OrgAPIDataSource.getGraphQLOrgFromId(org.id); + expect(result).toMatchObject({ + id: org.id, + name: org.name, + email: org.email, + }); + }, + ); testWithFixture( 'throws when the org does not exist (replaces Sequelize rejectOnEmpty)', @@ -51,16 +58,13 @@ describe('OrgAPI', () => { }); describe('updateOrgInfo', () => { - testWithFixture( - 'throws when the org does not exist', - async ({ deps }) => { - await expect( - deps.OrgAPIDataSource.updateOrgInfo(`missing-${uid()}`, { - name: 'whatever', - }), - ).rejects.toThrow(/Organization not found/); - }, - ); + testWithFixture('throws when the org does not exist', async ({ deps }) => { + await expect( + deps.OrgAPIDataSource.updateOrgInfo(`missing-${uid()}`, { + name: 'whatever', + }), + ).rejects.toThrow(/Organization not found/); + }); testWithFixture( 'returns the updated parent when the org exists', @@ -130,6 +134,167 @@ describe('OrgAPI', () => { ); }); + describe('getContentTypesForOrg', () => { + testWithFixture( + 'returns every item type for the org with the fields read by the ContentType resolver', + async ({ deps, org }) => { + const { itemTypes, cleanup } = await createContentItemTypes({ + moderationConfigService: deps.ModerationConfigService, + orgId: org.id, + numItemTypes: 1, + extra: {}, + }); + try { + const result = await deps.OrgAPIDataSource.getContentTypesForOrg( + org.id, + ); + const actualIds = new Set(result.map((it) => it.id)); + for (const created of itemTypes) { + expect(actualIds.has(created.id)).toBe(true); + } + for (const it of result) { + expect(it.orgId).toBe(org.id); + expect(typeof it.id).toBe('string'); + expect(typeof it.name).toBe('string'); + expect(['CONTENT', 'USER', 'THREAD']).toContain(it.kind); + expect(Array.isArray(it.schema)).toBe(true); + } + } finally { + await cleanup(); + } + }, + ); + + testWithFixture( + 'returns all item-type kinds, not just CONTENT (createOrg seeds a default USER)', + async ({ deps, org }) => { + const result = await deps.OrgAPIDataSource.getContentTypesForOrg( + org.id, + ); + expect(result.length).toBeGreaterThan(0); + expect(result.some((it) => it.kind === 'USER')).toBe(true); + }, + ); + + testWithFixture( + 'does not leak item types across orgs', + async ({ deps, org }) => { + const { org: otherOrg, cleanup: otherOrgCleanup } = await createOrg( + { + KyselyPg: deps.KyselyPg, + ModerationConfigService: deps.ModerationConfigService, + ApiKeyService: deps.ApiKeyService, + }, + uid(), + ); + try { + const result = await deps.OrgAPIDataSource.getContentTypesForOrg( + org.id, + ); + for (const it of result) { + expect(it.orgId).toBe(org.id); + expect(it.orgId).not.toBe(otherOrg.id); + } + } finally { + await otherOrgCleanup(); + } + }, + ); + }); + + describe('getOrgUsersForGraphQL', () => { + testWithFixture( + 'returns GraphQLUserParents for every user in the org with a working getPermissions() method', + async ({ deps, org }) => { + const adminId = uid(); + const analystId = uid(); + await kyselyUserInsert({ + db: deps.KyselyPg, + id: adminId, + orgId: org.id, + email: faker.internet.email(), + firstName: faker.name.firstName(), + lastName: faker.name.lastName(), + role: UserRole.ADMIN, + loginMethods: ['saml'], + password: null, + }); + await kyselyUserInsert({ + db: deps.KyselyPg, + id: analystId, + orgId: org.id, + email: faker.internet.email(), + firstName: faker.name.firstName(), + lastName: faker.name.lastName(), + role: UserRole.ANALYST, + loginMethods: ['saml'], + password: null, + }); + try { + const users = await deps.OrgAPIDataSource.getOrgUsersForGraphQL( + org.id, + ); + const ids = users.map((u) => u.id).sort(); + expect(ids).toEqual([adminId, analystId].sort()); + const admin = users.find((u) => u.id === adminId)!; + const analyst = users.find((u) => u.id === analystId)!; + expect(admin.getPermissions()).toEqual( + expect.arrayContaining(['EDIT_MRT_QUEUES']), + ); + expect(analyst.getPermissions()).not.toContain('EDIT_MRT_QUEUES'); + } finally { + await kyselyUserDeleteById(deps.KyselyPg, adminId); + await kyselyUserDeleteById(deps.KyselyPg, analystId); + } + }, + ); + + testWithFixture( + 'returns an empty array for an org with no users', + async ({ deps, org }) => { + const result = await deps.OrgAPIDataSource.getOrgUsersForGraphQL( + org.id, + ); + expect(result).toEqual([]); + }, + ); + + testWithFixture( + 'does not leak users across orgs', + async ({ deps, org }) => { + const { org: otherOrg, cleanup: otherOrgCleanup } = await createOrg( + { + KyselyPg: deps.KyselyPg, + ModerationConfigService: deps.ModerationConfigService, + ApiKeyService: deps.ApiKeyService, + }, + uid(), + ); + const otherUserId = uid(); + await kyselyUserInsert({ + db: deps.KyselyPg, + id: otherUserId, + orgId: otherOrg.id, + email: faker.internet.email(), + firstName: faker.name.firstName(), + lastName: faker.name.lastName(), + role: UserRole.ADMIN, + loginMethods: ['saml'], + password: null, + }); + try { + const result = await deps.OrgAPIDataSource.getOrgUsersForGraphQL( + org.id, + ); + expect(result.find((u) => u.id === otherUserId)).toBeUndefined(); + } finally { + await kyselyUserDeleteById(deps.KyselyPg, otherUserId); + await otherOrgCleanup(); + } + }, + ); + }); + describe('createOrg', () => { testWithFixture( 'throws a BadRequest with /input/website pointer for bad website', diff --git a/server/graphql/datasources/OrgApi.ts b/server/graphql/datasources/OrgApi.ts index 6e14981..2517481 100644 --- a/server/graphql/datasources/OrgApi.ts +++ b/server/graphql/datasources/OrgApi.ts @@ -33,6 +33,10 @@ import { validateOrgUpdatePatch, type OrgValidationFailure, } from './orgValidation.js'; +import { + type GraphQLUserParent, + kyselyUserListByOrg, +} from './userKyselyPersistence.js'; class OrgAPI { constructor( @@ -47,7 +51,6 @@ class OrgAPI { private readonly orgSettingsService: Dependencies['OrgSettingsService'], private readonly manualReviewToolService: Dependencies['ManualReviewToolService'], private readonly kysely: Dependencies['KyselyPg'], - private readonly sequelize: Dependencies['Sequelize'], ) {} async createOrg(params: GQLMutationCreateOrgArgs) { @@ -242,19 +245,12 @@ class OrgAPI { return updated; } - /** - * Legacy GraphQL `ContentType` parents still use Sequelize `getActions` on - * item types; load them from the ORM until item types are fully migrated. - */ - async getSequelizeContentTypesForOrg(orgId: string) { - return this.sequelize.ItemType.findAll({ - where: { orgId }, - }); + async getContentTypesForOrg(orgId: string) { + return this.moderationConfigService.getItemTypes({ orgId }); } - /** GraphQL `Org.users` / permission filters still use Sequelize `User` models. */ - async getOrgUsersForGraphQL(orgId: string) { - return this.sequelize.User.findAll({ where: { orgId } }); + async getOrgUsersForGraphQL(orgId: string): Promise { + return kyselyUserListByOrg(this.kysely, orgId); } // TODO: ApiKeyService should maybe be its own dataSource, @@ -378,7 +374,6 @@ export default inject( 'OrgSettingsService', 'ManualReviewToolService', 'KyselyPg', - 'Sequelize', ], OrgAPI, ); diff --git a/server/graphql/datasources/RuleApi.test.ts b/server/graphql/datasources/RuleApi.test.ts index 926c847..8907b7b 100644 --- a/server/graphql/datasources/RuleApi.test.ts +++ b/server/graphql/datasources/RuleApi.test.ts @@ -35,7 +35,7 @@ describe('RuleAPI', () => { uid(), ); const { user, cleanup: userCleanup } = await createUser( - deps.Sequelize, + deps.KyselyPg, org.id, ); const { itemTypes, cleanup: itemTypesCleanup } = @@ -264,14 +264,17 @@ describe('RuleAPI', () => { ); const now = new Date(); - await deps.Sequelize.Backtest.create({ - id: uid(), - ruleId: rule.id, - creatorId: user.id, - sampleDesiredSize: 10, - sampleStartAt: now, - sampleEndAt: now, - }); + await deps.KyselyPg.insertInto('public.backtests') + .values({ + id: uid(), + rule_id: rule.id, + creator_id: user.id, + sample_desired_size: 10, + sample_start_at: now, + sample_end_at: now, + updated_at: now, + }) + .execute(); await expect( deps.RuleAPIDataSource.updateContentRule({ diff --git a/server/graphql/datasources/RuleApi.ts b/server/graphql/datasources/RuleApi.ts index 88e3da2..0e25e52 100644 --- a/server/graphql/datasources/RuleApi.ts +++ b/server/graphql/datasources/RuleApi.ts @@ -2,17 +2,18 @@ import { type Exception } from '@opentelemetry/api'; import { makeEnumLike } from '@roostorg/types'; - import { type Kysely } from 'kysely'; import { uid } from 'uid'; import { inject, type Dependencies } from '../../iocContainer/index.js'; -import { type Backtest } from '../../models/rules/BacktestModel.js'; import { type ActionCountsInput } from '../../services/actionStatisticsService/index.js'; import { type AggregationClause } from '../../services/aggregationsService/index.js'; import { type ConditionSetWithResultAsLogged } from '../../services/analyticsLoggers/index.js'; import { type CombinedPg } from '../../services/combinedDbTypes.js'; import { + makeRuleHasRunningBacktestsError, + makeRuleIsMissingContentTypeError, + makeRuleNameExistsError, RuleType, type Condition, type ConditionInput, @@ -21,23 +22,27 @@ import { type LeafCondition, type RuleStatus, } from '../../services/moderationConfigService/index.js'; -import { - makeRuleHasRunningBacktestsError, - makeRuleIsMissingContentTypeError, - makeRuleNameExistsError, -} from '../../services/moderationConfigService/index.js'; import { isSignalId, signalIsExternal, type SignalId, } from '../../services/signalsService/index.js'; import { - type DataWarehousePublicSchema, warehouseDateToDate, + type DataWarehousePublicSchema, } from '../../storage/dataWarehouse/warehouseSchema.js'; import { toCorrelationId } from '../../utils/correlationIds.js'; -import { jsonParse, jsonStringify, tryJsonParse } from '../../utils/encoding.js'; +import { + jsonParse, + jsonStringify, + tryJsonParse, +} from '../../utils/encoding.js'; import { makeNotFoundError } from '../../utils/errors.js'; +import { isUniqueViolationError } from '../../utils/kysely.js'; +import { + makeKyselyTransactionWithRetry, + type KyselyTransactionWithRetry, +} from '../../utils/kyselyTransactionWithRetry.js'; import { assertUnreachable } from '../../utils/misc.js'; import { takeLast } from '../../utils/sql.js'; import { @@ -56,9 +61,11 @@ import { type GQLUpdateContentRuleInput, type GQLUpdateUserRuleInput, } from '../generated.js'; +import { unauthenticatedError } from '../utils/errors.js'; import { oneOfInputToTaggedUnion } from '../utils/inputHelpers.js'; import { type CursorInfo, type Edge } from '../utils/paginationHandler.js'; import { buildGraphqlRuleParent } from './buildGraphqlRuleParent.js'; +import { locationAreaInputToLocationArea } from './LocationBankApi.js'; import { kyselyCancelRunningBacktestsForRule, kyselyCreateRule, @@ -66,18 +73,12 @@ import { kyselyHasRunningBacktestsForRule, kyselyListBacktestsForRule, kyselyUpdateRule, + type GraphQLBacktestParent, } from './ruleKyselyPersistence.js'; import { - type GraphQLUserParent, kyselyUserFindByIdAndOrg, + type GraphQLUserParent, } from './userKyselyPersistence.js'; -import { locationAreaInputToLocationArea } from './LocationBankApi.js'; -import { unauthenticatedError } from '../utils/errors.js'; -import { isUniqueViolationError } from '../../utils/kysely.js'; -import { - makeKyselyTransactionWithRetry, - type KyselyTransactionWithRetry, -} from '../../utils/kyselyTransactionWithRetry.js'; /** * Normalize the GraphQL `expirationTime` input scalar into a shape our @@ -323,7 +324,10 @@ class RuleAPI { } async getGraphQLRuleFromId(id: string, orgId: string) { - const plain = await this.moderationConfigService.getRuleByIdAndOrg(id, orgId); + const plain = await this.moderationConfigService.getRuleByIdAndOrg( + id, + orgId, + ); if (plain == null) { throw unauthenticatedError('User not authenticated to fetch this rule'); } @@ -569,9 +573,13 @@ class RuleAPI { : e; } - const plain = await this.moderationConfigService.getRuleByIdAndOrg(id, orgId, { - readFromReplica: false, - }); + const plain = await this.moderationConfigService.getRuleByIdAndOrg( + id, + orgId, + { + readFromReplica: false, + }, + ); if (plain == null) { throw new Error('Rule was updated but could not be reloaded'); } @@ -670,7 +678,7 @@ class RuleAPI { async createBacktest( _input: GQLCreateBacktestInput, _user: GraphQLUserParent, - ): Promise { + ): Promise { throw new Error( 'createBacktest is temporarily disabled (TODO BACKTEST_RETROACTION: no UI / env to validate).', ); @@ -735,9 +743,7 @@ class RuleAPI { // have to use our helper that implements "takeLast" in SQL. const finalQuery = takeFrom === 'start' - ? filteredResultsQuery - .orderBy('ts', desiredSort.order) - .limit(count) + ? filteredResultsQuery.orderBy('ts', desiredSort.order).limit(count) : takeLast(this.warehouse, filteredResultsQuery, [desiredSort], count); const results = await finalQuery.execute(); @@ -751,7 +757,7 @@ class RuleAPI { itemTypeId: it.itemTypeId, userId: it.userId ?? undefined, userTypeId: it.userTypeId ?? undefined, - content: (it.content ?? '') as string, + content: it.content ?? '', result: it.result ? jsonParse(it.result) : null, environment: it.environment as RuleStatus, passed: it.passed, diff --git a/server/graphql/datasources/UserApi.ts b/server/graphql/datasources/UserApi.ts index 10585ba..2efa75c 100644 --- a/server/graphql/datasources/UserApi.ts +++ b/server/graphql/datasources/UserApi.ts @@ -3,7 +3,6 @@ import { type PassportContext } from 'graphql-passport'; import { uid } from 'uid'; import { inject, type Dependencies } from '../../iocContainer/index.js'; -import { type Rule } from '../../models/rules/RuleModel.js'; import { type LoginMethod } from '../../services/coreAppTables.js'; import { hashPassword, @@ -21,22 +20,23 @@ import { import { safePick } from '../../utils/misc.js'; import { WEEK_MS } from '../../utils/time.js'; import { buildGraphqlRuleParent } from './buildGraphqlRuleParent.js'; +import { type GraphQLRuleParent } from './ruleKyselyPersistence.js'; import { - type GraphQLUserParent, kyselyUserAddFavoriteRule, kyselyUserFindByEmail, - kyselyUserFindByIdAndOrg, kyselyUserFindById, + kyselyUserFindByIdAndOrg, kyselyUserFindByIds, kyselyUserInsert, kyselyUserListFavoriteRuleIds, kyselyUserRemoveFavoriteRule, kyselyUserUpdate, + type GraphQLUserParent, } from './userKyselyPersistence.js'; import { - type UserValidationFailure, validateUserCreateInput, validateUserUpdatePatch, + type UserValidationFailure, } from './userValidation.js'; /** @@ -142,7 +142,9 @@ class UserAPI { }); } - const loginMethodNormalized = String(loginMethod).toLowerCase() as LoginMethod; + const loginMethodNormalized = String( + loginMethod, + ).toLowerCase() as LoginMethod; const createInput = { email, firstName, @@ -328,7 +330,10 @@ class UserAPI { return true; } - async getFavoriteRules(id: string, orgId: string): Promise> { + async getFavoriteRules( + id: string, + orgId: string, + ): Promise> { // Make sure the requested user lives in the invoker's org (the caller // always passes the invoker's orgId), then scope rule lookups to that // org so cross-org data can't leak even if stale favorites exist. @@ -372,7 +377,9 @@ class UserAPI { } } -function userValidationFailureToBadRequestError(failure: UserValidationFailure) { +function userValidationFailureToBadRequestError( + failure: UserValidationFailure, +) { return makeBadRequestError(failure.message, { pointer: `/input/${failure.field}`, shouldErrorSpan: false, diff --git a/server/graphql/datasources/buildGraphqlRuleParent.ts b/server/graphql/datasources/buildGraphqlRuleParent.ts index 2245ec9..2a031f7 100644 --- a/server/graphql/datasources/buildGraphqlRuleParent.ts +++ b/server/graphql/datasources/buildGraphqlRuleParent.ts @@ -1,9 +1,8 @@ -import { type Rule as SequelizeRule } from '../../models/rules/RuleModel.js'; import { + type ModerationConfigService, type PlainRuleWithLatestVersion, - type Rule as RuleGraphqlParent, -} from '../../models/rules/ruleTypes.js'; -import { type ModerationConfigService } from '../../services/moderationConfigService/index.js'; +} from '../../services/moderationConfigService/index.js'; +import { type GraphQLRuleParent } from './ruleKyselyPersistence.js'; import { type GraphQLUserParent } from './userKyselyPersistence.js'; type FindUserByIdAndOrg = (opts: { @@ -13,16 +12,9 @@ type FindUserByIdAndOrg = (opts: { /** * Builds a GraphQL Rule parent (plain row fields + the three association - * getters our resolvers actually use) backed by ModerationConfigService - * reads and a Kysely-backed User lookup for the creator. - * - * The returned object only implements the {@link RuleGraphqlParent} contract - * (`getCreator` / `getActions` / `getPolicies`). We cast to `SequelizeRule` - * at the return to satisfy the GraphQL codegen parent type that still points - * at `RuleModel.Rule`; resolvers that reach for Sequelize-only methods like - * `save` / `destroy` / `getContentTypes` / `getBacktests` on this value will - * blow up at runtime. The cast will be removed once `codegen.yaml` is flipped - * to `ruleTypes.js#Rule` (see the TODO there). + * getters our Rule / ContentRule / UserRule / RuleInsights resolvers actually + * use) backed by ModerationConfigService reads and a Kysely-backed User + * lookup for the creator. */ export function buildGraphqlRuleParent( plain: PlainRuleWithLatestVersion, @@ -30,8 +22,8 @@ export function buildGraphqlRuleParent( moderationConfigService: ModerationConfigService; findUserByIdAndOrg: FindUserByIdAndOrg; }, -): SequelizeRule { - const parent: RuleGraphqlParent = { +): GraphQLRuleParent { + return { ...plain, async getCreator() { const user = await deps.findUserByIdAndOrg({ @@ -56,5 +48,4 @@ export function buildGraphqlRuleParent( return byRule[plain.id] ?? []; }, }; - return parent as unknown as SequelizeRule; } diff --git a/server/graphql/datasources/ruleKyselyPersistence.ts b/server/graphql/datasources/ruleKyselyPersistence.ts index c96f603..d0367b9 100644 --- a/server/graphql/datasources/ruleKyselyPersistence.ts +++ b/server/graphql/datasources/ruleKyselyPersistence.ts @@ -1,15 +1,49 @@ -import { type Insertable, type Kysely, type Updateable, sql } from 'kysely'; +import { sql, type Insertable, type Kysely, type Updateable } from 'kysely'; -import { computeRuleStatusFromRow } from '../../models/rules/ruleTypes.js'; import { type CombinedPg } from '../../services/combinedDbTypes.js'; -import { makeNotFoundError } from '../../utils/errors.js'; +import { type BacktestStatusDb } from '../../services/coreAppTables.js'; import { + computeRuleStatusFromRow, RuleAlarmStatus, RuleStatus, RuleType, + type Action, type ConditionSet, + type PlainRuleWithLatestVersion, + type Policy, } from '../../services/moderationConfigService/index.js'; -import { type Backtest } from '../../models/rules/BacktestModel.js'; +import { makeNotFoundError } from '../../utils/errors.js'; +import { type GraphQLUserParent } from './userKyselyPersistence.js'; + +/** + * GraphQL Rule parent: plain row fields plus the three resolver getters our + * Rule / ContentRule / UserRule / RuleInsights resolvers actually use. The + * codegen mapper for those four GraphQL types points here. + */ +export type GraphQLRuleParent = PlainRuleWithLatestVersion & { + getCreator(): Promise; + getActions(): Promise; + getPolicies(): Promise; +}; + +export type GraphQLBacktestParent = { + id: string; + ruleId: string; + creatorId: string; + sampleDesiredSize: number; + sampleActualSize: number; + sampleStartAt: Date; + sampleEndAt: Date; + samplingComplete: boolean; + contentItemsProcessed: number; + contentItemsMatched: number; + status: BacktestStatusDb; + createdAt: Date; + updatedAt: Date; + cancelationDate: Date | null; + correctedContentItemsProcessed: number; + correctedContentItemsMatched: number; +}; /** Matches `public.backtests.status` when generated value is RUNNING. */ const backtestRunningPredicate = sql`cancelation_date is null @@ -48,13 +82,18 @@ async function replaceRuleActions( ruleId: string, actionIds: readonly string[], ) { - await trx.deleteFrom('public.rules_and_actions').where('rule_id', '=', ruleId).execute(); + await trx + .deleteFrom('public.rules_and_actions') + .where('rule_id', '=', ruleId) + .execute(); if (actionIds.length === 0) { return; } await trx .insertInto('public.rules_and_actions') - .values(actionIds.map((actionId) => ({ rule_id: ruleId, action_id: actionId }))) + .values( + actionIds.map((actionId) => ({ rule_id: ruleId, action_id: actionId })), + ) .execute(); } @@ -63,7 +102,10 @@ async function replaceRulePolicies( ruleId: string, policyIds: readonly string[], ) { - await trx.deleteFrom('public.rules_and_policies').where('rule_id', '=', ruleId).execute(); + await trx + .deleteFrom('public.rules_and_policies') + .where('rule_id', '=', ruleId) + .execute(); if (policyIds.length === 0) { return; } @@ -86,7 +128,10 @@ async function replaceRuleItemTypes( ruleId: string, itemTypeIds: readonly string[], ) { - await trx.deleteFrom('public.rules_and_item_types').where('rule_id', '=', ruleId).execute(); + await trx + .deleteFrom('public.rules_and_item_types') + .where('rule_id', '=', ruleId) + .execute(); if (itemTypeIds.length === 0) { return; } @@ -156,10 +201,7 @@ export async function kyselyCreateRule( created_at: now, updated_at: now, }; - await trx - .insertInto('public.rules') - .values(ruleValues as Insertable) - .execute(); + await trx.insertInto('public.rules').values(ruleValues).execute(); await replaceRuleActions(trx, input.id, input.actionIds); await replaceRulePolicies(trx, input.id, input.policyIds); @@ -310,7 +352,11 @@ export async function kyselyDeleteRule( .deleteFrom('public.users_and_favorite_rules') .where('rule_id', '=', id) .execute(); - await trx.deleteFrom('public.rules').where('id', '=', id).where('org_id', '=', orgId).execute(); + await trx + .deleteFrom('public.rules') + .where('id', '=', id) + .where('org_id', '=', orgId) + .execute(); return true; } @@ -344,7 +390,7 @@ export async function kyselyListBacktestsForRule( kysely: Kysely, ruleId: string, backtestIds?: readonly string[] | null, -): Promise { +): Promise { let q = kysely .selectFrom('public.backtests') .selectAll() @@ -353,10 +399,10 @@ export async function kyselyListBacktestsForRule( q = q.where('id', 'in', [...backtestIds]); } const rows = await q.execute(); - return rows.map((r) => mapBacktestRowToGqlParent(r)) as unknown as Backtest[]; + return rows.map((r) => mapBacktestRowToGqlParent(r)); } -function mapBacktestRowToGqlParent(r: { +export function mapBacktestRowToGqlParent(r: { id: string; rule_id: string; creator_id: string; @@ -367,11 +413,13 @@ function mapBacktestRowToGqlParent(r: { sampling_complete: boolean; content_items_processed: number; content_items_matched: number; - status: string; + status: BacktestStatusDb; created_at: Date; updated_at: Date; cancelation_date: Date | null; -}) { +}): GraphQLBacktestParent { + // Queues deliver sampled items at-least-once, so processed/matched counters + // can rarely exceed sample_actual_size. Clamp the values exposed to clients. const correctedContentItemsProcessed = Math.min( r.sample_actual_size, r.content_items_processed, diff --git a/server/graphql/datasources/userKyselyPersistence.test.ts b/server/graphql/datasources/userKyselyPersistence.test.ts index 272207c..dd1158c 100644 --- a/server/graphql/datasources/userKyselyPersistence.test.ts +++ b/server/graphql/datasources/userKyselyPersistence.test.ts @@ -1,7 +1,7 @@ import { faker } from '@faker-js/faker'; import { uid } from 'uid'; -import { UserRole } from '../../models/types/permissioning.js'; +import { UserRole } from '../../services/userManagementService/index.js'; import createOrg from '../../test/fixtureHelpers/createOrg.js'; import createRule from '../../test/fixtureHelpers/createRule.js'; import { makeMockedServer } from '../../test/setupMockedServer.js'; @@ -131,7 +131,7 @@ describe('userKyselyPersistence', () => { ); testWithFixture( - "throws an invariant error when password/loginMethods disagree (CHECK constraint shape)", + 'throws an invariant error when password/loginMethods disagree (CHECK constraint shape)', async ({ deps, org }) => { await expect( kyselyUserInsert({ @@ -340,7 +340,7 @@ describe('userKyselyPersistence', () => { // constraint. This protects against regressions if we ever loosen // `validateUserUpdatePatch`. testWithFixture( - "clearing password on a password-login user violates password_null_when_not_present", + 'clearing password on a password-login user violates password_null_when_not_present', async ({ deps, org }) => { const input = { ...samlUserInput(org.id), @@ -364,8 +364,7 @@ describe('userKyselyPersistence', () => { const input = samlUserInput(org.id); await kyselyUserInsert({ db: deps.KyselyPg, ...input }); try { - const beforeRow = await deps.KyselyPg - .selectFrom('public.users') + const beforeRow = await deps.KyselyPg.selectFrom('public.users') .select(['updated_at']) .where('id', '=', input.id) .executeTakeFirstOrThrow(); @@ -379,8 +378,7 @@ describe('userKyselyPersistence', () => { expect(updated!.firstName).toBe('Updated'); expect(updated!.approvedByAdmin).toBe(true); - const afterRow = await deps.KyselyPg - .selectFrom('public.users') + const afterRow = await deps.KyselyPg.selectFrom('public.users') .select(['updated_at']) .where('id', '=', input.id) .executeTakeFirstOrThrow(); @@ -401,9 +399,8 @@ describe('userKyselyPersistence', () => { const input = samlUserInput(org.id); await kyselyUserInsert({ db: deps.KyselyPg, ...input }); // `users_and_favorite_rules.rule_id` has a FK to `public.rules`, so - // we need a real rule row. Reuse the existing Sequelize fixture until - // rule fixtures are themselves Kysely-backed. - const rule = await createRule(deps.Sequelize, org.id, { + // we need a real rule row. + const rule = await createRule(deps.KyselyPg, org.id, { creatorId: input.id, }); try { diff --git a/server/graphql/datasources/userKyselyPersistence.ts b/server/graphql/datasources/userKyselyPersistence.ts index 6b8680f..1acfe14 100644 --- a/server/graphql/datasources/userKyselyPersistence.ts +++ b/server/graphql/datasources/userKyselyPersistence.ts @@ -1,14 +1,12 @@ -import { type Kysely, sql } from 'kysely'; +import { sql, type Kysely } from 'kysely'; +import { type CombinedPg } from '../../services/combinedDbTypes.js'; +import { type LoginMethod } from '../../services/coreAppTables.js'; import { getPermissionsForRole, type UserPermission, type UserRole, -} from '../../models/types/permissioning.js'; -import { - type CoreAppTablesPg, - type LoginMethod, -} from '../../services/coreAppTables.js'; +} from '../../services/userManagementService/index.js'; import { validateUserCreateInput, validateUserUpdatePatch, @@ -39,7 +37,10 @@ export type GraphQLUserParent = { getPermissions(): UserPermission[]; }; -type UsersDb = Kysely; +// Aligns with `ruleKyselyPersistence.ts`: persistence helpers operate on the +// full app schema. Lets fixtures and `kyselyCreateRule` callers share a single +// `Kysely` handle without running into Kysely's invariant generic. +type UsersDb = Kysely; type UserRow = { id: string; diff --git a/server/graphql/datasources/userValidation.test.ts b/server/graphql/datasources/userValidation.test.ts index d91fd1f..2e8d6f5 100644 --- a/server/graphql/datasources/userValidation.test.ts +++ b/server/graphql/datasources/userValidation.test.ts @@ -79,7 +79,7 @@ describe('userValidation', () => { test('rejects unknown loginMethods entry', () => { const result = validateUserCreateInput({ ...validInput, - loginMethods: ['saml', 'oauth' as never], + loginMethods: ['saml', 'oauth'], }); expect(result.ok).toBe(false); if (!result.ok) { diff --git a/server/graphql/datasources/userValidation.ts b/server/graphql/datasources/userValidation.ts index 67d5ccb..92cc2fd 100644 --- a/server/graphql/datasources/userValidation.ts +++ b/server/graphql/datasources/userValidation.ts @@ -1,8 +1,8 @@ import { createRequire } from 'node:module'; import type { IsEmailOptions } from 'validator/lib/isEmail.js'; -import { UserRole } from '../../models/types/permissioning.js'; import { type LoginMethod } from '../../services/coreAppTables.js'; +import { UserRole } from '../../services/userManagementService/index.js'; // `validator` is CJS with UMD-style types whose `default` doesn't resolve to // a callable under `module: NodeNext`; `createRequire` gives us `module.exports` diff --git a/server/graphql/generated.ts b/server/graphql/generated.ts index 6f3c641..9a2abc5 100644 --- a/server/graphql/generated.ts +++ b/server/graphql/generated.ts @@ -8,6 +8,10 @@ import { JsonObject, JsonValue } from 'type-fest'; import type { UserHistoryForGQL } from '../graphql/datasources/InvestigationApi.js'; import type { GraphQLOrgParent } from '../graphql/datasources/orgKyselyPersistence.js'; +import type { + GraphQLBacktestParent, + GraphQLRuleParent, +} from '../graphql/datasources/ruleKyselyPersistence.js'; import type { GraphQLUserParent } from '../graphql/datasources/userKyselyPersistence.js'; import type { ContentItemTypeResolversParentType, @@ -18,12 +22,9 @@ import type { UserItemTypeResolversParentType, } from '../graphql/modules/itemType.js'; import type { ReportingInsights } from '../graphql/modules/reporting.js'; -import type { HashBank } from '../models/HashBankModel.js'; -import type { Backtest } from '../models/rules/BacktestModel.js'; -import type { ItemType } from '../models/rules/ItemTypeModel.js'; -import type { Rule } from '../models/rules/RuleModel.js'; import type { SignalWithScore } from '../services/analyticsQueries/RuleActionInsights.js'; import type { DerivedFieldSpecSource } from '../services/derivedFieldsService/helpers.js'; +import type { HashBank } from '../services/hmaService/index.js'; import type { ContentAppealReviewJobPayload, ContentManualReviewJobPayload, @@ -52,6 +53,7 @@ import type { EnqueueToMrtAction, EnqueueToNcmecAction, } from '../services/moderationConfigService/types/actions.js'; +import type { ItemType } from '../services/moderationConfigService/types/itemTypes.js'; import type { Notification } from '../services/notificationsService/notificationsService.js'; import type { ReportingRuleWithoutVersion } from '../services/reportingService/ReportingRules.js'; import type { Signal } from '../services/signalsService/index.js'; @@ -5467,7 +5469,7 @@ export type GQLResolversInterfaceTypes< | GQLSubmitNcmecReportDecisionComponent | GQLTransformJobAndRecreateInQueueDecisionComponent | GQLUserOrRelatedActionDecisionComponent; - Rule: Rule | Rule; + Rule: GraphQLRuleParent | GraphQLRuleParent; }; /** Mapping between all available schema types and the resolvers types */ @@ -5534,7 +5536,7 @@ export type GQLResolversTypes = { AppealSettings: ResolverTypeWrapper; AppealSettingsInput: GQLAppealSettingsInput; AutomaticCloseDecisionComponent: ResolverTypeWrapper; - Backtest: ResolverTypeWrapper; + Backtest: ResolverTypeWrapper; BacktestStatus: GQLBacktestStatus; BaseField: ResolverTypeWrapper; Boolean: ResolverTypeWrapper; @@ -5574,7 +5576,7 @@ export type GQLResolversTypes = { >; ContentItemType: ResolverTypeWrapper; ContentManualReviewJobPayload: ResolverTypeWrapper; - ContentRule: ResolverTypeWrapper; + ContentRule: ResolverTypeWrapper; ContentSchemaFieldRoles: ResolverTypeWrapper; ContentSchemaFieldRolesInput: GQLContentSchemaFieldRolesInput; ContentType: ResolverTypeWrapper; @@ -6106,7 +6108,7 @@ export type GQLResolversTypes = { RoutingRule: ResolverTypeWrapper; RoutingRuleNameExistsError: ResolverTypeWrapper; RoutingRuleStatus: GQLRoutingRuleStatus; - Rule: ResolverTypeWrapper; + Rule: ResolverTypeWrapper; RuleEnvironment: GQLRuleEnvironment; RuleExecutionEnqueueSourceInfo: ResolverTypeWrapper< Omit & { @@ -6132,7 +6134,7 @@ export type GQLResolversTypes = { } >; RuleHasRunningBacktestsError: ResolverTypeWrapper; - RuleInsights: ResolverTypeWrapper; + RuleInsights: ResolverTypeWrapper; RuleNameExistsError: ResolverTypeWrapper; RulePassRateData: ResolverTypeWrapper; RuleStatus: GQLRuleStatus; @@ -6308,7 +6310,7 @@ export type GQLResolversTypes = { UserPenaltySeverity: GQLUserPenaltySeverity; UserPermission: GQLUserPermission; UserRole: GQLUserRole; - UserRule: ResolverTypeWrapper; + UserRule: ResolverTypeWrapper; UserSchemaFieldRoles: ResolverTypeWrapper; UserSchemaFieldRolesInput: GQLUserSchemaFieldRolesInput; UserStrikeBucket: ResolverTypeWrapper; @@ -6368,7 +6370,7 @@ export type GQLResolversParentTypes = { AppealSettings: GQLAppealSettings; AppealSettingsInput: GQLAppealSettingsInput; AutomaticCloseDecisionComponent: GQLAutomaticCloseDecisionComponent; - Backtest: Backtest; + Backtest: GraphQLBacktestParent; BaseField: GQLBaseField; Boolean: Scalars['Boolean']['output']; CannotDeleteDefaultUserError: GQLCannotDeleteDefaultUserError; @@ -6397,7 +6399,7 @@ export type GQLResolversParentTypes = { }; ContentItemType: ContentItemTypeResolversParentType; ContentManualReviewJobPayload: ContentManualReviewJobPayload; - ContentRule: Rule; + ContentRule: GraphQLRuleParent; ContentSchemaFieldRoles: GQLContentSchemaFieldRoles; ContentSchemaFieldRolesInput: GQLContentSchemaFieldRolesInput; ContentType: ItemType; @@ -6790,7 +6792,7 @@ export type GQLResolversParentTypes = { RotateWebhookSigningKeySuccessResponse: GQLRotateWebhookSigningKeySuccessResponse; RoutingRule: RoutingRuleWithoutVersion; RoutingRuleNameExistsError: GQLRoutingRuleNameExistsError; - Rule: Rule; + Rule: GraphQLRuleParent; RuleExecutionEnqueueSourceInfo: Omit< GQLRuleExecutionEnqueueSourceInfo, 'rules' @@ -6814,7 +6816,7 @@ export type GQLResolversParentTypes = { edges: ReadonlyArray; }; RuleHasRunningBacktestsError: GQLRuleHasRunningBacktestsError; - RuleInsights: Rule; + RuleInsights: GraphQLRuleParent; RuleNameExistsError: GQLRuleNameExistsError; RulePassRateData: GQLRulePassRateData; RunRetroactionInput: GQLRunRetroactionInput; @@ -6941,7 +6943,7 @@ export type GQLResolversParentTypes = { edges: ReadonlyArray; }; UserOrRelatedActionDecisionComponent: GQLUserOrRelatedActionDecisionComponent; - UserRule: Rule; + UserRule: GraphQLRuleParent; UserSchemaFieldRoles: GQLUserSchemaFieldRoles; UserSchemaFieldRolesInput: GQLUserSchemaFieldRolesInput; UserStrikeBucket: GQLUserStrikeBucket; diff --git a/server/graphql/modules/action.ts b/server/graphql/modules/action.ts index 039daaa..62b51e9 100644 --- a/server/graphql/modules/action.ts +++ b/server/graphql/modules/action.ts @@ -1,3 +1,4 @@ +import { parseStoredParameters } from '../../services/moderationConfigService/index.js'; import { isCoopErrorOfType } from '../../utils/errors.js'; import { assertUnreachable } from '../../utils/misc.js'; import { @@ -11,9 +12,8 @@ import { type GQLMutationResolvers, type GQLQueryResolvers, } from '../generated.js'; -import { gqlErrorResult, gqlSuccessResult } from '../utils/gqlResult.js'; import { unauthenticatedError } from '../utils/errors.js'; -import { parseStoredParameters } from '../../services/moderationConfigService/index.js'; +import { gqlErrorResult, gqlSuccessResult } from '../utils/gqlResult.js'; const typeDefs = /* GraphQL */ ` interface ActionBase { @@ -51,20 +51,30 @@ const typeDefs = /* GraphQL */ ` webhook payload under the parameter's \`name\`. """ type ActionParameter { - """Key under which the value is sent in the webhook payload.""" + """ + Key under which the value is sent in the webhook payload. + """ name: String! displayName: String! description: String type: ActionParameterType! required: Boolean! options: [ActionParameterOption!] - """NUMBER only: inclusive minimum.""" + """ + NUMBER only: inclusive minimum. + """ min: Float - """NUMBER only: inclusive maximum.""" + """ + NUMBER only: inclusive maximum. + """ max: Float - """STRING only: inclusive maximum length in characters.""" + """ + STRING only: inclusive maximum length in characters. + """ maxLength: Int - """Pre-filled value shown to the moderator. Shape matches \`type\`.""" + """ + Pre-filled value shown to the moderator. Shape matches \`type\`. + """ defaultValue: JSON } @@ -142,7 +152,7 @@ const typeDefs = /* GraphQL */ ` } union Action = - EnqueueToMrtAction + | EnqueueToMrtAction | EnqueueToNcmecAction | CustomAction | EnqueueAuthorToMrtAction @@ -183,7 +193,7 @@ const typeDefs = /* GraphQL */ ` } union MutateActionResponse = - MutateActionSuccessResponse + | MutateActionSuccessResponse | ActionNameExistsError type MutateActionSuccessResponse { @@ -272,9 +282,11 @@ function projectParameters(value: unknown): GQLActionParameter[] { name: p.name, displayName: p.displayName, description: p.description ?? null, - type: p.type as GQLActionParameter['type'], + type: p.type, required: p.required, - options: p.options ? p.options.map((o) => ({ value: o.value, label: o.label })) : null, + options: p.options + ? p.options.map((o) => ({ value: o.value, label: o.label })) + : null, min: p.min ?? null, max: p.max ?? null, maxLength: p.maxLength ?? null, @@ -290,7 +302,9 @@ function projectParameters(value: unknown): GQLActionParameter[] { // defensively so the GraphQL projection works for all four action types. function readRawParameters(parent: unknown): unknown { if (typeof parent !== 'object' || parent === null) return null; - return (parent as { customMrtApiParams?: unknown }).customMrtApiParams ?? null; + return ( + (parent as { customMrtApiParams?: unknown }).customMrtApiParams ?? null + ); } const CustomAction: GQLCustomActionResolvers = { @@ -443,11 +457,10 @@ const Mutation: GQLMutationResolvers = { actorEmail: email, // GraphQL `JSONObject` arrives as a plain object; the datasource // narrows + validates per-action against each spec. - actionIdToParameters: - (params.input.parameters ?? null) as Record< - string, - Record - > | null, + actionIdToParameters: (params.input.parameters ?? null) as Record< + string, + Record + > | null, actorNote: params.input.note ?? null, }); diff --git a/server/graphql/modules/actionStatistics.ts b/server/graphql/modules/actionStatistics.ts index 0744237..68f90c7 100644 --- a/server/graphql/modules/actionStatistics.ts +++ b/server/graphql/modules/actionStatistics.ts @@ -1,7 +1,4 @@ -/* eslint-disable max-lines */ - import { type GQLQueryResolvers } from '../generated.js'; - import { unauthenticatedError } from '../utils/errors.js'; const typeDefs = /* GraphQL */ ` diff --git a/server/graphql/modules/backtest.resolver.test.ts b/server/graphql/modules/backtest.resolver.test.ts index 34ff677..4bf3b33 100644 --- a/server/graphql/modules/backtest.resolver.test.ts +++ b/server/graphql/modules/backtest.resolver.test.ts @@ -1,6 +1,128 @@ -import { UserRole } from '../../models/types/permissioning.js'; +import { UserRole } from '../../services/userManagementService/index.js'; +import { + mapBacktestRowToGqlParent, + type GraphQLBacktestParent, +} from '../datasources/ruleKyselyPersistence.js'; import { resolvers } from './backtest.js'; +function makeBacktestRow( + overrides: Partial<{ + id: string; + rule_id: string; + creator_id: string; + sample_desired_size: number; + sample_actual_size: number; + sample_start_at: Date; + sample_end_at: Date; + sampling_complete: boolean; + content_items_processed: number; + content_items_matched: number; + status: 'RUNNING' | 'COMPLETE' | 'CANCELED'; + created_at: Date; + updated_at: Date; + cancelation_date: Date | null; + }> = {}, +) { + const now = new Date('2026-01-01T00:00:00Z'); + return { + id: 'bt-1', + rule_id: 'rule-1', + creator_id: 'user-1', + sample_desired_size: 100, + sample_actual_size: 80, + sample_start_at: now, + sample_end_at: now, + sampling_complete: true, + content_items_processed: 50, + content_items_matched: 10, + status: 'COMPLETE' as const, + created_at: now, + updated_at: now, + cancelation_date: null, + ...overrides, + }; +} + +describe('mapBacktestRowToGqlParent', () => { + it('round-trips snake_case columns to camelCase fields', () => { + const row = makeBacktestRow(); + const parent = mapBacktestRowToGqlParent(row); + expect(parent).toMatchObject({ + id: row.id, + ruleId: row.rule_id, + creatorId: row.creator_id, + sampleDesiredSize: row.sample_desired_size, + sampleActualSize: row.sample_actual_size, + sampleStartAt: row.sample_start_at, + sampleEndAt: row.sample_end_at, + samplingComplete: row.sampling_complete, + contentItemsProcessed: row.content_items_processed, + contentItemsMatched: row.content_items_matched, + status: row.status, + createdAt: row.created_at, + updatedAt: row.updated_at, + cancelationDate: row.cancelation_date, + }); + }); + + it('clamps correctedContentItemsProcessed at sampleActualSize when items overshoot', () => { + // Queues deliver at-least-once, so processed can exceed actual size. + const parent = mapBacktestRowToGqlParent( + makeBacktestRow({ + sample_actual_size: 80, + content_items_processed: 95, + content_items_matched: 5, + }), + ); + expect(parent.correctedContentItemsProcessed).toBe(80); + }); + + it('clamps correctedContentItemsMatched at correctedContentItemsProcessed', () => { + const parent = mapBacktestRowToGqlParent( + makeBacktestRow({ + sample_actual_size: 80, + content_items_processed: 50, + content_items_matched: 60, + }), + ); + expect(parent.correctedContentItemsProcessed).toBe(50); + expect(parent.correctedContentItemsMatched).toBe(50); + }); + + it('passes through processed/matched values when below the clamp ceilings', () => { + const parent = mapBacktestRowToGqlParent( + makeBacktestRow({ + sample_actual_size: 80, + content_items_processed: 30, + content_items_matched: 5, + }), + ); + expect(parent.correctedContentItemsProcessed).toBe(30); + expect(parent.correctedContentItemsMatched).toBe(5); + }); +}); + +describe('Backtest field resolvers', () => { + function parent( + overrides: Partial = {}, + ): GraphQLBacktestParent { + return { + ...mapBacktestRowToGqlParent(makeBacktestRow()), + ...overrides, + }; + } + + it('Backtest.contentItemsProcessed returns the corrected (clamped) value', () => { + const source = parent({ correctedContentItemsProcessed: 42 }); + expect(resolvers.Backtest.contentItemsProcessed(source)).toBe(42); + }); + + it('Backtest.contentItemsMatched returns the corrected (clamped) value', () => { + const source = parent({ correctedContentItemsMatched: 7 }); + expect(resolvers.Backtest.contentItemsMatched(source)).toBe(7); + }); +}); + describe('backtest resolvers', () => { describe('Mutation.createBacktest', () => { it('does not call getRuleByIdAndOrg when the user lacks RUN_BACKTEST', async () => { @@ -22,19 +144,22 @@ describe('backtest resolvers', () => { }; await expect( - (resolvers.Mutation as { createBacktest: (...a: unknown[]) => Promise }) - .createBacktest( - {}, - { - input: { - ruleId: 'rule-1', - sampleDesiredSize: 10, - sampleStartAt: new Date().toISOString(), - sampleEndAt: new Date().toISOString(), - }, + ( + resolvers.Mutation as { + createBacktest: (...a: unknown[]) => Promise; + } + ).createBacktest( + {}, + { + input: { + ruleId: 'rule-1', + sampleDesiredSize: 10, + sampleStartAt: new Date().toISOString(), + sampleEndAt: new Date().toISOString(), }, - ctx as never, - ), + }, + ctx as never, + ), ).rejects.toThrow('User not authorized to create backtests.'); expect(getRuleByIdAndOrg).not.toHaveBeenCalled(); diff --git a/server/graphql/modules/backtest.ts b/server/graphql/modules/backtest.ts index 0bc41a8..01ad2c1 100644 --- a/server/graphql/modules/backtest.ts +++ b/server/graphql/modules/backtest.ts @@ -1,16 +1,16 @@ -import { type Backtest } from '../../models/rules/BacktestModel.js'; import { hasPermission, UserPermission, -} from '../../models/types/permissioning.js'; +} from '../../services/userManagementService/index.js'; import { type RuleExecutionResult } from '../datasources/RuleApi.js'; +import { type GraphQLBacktestParent } from '../datasources/ruleKyselyPersistence.js'; import { type GQLMutationCreateBacktestArgs } from '../generated.js'; import { type Context } from '../resolvers.js'; +import { forbiddenError, unauthenticatedError } from '../utils/errors.js'; import { makeConnectionResolver, type ConnectionArguments, } from '../utils/paginationHandler.js'; -import { forbiddenError, unauthenticatedError } from '../utils/errors.js'; const typeDefs = /* GraphQL */ ` enum BacktestStatus { @@ -77,14 +77,14 @@ const typeDefs = /* GraphQL */ ` const resolvers = { Backtest: { - contentItemsProcessed(source: Backtest) { + contentItemsProcessed(source: GraphQLBacktestParent) { return source.correctedContentItemsProcessed; }, - contentItemsMatched(source: Backtest) { + contentItemsMatched(source: GraphQLBacktestParent) { return source.correctedContentItemsMatched; }, results: makeConnectionResolver< - Backtest, + GraphQLBacktestParent, { ts: number }, RuleExecutionResult, Context, diff --git a/server/graphql/modules/contentType.ts b/server/graphql/modules/contentType.ts index dc6e963..6789abb 100644 --- a/server/graphql/modules/contentType.ts +++ b/server/graphql/modules/contentType.ts @@ -25,12 +25,12 @@ const ContentType: GQLContentTypeResolvers = { }); }, baseFields(contentType) { - return contentType.fields; + return contentType.schema; }, async derivedFields(contentType, _, context) { return context.services.DerivedFieldsService.getDerivedFields( contentType.id, - contentType.fields, + contentType.schema, contentType.orgId, ); }, diff --git a/server/graphql/modules/integration.ts b/server/graphql/modules/integration.ts index bd91a09..71ceced 100644 --- a/server/graphql/modules/integration.ts +++ b/server/graphql/modules/integration.ts @@ -3,17 +3,16 @@ import { Integration } from '../../services/signalsService/index.js'; import { isCoopErrorOfType } from '../../utils/errors.js'; import { makeIntegrationConfigUnsupportedIntegrationError, + type TIntegrationConfigWithMetadata, } from '../datasources/IntegrationApi.js'; -import type { TIntegrationConfigWithMetadata } from '../datasources/IntegrationApi.js'; import { type GQLIntegrationConfig, - type GQLIntegrationMetadata, type GQLMutationResolvers, type GQLQueryResolvers, } from '../generated.js'; import { type ResolverMap } from '../resolvers.js'; -import { gqlErrorResult, gqlSuccessResult } from '../utils/gqlResult.js'; import { unauthenticatedError } from '../utils/errors.js'; +import { gqlErrorResult, gqlSuccessResult } from '../utils/gqlResult.js'; const typeDefs = /* GraphQL */ ` enum Integration { @@ -41,7 +40,7 @@ const typeDefs = /* GraphQL */ ` } union IntegrationApiCredential = - GoogleContentSafetyApiIntegrationApiCredential + | GoogleContentSafetyApiIntegrationApiCredential | OpenAiIntegrationApiCredential | ZentropiIntegrationApiCredential | PluginIntegrationApiCredential @@ -155,7 +154,7 @@ const typeDefs = /* GraphQL */ ` } union SetIntegrationConfigResponse = - SetIntegrationConfigSuccessResponse + | SetIntegrationConfigSuccessResponse | IntegrationConfigTooManyCredentialsError | IntegrationNoInputCredentialsError | IntegrationEmptyInputCredentialsError @@ -174,7 +173,7 @@ const typeDefs = /* GraphQL */ ` } union IntegrationConfigQueryResponse = - IntegrationConfigSuccessResult + | IntegrationConfigSuccessResult | IntegrationConfigUnsupportedIntegrationError type Query { @@ -197,7 +196,9 @@ const typeDefs = /* GraphQL */ ` } `; -const IntegrationApiCredential: ResolverMap = { +const IntegrationApiCredential: ResolverMap< + TIntegrationConfigWithMetadata['apiCredential'] +> = { __resolveType(it) { const integrationName = (it as { name?: string }).name ?? ''; switch (integrationName) { @@ -252,13 +253,13 @@ const Query: GQLQueryResolvers = { if (user == null) { throw unauthenticatedError('Unauthenticated User'); } - return context.dataSources.integrationAPI.getAvailableIntegrations() as GQLIntegrationMetadata[]; + return context.dataSources.integrationAPI.getAvailableIntegrations(); }, }; const PluginIntegrationApiCredential = { credential(it: TIntegrationConfigWithMetadata['apiCredential']) { - return it as Record; + return it; }, }; @@ -301,7 +302,7 @@ const Mutation: GQLMutationResolvers = { const newConfig = await context.dataSources.integrationAPI.setConfigByIntegrationId( params.input.integrationId, - params.input.credential as Record, + params.input.credential, user.orgId, ); diff --git a/server/graphql/modules/investigation.ts b/server/graphql/modules/investigation.ts index d13c56f..22b0ff9 100644 --- a/server/graphql/modules/investigation.ts +++ b/server/graphql/modules/investigation.ts @@ -1,13 +1,12 @@ /* eslint-disable max-lines */ import { type DateString } from '@roostorg/types'; - import _ from 'lodash'; -import { type ConditionSetWithResult } from '../../services/moderationConfigService/index.js'; import { getFieldValueForRole, type ItemSubmission, } from '../../services/itemProcessingService/index.js'; +import { type ConditionSetWithResult } from '../../services/moderationConfigService/index.js'; import { asyncIterableToArray, asyncIterableToArrayWithTimeout, @@ -19,13 +18,12 @@ import { isCoopErrorOfType, makeNotFoundError } from '../../utils/errors.js'; import { MONTH_MS } from '../../utils/time.js'; import { type GQLQueryResolvers, - type GQLResolversTypes, type GQLRuleEnvironment, type GQLUserHistoryResolvers, } from '../generated.js'; import { formatItemSubmissionForGQL } from '../types.js'; -import { gqlErrorResult, gqlSuccessResult } from '../utils/gqlResult.js'; import { unauthenticatedError } from '../utils/errors.js'; +import { gqlErrorResult, gqlSuccessResult } from '../utils/gqlResult.js'; const typeDefs = /* GraphQL */ ` type Query { @@ -366,9 +364,7 @@ const Query: GQLQueryResolvers = { { item: formatItemSubmissionForGQL(item.latestSubmission), // TODO: Fix casting here - executions: itemExecutionHistory as ReadonlyArray< - GQLResolversTypes['RuleExecutionResult'] - >, + executions: itemExecutionHistory, }, 'ItemHistoryResult', ); diff --git a/server/graphql/modules/manualReviewTool.ts b/server/graphql/modules/manualReviewTool.ts index eac2ff4..eb067cd 100644 --- a/server/graphql/modules/manualReviewTool.ts +++ b/server/graphql/modules/manualReviewTool.ts @@ -1,12 +1,12 @@ /* eslint-disable max-lines */ import _ from 'lodash'; +import { itemSubmissionWithTypeIdentifierToItemSubmission } from '../../services/itemProcessingService/index.js'; +import { NCMECIncidentType as NCMECIncidentTypeValues } from '../../services/ncmecService/index.js'; import { getPermissionsForRole, UserPermission, -} from '../../models/types/permissioning.js'; -import { itemSubmissionWithTypeIdentifierToItemSubmission } from '../../services/itemProcessingService/index.js'; -import { NCMECIncidentType as NCMECIncidentTypeValues } from '../../services/ncmecService/index.js'; +} from '../../services/userManagementService/index.js'; import { asyncIterableToArray, filterNullOrUndefined, @@ -38,9 +38,9 @@ import { type GQLUserManualReviewJobPayloadResolvers, } from '../generated.js'; import { formatItemSubmissionForGQL } from '../types.js'; +import { forbiddenError, unauthenticatedError } from '../utils/errors.js'; import { gqlErrorResult, gqlSuccessResult } from '../utils/gqlResult.js'; import { oneOfInputToTaggedUnion } from '../utils/inputHelpers.js'; -import { forbiddenError, unauthenticatedError } from '../utils/errors.js'; const { omit, sumBy } = _; @@ -101,7 +101,7 @@ const typeDefs = /* GraphQL */ ` } union ManualReviewJobEnqueueSourceInfo = - ReportEnqueueSourceInfo + | ReportEnqueueSourceInfo | RuleExecutionEnqueueSourceInfo | MrtJobEnqueueSourceInfo | PostActionsEnqueueSourceInfo @@ -202,7 +202,7 @@ const typeDefs = /* GraphQL */ ` } union ManualReviewJobPayload = - ContentManualReviewJobPayload + | ContentManualReviewJobPayload | UserManualReviewJobPayload | ThreadManualReviewJobPayload | NcmecManualReviewJobPayload @@ -335,7 +335,7 @@ const typeDefs = /* GraphQL */ ` } union SubmitDecisionResponse = - SubmitDecisionSuccessResponse + | SubmitDecisionSuccessResponse | JobHasAlreadyBeenSubmittedError | SubmittedJobActionNotFoundError | NoJobWithIdInQueueError @@ -363,11 +363,11 @@ const typeDefs = /* GraphQL */ ` } union CreateManualReviewQueueResponse = - MutateManualReviewQueueSuccessResponse + | MutateManualReviewQueueSuccessResponse | ManualReviewQueueNameExistsError union UpdateManualReviewQueueQueueResponse = - MutateManualReviewQueueSuccessResponse + | MutateManualReviewQueueSuccessResponse | ManualReviewQueueNameExistsError | NotFoundError @@ -405,10 +405,10 @@ const typeDefs = /* GraphQL */ ` } union AddAccessibleQueuesToUserResponse = - MutateAccessibleQueuesForUserSuccessResponse + | MutateAccessibleQueuesForUserSuccessResponse union RemoveAccessibleQueuesToUserResponse = - MutateAccessibleQueuesForUserSuccessResponse + | MutateAccessibleQueuesForUserSuccessResponse | NotFoundError type DeleteAllJobsFromQueueSuccessResponse { @@ -425,7 +425,7 @@ const typeDefs = /* GraphQL */ ` } union DeleteAllJobsFromQueueResponse = - DeleteAllJobsFromQueueSuccessResponse + | DeleteAllJobsFromQueueSuccessResponse | DeleteAllJobsUnauthorizedError enum MetricsTimeDivisionOptions { @@ -674,7 +674,7 @@ const typeDefs = /* GraphQL */ ` } union ManualReviewChartSettings = - GetDecisionCountSettings + | GetDecisionCountSettings | GetJobCreationCountSettings input ManualReviewChartSettingsInput { @@ -742,7 +742,7 @@ const typeDefs = /* GraphQL */ ` } union ManualReviewDecisionComponent = - IgnoreDecisionComponent + | IgnoreDecisionComponent | UserOrRelatedActionDecisionComponent | SubmitNCMECReportDecisionComponent | TransformJobAndRecreateInQueueDecisionComponent @@ -856,7 +856,7 @@ const typeDefs = /* GraphQL */ ` comment: ManualReviewJobComment! } union AddManualReviewJobCommentResponse = - AddManualReviewJobCommentSuccessResponse + | AddManualReviewJobCommentSuccessResponse | NotFoundError type ManualReviewJobWithDecisions { @@ -1019,8 +1019,11 @@ const ContentManualReviewJobPayload: GQLContentManualReviewJobPayloadResolvers = throw new Error('Invalid item type in content item type resolver'); } - const itemSubmission = itemSubmissionWithTypeIdentifierToItemSubmission(it.item, type); - + const itemSubmission = itemSubmissionWithTypeIdentifierToItemSubmission( + it.item, + type, + ); + // Matched banks are now stored directly in the item data during submission return formatItemSubmissionForGQL(itemSubmission); }, @@ -1103,8 +1106,9 @@ const ContentManualReviewJobPayload: GQLContentManualReviewJobPayloadResolvers = case 'POST_ACTIONS': return { kind: enqueueSourceInfo.kind }; case 'RULE_EXECUTION': { - const rules = - await context.dataSources.ruleAPI.getGraphQLRulesForOrg(user.orgId); + const rules = await context.dataSources.ruleAPI.getGraphQLRulesForOrg( + user.orgId, + ); return { kind: enqueueSourceInfo.kind, rules: rules.filter((rule) => @@ -1318,8 +1322,9 @@ const UserManualReviewJobPayload: GQLUserManualReviewJobPayloadResolvers = { case 'POST_ACTIONS': return { kind: enqueueSourceInfo.kind }; case 'RULE_EXECUTION': { - const rules = - await context.dataSources.ruleAPI.getGraphQLRulesForOrg(user.orgId); + const rules = await context.dataSources.ruleAPI.getGraphQLRulesForOrg( + user.orgId, + ); return { kind: enqueueSourceInfo.kind, rules: rules.filter((rule) => @@ -1486,8 +1491,9 @@ const ThreadManualReviewJobPayload: GQLThreadManualReviewJobPayloadResolvers = { case 'POST_ACTIONS': return { kind: enqueueSourceInfo.kind }; case 'RULE_EXECUTION': { - const rules = - await context.dataSources.ruleAPI.getGraphQLRulesForOrg(user.orgId); + const rules = await context.dataSources.ruleAPI.getGraphQLRulesForOrg( + user.orgId, + ); return { kind: enqueueSourceInfo.kind, rules: rules.filter((rule) => @@ -1611,8 +1617,9 @@ const NcmecManualReviewJobPayload: GQLNcmecManualReviewJobPayloadResolvers = { case 'POST_ACTIONS': return { kind: enqueueSourceInfo.kind }; case 'RULE_EXECUTION': { - const rules = - await context.dataSources.ruleAPI.getGraphQLRulesForOrg(user.orgId); + const rules = await context.dataSources.ruleAPI.getGraphQLRulesForOrg( + user.orgId, + ); return { kind: enqueueSourceInfo.kind, rules: rules.filter((rule) => @@ -1956,20 +1963,20 @@ const Query: GQLQueryResolvers = { actionIds: it.userOrRelatedActionDecision.actionIds, } : it.ignoreDecision - ? { type: 'IGNORE' } - : it.submitNcmecReportDecision - ? { type: 'SUBMIT_NCMEC_REPORT' } - : it.transformJobAndRecreateInQueueDecision - ? { type: 'TRANSFORM_JOB_AND_RECREATE_IN_QUEUE' } - : it.acceptAppealDecision - ? { - type: 'ACCEPT_APPEAL', - } - : it.rejectAppealDecision - ? { - type: 'REJECT_APPEAL', - } - : undefined, + ? { type: 'IGNORE' } + : it.submitNcmecReportDecision + ? { type: 'SUBMIT_NCMEC_REPORT' } + : it.transformJobAndRecreateInQueueDecision + ? { type: 'TRANSFORM_JOB_AND_RECREATE_IN_QUEUE' } + : it.acceptAppealDecision + ? { + type: 'ACCEPT_APPEAL', + } + : it.rejectAppealDecision + ? { + type: 'REJECT_APPEAL', + } + : undefined, ), ) : undefined, @@ -2082,20 +2089,20 @@ const Query: GQLQueryResolvers = { actionIds: it.userOrRelatedActionDecision.actionIds, } : it.ignoreDecision - ? { type: 'IGNORE' } - : it.submitNcmecReportDecision - ? { type: 'SUBMIT_NCMEC_REPORT' } - : it.transformJobAndRecreateInQueueDecision - ? { type: 'TRANSFORM_JOB_AND_RECREATE_IN_QUEUE' } - : it.acceptAppealDecision - ? { - type: 'ACCEPT_APPEAL', - } - : it.rejectAppealDecision - ? { - type: 'REJECT_APPEAL', - } - : undefined, + ? { type: 'IGNORE' } + : it.submitNcmecReportDecision + ? { type: 'SUBMIT_NCMEC_REPORT' } + : it.transformJobAndRecreateInQueueDecision + ? { type: 'TRANSFORM_JOB_AND_RECREATE_IN_QUEUE' } + : it.acceptAppealDecision + ? { + type: 'ACCEPT_APPEAL', + } + : it.rejectAppealDecision + ? { + type: 'REJECT_APPEAL', + } + : undefined, ), ) : undefined, @@ -2190,7 +2197,8 @@ const Mutation: GQLMutationResolvers = { case 'SUBMIT_NCMEC_REPORT': return { ...decision, - escalateToHighPriority: decision.escalateToHighPriority ?? undefined, + escalateToHighPriority: + decision.escalateToHighPriority ?? undefined, }; default: @@ -2528,14 +2536,22 @@ const ManualReviewChartSettings: GQLManualReviewChartSettingsResolvers = { }; const NCMECIncidentType = { - CHILD_PORNOGRAPHY: NCMECIncidentTypeValues['Child Pornography (possession, manufacture, and distribution)'], + CHILD_PORNOGRAPHY: + NCMECIncidentTypeValues[ + 'Child Pornography (possession, manufacture, and distribution)' + ], CHILD_SEX_TRAFFICKING: NCMECIncidentTypeValues['Child Sex Trafficking'], CHILD_SEX_TOURISM: NCMECIncidentTypeValues['Child Sex Tourism'], CHILD_SEXUAL_MOLESTATION: NCMECIncidentTypeValues['Child Sexual Molestation'], MISLEADING_DOMAIN_NAME: NCMECIncidentTypeValues['Misleading Domain Name'], - MISLEADING_WORDS_OR_DIGITAL_IMAGES: NCMECIncidentTypeValues['Misleading Words or Digital Images on the Internet'], - ONLINE_ENTICEMENT_OF_CHILDREN: NCMECIncidentTypeValues['Online Enticement of Children for Sexual Acts'], - UNSOLICITED_OBSCENE_MATERIAL_TO_CHILD: NCMECIncidentTypeValues['Unsolicited Obscene Material Sent to a Child'], + MISLEADING_WORDS_OR_DIGITAL_IMAGES: + NCMECIncidentTypeValues[ + 'Misleading Words or Digital Images on the Internet' + ], + ONLINE_ENTICEMENT_OF_CHILDREN: + NCMECIncidentTypeValues['Online Enticement of Children for Sexual Acts'], + UNSOLICITED_OBSCENE_MATERIAL_TO_CHILD: + NCMECIncidentTypeValues['Unsolicited Obscene Material Sent to a Child'], }; const resolvers = { diff --git a/server/graphql/modules/org.ts b/server/graphql/modules/org.ts index be9dbd7..5a96c4a 100644 --- a/server/graphql/modules/org.ts +++ b/server/graphql/modules/org.ts @@ -239,7 +239,7 @@ const Org: GQLOrgResolvers = { if (!user || user.orgId !== org.id) { throw unauthenticatedError('User required.'); } - return context.dataSources.orgAPI.getSequelizeContentTypesForOrg(org.id); + return context.dataSources.orgAPI.getContentTypesForOrg(org.id); }, async itemTypes(org, _, context) { const user = context.getUser(); diff --git a/server/graphql/modules/retroaction.resolver.test.ts b/server/graphql/modules/retroaction.resolver.test.ts index f578dfa..b829aa5 100644 --- a/server/graphql/modules/retroaction.resolver.test.ts +++ b/server/graphql/modules/retroaction.resolver.test.ts @@ -1,4 +1,4 @@ -import { UserRole } from '../../models/types/permissioning.js'; +import { UserRole } from '../../services/userManagementService/index.js'; import { resolvers } from './retroaction.js'; describe('retroaction resolvers', () => { @@ -22,18 +22,21 @@ describe('retroaction resolvers', () => { }; await expect( - (resolvers.Mutation as { runRetroaction: (...a: unknown[]) => Promise }) - .runRetroaction( - {}, - { - input: { - ruleId: 'rule-1', - startAt: new Date(), - endAt: new Date(), - }, + ( + resolvers.Mutation as { + runRetroaction: (...a: unknown[]) => Promise; + } + ).runRetroaction( + {}, + { + input: { + ruleId: 'rule-1', + startAt: new Date(), + endAt: new Date(), }, - ctx as never, - ), + }, + ctx as never, + ), ).rejects.toThrow('User not authorized to run retroaction.'); expect(getRuleByIdAndOrg).not.toHaveBeenCalled(); diff --git a/server/graphql/modules/retroaction.ts b/server/graphql/modules/retroaction.ts index 7a32158..085f08b 100644 --- a/server/graphql/modules/retroaction.ts +++ b/server/graphql/modules/retroaction.ts @@ -1,7 +1,7 @@ import { hasPermission, UserPermission, -} from '../../models/types/permissioning.js'; +} from '../../services/userManagementService/index.js'; import { type GQLMutationRunRetroactionArgs } from '../generated.js'; import { type Context } from '../resolvers.js'; import { forbiddenError, unauthenticatedError } from '../utils/errors.js'; diff --git a/server/graphql/modules/routingRule.ts b/server/graphql/modules/routingRule.ts index fae19cf..dc31c3a 100644 --- a/server/graphql/modules/routingRule.ts +++ b/server/graphql/modules/routingRule.ts @@ -1,7 +1,7 @@ import { hasPermission, UserPermission, -} from '../../models/types/permissioning.js'; +} from '../../services/userManagementService/index.js'; import { isCoopErrorOfType } from '../../utils/errors.js'; import { isNonEmptyArray, @@ -15,8 +15,8 @@ import { type GQLQueryResolvers, type GQLRoutingRuleResolvers, } from '../generated.js'; -import { gqlErrorResult, gqlSuccessResult } from '../utils/gqlResult.js'; import { unauthenticatedError } from '../utils/errors.js'; +import { gqlErrorResult, gqlSuccessResult } from '../utils/gqlResult.js'; const typeDefs = /* GraphQL */ ` type RoutingRule { @@ -94,12 +94,12 @@ const typeDefs = /* GraphQL */ ` } union CreateRoutingRuleResponse = - MutateRoutingRuleSuccessResponse + | MutateRoutingRuleSuccessResponse | RoutingRuleNameExistsError | QueueDoesNotExistError union UpdateRoutingRuleResponse = - MutateRoutingRuleSuccessResponse + | MutateRoutingRuleSuccessResponse | RoutingRuleNameExistsError | NotFoundError | QueueDoesNotExistError diff --git a/server/graphql/modules/spotTest.ts b/server/graphql/modules/spotTest.ts index 1ad18eb..d7c7496 100644 --- a/server/graphql/modules/spotTest.ts +++ b/server/graphql/modules/spotTest.ts @@ -80,7 +80,7 @@ const Query: GQLQueryResolvers = { passed: result.passed, ruleId: rule.id, ruleName: rule.name, - policies: await rule.getPolicies(), + policies: (await rule.getPolicies()).map((policy) => policy.id), tags: rule.tags, }; }, diff --git a/server/iocContainer/index.ts b/server/iocContainer/index.ts index 84b82c0..aa07361 100644 --- a/server/iocContainer/index.ts +++ b/server/iocContainer/index.ts @@ -3,13 +3,12 @@ import { createRequire } from 'module'; import Bottle from '@ethanresnick/bottlejs'; import opentelemetry from '@opentelemetry/api'; import { makeDateString, type ItemIdentifier } from '@roostorg/types'; -import { types as scyllaTypes, type Host as ScyllaHost } from 'cassandra-driver'; -import IORedis, { type Cluster } from 'ioredis'; import { - Kysely, - PostgresDialect, - type PostgresCursorConstructor, -} from 'kysely'; + types as scyllaTypes, + type Host as ScyllaHost, +} from 'cassandra-driver'; +import IORedis, { type Cluster } from 'ioredis'; +import { Kysely, PostgresDialect } from 'kysely'; import _ from 'lodash'; import { DynamicPool } from 'node-worker-threads-pool'; import pg from 'pg'; @@ -17,7 +16,6 @@ import Cursor from 'pg-cursor'; import { type JsonObject, type ReadonlyDeep } from 'type-fest'; import { v1 as uuidv1 } from 'uuid'; -import makeDb from '../models/index.js'; import type { IActionExecutionsAdapter } from '../plugins/warehouse/queries/IActionExecutionsAdapter.js'; import type { IActionStatisticsAdapter } from '../plugins/warehouse/queries/IActionStatisticsAdapter.js'; import type { IContentApiRequestsAdapter } from '../plugins/warehouse/queries/IContentApiRequestsAdapter.js'; @@ -36,10 +34,6 @@ import { makeItemSubmissionBulkWrite, type ItemSubmissionBulkWrite, } from '../queues/itemSubmissionQueue.js'; -import { - getPolicyActionPenaltiesForOrg, - type PolicyActionPenalties, -} from '../services/policyActionPenalties.js'; import makeActionPublisher, { type ActionPublisher, type ActionTargetItem, @@ -173,6 +167,10 @@ import { makePlacesApiService, type PlacesApiService, } from '../services/placesApiService/index.js'; +import { + getPolicyActionPenaltiesForOrg, + type PolicyActionPenalties, +} from '../services/policyActionPenalties.js'; import { makeReportingService, type ReportingService, @@ -316,15 +314,6 @@ export interface Dependencies { close: () => Promise; }; - Sequelize: ReturnType; - OrgModel: ReturnType['Org']; - RuleModel: ReturnType['Rule']; - ActionModel: ReturnType['Action']; - PolicyModel: ReturnType['Policy']; - ItemTypeModel: ReturnType['ItemType']; - LocationBankModel: ReturnType['LocationBank']; - LocationBankLocationModel: ReturnType['LocationBankLocation']; - // Data Warehouse abstraction DataWarehouse: IDataWarehouse; DataWarehouseDialect: IDataWarehouseDialect; @@ -442,6 +431,25 @@ export type PublicInterface = { [K in keyof T]: T[K] }; * copies of the container as needed for selective rebinding. */ export default async function getBottle() { + // Pool / client tuning shared by both Kysely pools. Defaults preserve our + // pre-Kysely behavior; env var names are generic. + const getPgPoolTuning = () => ({ + // pg's default is 10s, which churns connections during quiet periods. + idleTimeoutMillis: parseInt( + process.env.DATABASE_POOL_IDLE_TIMEOUT_MS ?? '300000', + ), + // pg's default is 0 (wait forever); fail fast if the db is unreachable. + connectionTimeoutMillis: parseInt( + process.env.DATABASE_POOL_CONNECTION_TIMEOUT_MS ?? '15000', + ), + // Bound long-running queries instead of letting them hold a pool slot. + query_timeout: parseInt(process.env.DATABASE_QUERY_TIMEOUT_MS ?? '1000000'), + // Kill sessions sitting idle inside an open transaction (holding locks). + idle_in_transaction_session_timeout: parseInt( + process.env.DATABASE_IDLE_IN_TRANSACTION_TIMEOUT_MS ?? '300000', + ), + }); + // NB: this is a function because safeGetEnvVar can throw, so we only want to // try to look up the env vars (and throw if they're missing) _if someone // actually tries to fetch a service from bottle that needs these env vars_. @@ -461,8 +469,17 @@ export default async function getBottle() { application_name: getEnvVarOrWarn('OTEL_SERVICE_NAME') ?? 'unknown-coop-service', ssl: isEnvTrue('DATABASE_SSL') ? { rejectUnauthorized: false } : undefined, + ...getPgPoolTuning(), }); + // Kysely's default is `['error']`; opt-in to also logging every executed + // query (SQL, bound params, duration). + const kyselyLogLevels: ReadonlyArray<'query' | 'error'> = isEnvTrue( + 'DATABASE_PRINT_LOGS', + ) + ? ['query', 'error'] + : ['error']; + const bottle = new Bottle(); // Pg services. @@ -473,16 +490,15 @@ export default async function getBottle() { // // - KyselyPgReadReplica gives us the same type safety, but sends queries to our // replicas, for when we only need reads and we're ok w/ eventual consistency. - // - // - 'Sequelize' + the sequelize models are used to query pg through sequelize. bottle.factory( 'KyselyPg', () => new Kysely({ dialect: new PostgresDialect({ pool: new pg.Pool(getPgMasterConnectionInfo()), - cursor: Cursor as unknown as PostgresCursorConstructor, + cursor: Cursor, }), + log: kyselyLogLevels, }), ); @@ -496,8 +512,9 @@ export default async function getBottle() { max: parseInt(process.env.DATABASE_READ_POOL_MAX ?? '150'), host: safeGetEnvVar('DATABASE_READ_ONLY_HOST'), }), - cursor: Cursor as unknown as PostgresCursorConstructor, + cursor: Cursor, }), + log: kyselyLogLevels, }), ); @@ -536,21 +553,6 @@ export default async function getBottle() { }), ); - bottle.factory('Sequelize', () => makeDb()); - bottle.factory('OrgModel', ({ Sequelize }) => Sequelize.Org); - bottle.factory('RuleModel', ({ Sequelize }) => Sequelize.Rule); - bottle.factory('ActionModel', ({ Sequelize }) => Sequelize.Action); - bottle.factory('PolicyModel', ({ Sequelize }) => Sequelize.Policy); - bottle.factory('ItemTypeModel', ({ Sequelize }) => Sequelize.ItemType); - bottle.factory( - 'LocationBankModel', - ({ Sequelize }) => Sequelize.LocationBank, - ); - bottle.factory( - 'LocationBankLocationModel', - ({ Sequelize }) => Sequelize.LocationBankLocation, - ); - // Data Warehouse abstraction layer // // All warehouse operations use these interfaces. @@ -691,8 +693,7 @@ export default async function getBottle() { // server cert. Prefer an explicit `SCYLLA_SSL_SERVERNAME` (e.g., the // Keyspaces regional endpoint) over inferring one from `SCYLLA_HOSTS`, // which may contain multiple contact points with different cert names. - const sslServerName = - process.env.SCYLLA_SSL_SERVERNAME ?? contactPoints[0]; + const sslServerName = process.env.SCYLLA_SSL_SERVERNAME ?? contactPoints[0]; const scyllaDriver = new ScyllaClient({ contactPoints, credentials: { @@ -768,7 +769,9 @@ export default async function getBottle() { // but keep it bounded so a true runaway is still noticeable. const controlConnection = ( scyllaDriver as unknown as { - controlConnection?: { hosts?: { setMaxListeners?: (n: number) => void } }; + controlConnection?: { + hosts?: { setMaxListeners?: (n: number) => void }; + }; } ).controlConnection; controlConnection?.hosts?.setMaxListeners?.(15); @@ -1270,7 +1273,7 @@ export default async function getBottle() { ...{ reportIds: 'reportIds' in job.payload - ? job.payload.reportIds ?? [] + ? (job.payload.reportIds ?? []) : [], }, ...('reportedForReason' in job.payload @@ -1410,10 +1413,7 @@ export default async function getBottle() { return cached({ async producer(orgId) { - return getPolicyActionPenaltiesForOrg( - moderationConfigService, - orgId, - ); + return getPolicyActionPenaltiesForOrg(moderationConfigService, orgId); }, directives: { freshUntilAge: 60 }, }); @@ -1597,18 +1597,8 @@ export default async function getBottle() { }[CloseMethodName]; }[keyof Dependencies] >, - // Seqelize puts a close method on each model, but we only need to - // close the root sequelize instance. - | 'OrgModel' - | 'PolicyModel' - | 'RuleModel' - | 'ActionModel' - | 'ItemTypeModel' - | 'LocationBankModel' - | 'LocationBankLocationModel' // Services that don't need cleanup - | 'UserStatisticsService' - | 'HMAHashBankService' + 'UserStatisticsService' | 'HMAHashBankService' >; // This will be a type error if we forgot to close something. @@ -1624,7 +1614,6 @@ export default async function getBottle() { 'Scylla', 'itemSubmissionQueueBulkWrite', 'itemSubmissionRetryQueueBulkWrite', - 'Sequelize', 'IORedis', // Storage abstractions 'DataWarehouse', diff --git a/server/models/HashBankModel.ts b/server/models/HashBankModel.ts deleted file mode 100644 index 257f3f5..0000000 --- a/server/models/HashBankModel.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { type HashBank as HashBankType } from '../services/hmaService/index.js'; - -// Re-export the HashBank type for GraphQL generated types -export type HashBank = HashBankType; - -// This file exists to provide the HashBank type import that GraphQL codegen expects -// The actual HashBank functionality is implemented in the HMA service -export default HashBank; diff --git a/server/models/OrgModel.ts b/server/models/OrgModel.ts deleted file mode 100644 index 2d3deac..0000000 --- a/server/models/OrgModel.ts +++ /dev/null @@ -1,114 +0,0 @@ -import sequelize, { - type CreationOptional, - type HasManyGetAssociationsMixin, - type InferAttributes, - type InferCreationAttributes, - type Sequelize, -} from 'sequelize'; - -import { validateUrl } from '../utils/url.js'; -import { type LocationBank } from './banks/LocationBankModel.js'; -import { type DataTypes } from './index.js'; -import { type Policy } from './PolicyModel.js'; -import { type SequelizeAction } from './rules/ActionModel.js'; -import { type Rule } from './rules/RuleModel.js'; -import { type User } from './UserModel.js'; - -const { Model } = sequelize; - -export type Org = InstanceType>; - -/** - * Data Model for Organizations - */ -export default function makeOrgModel( - sequelize: Sequelize, - DataTypes: DataTypes, -) { - class Org extends Model< - InferAttributes, - InferCreationAttributes - > { - public declare id: string; - public declare email: string; - public declare name: string; - public declare websiteUrl: string; - public declare apiKeyId?: CreationOptional; - public declare onCallAlertEmail?: CreationOptional; - - public declare getRules: HasManyGetAssociationsMixin; - public declare getActions: HasManyGetAssociationsMixin; - // Has to use any below to avoid circular type errors. - // eslint-disable-next-line @typescript-eslint/no-explicit-any - public declare getContentTypes: HasManyGetAssociationsMixin; - public declare getLocationBanks: HasManyGetAssociationsMixin; - public declare getUsers: HasManyGetAssociationsMixin; - public declare getPolicies: HasManyGetAssociationsMixin; - public declare createdAt: Date; - public declare updatedAt: Date; - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - static associate(models: { [key: string]: any }) { - Org.hasMany(models.User, { as: 'Users' }); - Org.hasMany(models.Rule, { as: 'Rules' }); - Org.hasMany(models.Action, { as: 'Actions', foreignKey: 'orgId' }); - Org.hasMany(models.ItemType, { as: 'ContentTypes' }); - Org.hasMany(models.LocationBank, { as: 'LocationBanks' }); - Org.hasMany(models.Policy, { as: 'policies' }); - } - } - - /* Fields */ - Org.init( - { - id: { - type: DataTypes.STRING, - primaryKey: true, - }, - email: { - type: DataTypes.STRING, - unique: true, - allowNull: false, - validate: { - isEmail: true, - notEmpty: true, - }, - }, - name: { - type: DataTypes.STRING, - unique: true, - allowNull: false, - validate: { - notEmpty: true, - }, - }, - websiteUrl: { - type: DataTypes.STRING, - unique: true, - allowNull: false, - validate: { - isValidUrl: validateUrl, - }, - }, - // ID of the AWS API Key resource that stores the API key. Not actually - // used for anything at the moment (instead, the API key is looked up in - // but potentially useful. - apiKeyId: { - type: DataTypes.STRING, - }, - onCallAlertEmail: { - type: DataTypes.STRING, - validate: { - isEmail: true, - }, - }, - }, - { - sequelize, - modelName: 'org', - underscored: true, - }, - ); - - return Org; -} diff --git a/server/models/PolicyModel.ts b/server/models/PolicyModel.ts deleted file mode 100644 index dd10dc2..0000000 --- a/server/models/PolicyModel.ts +++ /dev/null @@ -1,163 +0,0 @@ -import _ from 'lodash'; -import sequelize, { - type BelongsToGetAssociationMixin, - type HasManyAddAssociationsMixin, - type HasManyGetAssociationsMixin, - type HasManySetAssociationsMixin, - type InferAttributes, - type InferCreationAttributes, - type Sequelize, -} from 'sequelize'; - -import { - PolicyType, - UserPenaltySeverity, -} from '../services/moderationConfigService/index.js'; -import { type DataTypes } from './index.js'; -import { type Rule } from './rules/RuleModel.js'; - -const { groupBy, mapValues } = _; -const { Model } = sequelize; - -export type Policy = InstanceType>; - -/** - * Data Model for Policies. These policies can represent overall - * policy areas (crime, safety) or more granular policy areas (e.g. - * selling dangerous goods, child safety). Policies are constructed as - * a tree - or rather, a set of trees, all of which can be thought of - * as child trees under an abstract root node. - * - * ROOT - * | | | - * Crime Safety Hate - * | | | | | | - * Weapons Drugs Children Self-harm Dehumanization Threats - * - * Each node (except the root) is a policy. - */ -const makePolicy = (sequelize: Sequelize, DataTypes: DataTypes) => { - class Policy extends Model< - InferAttributes, - InferCreationAttributes - > { - public declare id: string; - public declare name: string; - public declare policyText?: string | undefined; - - public declare orgId: string; - - public declare parentId: string | undefined; - public declare parent?: Policy; - public declare getParent: BelongsToGetAssociationMixin; - - public declare getChildren: HasManyGetAssociationsMixin; - public declare addChildren: HasManyAddAssociationsMixin; - public declare setChildren: HasManySetAssociationsMixin; - - public declare penalty: UserPenaltySeverity; - public declare userStrikeCount: number; - public declare applyUserStrikeCountConfigToChildren: boolean; - public declare semanticVersion: number; - public declare policyType: PolicyType | undefined; - - public declare rules?: Rule[]; - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - static associate(models: { [key: string]: any }) { - Policy.belongsTo(models.Org, { as: 'org' }); - Policy.hasMany(models.Policy, { - as: 'children', - foreignKey: 'parentId', - }); - Policy.belongsTo(models.Policy, { - as: 'parent', - foreignKey: 'parentId', - }); - Policy.belongsToMany(models.Rule, { - through: 'rules_and_policies', - as: 'rules', - }); - } - - static async getPoliciesForRuleIds(ruleIds: readonly string[]) { - const results = await Policy.findAll({ - where: { '$rules.id$': ruleIds }, - include: [{ association: 'rules', attributes: ['id'] }], - }); - - const ruleIdPolicyPairs = results.flatMap((policy) => - policy.rules!.map((rule) => [rule.id, policy] as const), - ); - - return mapValues( - groupBy(ruleIdPolicyPairs, ([ruleId]) => ruleId), - (pairs) => pairs.map(([, policy]) => policy), - ) as { [ruleId: string]: Policy[] | undefined }; - } - } - - /* Fields */ - Policy.init( - { - id: { - type: DataTypes.STRING, - primaryKey: true, - }, - orgId: { - type: DataTypes.STRING, - allowNull: false, - }, - name: { - type: DataTypes.STRING, - allowNull: true, - }, - policyText: { - type: DataTypes.STRING, - allowNull: true, - }, - parentId: { - allowNull: true, - type: DataTypes.STRING, - }, - policyType: { - type: DataTypes.ENUM(...Object.values(PolicyType)), - allowNull: true, - }, - penalty: { - type: DataTypes.STRING, - defaultValue: UserPenaltySeverity.NONE, - allowNull: false, - validate: { - notNull: true, - isIn: [Object.values(UserPenaltySeverity)], - }, - }, - userStrikeCount: { - allowNull: false, - type: DataTypes.INTEGER, - defaultValue: 1, - }, - applyUserStrikeCountConfigToChildren: { - allowNull: false, - type: DataTypes.BOOLEAN, - defaultValue: false, - }, - semanticVersion: { - allowNull: false, - type: DataTypes.INTEGER, - defaultValue: 1, - }, - }, - { - sequelize, - modelName: 'policy', - underscored: true, - tableName: 'policies', - }, - ); - - return Policy; -}; - -export default makePolicy; diff --git a/server/models/UserModel.ts b/server/models/UserModel.ts deleted file mode 100644 index a37a5ab..0000000 --- a/server/models/UserModel.ts +++ /dev/null @@ -1,156 +0,0 @@ -import { promisify } from 'util'; -import bcrypt from 'bcryptjs'; -import sequelize, { - type CreationOptional, - type HasManyAddAssociationsMixin, - type HasManyGetAssociationsMixin, - type HasManyRemoveAssociationsMixin, - type InferAttributes, - type InferCreationAttributes, - type Sequelize, -} from 'sequelize'; - -import { type DataTypes } from './index.js'; -import { getPermissionsForRole, UserRole } from './types/permissioning.js'; - -const { Model } = sequelize; -const bcryptCompare = promisify(bcrypt.compare); - -export type User = InstanceType>; - -/** - * Data Model for Users. Users are Coop users who have - * created profiles on our website. Actors (see ActorModel.js) - * are users on the organization's platforms that upload potentially - * problematic content. - */ -const makeUserModel = (sequelize: Sequelize, DataTypes: DataTypes) => { - class User extends Model< - InferAttributes, - InferCreationAttributes - > { - public declare id: string; - public declare email: string; - public declare password: string | null; - public declare firstName: string; - public declare lastName: string; - public declare orgId: string; - public declare role: CreationOptional; - public declare approvedByAdmin: CreationOptional; - public declare rejectedByAdmin: CreationOptional; - public declare createdAt: Date; - public declare updatedAt: Date; - public declare loginMethods: ('password' | 'saml')[]; - - // Have to use any below to avoid circular type errors - /* eslint-disable @typescript-eslint/no-explicit-any */ - public declare addFavoriteRules: HasManyAddAssociationsMixin; - public declare removeFavoriteRules: HasManyRemoveAssociationsMixin< - any, - string - >; - public declare getFavoriteRules: HasManyGetAssociationsMixin; - /* eslint-enable @typescript-eslint/no-explicit-any */ - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - static associate(models: { [key: string]: any }) { - User.belongsTo(models.Org, { as: 'Org' }); - User.hasMany(models.Rule, { as: 'Rules', foreignKey: 'creatorId' }); - User.hasMany(models.LocationBank, { - as: 'LocationBanks', - foreignKey: 'ownerId', - }); - User.hasMany(models.Backtest, { - as: 'Backtests', - foreignKey: 'creatorId', - }); - User.belongsToMany(models.Rule, { - as: 'FavoriteRules', - through: 'users_and_favorite_rules', - }); - } - - static async passwordMatchesHash(givenPassword: string, hash: string) { - return bcryptCompare(givenPassword, hash); - } - - public getPermissions() { - return getPermissionsForRole(this.role); - } - } - - /* Fields */ - User.init( - { - id: { - type: DataTypes.STRING, - primaryKey: true, - }, - orgId: { - type: DataTypes.STRING, - allowNull: false, - }, - email: { - type: DataTypes.STRING, - unique: true, - allowNull: false, - validate: { - isEmail: true, - notEmpty: true, - }, - }, - password: { - type: DataTypes.STRING, - unique: false, - }, - firstName: { - type: DataTypes.STRING, - unique: false, - allowNull: false, - validate: { - notEmpty: true, - }, - }, - lastName: { - type: DataTypes.STRING, - unique: false, - allowNull: false, - validate: { - notEmpty: true, - }, - }, - role: { - type: DataTypes.STRING, - unique: false, - defaultValue: UserRole.ADMIN, - validate: { - isIn: [Object.values(UserRole)], - }, - }, - // Has the user been approved by the admin as part of the org - approvedByAdmin: { - type: DataTypes.BOOLEAN, - defaultValue: false, - }, - // Has the user been rejected by the admin as part of the org - rejectedByAdmin: { - type: DataTypes.BOOLEAN, - defaultValue: false, - }, - loginMethods: { - type: DataTypes.ARRAY(DataTypes.ENUM('password', 'saml')), - defaultValue: ['password'], - allowNull: false, - }, - }, - { - sequelize, - modelName: 'user', - underscored: true, - }, - ); - - return User; -}; - -export default makeUserModel; diff --git a/server/models/banks/LocationBankLocationModel.ts b/server/models/banks/LocationBankLocationModel.ts deleted file mode 100644 index 183e9d4..0000000 --- a/server/models/banks/LocationBankLocationModel.ts +++ /dev/null @@ -1,80 +0,0 @@ -import sequelize, { - type InferAttributes, - type InferCreationAttributes, - type Sequelize, -} from 'sequelize'; - -import { type DataTypes } from '../index.js'; -import { type LocationArea } from '../types/locationArea.js'; - -const { Model } = sequelize; - -export type LocationBankLocation = InstanceType< - ReturnType ->; - -/** - * Data Model for Location Banks. Location Banks are sets of locations - * used for distance checks. - */ -const makeLocationBankLocationModel = ( - sequelize: Sequelize, - DataTypes: DataTypes, -) => { - class LocationBankLocation - extends Model< - InferAttributes, - InferCreationAttributes - > - implements LocationArea - { - public declare id: string; - public declare bankId: string; - public declare name?: string; - public declare geometry: LocationArea['geometry']; - public declare bounds: LocationArea['bounds'] | null; - public declare googlePlaceInfo: LocationArea['googlePlaceInfo'] | null; - } - - /* Fields */ - LocationBankLocation.init( - { - id: { - type: DataTypes.STRING, - primaryKey: true, - }, - bankId: { - allowNull: false, - type: DataTypes.STRING, - }, - geometry: { - allowNull: false, - type: DataTypes.JSONB, - }, - bounds: { - allowNull: true, - type: DataTypes.JSONB, - }, - name: { - type: DataTypes.STRING, - allowNull: true, - validate: { - notEmpty: true, - }, - }, - googlePlaceInfo: { - type: DataTypes.JSONB, - allowNull: true, - }, - }, - { - sequelize, - modelName: 'location_bank_locations', - underscored: true, - }, - ); - - return LocationBankLocation; -}; - -export default makeLocationBankLocationModel; diff --git a/server/models/banks/LocationBankModel.ts b/server/models/banks/LocationBankModel.ts deleted file mode 100644 index 149c047..0000000 --- a/server/models/banks/LocationBankModel.ts +++ /dev/null @@ -1,104 +0,0 @@ -import sequelize, { - type HasManyAddAssociationsMixin, - type HasManyGetAssociationsMixin, - type HasManyRemoveAssociationsMixin, - type HasManySetAssociationsMixin, - type HasOneGetAssociationMixin, - type InferAttributes, - type InferCreationAttributes, - type Sequelize, -} from 'sequelize'; - -import { type DataTypes } from '../index.js'; -import { type User } from '../UserModel.js'; -import { type LocationBankLocation } from './LocationBankLocationModel.js'; - -const { Model } = sequelize; - -export type LocationBank = InstanceType< - ReturnType ->; - -/** - * Data Model for Location Banks. Location Banks are sets of locations - * used for distance checks. - */ -const makeLocationBankModel = (sequelize: Sequelize, DataTypes: DataTypes) => { - class LocationBank extends Model< - InferAttributes, - InferCreationAttributes - > { - public declare id: string; - public declare name: string; - public declare description?: string | null; - public declare orgId: string; - public declare ownerId: string; - public declare locations?: LocationBankLocation[]; - - public declare getOwner: HasOneGetAssociationMixin; - public declare getLocations: HasManyGetAssociationsMixin; - public declare setLocations: HasManySetAssociationsMixin< - LocationBankLocation, - string - >; - public declare addLocations: HasManyAddAssociationsMixin< - LocationBankLocation, - string - >; - public declare removeLocations: HasManyRemoveAssociationsMixin< - LocationBankLocation, - string - >; - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - static associate(models: { [key: string]: any }) { - LocationBank.belongsTo(models.Org, { as: 'org' }); - LocationBank.belongsTo(models.User, { as: 'owner' }); - LocationBank.hasMany(models.LocationBankLocation, { - as: 'locations', - foreignKey: 'bank_id', - onDelete: 'CASCADE', - }); - } - } - - /* Fields */ - LocationBank.init( - { - id: { - type: DataTypes.STRING, - primaryKey: true, - }, - orgId: { - allowNull: false, - type: DataTypes.STRING, - }, - ownerId: { - allowNull: false, - type: DataTypes.STRING, - }, - // Name of the location bank -- this must be unique for each Org - // (i.e. an Org can't have two location banks with the same name) - name: { - type: DataTypes.STRING, - allowNull: true, - validate: { - notEmpty: true, - }, - }, - description: { - type: DataTypes.STRING, - allowNull: true, - }, - }, - { - sequelize, - modelName: 'location_bank', - underscored: true, - }, - ); - - return LocationBank; -}; - -export default makeLocationBankModel; diff --git a/server/models/errors.ts b/server/models/errors.ts deleted file mode 100644 index 2d371e3..0000000 --- a/server/models/errors.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { safeGet } from '../utils/misc.js'; - -export function isUniqueConstraintError(error: unknown): boolean { - return safeGet(error, ['name']) === 'SequelizeUniqueConstraintError'; -} - -export function isEmptyResultSetError(error: unknown): boolean { - return safeGet(error, ['name']) === 'SequelizeEmptyResultError'; -} diff --git a/server/models/index.ts b/server/models/index.ts deleted file mode 100644 index 9c81228..0000000 --- a/server/models/index.ts +++ /dev/null @@ -1,56 +0,0 @@ -import pkg from 'sequelize'; - -/* Bank models */ -import LocationBankLocation from './banks/LocationBankLocationModel.js'; -import LocationBank from './banks/LocationBankModel.js'; -/* High level models */ -import Org from './OrgModel.js'; -import Policy from './PolicyModel.js'; -/* Rules models */ -import Action from './rules/ActionModel.js'; -import Backtest from './rules/BacktestModel.js'; -import ItemType from './rules/ItemTypeModel.js'; -import RuleLatestVersion from './rules/RuleLatestVersionModel.js'; -import Rule from './rules/RuleModel.js'; -/* Other */ -import { makeSequelize, maketransactionWithRetry } from './sequelizeSetup.js'; -import User from './UserModel.js'; - -const { Sequelize } = pkg; - -// NB: this type includes a bunch of exports that are not the DataType constructors, -// but at least it also includes the DataTypes, so that we get autocomplete. -// I don't think the DataTypes type is actually exported on its own. -// eslint-disable-next-line @typescript-eslint/consistent-type-imports -export type DataTypes = typeof import('sequelize'); - -/* eslint-disable @typescript-eslint/no-explicit-any */ -const makeDb = () => { - const sequelize = makeSequelize(); - const db = { - sequelize, - Sequelize, - Action: Action(sequelize, Sequelize as any), - Backtest: Backtest(sequelize, Sequelize as any), - ItemType: ItemType(sequelize, Sequelize as any), - LocationBank: LocationBank(sequelize, Sequelize as any), - LocationBankLocation: LocationBankLocation(sequelize, Sequelize as any), - Org: Org(sequelize, Sequelize as any), - Policy: Policy(sequelize, Sequelize as any), - Rule: Rule(sequelize, Sequelize as any), - RuleLatestVersion: RuleLatestVersion(sequelize, Sequelize as any), - User: User(sequelize, Sequelize as any), - transactionWithRetry: maketransactionWithRetry(sequelize), - async close() { - await sequelize.close(); - }, - }; - Object.values(db).forEach((model) => { - if ('associate' in model) { - model.associate(db); - } - }); - return db; -}; - -export default makeDb; diff --git a/server/models/rules/ActionModel.ts b/server/models/rules/ActionModel.ts deleted file mode 100644 index 826219a..0000000 --- a/server/models/rules/ActionModel.ts +++ /dev/null @@ -1,220 +0,0 @@ -import sequelize, { - type HasManyAddAssociationsMixin, - type HasManyGetAssociationsMixin, - type HasManyGetAssociationsMixinOptions, - type HasManySetAssociationsMixin, - type InferAttributes, - type InferCreationAttributes, - type Sequelize, -} from 'sequelize'; -import { type JsonObject } from 'type-fest'; - -import { - ActionType, - ItemTypeKind, - UserPenaltySeverity, -} from '../../services/moderationConfigService/index.js'; -import { validateUrlOrNull } from '../../utils/url.js'; -import { type DataTypes } from '../index.js'; -import { type ItemType as TContentType } from './ItemTypeModel.js'; - -const { Model } = sequelize; - -// The default type an Action sequelize model instance. -// This type is vague, w/ more optional fields than we'll have at runtime, and -// not accounting for the rules that we've set up in pg for how different action -// type values constrain the values in other columns. -export type CollapsedSequelizeAction = InstanceType< - ReturnType ->; - -// These types handle the different constraints per action type, mirroring pg. -export type EnqueueToMrtAction = CollapsedSequelizeAction & { - actionType: (typeof ActionType)['ENQUEUE_TO_MRT']; - callbackUrl: null; -}; - -export type EnqueueToNcmecAction = CollapsedSequelizeAction & { - actionType: (typeof ActionType)['ENQUEUE_TO_NCMEC']; - callbackUrl: null; -}; - -export type CustomAction = CollapsedSequelizeAction & { - actionType: (typeof ActionType)['CUSTOM_ACTION']; - callbackUrl: string; -}; - -export type EnqueueAuthorToMrtAction = CollapsedSequelizeAction & { - actionType: (typeof ActionType)['ENQUEUE_AUTHOR_TO_MRT']; - callbackUrl: string; -}; - -// And this is the more precise replacement for UntypedAction, which we -// use outside this file. -export type SequelizeAction = - | EnqueueToMrtAction - | EnqueueToNcmecAction - | EnqueueAuthorToMrtAction - | CustomAction; - -/** - * Data Model for Actions. Actions are components - * of Rules that get executed if all Conditions are met. - * Examples of Actions are Delete, Enqueue, Log, etc. - */ -const makeActionModel = (sequelize: Sequelize, DataTypes: DataTypes) => { - class Action extends Model< - InferAttributes, - InferCreationAttributes - > { - public declare id: string; - public declare name: string; - public declare orgId: string; - public declare description: string | null; - public declare callbackUrl: string | null; - public declare callbackUrlHeaders: JsonObject | null; - public declare callbackUrlBody: JsonObject | null; - public declare customMrtApiParams: JsonObject | null; - - public declare penalty: UserPenaltySeverity; - public declare actionType: ActionType; - public declare appliesToAllItemsOfKind: ItemTypeKind[]; - public declare applyUserStrikes: boolean; - - public declare addContentTypes: HasManyAddAssociationsMixin< - unknown, - string - >; - public declare setContentTypes: HasManySetAssociationsMixin< - unknown, - string - >; - private declare getContentTypesSequelizeImpl: HasManyGetAssociationsMixin; - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - static associate(models: { [key: string]: any }) { - Action.belongsTo(models.Org, { as: 'org' }); - Action.belongsToMany(models.Rule, { - through: 'rules_and_actions', - as: 'rules', - }); - - // Assign the default sequelize getContentTypes function to another - // name so that we can use it in the actual implemented function. - // - const contentTypeAssoc = Action.belongsToMany(models.ItemType, { - through: 'actions_and_item_types', - as: 'ContentTypes', - otherKey: 'item_type_id', - }); - Object.defineProperty( - models.Action.prototype, - 'getContentTypesSequelizeImpl', - { - enumerable: false, - value(...params: unknown[]) { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - return (contentTypeAssoc as any)['get'](this, ...params); - }, - }, - ); - } - - async getContentTypes( - options?: HasManyGetAssociationsMixinOptions, - ): Promise { - const contentTypes = - this.appliesToAllItemsOfKind.length > 0 - ? await this.sequelize.model('content_type').findAll({ - ...options, - where: { - ...options?.where, - orgId: this.orgId, - kind: this.appliesToAllItemsOfKind, - }, - }) - : await this.getContentTypesSequelizeImpl(options); - return contentTypes as TContentType[]; - } - } - - /* Fields */ - Action.init( - { - id: { - type: DataTypes.STRING, - primaryKey: true, - }, - // Name of the action -- this must be unique for each Org (i.e. an Org can't - // have two actions with the same name) - name: { - type: DataTypes.STRING, - allowNull: false, - validate: { notEmpty: true }, - }, - orgId: { - type: DataTypes.STRING, - allowNull: false, - }, - description: { - type: DataTypes.STRING, - allowNull: true, - }, - callbackUrl: { - type: DataTypes.STRING, - allowNull: true, - validate: { - isValidUrl: validateUrlOrNull, - }, - }, - callbackUrlHeaders: { - type: DataTypes.JSONB, - allowNull: true, - }, - callbackUrlBody: { - type: DataTypes.JSONB, - allowNull: true, - }, - customMrtApiParams: { - type: DataTypes.ARRAY(DataTypes.JSONB), - allowNull: true, - }, - penalty: { - type: DataTypes.STRING, - defaultValue: UserPenaltySeverity.NONE, - allowNull: false, - validate: { - isIn: [Object.values(UserPenaltySeverity)], - }, - }, - actionType: { - type: DataTypes.STRING, - defaultValue: ActionType.CUSTOM_ACTION, - allowNull: false, - validate: { - notNull: true, - isIn: [Object.values(ActionType)], - }, - }, - appliesToAllItemsOfKind: { - field: 'applies_to_all_items_of_kind', - type: DataTypes.ARRAY(DataTypes.ENUM(...Object.values(ItemTypeKind))), - defaultValue: [], - }, - applyUserStrikes: { - type: DataTypes.BOOLEAN, - defaultValue: false, - allowNull: false, - }, - }, - { - sequelize, - modelName: 'action', - underscored: true, - }, - ); - - return Action; -}; - -export default makeActionModel; diff --git a/server/models/rules/BacktestModel.ts b/server/models/rules/BacktestModel.ts deleted file mode 100644 index 7afe6f7..0000000 --- a/server/models/rules/BacktestModel.ts +++ /dev/null @@ -1,168 +0,0 @@ -import sequelize, { - type HasOneGetAssociationMixin, - type InferAttributes, - type InferCreationAttributes, - type NonAttribute, - type Sequelize, -} from 'sequelize'; - -import { type DataTypes } from '../index.js'; -import { type User } from '../UserModel.js'; -import { type Rule } from './RuleModel.js'; - -const { Model } = sequelize; - -export type Backtest = InstanceType>; -export enum BacktestStatus { - RUNNING = 'RUNNING', - COMPLETE = 'COMPLETE', - CANCELED = 'CANCELED', -} - -const makeBacktestModel = (sequelize: Sequelize, DataTypes: DataTypes) => { - class Backtest extends Model< - InferAttributes, - InferCreationAttributes< - Backtest, - // fields that _cannot be set explicitly_ at creation time, b/c they have - // db defaults that must always apply (i.e., cannot be overriden) at creation. - { - omit: - | 'contentItemsProcessed' - | 'contentItemsMatched' - | 'status' - | 'createdAt' - | 'updatedAt' - | 'samplingComplete' - | 'sampleActualSize' - | 'cancelationDate'; - } - > - > { - public declare id: string; - - public declare ruleId: string; - public declare rule?: Rule; - - public declare creatorId: string; - public declare creator?: User; - - public declare sampleDesiredSize: number; - public declare sampleActualSize: number; - public declare sampleStartAt: Date; - public declare sampleEndAt: Date; - - public declare cancelationDate: Date; - public declare samplingComplete: boolean; - - public declare contentItemsProcessed: number; - public declare contentItemsMatched: number; - - public declare status: BacktestStatus; - - public declare createdAt: Date; - public declare updatedAt: Date; - - declare getRule: HasOneGetAssociationMixin; - declare getCreator: HasOneGetAssociationMixin; - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - static associate(models: { [key: string]: any }) { - Backtest.belongsTo(models.User, { as: 'Creator' }); - Backtest.belongsTo(models.Rule, { as: 'Rule' }); - } - - public static async hasRunningBacktestsForRule(ruleId: string) { - // ugh the built-in sequelize query builder sucks. - // https://github.com/sequelize/sequelize/issues/10187 - return Backtest.findOne({ - where: { ruleId, status: BacktestStatus.RUNNING }, - }).then((it) => it != null); - } - - public static async cancelRunningBacktestsForRule(ruleId: string) { - return Backtest.update( - { cancelationDate: new Date() }, - { where: { ruleId, status: BacktestStatus.RUNNING } }, - ); - } - - public async cancel() { - this.cancelationDate = new Date(); - await this.save(); - return this; - } - - /** - * Because our queues will deliver sampled content items to be processed - * _at least once_, it’s possible that, rarely, contentItemsProcessed will - * be greater than sampleActualSize. To mitigate this, we clamp the - * exposed value for contentItemsProcessed at sampleActualSize. - */ - public get correctedContentItemsProcessed(): NonAttribute { - return Math.min(this.sampleActualSize, this.contentItemsProcessed); - } - - /** - * Similar to {@see correctedContentItemsProcessed}, we clamp the exposed - * value of contentItemsMatched, since we can't logically have matched more - * items than we processed. - */ - public get correctedContentItemsMatched(): NonAttribute { - return Math.min( - this.correctedContentItemsProcessed, - this.contentItemsMatched, - ); - } - } - - /* Fields */ - Backtest.init( - { - id: { type: DataTypes.STRING, primaryKey: true }, - ruleId: { type: DataTypes.STRING, allowNull: false }, - creatorId: { type: DataTypes.STRING, allowNull: false }, - - sampleDesiredSize: { type: DataTypes.INTEGER, allowNull: false }, - sampleActualSize: { - type: DataTypes.INTEGER, - allowNull: false, - defaultValue: 0, - }, - sampleStartAt: { type: DataTypes.DATE, allowNull: false }, - sampleEndAt: { type: DataTypes.DATE, allowNull: false }, - - cancelationDate: { type: DataTypes.DATE }, - samplingComplete: { - type: DataTypes.BOOLEAN, - allowNull: false, - defaultValue: false, - }, - - contentItemsProcessed: { - type: DataTypes.INTEGER, - allowNull: false, - defaultValue: 0, - }, - contentItemsMatched: { - type: DataTypes.INTEGER, - allowNull: false, - defaultValue: 0, - }, - - status: { type: DataTypes.STRING }, - createdAt: { type: DataTypes.DATE, allowNull: false }, - updatedAt: { type: DataTypes.DATE, allowNull: false }, - }, - { - sequelize, - modelName: 'backtest', - underscored: true, - timestamps: true, - }, - ); - - return Backtest; -}; - -export default makeBacktestModel; diff --git a/server/models/rules/ItemTypeModel.ts b/server/models/rules/ItemTypeModel.ts deleted file mode 100644 index 25a3920..0000000 --- a/server/models/rules/ItemTypeModel.ts +++ /dev/null @@ -1,174 +0,0 @@ -import { ItemTypeKind } from '@roostorg/types'; -import _ from 'lodash'; -import sequelize, { - type BelongsToGetAssociationMixin, - type HasManyAddAssociationsMixin, - type HasManyGetAssociationsMixin, - type HasManyGetAssociationsMixinOptions, - type HasManySetAssociationsMixin, - type InferAttributes, - type InferCreationAttributes, - type Sequelize, -} from 'sequelize'; - -import { type ItemSchema } from '../../services/moderationConfigService/index.js'; -import { type DataTypes } from '../index.js'; -import { type Org } from '../OrgModel.js'; -import { type SequelizeAction } from './ActionModel.js'; -import { type Rule, type RuleWithLatestVersion } from './RuleModel.js'; - -const { Model } = sequelize; - -export type ItemType = InstanceType>; - -const makeItemTypeModel = (sequelize: Sequelize, DataTypes: DataTypes) => { - class ItemType extends Model< - InferAttributes, - InferCreationAttributes - > { - public declare id: string; - public declare name: string; - public declare description?: string | null; - public declare fields: ItemSchema; - public declare getRules: HasManyGetAssociationsMixin; - - public declare orgId: string; - public declare getOrg: BelongsToGetAssociationMixin; - public declare kind: ItemTypeKind; - - public declare addActions: HasManyAddAssociationsMixin< - SequelizeAction, - string - >; - public declare setActions: HasManySetAssociationsMixin< - SequelizeAction, - string - >; - private declare getActionsSequelizeImpl: HasManyGetAssociationsMixin; - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - static associate(models: { [key: string]: any }) { - ItemType.belongsTo(models.Org, { as: 'Org' }); - ItemType.belongsToMany(models.Rule, { - through: 'rules_and_item_types', - foreignKey: 'item_type_id', - as: 'Rules', - }); - - const actionAssoc = ItemType.belongsToMany(models.Action, { - through: 'actions_and_item_types', - foreignKey: 'item_type_id', - as: 'Actions', - }); - Object.defineProperty( - models.ItemType.prototype, - 'getActionsSequelizeImpl', - { - enumerable: false, - value(...params: unknown[]) { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - return (actionAssoc as any)['get'](this, ...params); - }, - }, - ); - } - - /** - * This function returns the list of rules that are "enabled", meaning that - * we'd run them against a new piece of content of this content type, if the - * content were submitted right now. "Running the rule" just means checking - * if its conditions pass on the content; whether we'd run the actions of - * each passing rule is a different question. - * - * This function is _highly_ impure. Its results will change as rules - * expire, or as daily limits on rules are reached, among other things. As - * we see more use cases, we might wanna refactor where this lives. - */ - async getEnabledRules() { - return this.getRules({ - scope: 'enabled', - include: ['latestVersion'], - }) as Promise; - } - - async getActions( - options?: HasManyGetAssociationsMixinOptions, - ): Promise { - return sequelize.transaction(async () => { - const [explicitlyAssociatedActions, allActions] = await Promise.all([ - this.getActionsSequelizeImpl(options), - this.sequelize.model('action').findAll({ - ...options, - where: { - ...options?.where, - orgId: this.orgId, - }, - }), - ]); - - return _.uniqBy( - [ - // Do this filter outside of the sequelize query because Sequelize - // assumes the name of the enum and doesn't allow you to set it - ...(allActions as SequelizeAction[]).filter((it) => - it.appliesToAllItemsOfKind.includes(this.kind), - ), - ...explicitlyAssociatedActions, - ], - (it) => it.id, - ); - }); - } - } - - /* Fields */ - ItemType.init( - { - id: { - type: DataTypes.STRING, - primaryKey: true, - }, - orgId: { - type: DataTypes.STRING, - allowNull: false, - }, - // Name of the item type, which must unique within each Org - name: { - type: DataTypes.STRING, - allowNull: false, - validate: { notEmpty: true }, - }, - description: { - type: DataTypes.STRING, - allowNull: true, - }, - fields: { - type: DataTypes.ARRAY(DataTypes.JSONB), - allowNull: false, - validate: { - notEmpty: true, - }, - }, - kind: { - type: DataTypes.ENUM(...Object.values(ItemTypeKind)), - allowNull: false, - defaultValue: ItemTypeKind.CONTENT, - }, - }, - { - sequelize, - // legacy name; left as-is in case changing it will break auto-generated - // methods added by sequelize and calls to sequelize.model('content_type') - // and possibly many other things on which don't have great typescript - // support to check us. - modelName: 'content_type', - underscored: true, - tableName: 'item_types', - updatedAt: false, - }, - ); - - return ItemType; -}; - -export default makeItemTypeModel; diff --git a/server/models/rules/RuleLatestVersionModel.ts b/server/models/rules/RuleLatestVersionModel.ts deleted file mode 100644 index 4ca12dd..0000000 --- a/server/models/rules/RuleLatestVersionModel.ts +++ /dev/null @@ -1,45 +0,0 @@ -import sequelize, { type Sequelize } from 'sequelize'; - -import { type DataTypes } from '../index.js'; - -const { Model } = sequelize; - -export type RuleLatestVersion = InstanceType< - ReturnType ->; - -const makeRuleLatestVersionModel = ( - sequelize: Sequelize, - DataTypes: DataTypes, -) => { - class RuleLatestVersion extends Model { - public declare readonly ruleId: string; - public declare readonly version: string; - } - - /* Fields */ - RuleLatestVersion.init( - { - ruleId: { - type: DataTypes.STRING, - primaryKey: true, - }, - version: { - // Read version into JS as a string, not a date, because the date holds - // more digits of precision than JS can store, and we can't throw those - // away when we write this field's value out elsewhere. - type: DataTypes.STRING, - }, - }, - { - sequelize, - tableName: 'rules_latest_versions', - underscored: true, - timestamps: false, - }, - ); - - return RuleLatestVersion; -}; - -export default makeRuleLatestVersionModel; diff --git a/server/models/rules/RuleModel.ts b/server/models/rules/RuleModel.ts deleted file mode 100644 index 4b2a5d0..0000000 --- a/server/models/rules/RuleModel.ts +++ /dev/null @@ -1,303 +0,0 @@ -import _ from 'lodash'; -import sequelize, { - Sequelize, - type CreationOptional, - type HasManyGetAssociationsMixin, - type HasManySetAssociationsMixin, - type HasOneGetAssociationMixin, - type InferAttributes, - type InferCreationAttributes, - type NonAttribute, -} from 'sequelize'; - -import { - RuleAlarmStatus, - RuleStatus, - RuleType, - type ConditionSet, -} from '../../services/moderationConfigService/index.js'; -import { getUtcDateOnlyString } from '../../utils/time.js'; -import { type DataTypes } from '../index.js'; -import { type User } from '../UserModel.js'; -import { type SequelizeAction } from './ActionModel.js'; -import { type RuleLatestVersion } from './RuleLatestVersionModel.js'; - -const { Model, Op } = sequelize; -const { without } = _; - -export type Rule = InstanceType>; -export type RuleWithLatestVersion = Rule & - Required>; - -/** - * Data Model for Rules. Rules are comprised of - * ContentType inputs, Conditions, and Actions. - */ -const makeRuleModel = (sequelize: Sequelize, DataTypes: DataTypes) => { - class Rule extends Model< - InferAttributes, - InferCreationAttributes - > { - public declare id: string; - public declare name: string; - public declare description?: string | null; - public declare expirationTime?: Date | null; - public declare statusIfUnexpired: CreationOptional< - Exclude - >; - public declare lastActionDate: string | null; - public declare maxDailyActions: number | null; - public declare dailyActionsRun: CreationOptional; - public declare status: RuleStatus; - - public declare orgId: string; - public declare creatorId: string; - public declare tags: string[]; - public declare conditionSet: ConditionSet; - public declare ruleType: RuleType; - - public declare alarmStatus: CreationOptional; - public declare alarmStatusSetAt: CreationOptional; - - public declare getActions: HasManyGetAssociationsMixin; - public declare setActions: HasManySetAssociationsMixin< - SequelizeAction, - string - >; - - // Have to use any below to avoid circular type errors - /* eslint-disable @typescript-eslint/no-explicit-any */ - public declare getContentTypes: HasManyGetAssociationsMixin; - public declare setContentTypes: HasManySetAssociationsMixin; - - public declare getPolicies: HasManyGetAssociationsMixin; - public declare setPolicies: HasManySetAssociationsMixin; - - public declare getBacktests: HasManyGetAssociationsMixin; - /* eslint-enable @typescript-eslint/no-explicit-any */ - - public declare getCreator: HasOneGetAssociationMixin; - - public declare getLatestVersion: HasOneGetAssociationMixin; - public declare latestVersion?: NonAttribute; - - public declare createdAt: Date; - public declare updatedAt: Date; - - public declare parentId?: string | null; - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - static associate(models: { [key: string]: any }) { - Rule.belongsTo(models.Org, { as: 'Org', foreignKey: 'orgId' }); - Rule.belongsTo(models.User, { as: 'Creator', foreignKey: 'creatorId' }); - Rule.belongsToMany(models.ItemType, { - through: 'rules_and_item_types', - as: 'contentTypes', - otherKey: 'item_type_id', - }); - Rule.belongsToMany(models.Action, { - through: 'rules_and_actions', - as: 'Actions', - }); - Rule.belongsToMany(models.Policy, { - through: 'rules_and_policies', - as: 'Policies', - }); - Rule.hasMany(models.Backtest, { as: 'Backtests' }); - Rule.hasOne(models.RuleLatestVersion, { - as: 'latestVersion', - foreignKey: 'ruleId', - }); - } - - /** - * Finds all enabled rules that are not associated with any content types. - * We call these "user rules" (or "pure user rules") because their input is - * solely data about a user -- rather than any content submission. - */ - static async findEnabledUserRules(): Promise { - return Rule.sequelize!.transaction(async (t) => { - return Rule.scope('enabled').findAll({ - where: { ruleType: RuleType.USER }, - include: ['latestVersion'], - transaction: t, - }) as Promise; - }); - } - } - - /* Fields */ - Rule.init( - { - id: { - type: DataTypes.STRING, - primaryKey: true, - }, - orgId: { - allowNull: false, - type: DataTypes.STRING, - }, - creatorId: { - allowNull: false, - type: DataTypes.STRING, - }, - // Name of the Rule -- this must be unique for each Org (i.e. an Org can't - // have two Rules with the same name) - name: { - type: DataTypes.STRING, - allowNull: false, - validate: { notEmpty: true }, - }, - description: { - type: DataTypes.STRING, - allowNull: true, - }, - statusIfUnexpired: { - type: DataTypes.STRING, - allowNull: false, - defaultValue: RuleStatus.DRAFT, - validate: { - notEmpty: true, - isIn: [without(Object.values(RuleStatus), RuleStatus.EXPIRED)], - }, - }, - status: { - type: DataTypes.VIRTUAL(DataTypes.STRING, [ - 'statusIfUnexpired', - 'expirationTime', - ]), - get() { - const { expirationTime, statusIfUnexpired } = this; - return expirationTime && expirationTime.valueOf() < Date.now() - ? RuleStatus.EXPIRED - : statusIfUnexpired; - }, - set(value: RuleStatus) { - const expirationTime = this.expirationTime; - if (value === RuleStatus.EXPIRED) { - this.expirationTime = expirationTime - ? new Date(Math.max(expirationTime.valueOf(), Date.now())) - : new Date(); - } else { - this.statusIfUnexpired = value; - } - }, - }, - tags: { - type: DataTypes.ARRAY(DataTypes.STRING), - defaultValue: [], - allowNull: false, - }, - /** - * Maximum number of times this rule's actions can apply per day. - * Useful for slowly rolling out rules. The field name is a bit of a - * misnomer, in that it doesn't record the maximum _number of actions_ - * that a rule can trigger in a day, but rather the maximum _number of - * times_ all of the rule's actions can be triggered. - * - * NB: we use DataTypes.INTEGER (which is an int32), rather than BIGINT - * (which is an int64) so that the value can be represented as a JS Number, - * without us having to parse it to a bigint. int32 supports > 2 billion - * positive values, so this should be enough lol. - */ - maxDailyActions: { - type: DataTypes.INTEGER, - allowNull: true, - }, - /** - * The number of times this rule's actions were run in the most recent day - * when this rule's actions actually ran. That date is stored in - * lastActionDate. This field is used to enforce maxDailyActions. - */ - dailyActionsRun: { - type: DataTypes.INTEGER, - defaultValue: 0, - }, - /** - * The last date when this rule's actions were run. This is used - * to enforce maxDailyActions. - */ - lastActionDate: { - type: DataTypes.DATEONLY, - allowNull: true, - }, - expirationTime: { - type: DataTypes.DATE, - allowNull: true, - }, - conditionSet: { - type: DataTypes.JSONB, - allowNull: false, - }, - ruleType: { type: DataTypes.STRING, allowNull: false }, - alarmStatus: { - type: DataTypes.STRING, - validate: { - isIn: [Object.values(RuleAlarmStatus)], - }, - defaultValue: RuleAlarmStatus.INSUFFICIENT_DATA, - allowNull: false, - }, - alarmStatusSetAt: { - type: DataTypes.DATE, - defaultValue: new Date(), - allowNull: false, - }, - parentId: { - type: DataTypes.STRING, - allowNull: true, - }, - }, - { - sequelize, - modelName: 'rule', - underscored: true, - tableName: 'rules', - }, - ); - - /** - * A scope for finding "enabled rules", where an an enabled rule is one that - * we'd run if a new piece of content (of one of the rule's content types) - * is submitted, or that we'd run against a user in the next user rule run. - * - * "Running the rule" just means checking if its conditions pass on the - * content; whether we'd run the actions of each passing rule is a different - * question. - * - * This function is _highly_ impure. Its results will change as rules - * expire, or as daily limits on rules are reached, among other things. - */ - Rule.addScope('enabled', () => ({ - // NB: this where query is brittle because it hardcodes the column name - // (max_daily_actions) for the maxDailyActions attribute of the Rule model. - // This hardcoding costs us type safety (once we set it up for sequelize) - // and automatic refactoring and makes it harder to find all uses of the - // attribute, so it's very bug-prone. But it seems to be the only thing - // that Sequelize supports for comparing two columns in a WHERE clause?!?! - // Meanwhile, we put `rule.max_daily_actions`, not just `max_daily_actions`, - // to make sure we get the right field, but this makes us reliant on even - // more details of the final query that we shouldn't have to know about. - where: { - // Keep rules that don't expire or haven't expired yet. - expirationTime: { [Op.or]: [null, { [Op.gt]: Sequelize.fn('now') }] }, - // And are in an enabled status. - statusIfUnexpired: { [Op.or]: [RuleStatus.LIVE, RuleStatus.BACKGROUND] }, - // And either don't have a daily actions quota, haven't run yet - // today (in which case they can't have exceeded the quota and the - // value in dailyActionsRun refers to a prior day), or have run - // today, but fewer times than their quota. - [Op.or]: [ - { maxDailyActions: null }, - { lastActionDate: { [Op.ne]: getUtcDateOnlyString() } }, - { - dailyActionsRun: { [Op.lt]: { [Op.col]: 'rule.max_daily_actions' } }, - }, - ], - }, - })); - - return Rule; -}; - -export default makeRuleModel; diff --git a/server/models/rules/ruleTypes.ts b/server/models/rules/ruleTypes.ts deleted file mode 100644 index 53a4475..0000000 --- a/server/models/rules/ruleTypes.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { - type RuleAlarmStatus, - RuleStatus, - type RuleType, - type ConditionSet, - type Action, - type Policy, -} from '../../services/moderationConfigService/index.js'; -import { type GraphQLUserParent } from '../../graphql/datasources/userKyselyPersistence.js'; - -export type RuleLatestVersionRow = { - ruleId: string; - version: string; -}; - -/** - * Rule row fields shared by the rule engine (no GraphQL resolver methods). - */ -export type PlainRuleWithLatestVersion = { - id: string; - name: string; - description: string | null; - statusIfUnexpired: Exclude; - status: RuleStatus; - tags: string[]; - maxDailyActions: number | null; - dailyActionsRun: number; - lastActionDate: string | null; - createdAt: Date; - updatedAt: Date; - orgId: string; - creatorId: string; - expirationTime: Date | null; - conditionSet: ConditionSet; - alarmStatus: RuleAlarmStatus; - alarmStatusSetAt: Date; - ruleType: RuleType; - parentId: string | null; - latestVersion: RuleLatestVersionRow; -}; - -export function computeRuleStatusFromRow( - expirationTime: Date | null, - statusIfUnexpired: Exclude, -): RuleStatus { - if (expirationTime && expirationTime.valueOf() < Date.now()) { - return RuleStatus.EXPIRED; - } - return statusIfUnexpired; -} - -export type RuleGraphqlMethods = { - getCreator(): Promise; - getActions(): Promise; - getPolicies(): Promise; -}; - -/** GraphQL parent for Rule / ContentRule / UserRule / RuleInsights. */ -export type Rule = PlainRuleWithLatestVersion & RuleGraphqlMethods; - -/** @deprecated Use {@link PlainRuleWithLatestVersion} directly. Remove after Kysely migration. */ -export type RuleWithLatestVersion = PlainRuleWithLatestVersion; diff --git a/server/models/sequelizeSetup.ts b/server/models/sequelizeSetup.ts deleted file mode 100644 index 261dc4c..0000000 --- a/server/models/sequelizeSetup.ts +++ /dev/null @@ -1,104 +0,0 @@ -/** - * @fileoverview Connects to pg via sequelize and exposes some helper functions - * for running queries in a transaction. - * - * Critically, this code is not in models/index.ts in order to avoid a circular - * dependency with model definitions that use the transactionWithRetry helper. - */ -import clsHooked from 'cls-hooked'; -import pkg, { type Transaction, type TransactionOptions } from 'sequelize'; - -import { isEnvTrue } from '../iocContainer/utils.js'; -import { safeGet } from '../utils/misc.js'; - -const { Sequelize } = pkg; -const { - DATABASE_HOST, - DATABASE_READ_ONLY_HOST, - DATABASE_PORT = 5432, - DATABASE_NAME = 'development', - DATABASE_USER = 'postgres', - DATABASE_PASSWORD, - SEQUELIZE_PRINT_LOGS, -} = process.env; - -// Set up CLS so that we can ambiently link queries into the same transaction. -// See https://sequelize.org/docs/v6/other-topics/transactions/ -Sequelize.useCLS(clsHooked.createNamespace('sequelize')); - -export const makeSequelize = () => - new Sequelize(DATABASE_NAME, DATABASE_USER, DATABASE_PASSWORD, { - port: Number(DATABASE_PORT), - // eslint-disable-next-line no-console - logging: SEQUELIZE_PRINT_LOGS === 'true' ? console.log : false, - dialect: 'postgres', - replication: { - read: [{ host: DATABASE_READ_ONLY_HOST }], - write: { host: DATABASE_HOST }, - }, - pool: { - max: Number(process.env.SEQUELIZE_POOL_MAX ?? 150), - acquire: 15_000, - // This timeout was made crazy long so that queries which take a long time - // to respond don't cause Sequelize to release the connection back to the - // pool and/or close it. We needed this for loading location bank - // locations, which were previously stored as 40mb json blobs, which pg - // could take ~1 minute to respond with (or more when the db was under - // heavy load). Going forward, we don't want to have idle connections for - // this long (and should warn if a connection is idle in connection for - // more than, idk, ~5s), but having a long idle timeout is still probably - // better than canceling the query and releasing the connection. However, - // we also don't want this to be too long, so that a server instance that - // briefly needs to open a lot of connections (e.g., to warm its caches on - // startup) doesn't hold those connections for longer than necessary, - // which'll add db load and [in future] potentially lead to hitting the - // db's max connection limit as new server instances are autoscaled in. - idle: 300_000, - // TODO: set maxUses once we start auto scaling the number of read replicas. - // See https://github.com/sequelize/sequelize-pool#using-maxuses-option - // Think about how/if we'll do this w/ our kysely connection pools. - }, - dialectOptions: { - ssl: isEnvTrue('DATABASE_SSL') ? { rejectUnauthorized: false } : undefined, - query_timeout: 1_000_000, - idle_in_transaction_session_timeout: 300_000, - }, - }); - -export function maketransactionWithRetry( - sequelize: pkg.Sequelize, -): TransactionWithRetry { - /** - * Run a Sequelize transaction, and auto-retry up to two times if it fails due - * to a serialization error. Sequelize's should really have this built-in See - * https://stackoverflow.com/questions/68427796/sequelize-transaction-retry-doenst-work-as-expected - * - * See https://www.postgresql.org/docs/current/transaction-iso.html - */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any - return async function transactionWithRetry(...args: any[]) { - let remainingTries = 3; - while (remainingTries > 0) { - try { - remainingTries -= 1; - return await sequelize.transaction(...args); - } catch (e: unknown) { - if (safeGet(e, ['original', 'code']) === '40001') { - await sequelize.query('ROLLBACK'); - } else { - throw e; - } - } - } - - throw new Error('Retry limit exceeded.'); - }; -} - -type TransactionWithRetry = { - ( - options: TransactionOptions, - autoCallback: (t: Transaction) => PromiseLike, - ): Promise; - (autoCallback: (t: Transaction) => PromiseLike): Promise; -}; diff --git a/server/package-lock.json b/server/package-lock.json index 17b29a4..7367c55 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -82,7 +82,6 @@ "pg-cursor": "^2.7.4", "pipeline-segment": "^0.0.6", "safe-stable-stringify": "^2.4.2", - "sequelize": "^6.37.8", "size-limited-map": "^2.0.0", "stream-json": "^1.8.0", "stream-to-async-iterator": "^1.0.0", @@ -13491,12 +13490,6 @@ "node": ">=10" } }, - "node_modules/dottie": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.7.tgz", - "integrity": "sha512-7lAK2A0b3zZr3UC5aE69CPdCFR4RHW1o2Dr74TqFykxkUCBXSRJum/yPc7g8zRHJqWKomPLHwFLLoUnn8PXXRg==", - "license": "MIT" - }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -16081,14 +16074,6 @@ "node": ">=0.8.19" } }, - "node_modules/inflection": { - "version": "1.13.4", - "resolved": "https://registry.npmjs.org/inflection/-/inflection-1.13.4.tgz", - "integrity": "sha512-6I/HUDeYFfuNCVS3td055BaXBwKYuzw7K3ExVMStBowKo9oOAMJIXIHvdyR3iboTCp1b+1i5DSkIZTcwIktuDw==", - "engines": [ - "node >= 0.4.0" - ] - }, "node_modules/inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -17995,25 +17980,6 @@ "node": ">=10" } }, - "node_modules/moment": { - "version": "2.29.4", - "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz", - "integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==", - "engines": { - "node": "*" - } - }, - "node_modules/moment-timezone": { - "version": "0.5.43", - "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.43.tgz", - "integrity": "sha512-72j3aNyuIsDxdF1i7CEgV2FfxM1r6aaqJyLB2vwb33mXYyoyLly+F1zbWqhA3/bVIoJ4szlUoMbUnVdid32NUQ==", - "dependencies": { - "moment": "^2.29.4" - }, - "engines": { - "node": "*" - } - }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -18788,11 +18754,6 @@ "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==", "optional": true }, - "node_modules/pg-connection-string": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.1.tgz", - "integrity": "sha512-w6ZzNu6oMmIzEAYVw+RLK0+nqHPt8K3ZnknKi+g48Ak2pr3dtljJW3o+D/n2zzCG07Zoe9VOX3aiKpj+BN0pjg==" - }, "node_modules/pg-cursor": { "version": "2.10.2", "resolved": "https://registry.npmjs.org/pg-cursor/-/pg-cursor-2.10.2.tgz", @@ -19482,11 +19443,6 @@ "node": ">= 4" } }, - "node_modules/retry-as-promised": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-7.0.4.tgz", - "integrity": "sha512-XgmCoxKWkDofwH8WddD0w85ZfqYz+ZHlr5yo+3YUCfycWawU56T5ckWXsScsj5B8tqUcIG67DxXByo3VUgiAdA==" - }, "node_modules/retry-axios": { "name": "@ethanresnick/retry-axios", "version": "2.6.1", @@ -19802,76 +19758,6 @@ "url": "https://opencollective.com/express" } }, - "node_modules/sequelize": { - "version": "6.37.8", - "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-6.37.8.tgz", - "integrity": "sha512-HJ0IQFqcTsTiqbEgiuioYFMSD00TP6Cz7zoTti+zVVBwVe9fEhev9cH6WnM3XU31+ABS356durAb99ZuOthnKw==", - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/sequelize" - } - ], - "license": "MIT", - "dependencies": { - "@types/debug": "^4.1.8", - "@types/validator": "^13.7.17", - "debug": "^4.3.4", - "dottie": "^2.0.6", - "inflection": "^1.13.4", - "lodash": "^4.17.21", - "moment": "^2.29.4", - "moment-timezone": "^0.5.43", - "pg-connection-string": "^2.6.1", - "retry-as-promised": "^7.0.4", - "semver": "^7.5.4", - "sequelize-pool": "^7.1.0", - "toposort-class": "^1.0.1", - "uuid": "^8.3.2", - "validator": "^13.9.0", - "wkx": "^0.5.0" - }, - "engines": { - "node": ">=10.0.0" - }, - "peerDependenciesMeta": { - "ibm_db": { - "optional": true - }, - "mariadb": { - "optional": true - }, - "mysql2": { - "optional": true - }, - "oracledb": { - "optional": true - }, - "pg": { - "optional": true - }, - "pg-hstore": { - "optional": true - }, - "snowflake-sdk": { - "optional": true - }, - "sqlite3": { - "optional": true - }, - "tedious": { - "optional": true - } - } - }, - "node_modules/sequelize-pool": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/sequelize-pool/-/sequelize-pool-7.1.0.tgz", - "integrity": "sha512-G9c0qlIWQSK29pR/5U2JF5dDQeqqHRragoyahj/Nx4KOOQ3CPPfzxnfqFPCSB7x5UgjOgnZ61nSxz+fjDpRlJg==", - "engines": { - "node": ">= 10.0.0" - } - }, "node_modules/serve-static": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", @@ -20797,11 +20683,6 @@ "node": ">=0.6" } }, - "node_modules/toposort-class": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/toposort-class/-/toposort-class-1.0.1.tgz", - "integrity": "sha512-OsLcGGbYF3rMjPUf8oKktyvCiUxSbqMMS39m33MAjLTC1DVIH6x3WSt63/M77ihI09+Sdfk1AXvfhCEeUmC7mg==" - }, "node_modules/tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", @@ -21742,14 +21623,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/wkx": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/wkx/-/wkx-0.5.0.tgz", - "integrity": "sha512-Xng/d4Ichh8uN4l0FToV/258EjMGU9MGcA0HV2d9B/ZpZB3lqQm7nkOdZdm5GhKtLLhAE7PiVQwN4eN+2YJJUg==", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/word-wrap": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", diff --git a/server/package.json b/server/package.json index 8029b15..bcddd03 100644 --- a/server/package.json +++ b/server/package.json @@ -96,7 +96,6 @@ "pg-cursor": "^2.7.4", "pipeline-segment": "^0.0.6", "safe-stable-stringify": "^2.4.2", - "sequelize": "^6.37.8", "size-limited-map": "^2.0.0", "stream-json": "^1.8.0", "stream-to-async-iterator": "^1.0.0", diff --git a/server/plugins/warehouse/utils/clickhouseSql.ts b/server/plugins/warehouse/utils/clickhouseSql.ts index fa867c9..42f6277 100644 --- a/server/plugins/warehouse/utils/clickhouseSql.ts +++ b/server/plugins/warehouse/utils/clickhouseSql.ts @@ -21,23 +21,22 @@ function formatValue(value: unknown): string { return `unhex('${value.toString('hex')}')`; } - const type = typeof value; - if (type === 'number') { - if (!Number.isFinite(value as number)) { + if (typeof value === 'number') { + if (!Number.isFinite(value)) { throw new Error('ClickHouse adapter does not support non-finite numbers'); } return String(value); } - if (type === 'bigint') { + if (typeof value === 'bigint') { return value.toString(); } - if (type === 'boolean') { - return (value as boolean) ? '1' : '0'; + if (typeof value === 'boolean') { + return value ? '1' : '0'; } - if (type === 'object') { + if (typeof value === 'object') { const json = safeStableStringify(value); return `'${escapeString(json)}'`; } @@ -89,4 +88,3 @@ function translateFunctions(statement: string): string { .replace(/\bDATE_TRUNC\b/gi, 'date_trunc') .replace(/\bDATE\(/gi, 'toDate('); } - diff --git a/server/routes/action/submitAction.test.ts b/server/routes/action/submitAction.test.ts index 8794f00..a770958 100644 --- a/server/routes/action/submitAction.test.ts +++ b/server/routes/action/submitAction.test.ts @@ -2,8 +2,6 @@ import { type Request, type Response } from 'express'; import submitAction from './submitAction.js'; -type Handler = ReturnType; - function makeDeps( overrides?: Partial<{ action: Record; @@ -48,7 +46,7 @@ function makeDeps( ModerationConfigService: { getActions, getPolicies }, getItemTypeEventuallyConsistent, UserAPIDataSource: { getGraphQLUserFromId }, - } as never) as Handler; + } as never); return { handler, publishActions, getActions }; } @@ -100,11 +98,7 @@ describe('submitAction (REST handler)', () => { const res = makeRes(); const next = jest.fn(); - await handler( - makeReq({ ...validBody, parameters: {} }), - res, - next, - ); + await handler(makeReq({ ...validBody, parameters: {} }), res, next); expect(publishActions).not.toHaveBeenCalled(); expect(next).toHaveBeenCalledTimes(1); diff --git a/server/routes/content/ContentRoutes.test.ts b/server/routes/content/ContentRoutes.test.ts index eeedb37..a4f2efa 100644 --- a/server/routes/content/ContentRoutes.test.ts +++ b/server/routes/content/ContentRoutes.test.ts @@ -7,6 +7,7 @@ import { type Dependencies } from '../../iocContainer/index.js'; import { serializeDerivedFieldSpec } from '../../services/derivedFieldsService/index.js'; import { type ContentItemType } from '../../services/moderationConfigService/index.js'; import createOrg from '../../test/fixtureHelpers/createOrg.js'; +import createUser from '../../test/fixtureHelpers/createUser.js'; import { makeMockedServer } from '../../test/setupMockedServer.js'; const { omit } = _; @@ -21,7 +22,7 @@ describe('POST Content', () => { shutdown: Awaited>['shutdown'], apiKey: Awaited>['apiKey'], orgCleanup: Awaited>['cleanup'], - models: Dependencies['Sequelize'], + userCleanup: Awaited>['cleanup'], ModerationConfigService: Dependencies['ModerationConfigService'], ApiKeyService: Dependencies['ApiKeyService'], analytics: Dependencies['DataWarehouseAnalytics'], @@ -32,7 +33,6 @@ describe('POST Content', () => { request, shutdown, deps: { - Sequelize: models, DataWarehouseAnalytics: analytics, ModerationConfigService, ApiKeyService, @@ -40,8 +40,6 @@ describe('POST Content', () => { }, } = await makeMockedServer()); - const { User } = models; - ({ apiKey, cleanup: orgCleanup } = await createOrg( { KyselyPg, ModerationConfigService, ApiKeyService }, orgId, @@ -81,19 +79,12 @@ describe('POST Content', () => { schemaFieldRoles: {}, }); - await User.create({ + ({ cleanup: userCleanup } = await createUser(KyselyPg, orgId, { id: userId, - orgId, - password: faker.random.alphaNumeric(), - firstName: faker.name.firstName(), - lastName: faker.name.lastName(), - email: faker.internet.email(), - loginMethods: ['password'], - }); + })); }); afterAll(async () => { - const { User } = models; await orgCleanup(); await ModerationConfigService.deleteItemType({ orgId, @@ -103,7 +94,7 @@ describe('POST Content', () => { orgId, itemTypeId: contentType2.id, }); - await User.destroy({ where: { id: userId } }); + await userCleanup(); await shutdown(); }); @@ -187,28 +178,31 @@ describe('POST Content', () => { // But I ran it manually once and it works. test.skip('should return the requested derived fields', async () => { const seedOrgId = 'e7c89ce7729'; - const contentTypeId = uid(); + const contentType = await ModerationConfigService.createContentType( + seedOrgId, + { + name: 'tes333t', + description: faker.datatype.string(), + schema: [ + { + name: 'video', + type: 'VIDEO', + required: false, + container: null, + }, + ], + schemaFieldRoles: {}, + }, + ); const fieldId = serializeDerivedFieldSpec({ - source: { type: 'CONTENT_FIELD', name: 'video', contentTypeId }, + source: { + type: 'CONTENT_FIELD', + name: 'video', + contentTypeId: contentType.id, + }, derivationType: 'VIDEO_TRANSCRIPTION', }); - const contentType = await models.ItemType.create({ - id: contentTypeId, - name: 'tes333t', - description: faker.datatype.string(), - orgId: seedOrgId, - fields: [ - { - name: 'video', - type: 'VIDEO', - required: false, - container: null, - }, - ], - kind: 'CONTENT', - }); - try { return await request .post(`/api/v1/content?includeDerivedField=${fieldId}`) @@ -226,7 +220,7 @@ describe('POST Content', () => { .expect(200) .expect(({ body }) => { expect(body.derivedFields[fieldId].field.source.contentTypeId).toBe( - contentTypeId, + contentType.id, ); expect(body.derivedFields[fieldId].value).toMatchInlineSnapshot( @@ -249,7 +243,10 @@ describe('POST Content', () => { throw e; }); } finally { - await contentType.destroy(); + await ModerationConfigService.deleteItemType({ + orgId: seedOrgId, + itemTypeId: contentType.id, + }); } }); diff --git a/server/routes/content/submitContent.ts b/server/routes/content/submitContent.ts index 88611bd..3017e4a 100644 --- a/server/routes/content/submitContent.ts +++ b/server/routes/content/submitContent.ts @@ -3,7 +3,6 @@ import { type UnwrapOpaque } from 'type-fest'; import { v1 as uuidv1 } from 'uuid'; import { type Dependencies } from '../../iocContainer/index.js'; -import { isTaggedItemData } from '../../models/rules/item-type-fields.js'; import { parseDerivedFieldSpec, type DerivedFieldSpec, @@ -16,8 +15,10 @@ import { } from '../../services/itemProcessingService/index.js'; import { CoopInput, + isTaggedItemData, type ItemType, } from '../../services/moderationConfigService/index.js'; +import { hasOrgId } from '../../utils/apiKeyMiddleware.js'; import { fromCorrelationId, toCorrelationId, @@ -34,7 +35,6 @@ import { import { safeGet, safePick, sleep } from '../../utils/misc.js'; import { type RequestHandlerWithBodies } from '../../utils/route-helpers.js'; import { instantiateOpaqueType } from '../../utils/typescript-types.js'; -import { hasOrgId } from '../../utils/apiKeyMiddleware.js'; import { type EvaluateContentInputCamelCase, type EvaluateContentOutput, @@ -113,7 +113,7 @@ Dependencies): RequestHandlerWithBodies< }), ); } - + const { orgId } = req; const activeSpan = Tracer.getActiveSpan(); if (activeSpan?.isRecording()) { diff --git a/server/routes/items/ItemRoutes.test.ts b/server/routes/items/ItemRoutes.test.ts index bcc7dea..4ee0a6e 100644 --- a/server/routes/items/ItemRoutes.test.ts +++ b/server/routes/items/ItemRoutes.test.ts @@ -5,6 +5,7 @@ import { uid } from 'uid'; import { type Dependencies } from '../../iocContainer/index.js'; import { type ContentItemType } from '../../services/moderationConfigService/index.js'; import createOrg from '../../test/fixtureHelpers/createOrg.js'; +import createUser from '../../test/fixtureHelpers/createUser.js'; import { makeMockedServer } from '../../test/setupMockedServer.js'; describe('POST Items', () => { @@ -16,7 +17,7 @@ describe('POST Items', () => { shutdown: Awaited>['shutdown'], apiKey: Awaited>['apiKey'], orgCleanup: Awaited>['cleanup'], - models: Dependencies['Sequelize'], + userCleanup: Awaited>['cleanup'], ModerationConfigService: Dependencies['ModerationConfigService'], ApiKeyService: Dependencies['ApiKeyService'], analytics: Dependencies['DataWarehouseAnalytics'], @@ -27,7 +28,6 @@ describe('POST Items', () => { request, shutdown, deps: { - Sequelize: models, DataWarehouseAnalytics: analytics, ModerationConfigService, ApiKeyService, @@ -35,8 +35,6 @@ describe('POST Items', () => { }, } = await makeMockedServer()); - const { User } = models; - ({ apiKey, cleanup: orgCleanup } = await createOrg( { KyselyPg, ModerationConfigService, ApiKeyService }, orgId, @@ -62,25 +60,18 @@ describe('POST Items', () => { schemaFieldRoles: {}, }); - await User.create({ + ({ cleanup: userCleanup } = await createUser(KyselyPg, orgId, { id: userId, - orgId, - password: faker.random.alphaNumeric(), - firstName: faker.name.firstName(), - lastName: faker.name.lastName(), - email: faker.internet.email(), - loginMethods: ['password'], - }); + })); }); afterAll(async () => { - const { User } = models; await orgCleanup(); await ModerationConfigService.deleteItemType({ orgId, itemTypeId: contentType.id, }); - await User.destroy({ where: { id: userId } }); + await userCleanup(); await shutdown(); }); diff --git a/server/routes/policies/PoliciesRoutes.test.ts b/server/routes/policies/PoliciesRoutes.test.ts index 29fbbfa..b9c25c0 100644 --- a/server/routes/policies/PoliciesRoutes.test.ts +++ b/server/routes/policies/PoliciesRoutes.test.ts @@ -6,15 +6,12 @@ import createPolicy from '../../test/fixtureHelpers/createPolicy.js'; import { makeMockedServer } from '../../test/setupMockedServer.js'; describe('GET policies', () => { - const orgId = uid(), - policyId1 = uid(), - policyId2 = uid(); + const orgId = uid(); let request: Awaited>['request'], shutdown: Awaited>['shutdown'], apiKey: Awaited>['apiKey'], orgCleanup: Awaited>['cleanup'], - models: Dependencies['Sequelize'], ModerationConfigService: Dependencies['ModerationConfigService'], ApiKeyService: Dependencies['ApiKeyService'], KyselyPg: Dependencies['KyselyPg']; @@ -23,12 +20,7 @@ describe('GET policies', () => { ({ request, shutdown, - deps: { - Sequelize: models, - ModerationConfigService, - ApiKeyService, - KyselyPg, - }, + deps: { ModerationConfigService, ApiKeyService, KyselyPg }, } = await makeMockedServer()); ({ apiKey, cleanup: orgCleanup } = await createOrg( @@ -38,9 +30,6 @@ describe('GET policies', () => { }); afterAll(async () => { - const { Policy } = models; - await Policy.destroy({ where: { id: policyId1 } }); - await Policy.destroy({ where: { id: policyId2 } }); await orgCleanup(); await shutdown(); }); diff --git a/server/routes/reporting/ReportingRoutes.test.ts b/server/routes/reporting/ReportingRoutes.test.ts index 265d92b..bfa96fb 100644 --- a/server/routes/reporting/ReportingRoutes.test.ts +++ b/server/routes/reporting/ReportingRoutes.test.ts @@ -1,8 +1,8 @@ /* eslint-disable max-lines */ import { faker } from '@faker-js/faker'; -import { type Dependencies } from '../../iocContainer/index.js'; import createOrg from '../../test/fixtureHelpers/createOrg.js'; +import createUser from '../../test/fixtureHelpers/createUser.js'; import { makeMockedServer } from '../../test/setupMockedServer.js'; describe('POST Report', () => { @@ -13,23 +13,18 @@ describe('POST Report', () => { let userTypeId: string; let threadTypeId: string; - let models: Dependencies['Sequelize'], - deps: Awaited>['deps'], + let deps: Awaited>['deps'], request: Awaited>['request'], shutdown: Awaited>['shutdown'], apiKey: Awaited>['apiKey'], - orgCleanup: Awaited>['cleanup']; + orgCleanup: Awaited>['cleanup'], + userCleanup: Awaited>['cleanup']; - const getBulkWriteMock = () => - deps.DataWarehouseAnalytics.bulkWrite as jest.MockedFunction< - Dependencies['DataWarehouseAnalytics']['bulkWrite'] - >; + const getBulkWriteMock = () => deps.DataWarehouseAnalytics.bulkWrite; beforeAll(async () => { ({ deps, request, shutdown } = await makeMockedServer()); - models = deps.Sequelize; - ({ apiKey, cleanup: orgCleanup } = await createOrg( { KyselyPg: deps.KyselyPg, @@ -110,24 +105,28 @@ describe('POST Report', () => { threadTypeId = threadType.id; - await models.User.create({ + ({ cleanup: userCleanup } = await createUser(deps.KyselyPg, orgId, { id: userId, - orgId, - password: faker.random.alphaNumeric(), - firstName: faker.name.firstName(), - lastName: faker.name.lastName(), - email: faker.internet.email(), - loginMethods: ['password'], - }); + })); }); afterAll(async () => { - const { User, ItemType } = models; await orgCleanup(); - await ItemType.destroy({ where: { id: contentTypeId } }); - await ItemType.destroy({ where: { id: userTypeId } }); - await ItemType.destroy({ where: { id: threadTypeId } }); - await User.destroy({ where: { id: userId } }); + await Promise.all([ + deps.ModerationConfigService.deleteItemType({ + orgId, + itemTypeId: contentTypeId, + }), + deps.ModerationConfigService.deleteItemType({ + orgId, + itemTypeId: userTypeId, + }), + deps.ModerationConfigService.deleteItemType({ + orgId, + itemTypeId: threadTypeId, + }), + ]); + await userCleanup(); await shutdown(); }); diff --git a/server/rule_engine/ActionPublisher.ts b/server/rule_engine/ActionPublisher.ts index 89b29b2..402f814 100644 --- a/server/rule_engine/ActionPublisher.ts +++ b/server/rule_engine/ActionPublisher.ts @@ -77,8 +77,8 @@ export function getUserFromActionTarget(it: ActionTargetItem) { return it.itemType.kind === 'USER' ? { id: it.itemId, typeId: it.itemType.id } : isFullSubmission(it) - ? it.creator - : undefined; + ? it.creator + : undefined; } /** @@ -100,8 +100,8 @@ export function getUserFromActionTargetItem(it: ActionTargetItem) { return it.itemType.kind === 'USER' ? { id: it.itemId, typeId: it.itemType.id } : isFullSubmission(it) - ? it.creator - : undefined; + ? it.creator + : undefined; } /** @@ -267,9 +267,7 @@ class ActionPublisher { // Audit-trail context: persist what the moderator supplied // alongside the action itself so reviewers can see why and // with what values it ran (PR 3 for #377). - parameterValues: customMrtApiParamDecisionPayload as - | Record - | undefined, + parameterValues: customMrtApiParamDecisionPayload, actorNote, }, ], diff --git a/server/rule_engine/RuleEngine.ts b/server/rule_engine/RuleEngine.ts index 2ad89a9..355d67a 100644 --- a/server/rule_engine/RuleEngine.ts +++ b/server/rule_engine/RuleEngine.ts @@ -7,16 +7,16 @@ import { } from '../condition_evaluator/conditionSet.js'; import { type Dependencies } from '../iocContainer/index.js'; import { inject } from '../iocContainer/utils.js'; -import { type PlainRuleWithLatestVersion } from '../models/rules/ruleTypes.js'; import { evaluateAggregationRuntimeArgsForItem } from '../services/aggregationsService/index.js'; +import { type RuleExecutionCorrelationId } from '../services/analyticsLoggers/index.js'; import { type ItemSubmission } from '../services/itemProcessingService/index.js'; import { - type Action, ConditionCompletionOutcome, - type ConditionSet, RuleStatus, + type Action, + type ConditionSet, + type PlainRuleWithLatestVersion, } from '../services/moderationConfigService/index.js'; -import { type RuleExecutionCorrelationId } from '../services/analyticsLoggers/index.js'; import { type CorrelationId, type CorrelationIdType, @@ -119,7 +119,6 @@ class RuleEngine { (rule) => rule.status === RuleStatus.LIVE, ); - const evaluationContext = this.makeRuleExecutionContext({ orgId: itemSubmission.itemType.orgId, input: itemSubmission, @@ -220,7 +219,6 @@ class RuleEngine { ), ); - const rulesToResults = new Map(equalLengthZip(rules, ruleResults)); const passingRules = [...rulesToResults.entries()] @@ -231,17 +229,14 @@ class RuleEngine { const actionableRulesToActions = new Map( await Promise.all( - actionableRules.map( - async (rule) => { - const actions = (await this.getRuleActionsEventuallyConsistent({ - orgId: evaluationContext.org.id, - ruleId: rule.id, - })) satisfies readonly ReadonlyDeep[] as readonly Action[]; - - - return [rule, actions] as const; - } - ), + actionableRules.map(async (rule) => { + const actions = (await this.getRuleActionsEventuallyConsistent({ + orgId: evaluationContext.org.id, + ruleId: rule.id, + })) satisfies readonly ReadonlyDeep[] as readonly Action[]; + + return [rule, actions] as const; + }), ), ); @@ -396,7 +391,6 @@ class RuleEngine { ); } } - } export default inject( diff --git a/server/rule_engine/ruleEngineQueries.ts b/server/rule_engine/ruleEngineQueries.ts index 133aff2..1985ddd 100644 --- a/server/rule_engine/ruleEngineQueries.ts +++ b/server/rule_engine/ruleEngineQueries.ts @@ -10,8 +10,8 @@ import { sql, type Kysely } from 'kysely'; import { inject } from '../iocContainer/index.js'; -import { type LocationArea } from '../models/types/locationArea.js'; import { type CombinedPg } from '../services/combinedDbTypes.js'; +import { type LocationArea } from '../services/moderationConfigService/index.js'; import { cached } from '../utils/caching.js'; import { jsonParse, jsonStringify } from '../utils/encoding.js'; import { makeKyselyTransactionWithRetry } from '../utils/kyselyTransactionWithRetry.js'; @@ -107,7 +107,8 @@ export const makeGetLocationBankLocationsEventuallyConsistent = inject( name: r.name ?? undefined, geometry: r.geometry as LocationArea['geometry'], bounds: r.bounds as LocationArea['bounds'], - googlePlaceInfo: r.google_place_info as LocationArea['googlePlaceInfo'], + googlePlaceInfo: + r.google_place_info as LocationArea['googlePlaceInfo'], }) satisfies LocationArea, ); }, diff --git a/server/services/analyticsLoggers/ruleExecutionLoggingUtils.ts b/server/services/analyticsLoggers/ruleExecutionLoggingUtils.ts index 87ef6fc..cc350ce 100644 --- a/server/services/analyticsLoggers/ruleExecutionLoggingUtils.ts +++ b/server/services/analyticsLoggers/ruleExecutionLoggingUtils.ts @@ -1,7 +1,6 @@ import { type ReadonlyDeep } from 'type-fest'; import { isConditionSet } from '../../condition_evaluator/condition.js'; -import { type LocationArea } from '../../models/types/locationArea.js'; import { type DerivedFieldSpec } from '../../services/derivedFieldsService/index.js'; import { type ConditionResult, @@ -20,6 +19,7 @@ import { type NonEmptyArray, type ReplaceDeep, } from '../../utils/typescript-types.js'; +import { type LocationArea } from '../moderationConfigService/index.js'; type ConditionResultAsLogged = ReplaceDeep< ConditionResult, diff --git a/server/services/coreAppTables.ts b/server/services/coreAppTables.ts index a36294f..dc27890 100644 --- a/server/services/coreAppTables.ts +++ b/server/services/coreAppTables.ts @@ -1,6 +1,6 @@ import { type Generated, type GeneratedAlways } from 'kysely'; -import { type UserRole } from '../models/types/permissioning.js'; +import { type UserRole } from './userManagementService/index.js'; /** Postgres enum for backtests.status (generated column — read-only in app). */ export type BacktestStatusDb = 'RUNNING' | 'COMPLETE' | 'CANCELED'; diff --git a/server/services/derivedFieldsService/helpers.ts b/server/services/derivedFieldsService/helpers.ts index 46c18e3..c54b2b4 100644 --- a/server/services/derivedFieldsService/helpers.ts +++ b/server/services/derivedFieldsService/helpers.ts @@ -4,7 +4,6 @@ import _ from 'lodash'; import { type ReadonlyDeep } from 'type-fest'; import { extractContentValueOrValues } from '../../condition_evaluator/leafCondition.js'; -import { type TaggedItemData } from '../../models/rules/item-type-fields.js'; import { b64UrlDecode, b64UrlEncode, @@ -21,7 +20,10 @@ import { everyAsync } from '../../utils/fp-helpers.js'; import { assertUnreachable } from '../../utils/misc.js'; import { type NonEmptyArray } from '../../utils/typescript-types.js'; import { type ItemSubmission } from '../itemProcessingService/makeItemSubmission.js'; -import { CoopInput } from '../moderationConfigService/index.js'; +import { + CoopInput, + type TaggedItemData, +} from '../moderationConfigService/index.js'; import { type TransientRunSignalWithCache } from '../orgAwareSignalExecutionService/signalExecutionService.js'; import { isSignalErrorResult, diff --git a/server/services/itemProcessingService/fieldTypeHandlers.test.ts b/server/services/itemProcessingService/fieldTypeHandlers.test.ts index cd241e8..36bcd60 100644 --- a/server/services/itemProcessingService/fieldTypeHandlers.test.ts +++ b/server/services/itemProcessingService/fieldTypeHandlers.test.ts @@ -31,7 +31,7 @@ describe('Content type schemas', () => { fc.property(dummyContainerFieldArb, (containerField) => { expect( (handlers as (typeof fieldTypeHandlers)[ContainerType]).coerce( - null as never, + null, [], containerField.container as never, ), diff --git a/server/services/manualReviewToolService/manualReviewToolQueries.ts b/server/services/manualReviewToolService/manualReviewToolQueries.ts index 921dee7..549f306 100644 --- a/server/services/manualReviewToolService/manualReviewToolQueries.ts +++ b/server/services/manualReviewToolService/manualReviewToolQueries.ts @@ -23,7 +23,7 @@ export const makeGetActionsByIdEventuallyConsistent = inject( orgId: actionIds.orgId, ids: actionIds.ids, readFromReplica: true, - }) as Promise[]>; + }); }, directives: { freshUntilAge: 10, maxStale: [0, 2, 2] }, }), diff --git a/server/services/manualReviewToolService/manualReviewToolService.ts b/server/services/manualReviewToolService/manualReviewToolService.ts index 0fd46c2..ba0d39a 100644 --- a/server/services/manualReviewToolService/manualReviewToolService.ts +++ b/server/services/manualReviewToolService/manualReviewToolService.ts @@ -8,10 +8,6 @@ import { type Opaque } from 'type-fest'; import { type Dependencies } from '../../iocContainer/index.js'; import { type ConsumerDirectives } from '../../lib/cache/index.js'; -import { - type Invoker, - type UserPermission, -} from '../../models/types/permissioning.js'; import { jsonStringify } from '../../utils/encoding.js'; import { isCoopErrorOfType } from '../../utils/errors.js'; import { isUniqueViolationError } from '../../utils/kysely.js'; @@ -24,6 +20,10 @@ import { import { type ItemSubmissionWithTypeIdentifier } from '../itemProcessingService/makeItemSubmissionWithTypeIdentifier.js'; import { type ModerationConfigService } from '../moderationConfigService/index.js'; import { type PartialItemsService } from '../partialItemsService/index.js'; +import { + type Invoker, + type UserPermission, +} from '../userManagementService/index.js'; import { type UserScore, type UserStatisticsService, @@ -553,9 +553,8 @@ export class ManualReviewToolService { `No item type for org ${input.orgId} with ID ${input.payload.item.itemTypeIdentifier.id}`, ); } - const enrichedJobPayload = await this.jobEnrichment.enrichAppealPayload( - input, - ); + const enrichedJobPayload = + await this.jobEnrichment.enrichAppealPayload(input); const attemptAppealEnqueue = async (): Promise< | { job: ManualReviewAppealJob; targetQueueForNewJob: string } @@ -688,10 +687,10 @@ export class ManualReviewToolService { ? _.uniqBy( [ ...('itemThreadContentItems' in newJob - ? newJob.itemThreadContentItems ?? [] + ? (newJob.itemThreadContentItems ?? []) : []), ...('itemThreadContentItems' in existingJob - ? existingJob.itemThreadContentItems ?? [] + ? (existingJob.itemThreadContentItems ?? []) : []), ], (it) => jsonStringify([it.itemId, it.itemTypeIdentifier.id]), @@ -704,10 +703,10 @@ export class ManualReviewToolService { ? _.uniqBy( [ ...('reportedItems' in newJob - ? newJob.reportedItems ?? [] + ? (newJob.reportedItems ?? []) : []), ...('reportedItems' in existingJob - ? existingJob.reportedItems ?? [] + ? (existingJob.reportedItems ?? []) : []), ], (it) => jsonStringify([it.id, it.typeId]), diff --git a/server/services/manualReviewToolService/modules/CommentOperations.test.ts b/server/services/manualReviewToolService/modules/CommentOperations.test.ts index 26091f1..c87edc8 100644 --- a/server/services/manualReviewToolService/modules/CommentOperations.test.ts +++ b/server/services/manualReviewToolService/modules/CommentOperations.test.ts @@ -1,10 +1,10 @@ import { v1 as uuidv1 } from 'uuid'; import getBottle from '../../../iocContainer/index.js'; -import { UserPermission } from '../../../models/types/permissioning.js'; import createOrg from '../../../test/fixtureHelpers/createOrg.js'; import createUser from '../../../test/fixtureHelpers/createUser.js'; import { makeTestWithFixture } from '../../../test/utils.js'; +import { UserPermission } from '../../userManagementService/index.js'; import CommentOperations from './CommentOperations.js'; describe('CommentOperations', () => { @@ -26,23 +26,24 @@ describe('CommentOperations', () => { // Create test user const { user, cleanup: userCleanup } = await createUser( - container.Sequelize, + container.KyselyPg, orgId, ); // Create a queue (required for job_creations foreign key) - const queue = await container.ManualReviewToolService.createManualReviewQueue({ - name: 'Test Queue', - description: null, - userIds: [user.id], - hiddenActionIds: [], - isAppealsQueue: false, - invokedBy: { - userId: user.id, - permissions: [UserPermission.EDIT_MRT_QUEUES], - orgId, - }, - }); + const queue = + await container.ManualReviewToolService.createManualReviewQueue({ + name: 'Test Queue', + description: null, + userIds: [user.id], + hiddenActionIds: [], + isAppealsQueue: false, + invokedBy: { + userId: user.id, + permissions: [UserPermission.EDIT_MRT_QUEUES], + orgId, + }, + }); // Create test item identifiers and jobs const itemId = uuidv1(); @@ -164,7 +165,10 @@ describe('CommentOperations', () => { }) .execute(); - const result = await commentOps.getComments({ orgId, jobId: singleJobId }); + const result = await commentOps.getComments({ + orgId, + jobId: singleJobId, + }); expect(result).toHaveLength(1); expect(result[0].commentText).toBe('Test comment'); @@ -262,7 +266,10 @@ describe('CommentOperations', () => { testWithFixtures( 'should return 0 when no comments found', async ({ commentOps, orgId, jobId1 }) => { - const result = await commentOps.getCommentCount({ orgId, jobId: jobId1 }); + const result = await commentOps.getCommentCount({ + orgId, + jobId: jobId1, + }); expect(result).toBe(0); }, @@ -302,7 +309,10 @@ describe('CommentOperations', () => { ]) .execute(); - const result = await commentOps.getCommentCount({ orgId, jobId: jobId1 }); + const result = await commentOps.getCommentCount({ + orgId, + jobId: jobId1, + }); expect(result).toBe(3); }, @@ -461,7 +471,10 @@ describe('CommentOperations', () => { authorId: userId, }); - const result = await commentOps.getCommentCount({ orgId, jobId: jobId2 }); + const result = await commentOps.getCommentCount({ + orgId, + jobId: jobId2, + }); expect(result).toBe(3); }, diff --git a/server/services/manualReviewToolService/modules/DecisionAnalytics.ts b/server/services/manualReviewToolService/modules/DecisionAnalytics.ts index 15adefc..4f20d6d 100644 --- a/server/services/manualReviewToolService/modules/DecisionAnalytics.ts +++ b/server/services/manualReviewToolService/modules/DecisionAnalytics.ts @@ -2,8 +2,8 @@ import { sql, type Kysely } from 'kysely'; import { type ReadonlyDeep } from 'type-fest'; -import { UserPermission } from '../../../models/types/permissioning.js'; import { MONTH_MS } from '../../../utils/time.js'; +import { UserPermission } from '../../userManagementService/index.js'; import { type ManualReviewToolServicePg } from '../dbTypes.js'; import { type ManualReviewJob, diff --git a/server/services/manualReviewToolService/modules/JobRouting.test.ts b/server/services/manualReviewToolService/modules/JobRouting.test.ts index 9ba7376..66ecb2f 100644 --- a/server/services/manualReviewToolService/modules/JobRouting.test.ts +++ b/server/services/manualReviewToolService/modules/JobRouting.test.ts @@ -3,7 +3,6 @@ import { ScalarTypes } from '@roostorg/types'; import { uid } from 'uid'; import getBottle from '../../../iocContainer/index.js'; -import { UserPermission } from '../../../models/types/permissioning.js'; import createContentItemTypes from '../../../test/fixtureHelpers/createContentItemTypes.js'; import createOrg from '../../../test/fixtureHelpers/createOrg.js'; import { makeTestWithFixture } from '../../../test/utils.js'; @@ -17,6 +16,7 @@ import { import { itemSubmissionToItemSubmissionWithTypeIdentifier } from '../../itemProcessingService/makeItemSubmissionWithTypeIdentifier.js'; import { toNormalizedItemDataOrErrors } from '../../itemProcessingService/toNormalizedItemDataOrErrors.js'; import { SignalType } from '../../signalsService/index.js'; +import { UserPermission } from '../../userManagementService/index.js'; describe('JobRouting tests', () => { const jobRoutingTestWithFixtures = makeTestWithFixture(async () => { diff --git a/server/services/manualReviewToolService/modules/QueueOperations.test.ts b/server/services/manualReviewToolService/modules/QueueOperations.test.ts index 5a67819..c7590db 100644 --- a/server/services/manualReviewToolService/modules/QueueOperations.test.ts +++ b/server/services/manualReviewToolService/modules/QueueOperations.test.ts @@ -45,7 +45,7 @@ describe('QueueOperations', () => { ); const { user, cleanup: userCleanup } = await createUser( - container.Sequelize, + container.KyselyPg, org.id, ); const { itemTypes, cleanup: itemTypesCleanup } = diff --git a/server/services/manualReviewToolService/modules/QueueOperations.ts b/server/services/manualReviewToolService/modules/QueueOperations.ts index 4ba335a..c42f98a 100644 --- a/server/services/manualReviewToolService/modules/QueueOperations.ts +++ b/server/services/manualReviewToolService/modules/QueueOperations.ts @@ -1,8 +1,7 @@ /* eslint-disable max-lines */ import { type ItemIdentifier } from '@roostorg/types'; -import { Queue, Worker } from 'bullmq'; -import { type Job } from 'bullmq'; +import { Queue, Worker, type Job } from 'bullmq'; import { type Cluster } from 'ioredis'; import type IORedis from 'ioredis'; import { type Kysely, type Transaction } from 'kysely'; @@ -11,10 +10,6 @@ import { type Opaque, type ReadonlyDeep } from 'type-fest'; import { v1 as uuidv1 } from 'uuid'; import { type Dependencies } from '../../../iocContainer/index.js'; -import { - UserPermission, - type Invoker, -} from '../../../models/types/permissioning.js'; import { cached, type Cached } from '../../../utils/caching.js'; import { filterNullOrUndefined } from '../../../utils/collections.js'; import { @@ -41,6 +36,10 @@ import { makeSubmissionId, } from '../../itemProcessingService/index.js'; import { type ItemSubmissionWithTypeIdentifier } from '../../itemProcessingService/makeItemSubmissionWithTypeIdentifier.js'; +import { + UserPermission, + type Invoker, +} from '../../userManagementService/index.js'; import { type ManualReviewToolServicePg } from '../dbTypes.js'; import { type AppealEnqueueSourceInfo, @@ -1010,8 +1009,8 @@ export default class QueueOperations { // putting the payload kind in a variable to help TS do some type narrowing. const jobKind = job.data.payload.kind; if (jobKind === 'DEFAULT') { - const { allMediaItems: _omitted, ...storedPayloadWithoutNcmec } = - job.data.payload as Record & { allMediaItems?: unknown }; + const { allMediaItems: _omitted, ...storedPayloadWithoutNcmec } = job.data + .payload as Record & { allMediaItems?: unknown }; payload = { ...storedPayloadWithoutNcmec, kind: 'DEFAULT', @@ -1105,8 +1104,8 @@ export default class QueueOperations { 'policyIds' in job.data ? job.data.policyIds : 'policyId' in job.data.payload && job.data.payload.policyId - ? [job.data.payload.policyId] - : []; + ? [job.data.payload.policyId] + : []; const convertedJobData = { ...safePick(job.data, [ diff --git a/server/services/moderationConfigService/index.ts b/server/services/moderationConfigService/index.ts index 2c0292c..6bb7995 100644 --- a/server/services/moderationConfigService/index.ts +++ b/server/services/moderationConfigService/index.ts @@ -27,6 +27,10 @@ export { ConditionSet, LeafCondition, ConditionSignalInfo, + PlainRuleWithLatestVersion, + RuleLatestVersionRow, + RuleWithLatestVersion, + computeRuleStatusFromRow, } from './types/rules.js'; export { @@ -52,6 +56,23 @@ export { Policy, PolicyType } from './types/policies.js'; export { UserPenaltySeverity } from './types/shared.js'; +export { LocationArea, LocationGeometry } from './types/locationArea.js'; + +export { + MatchingValueType, + MatchingValues, + getMatchingValuesType, + isLocationArea, +} from './types/matchingValues.js'; + +export { + TaggedItemData, + isTaggedItemData, + isTextValue, + isTranscribableType, + isTranscribableValue, +} from './types/itemTypeFields.js'; + export { ModerationConfigService, ModerationConfigErrorType, diff --git a/server/services/moderationConfigService/moderationConfigService.test.ts b/server/services/moderationConfigService/moderationConfigService.test.ts index d925751..e410f32 100644 --- a/server/services/moderationConfigService/moderationConfigService.test.ts +++ b/server/services/moderationConfigService/moderationConfigService.test.ts @@ -17,12 +17,11 @@ import { ErrorType } from '../../utils/errors.js'; import { type Satisfies } from '../../utils/typescript-types.js'; import { type ModerationConfigServicePg } from './dbTypes.js'; import { + RuleStatus, + RuleType, type Action, type ConditionSet, type ItemType, - RuleAlarmStatus, - RuleStatus, - RuleType, type Policy, type UserItemType, } from './index.js'; @@ -133,13 +132,11 @@ describe('ModerationConfigService', () => { }); afterAll(async () => { - const { Sequelize: models } = (await getBottle()).container; await dummyOrgCleanup(); await Promise.all([ container.KyselyPg.destroy(), container.KyselyPgReadReplica.destroy(), - await models.close(), ]); }); @@ -205,33 +202,23 @@ describe('ModerationConfigService', () => { uid(), ); const { user, cleanup: userCleanup } = await createUser( - container.Sequelize, + container.KyselyPg, org.id, ); - const ruleId = uid(); - await container.Sequelize.Rule.create({ - id: ruleId, - orgId: org.id, - creatorId: user.id, + const rule = await createRule(container.KyselyPg, org.id, { + creator: user, name: 'getRuleByIdAndOrg fixture rule', - description: null, + ruleType: RuleType.USER, status: RuleStatus.DRAFT, - statusIfUnexpired: RuleStatus.DRAFT, - tags: [], conditionSet: minimalRuleConditionSet, - ruleType: RuleType.USER, - alarmStatus: RuleAlarmStatus.INSUFFICIENT_DATA, }); return { org, user, - ruleId, + ruleId: rule.id, async cleanup() { - await container.Sequelize.Rule.destroy({ - where: { id: ruleId }, - force: true, - }); + await rule.destroy(); await userCleanup(); await orgCleanup(); }, @@ -572,31 +559,31 @@ describe('ModerationConfigService', () => { uid(), ); try { - const contentType = - await sutWithPrimary.createContentType(fresh.org.id, { + const contentType = await sutWithPrimary.createContentType( + fresh.org.id, + { schema: dummySchema, description: null, name: faker.random.alphaNumeric(16), schemaFieldRoles: { displayName: 'fakeField' }, - }); + }, + ); const forUser = await sutWithPrimary.getActionsForItemType({ orgId: fresh.org.id, itemTypeId: fresh.defaultUserItemType.id, itemTypeKind: 'USER', }); - expect( - forUser.map((it) => it.actionType).sort(), - ).toEqual(['ENQUEUE_TO_MRT', 'ENQUEUE_TO_NCMEC'].sort()); + expect(forUser.map((it) => it.actionType).sort()).toEqual( + ['ENQUEUE_TO_MRT', 'ENQUEUE_TO_NCMEC'].sort(), + ); const forContent = await sutWithPrimary.getActionsForItemType({ orgId: fresh.org.id, itemTypeId: contentType.id, itemTypeKind: 'CONTENT', }); - expect( - forContent.map((it) => it.actionType).sort(), - ).toEqual( + expect(forContent.map((it) => it.actionType).sort()).toEqual( [ 'ENQUEUE_AUTHOR_TO_MRT', 'ENQUEUE_TO_MRT', @@ -663,9 +650,7 @@ describe('ModerationConfigService', () => { (it) => it.actionType === 'CUSTOM_ACTION', ); expect(customActions).toHaveLength(createdActions.length); - expect(customActions).toEqual( - expect.arrayContaining(createdActions), - ); + expect(customActions).toEqual(expect.arrayContaining(createdActions)); }); it('should round-trip a non-null customMrtApiParams value', async () => { @@ -862,16 +847,18 @@ describe('ModerationConfigService', () => { await new Promise((resolve) => setTimeout(resolve, 5)); - const result = await sutWithPrimary.updateCustomAction( - dummyOrgId, - { actionId: action.id, patch: {} }, - ); + const result = await sutWithPrimary.updateCustomAction(dummyOrgId, { + actionId: action.id, + patch: {}, + }); const after = await container.KyselyPg.selectFrom('public.actions') .select(['updated_at']) .where('id', '=', action.id) .executeTakeFirstOrThrow(); - expect(after.updated_at.getTime()).toBe(before.updated_at.getTime()); + expect(after.updated_at.getTime()).toBe( + before.updated_at.getTime(), + ); expect(result.id).toBe(action.id); }, ); @@ -1196,11 +1183,9 @@ describe('ModerationConfigService', () => { schemaFieldRoles: { displayName: 'fakeField' }, }, ); - const rule = await createRule(container.Sequelize, dummyOrgId); + const rule = await createRule(container.KyselyPg, dummyOrgId); - await container.KyselyPg.insertInto( - 'public.actions_and_item_types', - ) + await container.KyselyPg.insertInto('public.actions_and_item_types') .values({ action_id: action.id, item_type_id: itemType.id }) .execute(); await container.KyselyPg.insertInto('public.rules_and_actions') @@ -1375,7 +1360,7 @@ describe('ModerationConfigService', () => { describe('#getActionsForRuleId', () => { const testWithRuleAndAction = makeTestWithFixture(async () => { - const rule = await createRule(container.Sequelize, dummyOrgId); + const rule = await createRule(container.KyselyPg, dummyOrgId); const action = await sutWithPrimary.createAction(dummyOrgId, { name: faker.random.alphaNumeric(16), description: null, @@ -1463,7 +1448,7 @@ describe('ModerationConfigService', () => { ); const { user, cleanup: userCleanup } = await createUser( - container.Sequelize, + container.KyselyPg, org.id, ); diff --git a/server/services/moderationConfigService/moderationConfigService.ts b/server/services/moderationConfigService/moderationConfigService.ts index 53ca087..3167fd1 100644 --- a/server/services/moderationConfigService/moderationConfigService.ts +++ b/server/services/moderationConfigService/moderationConfigService.ts @@ -3,9 +3,9 @@ import _ from 'lodash'; import { type ReadonlyDeep } from 'type-fest'; import { type ConsumerDirectives } from '../../lib/cache/index.js'; -import type { Invoker } from '../../models/types/permissioning.js'; -import { type RuleErrorType, type LocationBankErrorType } from './errors.js'; +import type { Invoker } from '../userManagementService/index.js'; import { type ModerationConfigServicePg } from './dbTypes.js'; +import { type LocationBankErrorType, type RuleErrorType } from './errors.js'; import { type Action, type CustomAction, type Policy } from './index.js'; import ActionOperations, { type ActionErrorType, @@ -31,7 +31,7 @@ import { type UserItemType, } from './types/itemTypes.js'; import type { PolicyType } from './types/policies.js'; -import { type PlainRuleWithLatestVersion } from '../../models/rules/ruleTypes.js'; +import { type PlainRuleWithLatestVersion } from './types/rules.js'; export type ModerationConfigErrorType = | 'AttemptingToDeleteDefaultUserType' @@ -477,4 +477,3 @@ export class ModerationConfigService implements ReturnsModerationConfigTypes { await this.itemTypeOps.close(); } } - diff --git a/server/services/moderationConfigService/modules/PolicyOperations.ts b/server/services/moderationConfigService/modules/PolicyOperations.ts index 3ae32c6..6fc3ff6 100644 --- a/server/services/moderationConfigService/modules/PolicyOperations.ts +++ b/server/services/moderationConfigService/modules/PolicyOperations.ts @@ -2,10 +2,6 @@ import { type Kysely } from 'kysely'; import { type Writable } from 'type-fest'; import { uid } from 'uid'; -import { - UserPermission, - type Invoker, -} from '../../../models/types/permissioning.js'; import { CoopError, ErrorType, @@ -15,9 +11,12 @@ import { import { isUniqueViolationError, type FixKyselyRowCorrelation, - } from '../../../utils/kysely.js'; import { removeUndefinedKeys } from '../../../utils/misc.js'; +import { + UserPermission, + type Invoker, +} from '../../userManagementService/index.js'; import { type ModerationConfigServicePg } from '../dbTypes.js'; import { type Policy } from '../index.js'; import type { PolicyType } from '../types/policies.js'; @@ -124,7 +123,7 @@ export default class PolicyOperations { const out: Record = {}; for (const row of rows) { const { ruleId, ...policyFields } = row; - const policy = this.#dbResultToPolicy(policyFields as PolicyDbResult); + const policy = this.#dbResultToPolicy(policyFields); (out[ruleId] ??= []).push(policy); } return out; @@ -142,8 +141,7 @@ export default class PolicyOperations { .select(policyDbSelection) .where('org_id', '=', orgId) .where('id', '=', policyId); - const result = - (await query.executeTakeFirst()) as PolicyDbResult; + const result = (await query.executeTakeFirst()) as PolicyDbResult; return this.#dbResultToPolicy(result); } diff --git a/server/services/moderationConfigService/modules/RuleReadOperations.ts b/server/services/moderationConfigService/modules/RuleReadOperations.ts index 7e2a2e1..79b0d71 100644 --- a/server/services/moderationConfigService/modules/RuleReadOperations.ts +++ b/server/services/moderationConfigService/modules/RuleReadOperations.ts @@ -1,17 +1,17 @@ -import { type Kysely, sql } from 'kysely'; +import { sql, type Kysely } from 'kysely'; +import { getUtcDateOnlyString } from '../../../utils/time.js'; +import { type ModerationConfigServicePg } from '../dbTypes.js'; import { - type RuleAlarmStatus, RuleStatus, RuleType, type ConditionSet, + type RuleAlarmStatus, } from '../index.js'; -import { type ModerationConfigServicePg } from '../dbTypes.js'; -import { getUtcDateOnlyString } from '../../../utils/time.js'; import { - type PlainRuleWithLatestVersion, computeRuleStatusFromRow, -} from '../../../models/rules/ruleTypes.js'; + type PlainRuleWithLatestVersion, +} from '../types/rules.js'; const ruleSelect = [ 'r.id', @@ -63,7 +63,10 @@ function enabledQuotaWhere(today: string) { } function rowToPlainRuleWithLatest(row: RuleRow): PlainRuleWithLatestVersion { - const status = computeRuleStatusFromRow(row.expirationTime, row.statusIfUnexpired); + const status = computeRuleStatusFromRow( + row.expirationTime, + row.statusIfUnexpired, + ); const version = row.latestVersionString ?? ''; return { id: row.id, diff --git a/server/services/moderationConfigService/types/conditionResults.ts b/server/services/moderationConfigService/types/conditionResults.ts index 70d5fba..54d32a2 100644 --- a/server/services/moderationConfigService/types/conditionResults.ts +++ b/server/services/moderationConfigService/types/conditionResults.ts @@ -1,11 +1,11 @@ import { type ScalarType, type TaggedScalar } from '@roostorg/types'; -import { type TaggedItemData } from '../../../models/rules/item-type-fields.js'; import { type SerializableError } from '../../../utils/errors.js'; import { type NonEmptyArray, type WithUndefined, } from '../../../utils/typescript-types.js'; +import { type TaggedItemData } from './itemTypeFields.js'; import { type ConditionSet, type LeafCondition } from './rules.js'; /** diff --git a/server/models/rules/item-type-fields.ts b/server/services/moderationConfigService/types/itemTypeFields.ts similarity index 82% rename from server/models/rules/item-type-fields.ts rename to server/services/moderationConfigService/types/itemTypeFields.ts index fc90b92..eabc36e 100644 --- a/server/models/rules/item-type-fields.ts +++ b/server/services/moderationConfigService/types/itemTypeFields.ts @@ -4,9 +4,9 @@ import { type TaggedScalar, } from '@roostorg/types'; -import { type NormalizedItemData } from '../../services/itemProcessingService/index.js'; -import { type ItemType } from '../../services/moderationConfigService/index.js'; -import { hasOwn } from '../../utils/misc.js'; +import { hasOwn } from '../../../utils/misc.js'; +import { type NormalizedItemData } from '../../itemProcessingService/index.js'; +import { type ItemType } from './itemTypes.js'; export type TaggedItemData = Readonly<{ itemType: ItemType; diff --git a/server/models/types/locationArea.ts b/server/services/moderationConfigService/types/locationArea.ts similarity index 100% rename from server/models/types/locationArea.ts rename to server/services/moderationConfigService/types/locationArea.ts diff --git a/server/models/rules/matchingValues.ts b/server/services/moderationConfigService/types/matchingValues.ts similarity index 95% rename from server/models/rules/matchingValues.ts rename to server/services/moderationConfigService/types/matchingValues.ts index a52f5b5..96e3a7a 100644 --- a/server/models/rules/matchingValues.ts +++ b/server/services/moderationConfigService/types/matchingValues.ts @@ -1,6 +1,6 @@ import { type ReadonlyDeep } from 'type-fest'; -import { type LocationArea } from '../types/locationArea.js'; +import { type LocationArea } from './locationArea.js'; export enum MatchingValueType { STRING = 'STRING', diff --git a/server/services/moderationConfigService/types/rules.ts b/server/services/moderationConfigService/types/rules.ts index cc4a609..71fb16a 100644 --- a/server/services/moderationConfigService/types/rules.ts +++ b/server/services/moderationConfigService/types/rules.ts @@ -1,6 +1,5 @@ import { makeEnumLike } from '@roostorg/types'; -import { type MatchingValues } from '../../../models/rules/matchingValues.js'; import { type JsonOf } from '../../../utils/encoding.js'; import { type NonEmptyArray, @@ -12,6 +11,7 @@ import { type SignalArgsByType, type SignalType, } from '../../signalsService/index.js'; +import { type MatchingValues } from './matchingValues.js'; export const RuleStatus = makeEnumLike([ 'BACKGROUND', @@ -32,6 +32,48 @@ export enum RuleAlarmStatus { INSUFFICIENT_DATA = 'INSUFFICIENT_DATA', } +export type RuleLatestVersionRow = { + ruleId: string; + version: string; +}; + +/** Rule row fields shared by the rule engine (no GraphQL resolver methods). */ +export type PlainRuleWithLatestVersion = { + id: string; + name: string; + description: string | null; + statusIfUnexpired: Exclude; + status: RuleStatus; + tags: string[]; + maxDailyActions: number | null; + dailyActionsRun: number; + lastActionDate: string | null; + createdAt: Date; + updatedAt: Date; + orgId: string; + creatorId: string; + expirationTime: Date | null; + conditionSet: ConditionSet; + alarmStatus: RuleAlarmStatus; + alarmStatusSetAt: Date; + ruleType: RuleType; + parentId: string | null; + latestVersion: RuleLatestVersionRow; +}; + +export function computeRuleStatusFromRow( + expirationTime: Date | null, + statusIfUnexpired: Exclude, +): RuleStatus { + if (expirationTime && expirationTime.valueOf() < Date.now()) { + return RuleStatus.EXPIRED; + } + return statusIfUnexpired; +} + +/** @deprecated Use {@link PlainRuleWithLatestVersion} directly. */ +export type RuleWithLatestVersion = PlainRuleWithLatestVersion; + // TODO: we really shouldn't be storing the signal name in the condition, as // that's derived state liable to come out of sync. Instead, we just wanna store // the SignalId, plus the args that the condition feeds to the signal (which, @@ -161,4 +203,3 @@ export type LeafCondition = matchingValues?: MatchingValues | null; threshold?: string | number | null; }; - diff --git a/server/services/orgAwareSignalExecutionService/signalExecutionService.ts b/server/services/orgAwareSignalExecutionService/signalExecutionService.ts index 89ad951..4872a13 100644 --- a/server/services/orgAwareSignalExecutionService/signalExecutionService.ts +++ b/server/services/orgAwareSignalExecutionService/signalExecutionService.ts @@ -4,9 +4,6 @@ import stringify from 'safe-stable-stringify'; import { type ReadonlyDeep } from 'type-fest'; import { inject } from '../../iocContainer/utils.js'; -import { type PolicyActionPenalties } from '../policyActionPenalties.js'; -import { type MatchingValues } from '../../models/rules/matchingValues.js'; -import { type LocationArea } from '../../models/types/locationArea.js'; import { jsonStringify } from '../../utils/encoding.js'; import { CoopError, ErrorType } from '../../utils/errors.js'; import type SafeTracer from '../../utils/SafeTracer.js'; @@ -14,6 +11,12 @@ import { isNonEmptyArray, type NonEmptyArray, } from '../../utils/typescript-types.js'; +import { type HashBank } from '../hmaService/index.js'; +import { + type LocationArea, + type MatchingValues, +} from '../moderationConfigService/index.js'; +import { type PolicyActionPenalties } from '../policyActionPenalties.js'; import { type SignalId, type SignalInput, @@ -24,7 +27,6 @@ import { type SignalType, type SignalTypesToRunInputTypes, } from '../signalsService/index.js'; -import { type HashBank } from '../hmaService/index.js'; const { memoize } = _; @@ -68,8 +70,13 @@ export default inject( orgId: string; bankId: string; }) => Promise, - getPolicyActionPenalties: (orgId: string) => Promise>, - getImageBank: (input: { orgId: string; bankId: string }) => Promise, + getPolicyActionPenalties: ( + orgId: string, + ) => Promise>, + getImageBank: (input: { + orgId: string; + bankId: string; + }) => Promise, signalsService: SignalsService, tracer: SafeTracer, ) => @@ -112,12 +119,9 @@ export default inject( bankIds: readonly string[], ) => Promise.all( - bankIds.map(async (bankId) => - getImageBank({ orgId, bankId }), - ), - ).then( - (bankResults) => - bankResults.filter((it): it is HashBank => it !== null), + bankIds.map(async (bankId) => getImageBank({ orgId, bankId })), + ).then((bankResults) => + bankResults.filter((it): it is HashBank => it !== null), ); // For running a signal for now with caching, we use a memoized function @@ -184,7 +188,8 @@ async function runSignal( return undefined; } - const { locationBankIds, textBankIds, imageBankIds } = matchingValues ?? {}; + const { locationBankIds, textBankIds, imageBankIds } = + matchingValues ?? {}; // A condition can have both strings and text banks as matching // values simultaneously (and same with locations and location @@ -194,9 +199,16 @@ async function runSignal( const scalarMatchingValues = matchingValues?.strings ?? matchingValues?.locations ?? []; - let matchingValuesFromBanks: readonly (string | ReadonlyDeep | HashBank)[] = []; + let matchingValuesFromBanks: readonly ( + | string + | ReadonlyDeep + | HashBank + )[] = []; if (textBankIds?.length) { - matchingValuesFromBanks = await textBanksStringsLoader(orgId, textBankIds); + matchingValuesFromBanks = await textBanksStringsLoader( + orgId, + textBankIds, + ); } else if (locationBankIds?.length) { matchingValuesFromBanks = await Promise.all( locationBankIds.map(async (id) => locationsLoader(id)), @@ -208,7 +220,10 @@ async function runSignal( .flat(), ); } else if (imageBankIds?.length) { - matchingValuesFromBanks = await imageBanksLoader(orgId, imageBankIds); + matchingValuesFromBanks = await imageBanksLoader( + orgId, + imageBankIds, + ); } const loadedMatchingValues = [ diff --git a/server/services/reportingService/ReportingRules.ts b/server/services/reportingService/ReportingRules.ts index 458f276..a5fb947 100644 --- a/server/services/reportingService/ReportingRules.ts +++ b/server/services/reportingService/ReportingRules.ts @@ -1,9 +1,9 @@ -import { type ConsumerDirectives } from '../../lib/cache/index.js'; import { makeEnumLike } from '@roostorg/types'; import { sql, type Kysely, type Transaction } from 'kysely'; import { type ReadonlyDeep } from 'type-fest'; import { v1 as uuidv1 } from 'uuid'; +import { type ConsumerDirectives } from '../../lib/cache/index.js'; import { cached } from '../../utils/caching.js'; import { filterNullOrUndefined } from '../../utils/collections.js'; import { @@ -162,7 +162,7 @@ export default class ReportingRules { return { ...reportingRule, - itemTypeIds: itemTypeIds as string[], + itemTypeIds, actionIds, policyIds, }; diff --git a/server/services/ruleAnomalyDetectionService/detectRulePassRateAnomaliesJob.test.ts b/server/services/ruleAnomalyDetectionService/detectRulePassRateAnomaliesJob.test.ts index 64075b0..d6ab826 100644 --- a/server/services/ruleAnomalyDetectionService/detectRulePassRateAnomaliesJob.test.ts +++ b/server/services/ruleAnomalyDetectionService/detectRulePassRateAnomaliesJob.test.ts @@ -86,12 +86,9 @@ describe('Detect Rule Anomalies', () => { beforeAll(async () => { /* eslint-disable functional/immutable-data */ - const { - Sequelize: models, - ModerationConfigService, - ApiKeyService, - KyselyPg, - } = (await getBottle()).container; + const { ModerationConfigService, ApiKeyService, KyselyPg } = ( + await getBottle() + ).container; // make some fake rules (w/ stable ids so we can match them in a snapshot) // in different initial alarm statuses, to test all 9 combinations [i.e., @@ -111,56 +108,60 @@ describe('Detect Rule Anomalies', () => { undefined, { onCallAlertEmail: 'test@gmail.com' }, ); - const { user: ruleOwner } = await createUser(models, org.id, { - id: 'cb34377bcc3', - }); - const { user: ruleOwner2 } = await createUser(models, org.id, { - id: 'cb34377bcc4', - }); + const { user: ruleOwner, cleanup: ruleOwnerCleanup } = await createUser( + KyselyPg, + org.id, + { id: 'cb34377bcc3' }, + ); + const { user: ruleOwner2, cleanup: ruleOwner2Cleanup } = await createUser( + KyselyPg, + org.id, + { id: 'cb34377bcc4' }, + ); const fakeRules = await Promise.all([ - createRule(models, org.id, { + createRule(KyselyPg, org.id, { alarmStatus: RuleAlarmStatus.ALARM, id: '9d237a650c1', creator: ruleOwner, }), - createRule(models, org.id, { + createRule(KyselyPg, org.id, { alarmStatus: RuleAlarmStatus.ALARM, id: '386da8abc3b', creator: ruleOwner, }), - createRule(models, org.id, { + createRule(KyselyPg, org.id, { alarmStatus: RuleAlarmStatus.ALARM, id: 'd237a650c13', creator: ruleOwner, }), - createRule(models, org.id, { + createRule(KyselyPg, org.id, { alarmStatus: RuleAlarmStatus.OK, id: '86da8abc3b6', creator: ruleOwner, }), - createRule(models, org.id, { + createRule(KyselyPg, org.id, { alarmStatus: RuleAlarmStatus.OK, id: 'fdb4ee86f93', creator: ruleOwner, }), - createRule(models, org.id, { + createRule(KyselyPg, org.id, { alarmStatus: RuleAlarmStatus.OK, id: '237a650c134', creator: ruleOwner, }), - createRule(models, org2.id, { + createRule(KyselyPg, org2.id, { alarmStatus: RuleAlarmStatus.INSUFFICIENT_DATA, id: 'db4ee86f938', creator: ruleOwner2, }), - createRule(models, org2.id, { + createRule(KyselyPg, org2.id, { alarmStatus: RuleAlarmStatus.INSUFFICIENT_DATA, id: '37a650c1342', creator: ruleOwner2, }), - createRule(models, org2.id, { + createRule(KyselyPg, org2.id, { alarmStatus: RuleAlarmStatus.INSUFFICIENT_DATA, id: 'b4ee86f9386', creator: ruleOwner2, @@ -210,10 +211,9 @@ describe('Detect Rule Anomalies', () => { deleteMockData = async () => { await Promise.all(fakeRules.map(async (it) => it.destroy())); - await Promise.all([ruleOwner.destroy(), ruleOwner2.destroy()]); + await Promise.all([ruleOwnerCleanup(), ruleOwner2Cleanup()]); await orgCleanup(); await org2Cleanup(); - await models.sequelize.close(); }; /* eslint-enable functional/immutable-data */ }); diff --git a/server/services/signalsService/SignalsService.ts b/server/services/signalsService/SignalsService.ts index 6234295..cdc5348 100644 --- a/server/services/signalsService/SignalsService.ts +++ b/server/services/signalsService/SignalsService.ts @@ -2,12 +2,12 @@ import { type ReadonlyDeep, type Simplify } from 'type-fest'; import { inject, type Dependencies } from '../../iocContainer/index.js'; import { type ConsumerDirectives } from '../../lib/cache/index.js'; -import { isTaggedItemData } from '../../models/rules/item-type-fields.js'; import { jsonStringify } from '../../utils/encoding.js'; import { CoopError, ErrorType, makeNotFoundError } from '../../utils/errors.js'; import { __throw, assertUnreachable } from '../../utils/misc.js'; import { type CollapseCases } from '../../utils/typescript-types.js'; import { getIntegrationRegistry } from '../integrationRegistry/index.js'; +import { isTaggedItemData } from '../moderationConfigService/index.js'; import { instantiateBuiltInSignals } from './helpers/instantiateBuiltInSignals.js'; import { loadPluginSignals } from './helpers/loadPluginSignals.js'; import { makeCachedCredentialGetters } from './helpers/makeCachedCredentialsGetters.js'; @@ -294,8 +294,8 @@ export class SignalsService { 'value' in input.value ? input.value.type : isTaggedItemData(input.value) - ? 'FULL_ITEM' - : assertUnreachable(input.value, 'Unknown signal input...'); + ? 'FULL_ITEM' + : assertUnreachable(input.value, 'Unknown signal input...'); if (!signal.eligibleInputs.includes(signalInputType)) { throw new CoopError({ @@ -311,16 +311,13 @@ export class SignalsService { }); } - // When we look up the signal with `#getSignalInstance` above, TS thinks - // (correctly) that we could get any signal back, but it loses track of the - // relationship between the signal we get back and the type of the `input`. - // So, when we call `signal.run()` it tries to check that the `input` is a - // type that _any_ signal would accept. However, there is no type that'll - // work as every signal's input (e.g, because some signals demand text input - // and some demand an image reference), so it'll only let us make this call - // if we cast the input to `never` (which is the intersection of every - // signal's input type). - return signal.run(input as never) as Promise< + // `#getSignalInstance` returns a signal whose input type is the union of + // every signal's input, so TS loses the relationship between the looked-up + // signal and the caller's `input`. The cast on the return value restores + // the expected output type for this signal type `T`; the runtime call is + // sound because we already validated `signalInputType` against + // `signal.eligibleInputs` above. + return signal.run(input) as Promise< Awaited >; } diff --git a/server/services/signalsService/signals/GeoContainedWithinSignal.ts b/server/services/signalsService/signals/GeoContainedWithinSignal.ts index dce3b4f..17854ca 100644 --- a/server/services/signalsService/signals/GeoContainedWithinSignal.ts +++ b/server/services/signalsService/signals/GeoContainedWithinSignal.ts @@ -1,8 +1,10 @@ import { ScalarTypes } from '@roostorg/types'; import Geohash from 'latlon-geohash'; -import { isLocationArea } from '../../../models/rules/matchingValues.js'; -import { type LocationArea } from '../../../models/types/locationArea.js'; +import { + isLocationArea, + type LocationArea, +} from '../../moderationConfigService/index.js'; import { SignalPricingStructure as SignalPricingStructureType } from '../types/SignalPricingStructure.js'; import { SignalType } from '../types/SignalType.js'; import SignalBase, { type SignalInput } from './SignalBase.js'; diff --git a/server/services/signalsService/signals/SignalBase.ts b/server/services/signalsService/signals/SignalBase.ts index 7a51a41..1f37cd7 100644 --- a/server/services/signalsService/signals/SignalBase.ts +++ b/server/services/signalsService/signals/SignalBase.ts @@ -6,11 +6,11 @@ import { } from '@roostorg/types'; import { type ReadonlyDeep, type Simplify } from 'type-fest'; -import { type PolicyActionPenalties } from '../../policyActionPenalties.js'; -import { type TaggedItemData } from '../../../models/rules/item-type-fields.js'; import { type CoopError } from '../../../utils/errors.js'; import { type Language } from '../../../utils/language.js'; import { type NonEmptyArray } from '../../../utils/typescript-types.js'; +import { type TaggedItemData } from '../../moderationConfigService/index.js'; +import { type PolicyActionPenalties } from '../../policyActionPenalties.js'; import { type Integration } from '../types/Integration.js'; import { type RecommendedThresholds } from '../types/RecommendedThresholds.js'; import { @@ -70,8 +70,8 @@ export type SignalInput< value: T extends 'FULL_ITEM' ? TaggedItemData : T extends ScalarType - ? TaggedScalar - : TaggedItemData | TaggedScalar>; + ? TaggedScalar + : TaggedItemData | TaggedScalar>; matchingValues: | NonEmptyArray | (NeedsMatchingValues extends true ? never : undefined); diff --git a/server/services/signalsService/signals/aggregation/AggregationSignal.test.ts b/server/services/signalsService/signals/aggregation/AggregationSignal.test.ts index 7062b0c..66b631f 100644 --- a/server/services/signalsService/signals/aggregation/AggregationSignal.test.ts +++ b/server/services/signalsService/signals/aggregation/AggregationSignal.test.ts @@ -21,7 +21,6 @@ describe('AggregationSignal', () => { const { server, deps, shutdown } = await makeMockedServer(); const { - Sequelize: models, ModerationConfigService, AggregationsService, RuleAPIDataSource, @@ -35,7 +34,7 @@ describe('AggregationSignal', () => { uid(), ); - const { user, cleanup: userCleanup } = await createUser(models, org.id, {}); + const { user, cleanup: userCleanup } = await createUser(KyselyPg, org.id); const { itemTypes, cleanup: itemTypesCleanup } = await createContentItemTypes({ diff --git a/server/services/signalsService/types/SignalId.ts b/server/services/signalsService/types/SignalId.ts index 7c70237..4146a3f 100644 --- a/server/services/signalsService/types/SignalId.ts +++ b/server/services/signalsService/types/SignalId.ts @@ -78,7 +78,7 @@ export function isSignalId(it: unknown): it is SignalId { typeof it === 'object' && it !== null && 'type' in it && - typeof (it as { type: unknown }).type === 'string' && + typeof it.type === 'string' && (it.type === SignalType.CUSTOM ? 'id' in it && isNonEmptyString(it.id) : true) diff --git a/server/services/signingKeyPairService/postgresSigningKeyPairStorage.ts b/server/services/signingKeyPairService/postgresSigningKeyPairStorage.ts index e850c61..3ffdf5b 100644 --- a/server/services/signingKeyPairService/postgresSigningKeyPairStorage.ts +++ b/server/services/signingKeyPairService/postgresSigningKeyPairStorage.ts @@ -52,11 +52,10 @@ export class PostgresSigningKeyPairStorage implements SigningKeyPairStorage { org_id: keyId.orgId, key_data: jsonStringify(keyData), }) - .onConflict((oc) => oc - .column('org_id') - .doUpdateSet({ + .onConflict((oc) => + oc.column('org_id').doUpdateSet({ key_data: jsonStringify(keyData), - }) + }), ) .execute(); } @@ -81,10 +80,8 @@ export class PostgresSigningKeyPairStorage implements SigningKeyPairStorage { const keyData: JWTCryptoKeyPairWithAlgorithm = typeof result.key_data === 'string' - ? jsonParse( - result.key_data as JsonOf, - ) - : (result.key_data as JWTCryptoKeyPairWithAlgorithm); + ? jsonParse(result.key_data as JsonOf) + : result.key_data; const { privateKeyWithAlgorithm, publicKeyWithAlgorithm } = keyData; return { diff --git a/server/services/userManagementService/dbTypes.ts b/server/services/userManagementService/dbTypes.ts index ce93844..f26a8e3 100644 --- a/server/services/userManagementService/dbTypes.ts +++ b/server/services/userManagementService/dbTypes.ts @@ -1,12 +1,12 @@ import type { ColumnType, GeneratedAlways } from 'kysely'; -import type { UserRole } from '../../models/types/permissioning.js'; import { type CoreAppTablesPg } from '../coreAppTables.js'; import type { DecisionCountsInput, JobCreationsInput, } from '../manualReviewToolService/modules/DecisionAnalytics.js'; import { type OrgSettingsPg } from '../orgSettingsService/index.js'; +import type { UserRole } from './permissioning.js'; export type MrtChartConfig = { title: string; diff --git a/server/services/userManagementService/index.ts b/server/services/userManagementService/index.ts index a37c59f..7b52fe5 100644 --- a/server/services/userManagementService/index.ts +++ b/server/services/userManagementService/index.ts @@ -4,3 +4,11 @@ export { type UserManagementService, } from './userManagementService.js'; export { hashPassword, passwordMatchesHash } from './utils.js'; +export { + Invoker, + UserPermission, + UserPermissionsForRole, + UserRole, + getPermissionsForRole, + hasPermission, +} from './permissioning.js'; diff --git a/server/models/types/permissioning.ts b/server/services/userManagementService/permissioning.ts similarity index 100% rename from server/models/types/permissioning.ts rename to server/services/userManagementService/permissioning.ts diff --git a/server/services/userManagementService/userManagementService.ts b/server/services/userManagementService/userManagementService.ts index cb8d681..0886212 100644 --- a/server/services/userManagementService/userManagementService.ts +++ b/server/services/userManagementService/userManagementService.ts @@ -3,11 +3,6 @@ import { type Kysely } from 'kysely'; import type { Dependencies } from '../../iocContainer/index.js'; import { inject } from '../../iocContainer/utils.js'; -import { - UserPermission, - type Invoker, - type UserRole, -} from '../../models/types/permissioning.js'; import { makeNotFoundError, makeUnauthorizedError, @@ -17,6 +12,11 @@ import { HOUR_MS } from '../../utils/time.js'; import { CoopEmailAddress } from '../sendEmailService/sendEmailService.js'; import type { MrtChartConfig } from './dbTypes.js'; import type { UserManagementPg } from './index.js'; +import { + UserPermission, + type Invoker, + type UserRole, +} from './permissioning.js'; import { hashPassword } from './utils.js'; class UserManagementService { diff --git a/server/storage/dataWarehouse/DataWarehouseFactory.ts b/server/storage/dataWarehouse/DataWarehouseFactory.ts index d1e1b57..fd2799a 100644 --- a/server/storage/dataWarehouse/DataWarehouseFactory.ts +++ b/server/storage/dataWarehouse/DataWarehouseFactory.ts @@ -3,10 +3,11 @@ */ /* eslint-disable max-classes-per-file */ +import { type Kysely } from 'kysely'; + import { ClickhouseAnalyticsAdapter as ClickhouseAnalyticsPlugin, NoOpAnalyticsAdapter, - type AnalyticsEventInput, type IAnalyticsAdapter, } from '../../plugins/analytics/index.js'; import { @@ -27,7 +28,6 @@ import { type DataWarehouseProvider as IDataWarehouseProvider, type TransactionFunction, } from './IDataWarehouse.js'; -import { type Kysely } from 'kysely'; import type { AnalyticsSchema, BulkWriteConfig, @@ -119,7 +119,7 @@ class WarehouseAdapterBridge implements IDataWarehouse { return this.adapter.transaction(async (warehouseQuery) => { return fn(async (statement, parameters = []) => { const rows = await warehouseQuery(statement, parameters); - return Array.from(rows) as unknown[]; + return Array.from(rows); }); }); } @@ -154,7 +154,7 @@ class AnalyticsAdapterBridge implements IDataWarehouseAnalytics { ): Promise { await this.adapter.writeEvents( tableName, - rows as readonly AnalyticsEventInput[], + rows, config?.batchTimeout !== undefined ? { batchTimeout: config.batchTimeout } : undefined, @@ -263,8 +263,7 @@ export class DataWarehouseFactory { config: DataWarehouseConfig, dialect?: IDataWarehouseDialect, ): IDataWarehouseAnalytics { - const analyticsProvider = - config.analyticsProvider ?? (config.provider as AnalyticsProvider); + const analyticsProvider = config.analyticsProvider ?? config.provider; switch (analyticsProvider) { case 'noop': diff --git a/server/test/fixtureHelpers/createMrtQueue.ts b/server/test/fixtureHelpers/createMrtQueue.ts index eced985..551ca81 100644 --- a/server/test/fixtureHelpers/createMrtQueue.ts +++ b/server/test/fixtureHelpers/createMrtQueue.ts @@ -1,5 +1,5 @@ import type { Dependencies } from '../../iocContainer/index.js'; -import { UserPermission } from '../../models/types/permissioning.js'; +import { UserPermission } from '../../services/userManagementService/index.js'; export default async function (opts: { orgId: string; diff --git a/server/test/fixtureHelpers/createPolicy.ts b/server/test/fixtureHelpers/createPolicy.ts index 331d537..94718e1 100644 --- a/server/test/fixtureHelpers/createPolicy.ts +++ b/server/test/fixtureHelpers/createPolicy.ts @@ -1,7 +1,7 @@ import { faker } from '@faker-js/faker'; import { type Dependencies } from '../../iocContainer/index.js'; -import { UserPermission } from '../../models/types/permissioning.js'; +import { UserPermission } from '../../services/userManagementService/index.js'; export default async function (opts: { moderationConfigService: Dependencies['ModerationConfigService']; diff --git a/server/test/fixtureHelpers/createRule.ts b/server/test/fixtureHelpers/createRule.ts index 67698af..e2fc9fb 100644 --- a/server/test/fixtureHelpers/createRule.ts +++ b/server/test/fixtureHelpers/createRule.ts @@ -1,57 +1,103 @@ +import { type Kysely } from 'kysely'; import { uid } from 'uid'; -import { type Dependencies } from '../../iocContainer/index.js'; -import { type User } from '../../models/UserModel.js'; +import { + kyselyCreateRule, + kyselyDeleteRule, +} from '../../graphql/datasources/ruleKyselyPersistence.js'; +import { type CombinedPg } from '../../services/combinedDbTypes.js'; import { ConditionConjunction, + RuleAlarmStatus, RuleStatus, - type RuleAlarmStatus, + RuleType, + type ConditionSet, } from '../../services/moderationConfigService/index.js'; import { SignalType } from '../../services/signalsService/index.js'; import { jsonStringify } from '../../utils/encoding.js'; import { logErrorAndThrow } from '../utils.js'; import createUser from './createUser.js'; -export default async function ( - models: Dependencies['Sequelize'], +const DEFAULT_CONDITION_SET: ConditionSet = { + conditions: [ + { + input: { + type: 'CONTENT_FIELD', + name: 'text', + contentTypeId: '6f8f8612205', + }, + signal: { + id: jsonStringify({ type: SignalType.TEXT_MATCHING_CONTAINS_TEXT }), + type: SignalType.TEXT_MATCHING_CONTAINS_TEXT, + }, + matchingValues: { strings: ['test'] }, + }, + ], + conjunction: ConditionConjunction.AND, +}; + +export default async function createRule( + db: Kysely, orgId: string, extra: { creatorId?: string; - creator?: User; + creator?: { id: string }; id?: string; alarmStatus?: RuleAlarmStatus; name?: string; + ruleType?: RuleType; + status?: RuleStatus; + conditionSet?: ConditionSet; } = {}, ) { - const finalId = extra.id ?? uid(); - return models.Rule.create({ - id: finalId, - name: extra.name ?? `Dummy_Rule_Name_${finalId}`, - status: RuleStatus.LIVE, - alarmStatus: extra.alarmStatus, + const ruleId = extra.id ?? uid(); + const name = extra.name ?? `Dummy_Rule_Name_${ruleId}`; + const ruleType = extra.ruleType ?? RuleType.CONTENT; + const status = extra.status ?? RuleStatus.LIVE; + const creatorId = + extra.creator?.id ?? + extra.creatorId ?? + (await createUser(db, orgId)).user.id; + + await kyselyCreateRule(db, { + id: ruleId, + name, + description: null, + status, + conditionSet: extra.conditionSet ?? DEFAULT_CONDITION_SET, tags: [], + maxDailyActions: null, + expirationTime: null, + creatorId, orgId, - ruleType: 'CONTENT', - creatorId: - extra.creator?.id ?? - extra.creatorId ?? - (await createUser(models, orgId)).user.id, - conditionSet: { - conditions: [ - { - input: { - type: 'CONTENT_FIELD', - name: 'text', - contentTypeId: '6f8f8612205', - }, - signal: { - id: jsonStringify({ type: SignalType.TEXT_MATCHING_CONTAINS_TEXT }), - type: SignalType.TEXT_MATCHING_CONTAINS_TEXT, - }, - matchingValues: { strings: ['test'] }, - }, - ], - conjunction: ConditionConjunction.AND, - }, + ruleType, + parentId: null, + actionIds: [], + policyIds: [], + contentTypeIds: [], }).catch(logErrorAndThrow); + + // `kyselyCreateRule` always seeds `INSUFFICIENT_DATA`; patch when callers + // seed a different alarm status (e.g. anomaly-detection snapshot fixtures). + const alarmStatus = extra.alarmStatus ?? RuleAlarmStatus.INSUFFICIENT_DATA; + if (alarmStatus !== RuleAlarmStatus.INSUFFICIENT_DATA) { + await db + .updateTable('public.rules') + .set({ alarm_status: alarmStatus, alarm_status_set_at: new Date() }) + .where('id', '=', ruleId) + .where('org_id', '=', orgId) + .execute(); + } + + return { + id: ruleId, + orgId, + creatorId, + name, + alarmStatus, + statusIfUnexpired: status, + async destroy() { + await kyselyDeleteRule(db, ruleId, orgId); + }, + }; } diff --git a/server/test/fixtureHelpers/createUser.ts b/server/test/fixtureHelpers/createUser.ts index ede6cb4..4d6e976 100644 --- a/server/test/fixtureHelpers/createUser.ts +++ b/server/test/fixtureHelpers/createUser.ts @@ -1,28 +1,50 @@ import { faker } from '@faker-js/faker'; +import { type Kysely } from 'kysely'; import { uid } from 'uid'; -import { type Dependencies } from '../../iocContainer/index.js'; +import { + kyselyUserDeleteById, + kyselyUserInsert, +} from '../../graphql/datasources/userKyselyPersistence.js'; +import { type CombinedPg } from '../../services/combinedDbTypes.js'; +import { type LoginMethod } from '../../services/coreAppTables.js'; +import { UserRole } from '../../services/userManagementService/index.js'; import { logErrorAndThrow } from '../utils.js'; -export default async function ( - models: Dependencies['Sequelize'], +// SAML-only by default keeps the `password_null_when_not_present` CHECK +// satisfied without a placeholder password. +const DEFAULT_LOGIN_METHODS: readonly LoginMethod[] = ['saml']; + +export default async function createUser( + db: Kysely, orgId: string, - extra: { id?: string } = {}, + extra: { + id?: string; + role?: UserRole; + loginMethods?: readonly LoginMethod[]; + password?: string | null; + } = {}, ) { - const user = await models.User.create({ + const userId = extra.id ?? uid(); + const loginMethods = extra.loginMethods ?? DEFAULT_LOGIN_METHODS; + const password = extra.password ?? null; + + const user = await kyselyUserInsert({ + db, + id: userId, orgId, - id: extra.id ?? uid(), email: faker.internet.email(), - password: '', + password, firstName: faker.name.firstName(), lastName: faker.name.lastName(), - loginMethods: ['password'], + role: extra.role ?? UserRole.ADMIN, + loginMethods, }).catch(logErrorAndThrow); return { user, async cleanup() { - await user.destroy(); + await kyselyUserDeleteById(db, userId); }, }; } diff --git a/server/test/fixtureHelpers/fixtureHelpers.test.ts b/server/test/fixtureHelpers/fixtureHelpers.test.ts new file mode 100644 index 0000000..b4526e8 --- /dev/null +++ b/server/test/fixtureHelpers/fixtureHelpers.test.ts @@ -0,0 +1,178 @@ +import { uid } from 'uid'; + +import { kyselyUserFindById } from '../../graphql/datasources/userKyselyPersistence.js'; +import { + RuleAlarmStatus, + RuleStatus, + RuleType, +} from '../../services/moderationConfigService/index.js'; +import { UserRole } from '../../services/userManagementService/index.js'; +import { makeMockedServer } from '../setupMockedServer.js'; +import { makeTestWithFixture } from '../utils.js'; +import createOrg from './createOrg.js'; +import createRule from './createRule.js'; +import createUser from './createUser.js'; + +describe('fixtureHelpers', () => { + const testWithOrg = makeTestWithFixture(async () => { + const { deps, shutdown } = await makeMockedServer(); + const { org, cleanup: orgCleanup } = await createOrg( + { + KyselyPg: deps.KyselyPg, + ModerationConfigService: deps.ModerationConfigService, + ApiKeyService: deps.ApiKeyService, + }, + uid(), + ); + return { + deps, + org, + async cleanup() { + await orgCleanup(); + await shutdown(); + }, + }; + }); + + describe('createUser', () => { + testWithOrg( + 'defaults: SAML-only loginMethods, ADMIN role, null password, override id honored', + async ({ deps, org }) => { + const overrideId = uid(); + const { user, cleanup } = await createUser(deps.KyselyPg, org.id, { + id: overrideId, + }); + try { + expect(user).toMatchObject({ + id: overrideId, + orgId: org.id, + role: UserRole.ADMIN, + loginMethods: ['saml'], + password: null, + }); + } finally { + await cleanup(); + } + }, + ); + + testWithOrg('cleanup() actually deletes the row', async ({ deps, org }) => { + const { user, cleanup } = await createUser(deps.KyselyPg, org.id); + expect(await kyselyUserFindById(deps.KyselyPg, user.id)).toBeDefined(); + await cleanup(); + expect(await kyselyUserFindById(deps.KyselyPg, user.id)).toBeUndefined(); + }); + }); + + describe('createRule', () => { + testWithOrg( + 'defaults: CONTENT type, LIVE status, INSUFFICIENT_DATA alarm, override id/name honored', + async ({ deps, org }) => { + const overrideId = uid(); + const overrideName = `RuleFixture_${overrideId}`; + const rule = await createRule(deps.KyselyPg, org.id, { + id: overrideId, + name: overrideName, + }); + try { + expect(rule).toMatchObject({ + id: overrideId, + orgId: org.id, + name: overrideName, + alarmStatus: RuleAlarmStatus.INSUFFICIENT_DATA, + statusIfUnexpired: RuleStatus.LIVE, + }); + + const row = await deps.KyselyPg.selectFrom('public.rules') + .select(['rule_type', 'status_if_unexpired', 'alarm_status']) + .where('id', '=', overrideId) + .executeTakeFirstOrThrow(); + expect(row.rule_type).toBe(RuleType.CONTENT); + expect(row.status_if_unexpired).toBe(RuleStatus.LIVE); + expect(row.alarm_status).toBe(RuleAlarmStatus.INSUFFICIENT_DATA); + } finally { + await rule.destroy(); + } + }, + ); + + testWithOrg( + 'extra.alarmStatus !== INSUFFICIENT_DATA triggers the follow-up UPDATE', + async ({ deps, org }) => { + const rule = await createRule(deps.KyselyPg, org.id, { + alarmStatus: RuleAlarmStatus.ALARM, + }); + try { + expect(rule.alarmStatus).toBe(RuleAlarmStatus.ALARM); + const row = await deps.KyselyPg.selectFrom('public.rules') + .select('alarm_status') + .where('id', '=', rule.id) + .executeTakeFirstOrThrow(); + expect(row.alarm_status).toBe(RuleAlarmStatus.ALARM); + } finally { + await rule.destroy(); + } + }, + ); + + testWithOrg( + 'extra.ruleType: USER persists rule_type=USER (and skips item-type junctions)', + async ({ deps, org }) => { + const rule = await createRule(deps.KyselyPg, org.id, { + ruleType: RuleType.USER, + }); + try { + const row = await deps.KyselyPg.selectFrom('public.rules') + .select('rule_type') + .where('id', '=', rule.id) + .executeTakeFirstOrThrow(); + expect(row.rule_type).toBe(RuleType.USER); + + const junctions = await deps.KyselyPg.selectFrom( + 'public.rules_and_item_types', + ) + .select('item_type_id') + .where('rule_id', '=', rule.id) + .execute(); + expect(junctions).toEqual([]); + } finally { + await rule.destroy(); + } + }, + ); + + testWithOrg('destroy() removes the row', async ({ deps, org }) => { + const rule = await createRule(deps.KyselyPg, org.id); + const before = await deps.KyselyPg.selectFrom('public.rules') + .select('id') + .where('id', '=', rule.id) + .executeTakeFirst(); + expect(before).toBeDefined(); + + await rule.destroy(); + + const after = await deps.KyselyPg.selectFrom('public.rules') + .select('id') + .where('id', '=', rule.id) + .executeTakeFirst(); + expect(after).toBeUndefined(); + }); + + testWithOrg( + 'auto-creates a creator user when none is supplied', + async ({ deps, org }) => { + const rule = await createRule(deps.KyselyPg, org.id); + try { + const creator = await kyselyUserFindById( + deps.KyselyPg, + rule.creatorId, + ); + expect(creator).toBeDefined(); + expect(creator?.orgId).toBe(org.id); + } finally { + await rule.destroy(); + } + }, + ); + }); +}); diff --git a/server/test/setupMockedServer.ts b/server/test/setupMockedServer.ts index 59cf319..644c359 100644 --- a/server/test/setupMockedServer.ts +++ b/server/test/setupMockedServer.ts @@ -51,9 +51,7 @@ export async function getBottleContainerWithIOMocks() { ) as jest.MockedFunction; const transactionImpl: IDataWarehouse['transaction'] = async (fn) => - fn(async (sql, binds) => - queryMock(sql, tracer, binds as readonly unknown[] | undefined), - ); + fn(async (sql, binds) => queryMock(sql, tracer, binds)); const startMock = jest.fn(() => {}) as IDataWarehouse['start']; const closeMock = jest.fn(async () => {}) as IDataWarehouse['close']; diff --git a/server/utils/apiKeyMiddleware.ts b/server/utils/apiKeyMiddleware.ts index 1336373..20a96ee 100644 --- a/server/utils/apiKeyMiddleware.ts +++ b/server/utils/apiKeyMiddleware.ts @@ -2,11 +2,8 @@ import { type JsonObject, type JsonValue, type ReadonlyDeep } from 'type-fest'; import { v1 as uuidv1 } from 'uuid'; import { type Dependencies } from '../iocContainer/index.js'; -import { - fromCorrelationId, - toCorrelationId, -} from './correlationIds.js'; -import { ErrorType, CoopError } from './errors.js'; +import { fromCorrelationId, toCorrelationId } from './correlationIds.js'; +import { makeUnauthenticatedError } from './errors.js'; import { type RequestHandlerWithBodies } from './route-helpers.js'; /** @@ -22,41 +19,44 @@ export interface RequestWithOrgId { */ export function createApiKeyMiddleware< ReqBody extends JsonObject = JsonObject, - ResBody extends ReadonlyDeep | undefined = ReadonlyDeep | undefined + ResBody extends ReadonlyDeep | undefined = + | ReadonlyDeep + | undefined, >({ ApiKeyService, -}: Pick): RequestHandlerWithBodies { +}: Pick): RequestHandlerWithBodies< + ReqBody, + ResBody +> { return async (req, _res, next) => { const providedKey = req.header('x-api-key'); let orgId: string | null; - + try { - orgId = providedKey && !Array.isArray(providedKey) - ? await ApiKeyService.validateApiKey(providedKey) - : null; + orgId = + providedKey && !Array.isArray(providedKey) + ? await ApiKeyService.validateApiKey(providedKey) + : null; } catch (_error) { // If API key validation throws an error, treat it as invalid orgId = null; } if (!orgId) { - // Invalid API key is a client-side error, so return a 400. - const requestId = toCorrelationId({ - type: 'api-key-validation', - id: uuidv1() + const requestId = toCorrelationId({ + type: 'api-key-validation', + id: uuidv1(), }); - - return next(new CoopError({ - status: 401, - type: [ErrorType.Unauthorized], - title: 'Invalid API Key', - detail: - 'Something went wrong finding or validating your API key. ' + - 'Make sure the proper key is provided in the x-api-key header.', - requestId: fromCorrelationId(requestId), - name: 'UnauthorizedError', - shouldErrorSpan: true, - })); + + return next( + makeUnauthenticatedError('Invalid API Key', { + detail: + 'Something went wrong finding or validating your API key. ' + + 'Make sure the proper key is provided in the x-api-key header.', + requestId: fromCorrelationId(requestId), + shouldErrorSpan: true, + }), + ); } // Store orgId on the request for use by route handlers @@ -69,5 +69,10 @@ export function createApiKeyMiddleware< * Type guard to check if request has orgId set by the API key middleware */ export function hasOrgId(req: unknown): req is RequestWithOrgId { - return typeof req === 'object' && req !== null && 'orgId' in req && typeof (req as { orgId: unknown }).orgId === 'string'; + return ( + typeof req === 'object' && + req !== null && + 'orgId' in req && + typeof req.orgId === 'string' + ); } diff --git a/server/utils/errors.ts b/server/utils/errors.ts index 891a1cc..351c11d 100644 --- a/server/utils/errors.ts +++ b/server/utils/errors.ts @@ -259,6 +259,7 @@ export type CoopErrorName = | 'NotFoundError' | 'InternalServerError' | 'BadRequestError' + | 'UnauthenticatedError' | 'UnauthorizedError' // gql mutation errors | UserErrorType @@ -316,6 +317,21 @@ export const makeNotFoundError = (title: string, data: ErrorInstanceData) => name: 'NotFoundError', }); +// 401: caller did not provide valid credentials. Pair with `Unauthenticated` +// so the GraphQL layer (`server/api.ts`) maps it to `code: 'UNAUTHENTICATED'`. +export const makeUnauthenticatedError = ( + title: string, + data: ErrorInstanceData, +) => + new CoopError({ + ...data, + status: 401, + type: [...(data.type ?? []), ErrorType.Unauthenticated], + title, + name: 'UnauthenticatedError', + }); + +// 403: caller is authenticated but lacks permission for the requested action. export const makeUnauthorizedError = (title: string, data: ErrorInstanceData) => new CoopError({ ...data, @@ -357,8 +373,8 @@ export const sanitizeError = exposeUnsafeErrorDetails typeof err !== 'object' ? { title: String(err), status: 500, type: [] } : err instanceof CoopError - ? err - : { title: String(err), status: 500, type: [], ...err } + ? err + : { title: String(err), status: 500, type: [], ...err } : (err: unknown) => { // eslint-disable-next-line no-console console.error('Sanitizing error:', err); @@ -382,10 +398,10 @@ export const sanitizeError = exposeUnsafeErrorDetails function isSerializableError(it: unknown): it is SerializableError { return Boolean( typeof it === 'object' && - it && - 'status' in it && - 'type' in it && - 'title' in it, + it && + 'status' in it && + 'type' in it && + 'title' in it, ); } @@ -401,10 +417,10 @@ export function getMessageFromAggregateError(it: AggregateError): string { it instanceof AggregateError ? getMessageFromAggregateError(it) : it instanceof CoopError - ? it.title + (it.detail ? `: ${it.detail}` : '') - : it instanceof Error - ? it.message - : undefined, + ? it.title + (it.detail ? `: ${it.detail}` : '') + : it instanceof Error + ? it.message + : undefined, ), ).join('\n'); } @@ -416,7 +432,7 @@ export function getErrorsFromAggregateError( it instanceof AggregateError ? getErrorsFromAggregateError(it) : it instanceof Error - ? [it] - : [], + ? [it] + : [], ); } diff --git a/server/utils/sql.test.ts b/server/utils/sql.test.ts index 2703671..1f11a80 100644 --- a/server/utils/sql.test.ts +++ b/server/utils/sql.test.ts @@ -1,8 +1,10 @@ -import { Kysely, PostgresDialect, type PostgresQueryResult } from 'kysely'; +import { Kysely, PostgresDialect } from 'kysely'; import { takeLast } from './sql.js'; -function makeCompileOnlyDb>>() { +function makeCompileOnlyDb< + T extends Record>, +>() { return new Kysely({ dialect: new PostgresDialect({ pool: { @@ -12,7 +14,7 @@ function makeCompileOnlyDb>>() rows: [], command: 'SELECT', rowCount: 0, - } as PostgresQueryResult), + }), async release() {}, }; }, diff --git a/server/utils/sql.ts b/server/utils/sql.ts index 5285183..3b66f1b 100644 --- a/server/utils/sql.ts +++ b/server/utils/sql.ts @@ -47,22 +47,23 @@ export function takeLast< ) { let inner = unsortedSelectQuery.clearOrderBy(); for (const it of sortCriteria) { - inner = inner.orderBy( - it.column, - it.order === 'desc' ? 'asc' : 'desc', - ); + inner = inner.orderBy(it.column, it.order === 'desc' ? 'asc' : 'desc'); } inner = inner.limit(size); - // Chaining `orderBy` in a loop widens `outer` to an incompatible union; the - // builder is still the same concrete Kysely select at runtime. - let outer = db.selectFrom(inner.as(SUBQUERY_ALIAS)).selectAll() as SelectQueryBuilder< + // The initial cast pins `outer` to the concrete `SelectQueryBuilder` shape + // we want to return; chaining `orderBy` in the loop below otherwise widens + // it to an incompatible union, and the final cast restores that shape for + // callers. The builder is the same concrete Kysely select at runtime. + let outer = db + .selectFrom(inner.as(SUBQUERY_ALIAS)) + .selectAll() as SelectQueryBuilder< DB & { [K in typeof SUBQUERY_ALIAS]: O }, typeof SUBQUERY_ALIAS, O >; for (const it of sortCriteria) { - outer = outer.orderBy(it.column, it.order) as typeof outer; + outer = outer.orderBy(it.column, it.order); } return outer as SelectQueryBuilder< DB & { [K in typeof SUBQUERY_ALIAS]: O },