diff --git a/client/package-lock.json b/client/package-lock.json index 53e53b5..dc54525 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -29,7 +29,7 @@ "@radix-ui/react-slider": "^1.2.0", "@radix-ui/react-switch": "^1.1.0", "@radix-ui/react-tooltip": "^1.1.2", - "@roostorg/coop-types": "^2.3.0", + "@roostorg/coop-types": "^2.4.0", "@tailwindcss/container-queries": "^0.1.1", "@tailwindcss/forms": "^0.5.7", "@tailwindcss/typography": "^0.5.13", @@ -3990,9 +3990,9 @@ ] }, "node_modules/@roostorg/coop-types": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@roostorg/coop-types/-/coop-types-2.3.0.tgz", - "integrity": "sha512-aOHomJIoqNUUoHxewxjOe6x2Y7ALcVgweUX7SxD18DwfylYK1KJZi4Q5nZB0SMm6q7V5H6btJfo4jRQMLxUT+g==", + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/@roostorg/coop-types/-/coop-types-2.4.0.tgz", + "integrity": "sha512-N6apd8CebHsC1cvzm7YuVaGfUEi4RG5Lth785cDoh9KdOqC0MO6AKjPFR76GY6lAE/CCUPzFEqoFQYMWQt05qg==", "license": "ISC", "dependencies": { "date-fns": "^2.29.3", diff --git a/client/package.json b/client/package.json index 2134228..504c3df 100644 --- a/client/package.json +++ b/client/package.json @@ -37,7 +37,7 @@ "@radix-ui/react-slider": "^1.2.0", "@radix-ui/react-switch": "^1.1.0", "@radix-ui/react-tooltip": "^1.1.2", - "@roostorg/coop-types": "^2.3.0", + "@roostorg/coop-types": "^2.4.0", "@tailwindcss/container-queries": "^0.1.1", "@tailwindcss/forms": "^0.5.7", "@tailwindcss/typography": "^0.5.13", diff --git a/client/src/graphql/generated.ts b/client/src/graphql/generated.ts index f25d961..afc2273 100644 --- a/client/src/graphql/generated.ts +++ b/client/src/graphql/generated.ts @@ -1278,6 +1278,7 @@ export const GQLFieldType = { Audio: 'AUDIO', Boolean: 'BOOLEAN', Datetime: 'DATETIME', + EmailAddress: 'EMAIL_ADDRESS', Geohash: 'GEOHASH', Id: 'ID', Image: 'IMAGE', @@ -4262,6 +4263,7 @@ export const GQLScalarType = { Audio: 'AUDIO', Boolean: 'BOOLEAN', Datetime: 'DATETIME', + EmailAddress: 'EMAIL_ADDRESS', Geohash: 'GEOHASH', Id: 'ID', Image: 'IMAGE', @@ -4403,6 +4405,7 @@ export const GQLSignalInputType = { Audio: 'AUDIO', Boolean: 'BOOLEAN', Datetime: 'DATETIME', + EmailAddress: 'EMAIL_ADDRESS', FullItem: 'FULL_ITEM', Geohash: 'GEOHASH', Id: 'ID', diff --git a/client/src/models/signal.ts b/client/src/models/signal.ts index ec9f57f..24f06f6 100644 --- a/client/src/models/signal.ts +++ b/client/src/models/signal.ts @@ -96,6 +96,7 @@ export function outputTypeToComparators(outputType: GQLSignalOutputType) { case GQLScalarType.Url: case GQLScalarType.String: case GQLScalarType.IpAddress: + case GQLScalarType.EmailAddress: return outputType.__typename === 'EnumSignalOutputType' && outputType.ordered ? orderedComparators diff --git a/client/src/webpages/dashboard/item_types/itemTypeUtils.ts b/client/src/webpages/dashboard/item_types/itemTypeUtils.ts index d6a59dd..73e0d1a 100644 --- a/client/src/webpages/dashboard/item_types/itemTypeUtils.ts +++ b/client/src/webpages/dashboard/item_types/itemTypeUtils.ts @@ -50,7 +50,7 @@ export const schemaFieldRolesFieldTypes = { [SchemaFieldRoles.BACKGROUND_IMAGE]: GQLScalarType.Image, [SchemaFieldRoles.IS_DELETED]: GQLScalarType.Boolean, [SchemaFieldRoles.IP_ADDRESS]: GQLScalarType.IpAddress, - [SchemaFieldRoles.EMAIL]: GQLScalarType.String, + [SchemaFieldRoles.EMAIL]: GQLScalarType.EmailAddress, } satisfies Omit< { [key in SchemaFieldRoles]: GQLScalarType }, SchemaFieldRoles.NONE @@ -229,6 +229,8 @@ export function generateFakeScalarFieldValue(fieldType: ScalarType) { return `https://url.com/some-path/${Math.floor(100 * Math.random())}`; case 'IP_ADDRESS': return `192.0.2.${Math.floor(255 * Math.random())}`; + case 'EMAIL_ADDRESS': + return `user${Math.floor(1000 * Math.random())}@example.com`; case 'MEDIA': { // Like AUDIO/IMAGE/VIDEO, the fake value is the *input* (pre-coercion) form // — a raw URL string. The server's MEDIA coercion resolves the kind from diff --git a/client/src/webpages/dashboard/mrt/manual_review_job/v2/ManualReviewJobFieldsComponent.tsx b/client/src/webpages/dashboard/mrt/manual_review_job/v2/ManualReviewJobFieldsComponent.tsx index 314a6fc..3cf58a5 100644 --- a/client/src/webpages/dashboard/mrt/manual_review_job/v2/ManualReviewJobFieldsComponent.tsx +++ b/client/src/webpages/dashboard/mrt/manual_review_job/v2/ManualReviewJobFieldsComponent.tsx @@ -195,7 +195,10 @@ function TableRowComponent(props: { case 'ID': case 'NUMBER': case 'POLICY_ID': - case 'STRING': { + case 'STRING': + case 'EMAIL_ADDRESS': { + // EMAIL_ADDRESS renders as plain text for now; a follow-up could make + // it a mailto/pivot link the way IP_ADDRESS pivots on the IP. return (
{label ? ( @@ -520,6 +523,7 @@ function FieldComponent(props: { case 'URL': case 'POLICY_ID': case 'IP_ADDRESS': + case 'EMAIL_ADDRESS': case 'DATETIME': return (
@@ -576,6 +580,7 @@ function ContainerComponent(props: { case 'DATETIME': case 'POLICY_ID': case 'IP_ADDRESS': + case 'EMAIL_ADDRESS': return true; case 'AUDIO': case 'IMAGE': @@ -618,6 +623,7 @@ function ContainerComponent(props: { case 'URL': case 'POLICY_ID': case 'IP_ADDRESS': + case 'EMAIL_ADDRESS': case 'MEDIA': case 'VIDEO': { throw Error('Cannot call container component with scalar field'); diff --git a/client/src/webpages/dashboard/rules/info/RuleTestModal.tsx b/client/src/webpages/dashboard/rules/info/RuleTestModal.tsx index 8c8c374..7d3f201 100644 --- a/client/src/webpages/dashboard/rules/info/RuleTestModal.tsx +++ b/client/src/webpages/dashboard/rules/info/RuleTestModal.tsx @@ -140,6 +140,8 @@ export default function RuleTestModal(props: { return 'user-id'; case 'IP_ADDRESS': return '192.0.2.1'; + case 'EMAIL_ADDRESS': + return 'user@example.com'; case 'ARRAY': return `${getPlaceholder( containerValueScalarType!, diff --git a/client/src/webpages/dashboard/rules/info/insights/RuleInsightsSamplesTable.tsx b/client/src/webpages/dashboard/rules/info/insights/RuleInsightsSamplesTable.tsx index 1caf6c9..a7e4ad5 100644 --- a/client/src/webpages/dashboard/rules/info/insights/RuleInsightsSamplesTable.tsx +++ b/client/src/webpages/dashboard/rules/info/insights/RuleInsightsSamplesTable.tsx @@ -764,6 +764,7 @@ export function getStringFromContent( case GQLFieldType.Map: case GQLFieldType.PolicyId: case GQLFieldType.IpAddress: + case GQLFieldType.EmailAddress: return content.toString(); case GQLFieldType.Image: case GQLFieldType.Video: diff --git a/db/src/configs/pg-base.ts b/db/src/configs/pg-base.ts index 9b642b2..d6cca26 100644 --- a/db/src/configs/pg-base.ts +++ b/db/src/configs/pg-base.ts @@ -28,7 +28,47 @@ export function makePostgresDatabaseConfig(opts: { driverOpts: Options & { schema: string }; maintenanceDatabase?: string; }): DatabaseConfig { - const { driverOpts, scriptsDirectory, defaultScriptFormat } = opts; + const { scriptsDirectory, defaultScriptFormat } = opts; + // Forward server-side `RAISE NOTICE` / `RAISE WARNING` from migrations + // (e.g. PL/pgSQL DO blocks that report rows they modified) to stderr. + // Sequelize sets `client_min_messages` to WARNING by default, which makes + // postgres filter NOTICE-severity messages server-side before they reach + // the client; we lower it to NOTICE on each connection so RAISE NOTICE + // actually surfaces, then attach a listener that prints each one. + type PgClientLike = { + on: (event: string, cb: (msg: unknown) => void) => void; + query: (sql: string, cb: (err: unknown) => void) => void; + }; + const isPgClientLike = (c: unknown): c is PgClientLike => + c !== null && + typeof c === 'object' && + 'on' in c && + typeof (c as { on: unknown }).on === 'function' && + 'query' in c && + typeof (c as { query: unknown }).query === 'function'; + const driverOpts: Options & { schema: string } = { + ...opts.driverOpts, + hooks: { + ...opts.driverOpts.hooks, + afterConnect(connection: unknown) { + if (!isPgClientLike(connection)) return; + connection.on('notice', (msg) => { + const m = msg as { message?: string; severity?: string }; + const severity = m.severity ?? 'NOTICE'; + // eslint-disable-next-line no-console + console.warn(`[postgres ${severity}] ${m.message ?? ''}`); + }); + connection.query(`SET client_min_messages = 'notice'`, (err) => { + if (err) { + // eslint-disable-next-line no-console + console.warn( + `[postgres] failed to lower client_min_messages: ${String(err)}`, + ); + } + }); + }, + }, + }; // DB used for CREATE/DROP DATABASE (can't be the target DB itself). // Defaults to `postgres`; some managed providers use a different name // (e.g. `defaultdb`). diff --git a/db/src/scripts/api-server-pg/2026.06.26T03.32.16.tighten_email_field_role_to_email_address.sql b/db/src/scripts/api-server-pg/2026.06.26T03.32.16.tighten_email_field_role_to_email_address.sql new file mode 100644 index 0000000..be8b775 --- /dev/null +++ b/db/src/scripts/api-server-pg/2026.06.26T03.32.16.tighten_email_field_role_to_email_address.sql @@ -0,0 +1,61 @@ +-- Follow-up to #840: now that `@roostorg/coop-types@2.4.0` ships a dedicated +-- `EMAIL_ADDRESS` scalar (#841), tighten the `email_field` field-role CHECK +-- constraint from STRING to EMAIL_ADDRESS so adopters can't map arbitrary +-- string fields (e.g. bio, displayName) into NCMEC reports. +-- +-- Deployments between #840 and this migration may have configured the +-- `email` role to point at a STRING field. The new CHECK would reject +-- those rows and block the migration, so the data step below clears any +-- such mapping (setting `email_field` back to NULL) and emits a NOTICE +-- per affected row so operators can see which item type's mapping was +-- dropped and reconfigure via the admin UI if needed. The actual data in +-- the underlying STRING field is untouched; only the role pointer is +-- cleared. +-- +-- Operators: after applying this migration, scan the `db:update` output +-- for lines beginning with `[postgres NOTICE]`. Each one names the item +-- type (id, name, org_id) whose `email_field` mapping was cleared. +-- Reconfigure those item types in the admin UI if NCMEC reporting was +-- relying on them. The forwarding of pg NOTICE events to the migration +-- runner output is provided by the `afterConnect` hook in +-- `db/src/configs/pg-base.ts` (added in this same PR). + +BEGIN; + +DO $$ +DECLARE + affected RECORD; +BEGIN + FOR affected IN + SELECT id, org_id, name, email_field + FROM public.item_types + WHERE email_field IS NOT NULL + AND NOT jsonb_path_exists( + (array_to_json(fields))::jsonb, + '$[*]?(@."name" == $"name" && @."type" == "EMAIL_ADDRESS")'::jsonpath, + jsonb_build_object('name', email_field) + ) + LOOP + RAISE NOTICE 'Clearing email_field=% from item_type id=% (name=%, org=%): no EMAIL_ADDRESS field of that name exists. Reconfigure via admin UI if needed.', + affected.email_field, affected.id, affected.name, affected.org_id; + + UPDATE public.item_types + SET email_field = NULL + WHERE id = affected.id; + END LOOP; +END $$; + +ALTER TABLE public.item_types + DROP CONSTRAINT valid_email_field_field_type; + +ALTER TABLE public.item_types + ADD CONSTRAINT valid_email_field_field_type CHECK ( + (email_field IS NULL) + OR jsonb_path_exists( + (array_to_json(fields))::jsonb, + '$[*]?(@."name" == $"name" && @."type" == "EMAIL_ADDRESS")'::jsonpath, + jsonb_build_object('name', email_field) + ) + ); + +COMMIT; diff --git a/server/graphql/generated.ts b/server/graphql/generated.ts index 2210da2..4920888 100644 --- a/server/graphql/generated.ts +++ b/server/graphql/generated.ts @@ -1346,6 +1346,7 @@ export const GQLFieldType = { Audio: 'AUDIO', Boolean: 'BOOLEAN', Datetime: 'DATETIME', + EmailAddress: 'EMAIL_ADDRESS', Geohash: 'GEOHASH', Id: 'ID', Image: 'IMAGE', @@ -4330,6 +4331,7 @@ export const GQLScalarType = { Audio: 'AUDIO', Boolean: 'BOOLEAN', Datetime: 'DATETIME', + EmailAddress: 'EMAIL_ADDRESS', Geohash: 'GEOHASH', Id: 'ID', Image: 'IMAGE', @@ -4471,6 +4473,7 @@ export const GQLSignalInputType = { Audio: 'AUDIO', Boolean: 'BOOLEAN', Datetime: 'DATETIME', + EmailAddress: 'EMAIL_ADDRESS', FullItem: 'FULL_ITEM', Geohash: 'GEOHASH', Id: 'ID', diff --git a/server/graphql/schema.ts b/server/graphql/schema.ts index 194c123..00affb1 100644 --- a/server/graphql/schema.ts +++ b/server/graphql/schema.ts @@ -67,6 +67,7 @@ const typeDefs = /* GraphQL */ ` URL POLICY_ID IP_ADDRESS + EMAIL_ADDRESS } # This is equivalent to ScalarType, but with 'FULL_ITEM' added @@ -87,6 +88,7 @@ const typeDefs = /* GraphQL */ ` FULL_ITEM POLICY_ID IP_ADDRESS + EMAIL_ADDRESS } # !! IMPORTANT: when you add a value here, also add it to FieldType !! @@ -113,6 +115,7 @@ const typeDefs = /* GraphQL */ ` URL POLICY_ID IP_ADDRESS + EMAIL_ADDRESS } enum Language { diff --git a/server/package-lock.json b/server/package-lock.json index c46fbb8..77d2fac 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -25,7 +25,7 @@ "@opentelemetry/api": "^1.8.0", "@opentelemetry/semantic-conventions": "^1.22.0", "@roostorg/coop-integration-example": "^2.0.0", - "@roostorg/coop-types": "^2.3.0", + "@roostorg/coop-types": "^2.4.0", "@sendgrid/mail": "^8.1.6", "@stdlib/stats-binomial-test": "^0.0.7", "@total-typescript/ts-reset": "^0.3.7", @@ -3624,9 +3624,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3644,9 +3641,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3664,9 +3658,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3684,9 +3675,6 @@ "riscv64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3704,9 +3692,6 @@ "riscv64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3724,9 +3709,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3744,9 +3726,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3764,9 +3743,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3979,9 +3955,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3996,9 +3969,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -4013,9 +3983,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4030,9 +3997,6 @@ "riscv64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4047,9 +4011,6 @@ "riscv64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -4064,9 +4025,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4081,9 +4039,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4098,9 +4053,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -4275,9 +4227,9 @@ } }, "node_modules/@roostorg/coop-types": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@roostorg/coop-types/-/coop-types-2.3.0.tgz", - "integrity": "sha512-aOHomJIoqNUUoHxewxjOe6x2Y7ALcVgweUX7SxD18DwfylYK1KJZi4Q5nZB0SMm6q7V5H6btJfo4jRQMLxUT+g==", + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/@roostorg/coop-types/-/coop-types-2.4.0.tgz", + "integrity": "sha512-N6apd8CebHsC1cvzm7YuVaGfUEi4RG5Lth785cDoh9KdOqC0MO6AKjPFR76GY6lAE/CCUPzFEqoFQYMWQt05qg==", "license": "ISC", "dependencies": { "date-fns": "^2.29.3", diff --git a/server/package.json b/server/package.json index aff7cce..cd0d330 100644 --- a/server/package.json +++ b/server/package.json @@ -44,7 +44,7 @@ "@opentelemetry/api": "^1.8.0", "@opentelemetry/semantic-conventions": "^1.22.0", "@roostorg/coop-integration-example": "^2.0.0", - "@roostorg/coop-types": "^2.3.0", + "@roostorg/coop-types": "^2.4.0", "@sendgrid/mail": "^8.1.6", "@stdlib/stats-binomial-test": "^0.0.7", "@total-typescript/ts-reset": "^0.3.7", diff --git a/server/services/itemProcessingService/fieldTypeHandlers.ts b/server/services/itemProcessingService/fieldTypeHandlers.ts index 47a2c89..f919d6d 100644 --- a/server/services/itemProcessingService/fieldTypeHandlers.ts +++ b/server/services/itemProcessingService/fieldTypeHandlers.ts @@ -246,6 +246,22 @@ export const fieldTypeHandlers: Handlers = { }, getValues: scalarGetValues, }, + [ScalarTypes.EMAIL_ADDRESS]: { + // Leading/trailing whitespace stripped; all-whitespace/empty becomes + // "field omitted". Format validation (RFC 5321/5322 shape) is deferred + // to a follow-up issue per PR #840 reviewer note; for now this just + // enforces "string" so the type system can rely on it. + coerce: (v) => { + if (typeof v !== 'string') { + return new Error( + 'This field, if given, must be a string email address.', + ); + } + const trimmed = v.trim(); + return trimmed === '' ? null : trimmed; + }, + getValues: scalarGetValues, + }, [ContainerTypes.ARRAY]: { coerce(value, itemTypeIds, container) { if (!Array.isArray(value)) { diff --git a/server/services/moderationConfigService/types/itemTypes.ts b/server/services/moderationConfigService/types/itemTypes.ts index 076386b..c38c4e9 100644 --- a/server/services/moderationConfigService/types/itemTypes.ts +++ b/server/services/moderationConfigService/types/itemTypes.ts @@ -128,7 +128,7 @@ export type FieldRoleToScalarType = { backgroundImage: ScalarTypes['IMAGE']; isDeleted: ScalarTypes['BOOLEAN']; ipAddress: ScalarTypes['IP_ADDRESS']; - email: ScalarTypes['STRING']; + email: ScalarTypes['EMAIL_ADDRESS']; }; export function getPartialSchemaFromOriginal(schema: ItemSchema) { diff --git a/server/services/ncmecService/buildSubmitReportParamsFromDecision.test.ts b/server/services/ncmecService/buildSubmitReportParamsFromDecision.test.ts index f8b9315..6f161e6 100644 --- a/server/services/ncmecService/buildSubmitReportParamsFromDecision.test.ts +++ b/server/services/ncmecService/buildSubmitReportParamsFromDecision.test.ts @@ -376,4 +376,88 @@ describe('buildSubmitReportParamsFromDecision', () => { expect(result.reportedUser).not.toHaveProperty('email'); }); }); + + describe('HMA hash extraction on media', () => { + it('attaches hashes from the matching image in item data', async () => { + const result = await buildSubmitReportParamsFromDecision( + makeInput({ + reportedUserItemType: makeUserItemType({}), + reportedUserData: asNormalizedData({ display_name: 'Alice' }), + contentItemType: makeContentItemType({}), + contentData: asNormalizedData({ + created_at: FIXED_NOW, + image: { + url: 'https://example.com/m1.png', + hashes: { md5: 'abc123', pdq: 'def456' }, + }, + }), + }), + ); + + expect(result.media[0]).toMatchObject({ + url: 'https://example.com/m1.png', + hashes: { md5: 'abc123', pdq: 'def456' }, + }); + }); + + it('finds the matching image inside an ARRAY-of-IMAGE container', async () => { + const result = await buildSubmitReportParamsFromDecision( + makeInput({ + reportedUserItemType: makeUserItemType({}), + reportedUserData: asNormalizedData({ display_name: 'Alice' }), + contentItemType: makeContentItemType({}), + contentData: asNormalizedData({ + created_at: FIXED_NOW, + images: [ + { + url: 'https://example.com/other.png', + hashes: { md5: 'wrong' }, + }, + { + url: 'https://example.com/m1.png', + hashes: { md5: 'right' }, + }, + ], + }), + }), + ); + + expect(result.media[0].hashes).toEqual({ md5: 'right' }); + }); + + it('omits `hashes` when no image in the data matches the reported URL', async () => { + const result = await buildSubmitReportParamsFromDecision( + makeInput({ + reportedUserItemType: makeUserItemType({}), + reportedUserData: asNormalizedData({ display_name: 'Alice' }), + contentItemType: makeContentItemType({}), + contentData: asNormalizedData({ + created_at: FIXED_NOW, + image: { + url: 'https://example.com/different.png', + hashes: { md5: 'abc' }, + }, + }), + }), + ); + + expect(result.media[0]).not.toHaveProperty('hashes'); + }); + + it('omits `hashes` when the matching image has no hashes attached', async () => { + const result = await buildSubmitReportParamsFromDecision( + makeInput({ + reportedUserItemType: makeUserItemType({}), + reportedUserData: asNormalizedData({ display_name: 'Alice' }), + contentItemType: makeContentItemType({}), + contentData: asNormalizedData({ + created_at: FIXED_NOW, + image: { url: 'https://example.com/m1.png' }, + }), + }), + ); + + expect(result.media[0]).not.toHaveProperty('hashes'); + }); + }); }); diff --git a/server/services/ncmecService/buildSubmitReportParamsFromDecision.ts b/server/services/ncmecService/buildSubmitReportParamsFromDecision.ts index f002677..43b2a0e 100644 --- a/server/services/ncmecService/buildSubmitReportParamsFromDecision.ts +++ b/server/services/ncmecService/buildSubmitReportParamsFromDecision.ts @@ -168,6 +168,7 @@ export async function buildSubmitReportParamsFromDecision( 'ipAddress', reportedItem.contentItem.data, ); + const hashes = extractHashesForUrl(reportedItem.contentItem.data, it.url); return { id: it.id, typeId: it.typeId, @@ -176,6 +177,7 @@ export async function buildSubmitReportParamsFromDecision( industryClassification: it.industryClassification, fileAnnotations: it.fileAnnotations, ...(mediaIp ? { ipAddress: mediaIp } : {}), + ...(hashes ? { hashes } : {}), }; }), ); @@ -214,3 +216,48 @@ export async function buildSubmitReportParamsFromDecision( ...(jobId !== undefined ? { jobId } : {}), }; } + +/** Walk an item's data looking for an image-shaped value (`{ url, hashes }`) + * whose `url` matches the target. Returns the `hashes` map (typically + * populated by HMA at item-submission time, e.g. `{ md5: '...', pdq: '...' }`) + * or undefined when no match is found. + * + * Recurses into arrays and plain objects so ARRAY-of-IMAGE and MAP-of-IMAGE + * containers are covered, not just scalar IMAGE fields. Returns on the first + * match — duplicate URLs across fields would only ever yield the same hashes + * since HMA is deterministic per URL. */ +export function extractHashesForUrl( + data: NormalizedItemData, + url: string, +): Record | undefined { + const visit = (value: unknown): Record | undefined => { + if (Array.isArray(value)) { + for (const item of value) { + const found = visit(item); + if (found) return found; + } + return undefined; + } + if (typeof value !== 'object' || value === null) return undefined; + const obj = value as Record; + if ( + typeof obj.url === 'string' && + obj.url === url && + typeof obj.hashes === 'object' && + obj.hashes !== null + ) { + const hashes = obj.hashes as Record; + const stringHashes: Record = {}; + for (const [k, v] of Object.entries(hashes)) { + if (typeof v === 'string') stringHashes[k] = v; + } + return Object.keys(stringHashes).length > 0 ? stringHashes : undefined; + } + for (const inner of Object.values(obj)) { + const found = visit(inner); + if (found) return found; + } + return undefined; + }; + return visit(data); +} diff --git a/server/services/ncmecService/ncmecReporting.test.ts b/server/services/ncmecService/ncmecReporting.test.ts index 71b5659..9ab5040 100644 --- a/server/services/ncmecService/ncmecReporting.test.ts +++ b/server/services/ncmecService/ncmecReporting.test.ts @@ -350,6 +350,9 @@ describe('NCMEC reporting', () => { }); }); + // toOriginalFileHashes tests moved to ./toOriginalFileHashes.test.ts + // (this file was over the 500-line max-lines limit after expansion). + describe('summarizeCyberTipFailure', () => { const previousDebug = process.env.NCMEC_DEBUG; const previousNodeEnv = process.env.NODE_ENV; diff --git a/server/services/ncmecService/ncmecReporting.ts b/server/services/ncmecService/ncmecReporting.ts index 4f95fc7..c68e67e 100644 --- a/server/services/ncmecService/ncmecReporting.ts +++ b/server/services/ncmecService/ncmecReporting.ts @@ -133,6 +133,7 @@ type FileDetails = { fileRelevance?: 'Reported' | 'Supplemental Reported'; fileAnnotations?: FileAnnotations; industryClassification?: NCMECIndustryClassificationType; + originalFileHash?: OriginalFileHash[]; ipCaptureEvent?: IPNCMECEvent[]; deviceId?: DeviceId[]; details?: Detail[]; @@ -140,6 +141,13 @@ type FileDetails = { }; }; +type OriginalFileHash = { + _text: string; + _attributes: { + hashType: string; + }; +}; + type FileAnnotations = { animeDrawingVirtualHentai?: undefined; potentialMeme?: undefined; @@ -174,6 +182,12 @@ type Media = { * `Upload` event. */ ipAddress?: string; deviceId?: DeviceNCMECEvent[]; + /** Hashes computed for this URL (typically by HMA at item submission + * time). Keyed by hash algorithm name (e.g. `md5`, `pdq`); the value is + * the hex-encoded hash. Forwarded to NCMEC as `originalFileHash` + * entries with the algorithm name uppercased into the `hashType` + * attribute. */ + hashes?: Record; }; type NCMECUserParams = { @@ -547,6 +561,42 @@ export function mergeFieldRoleIpIntoEvents( return events.length > 0 ? events : undefined; } +/** Build the NCMEC `originalFileHash[]` shape from both hash sources Coop + * has: HMA-computed hashes stored on the item data (keyed by algorithm), + * and any single `{ hash, hashType }` returned by the additional-info + * webhook for this media. Trims blanks, uppercases the algorithm name into + * the `hashType` attribute, drops empty entries, and dedupes on + * (`hashType`, hash value) so a webhook that returns the same algorithm as + * HMA doesn't produce duplicate entries. Returns undefined when no usable + * hashes survive filtering; callers should branch on that to omit the key + * entirely rather than serialise an empty array. */ +export function toOriginalFileHashes(opts: { + hmaHashes?: Record; + webhookFileDetails?: { hash: string; hashType: string }; +}): OriginalFileHash[] | undefined { + const result: OriginalFileHash[] = []; + const seen = new Set(); + const push = (algorithm: string, hash: string) => { + const trimmedHash = typeof hash === 'string' ? hash.trim() : ''; + const trimmedAlgorithm = algorithm.trim(); + if (trimmedHash === '' || trimmedAlgorithm === '') return; + const hashType = trimmedAlgorithm.toUpperCase(); + const key = `${hashType}${trimmedHash}`; + if (seen.has(key)) return; + seen.add(key); + result.push({ _text: trimmedHash, _attributes: { hashType } }); + }; + if (opts.hmaHashes) { + for (const [algorithm, hash] of Object.entries(opts.hmaHashes)) { + push(algorithm, hash); + } + } + if (opts.webhookFileDetails) { + push(opts.webhookFileDetails.hashType, opts.webhookFileDetails.hash); + } + return result.length > 0 ? result : undefined; +} + /** Resolve the email(s) for `personOrUserReportedPerson`. Prefers the * webhook's enriched response (carries NCMEC `type` / `verified` attributes); * falls back to a bare field-role email otherwise. Returns undefined when @@ -713,6 +763,15 @@ type MediaAdditionalInfo = { fileName?: string; /** When set, sent to NCMEC in file details (whether the content was publicly viewable). */ publiclyAvailable?: boolean; + /** Optional single hash from the additional-info webhook response. + * Merged with HMA-sourced hashes (see `toOriginalFileHashes`); deduped + * on (`hashType` uppercase, trimmed hash value) so a webhook that + * returns the same algorithm as HMA doesn't produce duplicate entries + * in the outgoing `originalFileHash[]` list. */ + fileDetails?: { + hash: string; + hashType: string; + }; }; type FileAdditionalInfo = { @@ -1102,9 +1161,6 @@ export default class NcmecReporting { media: reportedMedia.map((media) => ({ id: media.id, typeId: media.typeId, - fileDetails: { - ipCaptureEvent: [], - }, })), }; } @@ -1958,6 +2014,10 @@ export default class NcmecReporting { const fileAnnotations = this.#fileAnnotationArrayToNCMECFileAnnotation( media.fileAnnotations, ); + const originalFileHash = toOriginalFileHashes({ + hmaHashes: media.hashes, + webhookFileDetails: additionalInfo.fileDetails, + }); const xml = await this.#uploadFileDetails( { fileDetails: { @@ -1970,6 +2030,9 @@ export default class NcmecReporting { : {}), ...(fileAnnotations ? { fileAnnotations } : {}), industryClassification: media.industryClassification, + ...(originalFileHash && originalFileHash.length > 0 + ? { originalFileHash } + : {}), ...(additionalInfo.ipCaptureEvent && additionalInfo.ipCaptureEvent.length > 0 ? { diff --git a/server/services/ncmecService/toOriginalFileHashes.test.ts b/server/services/ncmecService/toOriginalFileHashes.test.ts new file mode 100644 index 0000000..e0fd780 --- /dev/null +++ b/server/services/ncmecService/toOriginalFileHashes.test.ts @@ -0,0 +1,104 @@ +import { toOriginalFileHashes } from './ncmecReporting.js'; + +describe('toOriginalFileHashes', () => { + it('uppercases the algorithm into the `hashType` attribute', () => { + expect( + toOriginalFileHashes({ hmaHashes: { md5: 'abc', pdq: 'def' } }), + ).toEqual([ + { _text: 'abc', _attributes: { hashType: 'MD5' } }, + { _text: 'def', _attributes: { hashType: 'PDQ' } }, + ]); + }); + + it('trims surrounding whitespace from hash values', () => { + expect(toOriginalFileHashes({ hmaHashes: { md5: ' abc ' } })).toEqual([ + { _text: 'abc', _attributes: { hashType: 'MD5' } }, + ]); + }); + + it('drops entries with empty or whitespace-only hash values', () => { + // NCMEC requires non-empty `hash` text and `hashType` attribute; an + // entry with whitespace would fail XSD validation on receipt. + expect( + toOriginalFileHashes({ + hmaHashes: { md5: '', sha1: ' ', sha256: 'kept' }, + }), + ).toEqual([{ _text: 'kept', _attributes: { hashType: 'SHA256' } }]); + }); + + it('returns undefined when no entries survive filtering', () => { + // Caller branches on undefined to omit the `originalFileHash` key + // entirely rather than serialise an empty array. + expect(toOriginalFileHashes({})).toBeUndefined(); + expect(toOriginalFileHashes({ hmaHashes: {} })).toBeUndefined(); + expect( + toOriginalFileHashes({ hmaHashes: { md5: '', sha1: ' ' } }), + ).toBeUndefined(); + }); + + it('includes the webhook hash when only the webhook source has data', () => { + expect( + toOriginalFileHashes({ + webhookFileDetails: { hash: 'webhook-abc', hashType: 'md5' }, + }), + ).toEqual([{ _text: 'webhook-abc', _attributes: { hashType: 'MD5' } }]); + }); + + it('combines HMA and webhook hashes when both sources provide data', () => { + expect( + toOriginalFileHashes({ + hmaHashes: { md5: 'hma-md5', pdq: 'hma-pdq' }, + webhookFileDetails: { hash: 'webhook-sha1', hashType: 'sha1' }, + }), + ).toEqual([ + { _text: 'hma-md5', _attributes: { hashType: 'MD5' } }, + { _text: 'hma-pdq', _attributes: { hashType: 'PDQ' } }, + { _text: 'webhook-sha1', _attributes: { hashType: 'SHA1' } }, + ]); + }); + + it('dedupes when HMA and webhook return the same (algorithm, hash) pair', () => { + // Common case: both sources independently compute MD5 of the same + // content. Single entry in the outgoing report instead of two. + expect( + toOriginalFileHashes({ + hmaHashes: { md5: 'same-value' }, + webhookFileDetails: { hash: 'same-value', hashType: 'md5' }, + }), + ).toEqual([{ _text: 'same-value', _attributes: { hashType: 'MD5' } }]); + }); + + it('keeps both entries when HMA and webhook return the same algorithm but different hashes', () => { + // Real conflict (different MD5 values for the same content) is signal + // NCMEC investigators may care about; surface both rather than picking + // a winner. + expect( + toOriginalFileHashes({ + hmaHashes: { md5: 'hma-md5' }, + webhookFileDetails: { hash: 'webhook-md5', hashType: 'md5' }, + }), + ).toEqual([ + { _text: 'hma-md5', _attributes: { hashType: 'MD5' } }, + { _text: 'webhook-md5', _attributes: { hashType: 'MD5' } }, + ]); + }); + + it('drops a whitespace-only webhook hash', () => { + expect( + toOriginalFileHashes({ + webhookFileDetails: { hash: ' ', hashType: 'md5' }, + }), + ).toBeUndefined(); + }); + + it('dedupe is case-insensitive on the algorithm name', () => { + // Webhook returns `MD5` uppercase; HMA returns `md5` lowercase. Same + // algorithm, should dedupe on identical hash value. + expect( + toOriginalFileHashes({ + hmaHashes: { md5: 'shared' }, + webhookFileDetails: { hash: 'shared', hashType: 'MD5' }, + }), + ).toEqual([{ _text: 'shared', _attributes: { hashType: 'MD5' } }]); + }); +}); diff --git a/server/test/arbitraries/ContentType.ts b/server/test/arbitraries/ContentType.ts index 1b4cf8c..70fb29b 100644 --- a/server/test/arbitraries/ContentType.ts +++ b/server/test/arbitraries/ContentType.ts @@ -87,6 +87,16 @@ export const IpAddressArbitrary = fc.oneof( ), ); +// RFC 2606 reserved domains for example purposes. +export const EmailAddressArbitrary = fc.constantFrom( + 'yuki@example.com', + 'amara@example.org', + 'priya@example.net', + 'mateo@example.com', + 'aaliyah@example.org', + 'jian@example.net', +); + export const ScalarValidValuesArbitraries = { [ScalarTypes.AUDIO]: MediaUrlArbitrary, [ScalarTypes.BOOLEAN]: fc.boolean(), @@ -106,6 +116,7 @@ export const ScalarValidValuesArbitraries = { [ScalarTypes.RELATED_ITEM]: RelatedItemArbitrary, [ScalarTypes.POLICY_ID]: IdLikeArbitrary, [ScalarTypes.IP_ADDRESS]: IpAddressArbitrary, + [ScalarTypes.EMAIL_ADDRESS]: EmailAddressArbitrary, }; export const ScalarFieldArbitrary = fc