diff --git a/client/src/graphql/generated.ts b/client/src/graphql/generated.ts index df1648e..f25d961 100644 --- a/client/src/graphql/generated.ts +++ b/client/src/graphql/generated.ts @@ -5179,6 +5179,7 @@ export type GQLUserSchemaFieldRoles = { readonly backgroundImage?: Maybe; readonly createdAt?: Maybe; readonly displayName?: Maybe; + readonly email?: Maybe; readonly ipAddress?: Maybe; readonly isDeleted?: Maybe; readonly profileIcon?: Maybe; @@ -5188,6 +5189,7 @@ export type GQLUserSchemaFieldRolesInput = { readonly backgroundImage?: InputMaybe; readonly createdAt?: InputMaybe; readonly displayName?: InputMaybe; + readonly email?: InputMaybe; readonly ipAddress?: InputMaybe; readonly isDeleted?: InputMaybe; readonly profileIcon?: InputMaybe; diff --git a/client/src/webpages/dashboard/item_types/ItemTypeForm.tsx b/client/src/webpages/dashboard/item_types/ItemTypeForm.tsx index 222e33e..ad6c954 100644 --- a/client/src/webpages/dashboard/item_types/ItemTypeForm.tsx +++ b/client/src/webpages/dashboard/item_types/ItemTypeForm.tsx @@ -786,6 +786,7 @@ function availableRolesForItemKind(kind: ItemTypeKind): SchemaFieldRoles[] { SchemaFieldRoles.BACKGROUND_IMAGE, SchemaFieldRoles.IS_DELETED, SchemaFieldRoles.IP_ADDRESS, + SchemaFieldRoles.EMAIL, SchemaFieldRoles.NONE, ]; } diff --git a/client/src/webpages/dashboard/item_types/itemTypeUtils.ts b/client/src/webpages/dashboard/item_types/itemTypeUtils.ts index 03bf3ef..d6a59dd 100644 --- a/client/src/webpages/dashboard/item_types/itemTypeUtils.ts +++ b/client/src/webpages/dashboard/item_types/itemTypeUtils.ts @@ -37,6 +37,7 @@ export enum SchemaFieldRoles { BACKGROUND_IMAGE = 'backgroundImage', IS_DELETED = 'isDeleted', IP_ADDRESS = 'ipAddress', + EMAIL = 'email', } export const schemaFieldRolesFieldTypes = { @@ -49,6 +50,7 @@ export const schemaFieldRolesFieldTypes = { [SchemaFieldRoles.BACKGROUND_IMAGE]: GQLScalarType.Image, [SchemaFieldRoles.IS_DELETED]: GQLScalarType.Boolean, [SchemaFieldRoles.IP_ADDRESS]: GQLScalarType.IpAddress, + [SchemaFieldRoles.EMAIL]: GQLScalarType.String, } satisfies Omit< { [key in SchemaFieldRoles]: GQLScalarType }, SchemaFieldRoles.NONE @@ -77,6 +79,8 @@ export function getDisplayStringForRole( return 'Is Deleted'; case SchemaFieldRoles.IP_ADDRESS: return 'IP Address'; + case SchemaFieldRoles.EMAIL: + return 'Email'; case SchemaFieldRoles.NONE: return 'None'; } diff --git a/db/src/scripts/api-server-pg/2026.06.25T20.44.03.add_email_field_role_to_item_types.sql b/db/src/scripts/api-server-pg/2026.06.25T20.44.03.add_email_field_role_to_item_types.sql new file mode 100644 index 0000000..289cefb --- /dev/null +++ b/db/src/scripts/api-server-pg/2026.06.25T20.44.03.add_email_field_role_to_item_types.sql @@ -0,0 +1,140 @@ +-- Issue #839: add an `email` schema field role to user item types so adopters +-- can tag a string email field and have Coop populate +-- `personOrUserReportedPerson.email` on NCMEC reports when the additional-info +-- webhook is not configured (or returns no email). Today, email has no +-- field-role path, which causes NCMEC to reject reports as "incomplete." +-- +-- `email_field` lives on `public.item_types` and its temporal mirror +-- `public.item_types_history`. The materialized view `item_type_versions` +-- joins both tables, so we drop and recreate it (with its indexes and the +-- dependent `item_type_latest_versions` view) so the new column is +-- selectable. Postgres handles cascading via DROP ... CASCADE. + +BEGIN; + +ALTER TABLE public.item_types + ADD COLUMN email_field character varying(255); + +ALTER TABLE public.item_types_history + ADD COLUMN email_field character varying(255); + +-- Mirrors the per-role STRING check in `valid_field_role_field_type` +-- (which already covers `display_name_field`), but as a standalone +-- constraint so we don't touch the existing one. +ALTER TABLE public.item_types + ADD CONSTRAINT valid_email_field_field_type CHECK ( + (email_field IS NULL) + OR jsonb_path_exists( + (array_to_json(fields))::jsonb, + '$[*]?(@."name" == $"name" && @."type" == "STRING")'::jsonpath, + jsonb_build_object('name', email_field) + ) + ); + +-- CASCADE drops the four indexes on item_type_versions and the +-- item_type_latest_versions view; both are recreated below. +DROP MATERIALIZED VIEW public.item_type_versions CASCADE; + +CREATE MATERIALIZED VIEW public.item_type_versions AS +WITH item_type_versions AS ( + SELECT + item_types.id, + item_types.name, + item_types.description, + item_types.fields, + item_types.org_id, + item_types.sys_period, + item_types.kind, + item_types.display_name_field, + item_types.creator_id_field, + item_types.thread_id_field, + item_types.parent_id_field, + item_types.created_at_field, + item_types.is_deleted_field, + item_types.profile_icon_field, + item_types.background_image_field, + item_types.ip_address_field, + item_types.email_field, + item_types.is_default_user + FROM public.item_types + UNION ALL + SELECT + item_types_history.id, + item_types_history.name, + item_types_history.description, + item_types_history.fields, + item_types_history.org_id, + item_types_history.sys_period, + item_types_history.kind, + item_types_history.display_name_field, + item_types_history.creator_id_field, + item_types_history.thread_id_field, + item_types_history.parent_id_field, + item_types_history.created_at_field, + item_types_history.is_deleted_field, + item_types_history.profile_icon_field, + item_types_history.background_image_field, + item_types_history.ip_address_field, + item_types_history.email_field, + item_types_history.is_default_user + FROM public.item_types_history +), item_type_max_period_starts AS ( + SELECT + item_type_versions_1.id, + max(lower(item_type_versions_1.sys_period)) AS max_period_start + FROM item_type_versions item_type_versions_1 + GROUP BY item_type_versions_1.id +) +SELECT + item_type_versions.id, + item_type_versions.name, + item_type_versions.description, + item_type_versions.fields, + item_type_versions.org_id, + item_type_versions.kind, + item_type_versions.display_name_field, + item_type_versions.creator_id_field, + item_type_versions.thread_id_field, + item_type_versions.parent_id_field, + item_type_versions.created_at_field, + item_type_versions.is_deleted_field, + item_type_versions.profile_icon_field, + item_type_versions.background_image_field, + item_type_versions.ip_address_field, + item_type_versions.email_field, + item_type_versions.is_default_user, + lower(item_type_versions.sys_period) AS version, + ( + (item_type_max_period_starts.max_period_start = lower(item_type_versions.sys_period)) + AND upper_inf(item_type_versions.sys_period) + ) AS is_current +FROM item_type_versions +JOIN item_type_max_period_starts + ON ((item_type_max_period_starts.id)::text = (item_type_versions.id)::text) +WITH DATA; + +ALTER TABLE public.item_type_versions OWNER TO CURRENT_USER; + +CREATE INDEX item_type_versions_id_idx + ON public.item_type_versions USING btree (id); + +CREATE UNIQUE INDEX item_type_versions_id_is_current_idx + ON public.item_type_versions USING btree (id, is_current) + WHERE (is_current = true); + +CREATE INDEX item_type_versions_is_current_idx + ON public.item_type_versions USING btree (is_current); + +CREATE INDEX item_type_versions_version_idx + ON public.item_type_versions USING btree (version); + +CREATE VIEW public.item_type_latest_versions AS + SELECT + item_type_versions.id AS item_type_id, + to_char((item_type_versions.version AT TIME ZONE 'UTC'::text), 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"'::text) AS version + FROM public.item_type_versions + WHERE (item_type_versions.is_current = true); + +ALTER TABLE public.item_type_latest_versions OWNER TO CURRENT_USER; + +COMMIT; diff --git a/server/graphql/generated.ts b/server/graphql/generated.ts index 0c9921b..2210da2 100644 --- a/server/graphql/generated.ts +++ b/server/graphql/generated.ts @@ -5247,6 +5247,7 @@ export type GQLUserSchemaFieldRoles = { readonly backgroundImage?: Maybe; readonly createdAt?: Maybe; readonly displayName?: Maybe; + readonly email?: Maybe; readonly ipAddress?: Maybe; readonly isDeleted?: Maybe; readonly profileIcon?: Maybe; @@ -5256,6 +5257,7 @@ export type GQLUserSchemaFieldRolesInput = { readonly backgroundImage?: InputMaybe; readonly createdAt?: InputMaybe; readonly displayName?: InputMaybe; + readonly email?: InputMaybe; readonly ipAddress?: InputMaybe; readonly isDeleted?: InputMaybe; readonly profileIcon?: InputMaybe; @@ -14974,6 +14976,7 @@ export type GQLUserSchemaFieldRolesResolvers< ParentType, ContextType >; + email?: Resolver, ParentType, ContextType>; ipAddress?: Resolver< Maybe, ParentType, diff --git a/server/graphql/modules/itemType.ts b/server/graphql/modules/itemType.ts index e89e6bd..021c52d 100644 --- a/server/graphql/modules/itemType.ts +++ b/server/graphql/modules/itemType.ts @@ -221,6 +221,7 @@ const typeDefs = /* GraphQL */ ` backgroundImage: String isDeleted: String ipAddress: String + email: String } type ThreadSchemaFieldRoles { @@ -284,6 +285,7 @@ const typeDefs = /* GraphQL */ ` backgroundImage: String isDeleted: String ipAddress: String + email: String } input ThreadSchemaFieldRolesInput { diff --git a/server/services/moderationConfigService/dbTypes.ts b/server/services/moderationConfigService/dbTypes.ts index a1408f7..27a4628 100644 --- a/server/services/moderationConfigService/dbTypes.ts +++ b/server/services/moderationConfigService/dbTypes.ts @@ -33,6 +33,7 @@ export type ModerationConfigServicePg = { background_image_field: string | null; is_deleted_field: string | null; ip_address_field: string | null; + email_field: string | null; }; // TODO: redefine as a union to capture the correlation of the nulls, // then leverage FixKyselyRowCorrelation in the ItemTypesDbResult type. @@ -42,7 +43,6 @@ export type ModerationConfigServicePg = { name: GeneratedAlways; description: GeneratedAlways; org_id: GeneratedAlways; - created_at: GeneratedAlways; kind: GeneratedAlways; fields: GeneratedAlways; is_default_user: GeneratedAlways; @@ -55,6 +55,7 @@ export type ModerationConfigServicePg = { background_image_field: GeneratedAlways; is_deleted_field: GeneratedAlways; ip_address_field: GeneratedAlways; + email_field: GeneratedAlways; version: GeneratedAlways; is_current: GeneratedAlways; }; diff --git a/server/services/moderationConfigService/moderationConfigService.test.ts b/server/services/moderationConfigService/moderationConfigService.test.ts index fef4819..b425296 100644 --- a/server/services/moderationConfigService/moderationConfigService.test.ts +++ b/server/services/moderationConfigService/moderationConfigService.test.ts @@ -409,6 +409,7 @@ describe('ModerationConfigService', () => { "backgroundImage": undefined, "createdAt": undefined, "displayName": "fakeField", + "email": undefined, "ipAddress": undefined, "isDeleted": undefined, "profileIcon": undefined, diff --git a/server/services/moderationConfigService/modules/ItemTypeOperations.ts b/server/services/moderationConfigService/modules/ItemTypeOperations.ts index d136140..4fc467f 100644 --- a/server/services/moderationConfigService/modules/ItemTypeOperations.ts +++ b/server/services/moderationConfigService/modules/ItemTypeOperations.ts @@ -55,6 +55,7 @@ const itemTypeDbSelection = [ 'profile_icon_field as profileIconField', 'background_image_field as backgroundImageField', 'ip_address_field as ipAddressField', + 'email_field as emailField', 'org_id as orgId', 'is_default_user as isDefaultUserType', ] as const; @@ -465,6 +466,7 @@ export default class ItemTypeOperations { displayName?: string | null; isDeleted?: string | null; ipAddress?: string | null; + email?: string | null; }; }, ) { @@ -483,6 +485,7 @@ export default class ItemTypeOperations { display_name_field: input.schemaFieldRoles.displayName, is_deleted_field: input.schemaFieldRoles.isDeleted, ip_address_field: input.schemaFieldRoles.ipAddress, + email_field: input.schemaFieldRoles.email, }) .returning('id') .executeTakeFirstOrThrow(); @@ -508,6 +511,7 @@ export default class ItemTypeOperations { displayName?: string | null; isDeleted?: string | null; ipAddress?: string | null; + email?: string | null; }; }, ): Promise { @@ -536,6 +540,7 @@ export default class ItemTypeOperations { ip_address_field: replaceEmptyStringWithNull( input.schemaFieldRoles.ipAddress, ), + email_field: replaceEmptyStringWithNull(input.schemaFieldRoles.email), }), ) .where('id', '=', input.id) @@ -764,6 +769,7 @@ function dbResultToItemType( profileIcon: input.profileIconField ?? undefined, isDeleted: input.isDeletedField ?? undefined, ipAddress: input.ipAddressField ?? undefined, + email: input.emailField ?? undefined, } satisfies UserItemType['schemaFieldRoles']; default: assertUnreachable(input.kind); diff --git a/server/services/moderationConfigService/types/itemTypes.ts b/server/services/moderationConfigService/types/itemTypes.ts index d48e4c2..076386b 100644 --- a/server/services/moderationConfigService/types/itemTypes.ts +++ b/server/services/moderationConfigService/types/itemTypes.ts @@ -53,6 +53,7 @@ export type UserSchemaFieldRoles = { createdAt?: string; isDeleted?: string; ipAddress?: string; + email?: string; }; export type ThreadSchemaFieldRoles = { @@ -127,6 +128,7 @@ export type FieldRoleToScalarType = { backgroundImage: ScalarTypes['IMAGE']; isDeleted: ScalarTypes['BOOLEAN']; ipAddress: ScalarTypes['IP_ADDRESS']; + email: ScalarTypes['STRING']; }; export function getPartialSchemaFromOriginal(schema: ItemSchema) { diff --git a/server/services/ncmecService/buildSubmitReportParamsFromDecision.test.ts b/server/services/ncmecService/buildSubmitReportParamsFromDecision.test.ts index 62f8427..f8b9315 100644 --- a/server/services/ncmecService/buildSubmitReportParamsFromDecision.test.ts +++ b/server/services/ncmecService/buildSubmitReportParamsFromDecision.test.ts @@ -46,16 +46,22 @@ const datetimeField = (name: string): Field => ({ function makeUserItemType(overrides: { ipAddressField?: string; ipAddressFieldName?: string; + emailField?: string; data?: NormalizedItemData; }): UserItemType { const ipFieldName = overrides.ipAddressFieldName ?? 'client_ip'; // The schema is built immutably (no .push) to satisfy // functional/immutable-data; we then cast to the non-empty `ItemSchema` // brand because the constructor is internal. - const fields: readonly Field[] = - overrides.ipAddressField !== undefined - ? [stringField('display_name'), ipAddressField(ipFieldName)] - : [stringField('display_name')]; + const fields: readonly Field[] = [ + stringField('display_name'), + ...(overrides.ipAddressField !== undefined + ? [ipAddressField(ipFieldName)] + : []), + ...(overrides.emailField !== undefined + ? [stringField(overrides.emailField)] + : []), + ]; return { id: 'user-type-1', kind: 'USER', @@ -71,6 +77,9 @@ function makeUserItemType(overrides: { ...(overrides.ipAddressField !== undefined ? { ipAddress: overrides.ipAddressField } : {}), + ...(overrides.emailField !== undefined + ? { email: overrides.emailField } + : {}), }, }; } @@ -306,4 +315,65 @@ describe('buildSubmitReportParamsFromDecision', () => { expect(result.reportedUser.ipAddress).toBe('2001:db8::1'); }); }); + + // Regression: without this, adopters who don't run an external + // additional-info endpoint submit NCMEC reports with empty email, which + // NCMEC rejects as "incomplete." + describe('email field-role propagation', () => { + it('reads the user email from the schema field role and surfaces it on `reportedUser.email`', async () => { + const userItemType = makeUserItemType({ + emailField: 'user_email', + }); + const result = await buildSubmitReportParamsFromDecision( + makeInput({ + reportedUserItemType: userItemType, + reportedUserData: asNormalizedData({ + display_name: 'Alice', + user_email: 'alice@example.com', + }), + contentItemType: makeContentItemType({}), + contentData: asNormalizedData({ created_at: FIXED_NOW }), + }), + ); + + expect(result.reportedUser).toMatchObject({ + id: 'user-1', + typeId: 'user-type-1', + displayName: 'Alice', + email: 'alice@example.com', + }); + }); + + it('omits `reportedUser.email` when the role is not mapped', async () => { + const result = await buildSubmitReportParamsFromDecision( + makeInput({ + reportedUserItemType: makeUserItemType({}), + reportedUserData: asNormalizedData({ display_name: 'Alice' }), + contentItemType: makeContentItemType({}), + contentData: asNormalizedData({ created_at: FIXED_NOW }), + }), + ); + + // NCMEC validates email shape on receipt; an empty string here would + // produce the same "incomplete" rejection the bug repros. Missing key + // is the only safe encoding. + expect(result.reportedUser).not.toHaveProperty('email'); + }); + + it('omits `reportedUser.email` when the field is mapped but absent in the data', async () => { + const userItemType = makeUserItemType({ + emailField: 'user_email', + }); + const result = await buildSubmitReportParamsFromDecision( + makeInput({ + reportedUserItemType: userItemType, + reportedUserData: asNormalizedData({ display_name: 'Alice' }), + contentItemType: makeContentItemType({}), + contentData: asNormalizedData({ created_at: FIXED_NOW }), + }), + ); + + expect(result.reportedUser).not.toHaveProperty('email'); + }); + }); }); diff --git a/server/services/ncmecService/buildSubmitReportParamsFromDecision.ts b/server/services/ncmecService/buildSubmitReportParamsFromDecision.ts index 4e11086..f002677 100644 --- a/server/services/ncmecService/buildSubmitReportParamsFromDecision.ts +++ b/server/services/ncmecService/buildSubmitReportParamsFromDecision.ts @@ -110,6 +110,12 @@ export async function buildSubmitReportParamsFromDecision( 'ipAddress', reportedUserData, ); + const reportedUserEmail = getFieldValueForRole( + reportedUserItemType.schema, + reportedUserItemType.schemaFieldRoles, + 'email', + reportedUserData, + ); // Pre-index allMediaItems by (itemId, typeId) so the per-decisionComponent // lookup below is O(1) instead of O(n) for every reportedMedia entry. The @@ -187,6 +193,7 @@ export async function buildSubmitReportParamsFromDecision( ...(displayName ? { displayName } : {}), ...(profilePicUrl ? { profilePicture: profilePicUrl.url } : {}), ...(reportedUserIp ? { ipAddress: reportedUserIp } : {}), + ...(reportedUserEmail ? { email: reportedUserEmail } : {}), }, orgId, media, diff --git a/server/services/ncmecService/ncmecReporting.test.ts b/server/services/ncmecService/ncmecReporting.test.ts index 56ee6c0..71b5659 100644 --- a/server/services/ncmecService/ncmecReporting.test.ts +++ b/server/services/ncmecService/ncmecReporting.test.ts @@ -3,6 +3,7 @@ import { clampIncidentDateTimeToPast, mergeFieldRoleIpIntoEvents, NCMECEvent, + resolveReportedPersonEmail, summarizeCyberTipFailure, } from './ncmecReporting.js'; @@ -307,6 +308,48 @@ describe('NCMEC reporting', () => { }); }); + describe('resolveReportedPersonEmail', () => { + it('returns the webhook emails when present', () => { + const webhook = [ + { _text: 'verified@example.com', _attributes: { verified: true } }, + ]; + expect(resolveReportedPersonEmail(webhook, 'role@example.com')).toEqual( + webhook, + ); + }); + + it('falls back to the field-role email when the webhook returned an empty array', () => { + expect(resolveReportedPersonEmail([], 'role@example.com')).toEqual([ + { _text: 'role@example.com' }, + ]); + }); + + it('falls back to the field-role email when the webhook returned undefined', () => { + expect(resolveReportedPersonEmail(undefined, 'role@example.com')).toEqual( + [{ _text: 'role@example.com' }], + ); + }); + + it('returns undefined when neither source has data', () => { + expect(resolveReportedPersonEmail(undefined, undefined)).toBeUndefined(); + expect(resolveReportedPersonEmail([], undefined)).toBeUndefined(); + }); + + it('treats whitespace-only field-role email as absent', () => { + // NCMEC validates the email shape on receipt; a `{ _text: " " }` + // submission would fail the same way the original incomplete-report + // bug did. Trim and drop rather than ship whitespace. + expect(resolveReportedPersonEmail(undefined, ' ')).toBeUndefined(); + expect(resolveReportedPersonEmail([], '\t\n')).toBeUndefined(); + }); + + it('trims surrounding whitespace from a valid field-role email', () => { + expect( + resolveReportedPersonEmail(undefined, ' role@example.com '), + ).toEqual([{ _text: 'role@example.com' }]); + }); + }); + describe('summarizeCyberTipFailure', () => { const previousDebug = process.env.NCMEC_DEBUG; const previousNodeEnv = process.env.NODE_ENV; diff --git a/server/services/ncmecService/ncmecReporting.ts b/server/services/ncmecService/ncmecReporting.ts index 82c3b67..4f95fc7 100644 --- a/server/services/ncmecService/ncmecReporting.ts +++ b/server/services/ncmecService/ncmecReporting.ts @@ -186,6 +186,10 @@ type NCMECUserParams = { /** Bare IP from the `ipAddress` field role; appended as a synthesised * `Unknown` event. */ ipAddress?: string; + /** Bare email from the `email` field role. Used as the + * `personOrUserReportedPerson.email` when no external additional-info + * endpoint provided one. */ + email?: string; }; export type NCMECReportParams = { @@ -543,6 +547,20 @@ export function mergeFieldRoleIpIntoEvents( return events.length > 0 ? events : undefined; } +/** Resolve the email(s) for `personOrUserReportedPerson`. Prefers the + * webhook's enriched response (carries NCMEC `type` / `verified` attributes); + * falls back to a bare field-role email otherwise. Returns undefined when + * neither source has data. */ +export function resolveReportedPersonEmail( + webhookEmails: Email[] | undefined, + fieldRoleEmail: string | undefined, +): Email[] | undefined { + if (webhookEmails && webhookEmails.length > 0) return webhookEmails; + const trimmed = fieldRoleEmail?.trim(); + if (trimmed) return [{ _text: trimmed }]; + return undefined; +} + export function buildInternetDetailsFromOrgSetting( defaultInternetDetailType: string | null | undefined, moreInfoUrl: string | null | undefined, @@ -1576,10 +1594,10 @@ export default class NcmecReporting { ? queryResponse.termsOfService.trim() : undefined; - const reportedPersonEmail = - userAdditionalInfo.email && userAdditionalInfo.email.length > 0 - ? userAdditionalInfo.email - : undefined; + const reportedPersonEmail = resolveReportedPersonEmail( + userAdditionalInfo.email, + reportParams.reportedUser.email, + ); const personOrUserReportedPerson = reportedPersonEmail ? { email: reportedPersonEmail } : undefined;