diff --git a/client/package-lock.json b/client/package-lock.json index c015fc7..c64d621 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -29,7 +29,7 @@ "@radix-ui/react-slider": "^1.2.0", "@radix-ui/react-switch": "^1.1.0", "@radix-ui/react-tooltip": "^1.1.2", - "@roostorg/coop-types": "^2.2.0", + "@roostorg/coop-types": "^2.3.0", "@tailwindcss/container-queries": "^0.1.1", "@tailwindcss/forms": "^0.5.7", "@tailwindcss/typography": "^0.5.13", @@ -3146,9 +3146,9 @@ ] }, "node_modules/@roostorg/coop-types": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@roostorg/coop-types/-/coop-types-2.2.0.tgz", - "integrity": "sha512-qq7WmaGIPZo/bzJfhIme5yVU/SRtFeifEzq3NJi/PFjOEh2lgj30vUZVCeZHsi8sEjmj63sFeUyrM+kiHZKljg==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@roostorg/coop-types/-/coop-types-2.3.0.tgz", + "integrity": "sha512-aOHomJIoqNUUoHxewxjOe6x2Y7ALcVgweUX7SxD18DwfylYK1KJZi4Q5nZB0SMm6q7V5H6btJfo4jRQMLxUT+g==", "license": "ISC", "dependencies": { "date-fns": "^2.29.3", diff --git a/client/package.json b/client/package.json index 210874b..30521c4 100644 --- a/client/package.json +++ b/client/package.json @@ -36,7 +36,7 @@ "@radix-ui/react-slider": "^1.2.0", "@radix-ui/react-switch": "^1.1.0", "@radix-ui/react-tooltip": "^1.1.2", - "@roostorg/coop-types": "^2.2.0", + "@roostorg/coop-types": "^2.3.0", "@tailwindcss/container-queries": "^0.1.1", "@tailwindcss/forms": "^0.5.7", "@tailwindcss/typography": "^0.5.13", @@ -88,15 +88,15 @@ "@testing-library/jest-dom": "^5.11.9", "@testing-library/react": "^11.2.3", "@testing-library/user-event": "^12.6.0", + "@types/google.maps": "^3.48.7", + "@types/latlon-geohash": "^2.0.0", "@types/lodash": "^4.14.181", + "@types/papaparse": "^5.3.14", "@types/react": "^18.0.0", "@types/react-beautiful-dnd": "^13.1.4", + "@types/react-csv": "^1.1.3", "@types/react-dom": "^18.0.0", "@types/react-router-hash-link": "^2.4.5", - "@types/google.maps": "^3.48.7", - "@types/latlon-geohash": "^2.0.0", - "@types/papaparse": "^5.3.14", - "@types/react-csv": "^1.1.3", "@types/react-syntax-highlighter": "^13.5.2", "@types/react-table": "^7.7.5", "@types/recharts": "^1.8.23", diff --git a/client/src/graphql/generated.ts b/client/src/graphql/generated.ts index e5b5c66..bea697c 100644 --- a/client/src/graphql/generated.ts +++ b/client/src/graphql/generated.ts @@ -1277,6 +1277,7 @@ export const GQLFieldType = { Image: 'IMAGE', IpAddress: 'IP_ADDRESS', Map: 'MAP', + Media: 'MEDIA', Number: 'NUMBER', PolicyId: 'POLICY_ID', RelatedItem: 'RELATED_ITEM', @@ -4077,6 +4078,7 @@ export const GQLScalarType = { Id: 'ID', Image: 'IMAGE', IpAddress: 'IP_ADDRESS', + Media: 'MEDIA', Number: 'NUMBER', PolicyId: 'POLICY_ID', RelatedItem: 'RELATED_ITEM', @@ -4217,6 +4219,7 @@ export const GQLSignalInputType = { Id: 'ID', Image: 'IMAGE', IpAddress: 'IP_ADDRESS', + Media: 'MEDIA', Number: 'NUMBER', PolicyId: 'POLICY_ID', RelatedItem: 'RELATED_ITEM', diff --git a/client/src/models/signal.ts b/client/src/models/signal.ts index d0e77e0..31711a5 100644 --- a/client/src/models/signal.ts +++ b/client/src/models/signal.ts @@ -112,6 +112,7 @@ export function outputTypeToComparators(outputType: GQLSignalOutputType) { case GQLScalarType.Audio: case GQLScalarType.Image: case GQLScalarType.Video: + case GQLScalarType.Media: case GQLScalarType.Geohash: case GQLScalarType.Boolean: case GQLScalarType.RelatedItem: diff --git a/client/src/webpages/dashboard/item_types/itemTypeUtils.ts b/client/src/webpages/dashboard/item_types/itemTypeUtils.ts index 2d60bd2..03bf3ef 100644 --- a/client/src/webpages/dashboard/item_types/itemTypeUtils.ts +++ b/client/src/webpages/dashboard/item_types/itemTypeUtils.ts @@ -225,6 +225,15 @@ export function generateFakeScalarFieldValue(fieldType: ScalarType) { return `https://url.com/some-path/${Math.floor(100 * Math.random())}`; case 'IP_ADDRESS': return `192.0.2.${Math.floor(255 * Math.random())}`; + case 'MEDIA': { + // Like AUDIO/IMAGE/VIDEO, the fake value is the *input* (pre-coercion) form + // — a raw URL string. The server's MEDIA coercion resolves the kind from + // the extension, so cycle through one of each so the IMAGE/VIDEO/AUDIO + // render paths all get fake-data coverage. + const extensions = ['jpg', 'mp4', 'mp3']; + const ext = extensions[Math.floor(Math.random() * extensions.length)]; + return `https://url.com/${Math.floor(1000 * Math.random())}.${ext}`; + } } } diff --git a/client/src/webpages/dashboard/mrt/manual_review_job/v2/ManualReviewJobFieldsComponent.tsx b/client/src/webpages/dashboard/mrt/manual_review_job/v2/ManualReviewJobFieldsComponent.tsx index 963847d..52080ad 100644 --- a/client/src/webpages/dashboard/mrt/manual_review_job/v2/ManualReviewJobFieldsComponent.tsx +++ b/client/src/webpages/dashboard/mrt/manual_review_job/v2/ManualReviewJobFieldsComponent.tsx @@ -244,6 +244,78 @@ function TableRowComponent(props: { ); } + case 'MEDIA': { + // Polymorphic field — render with the kind detected at coercion time, + // falling back to a plain link when detection didn't resolve. + const url = value?.url; + if (url == null) { + return ; + } + if (value.mediaType === 'IMAGE') { + return ( +
+ + {label ?
{label}
: null} +
+ ); + } + if (value.mediaType === 'VIDEO') { + return ( +
+ + {label ?
{label}
: null} +
+ ); + } + if (value.mediaType === 'AUDIO') { + return ( +
+ {label ?
{label}
: null} + +
+ ); + } + return ( +
+ {label ?
{label}
: null} + + {url} + +
+ ); + } case 'DATETIME': { return (
@@ -348,6 +420,7 @@ function FieldComponent(props: { case 'STRING': case 'USER_ID': case 'VIDEO': + case 'MEDIA': case 'RELATED_ITEM': case 'URL': case 'POLICY_ID': @@ -411,6 +484,7 @@ function ContainerComponent(props: { return true; case 'AUDIO': case 'IMAGE': + case 'MEDIA': case 'RELATED_ITEM': case 'URL': case 'VIDEO': @@ -449,6 +523,7 @@ function ContainerComponent(props: { case 'URL': case 'POLICY_ID': case 'IP_ADDRESS': + case 'MEDIA': case 'VIDEO': { throw Error('Cannot call container component with scalar field'); } @@ -514,7 +589,8 @@ function ContainerComponent(props: { className={` ${ data.container!.valueScalarType === 'IMAGE' || data.container!.valueScalarType === 'VIDEO' || - data.container!.valueScalarType === 'AUDIO' + data.container!.valueScalarType === 'AUDIO' || + data.container!.valueScalarType === 'MEDIA' ? '' : 'flex-col' } flex overflow-x-scroll border-slate-200 rounded p-1.5 ${ diff --git a/client/src/webpages/dashboard/rules/info/RuleTestModal.tsx b/client/src/webpages/dashboard/rules/info/RuleTestModal.tsx index 88db331..8c8c374 100644 --- a/client/src/webpages/dashboard/rules/info/RuleTestModal.tsx +++ b/client/src/webpages/dashboard/rules/info/RuleTestModal.tsx @@ -130,6 +130,8 @@ export default function RuleTestModal(props: { return 'https://test.com/image.jpg'; case 'VIDEO': return 'https://test.com/video.mp4'; + case 'MEDIA': + return 'https://test.com/media.jpg'; case 'STRING': return 'Some text...'; case 'URL': diff --git a/client/src/webpages/dashboard/rules/info/insights/RuleInsightsSamplesTable.tsx b/client/src/webpages/dashboard/rules/info/insights/RuleInsightsSamplesTable.tsx index 49583b1..1caf6c9 100644 --- a/client/src/webpages/dashboard/rules/info/insights/RuleInsightsSamplesTable.tsx +++ b/client/src/webpages/dashboard/rules/info/insights/RuleInsightsSamplesTable.tsx @@ -767,6 +767,7 @@ export function getStringFromContent( return content.toString(); case GQLFieldType.Image: case GQLFieldType.Video: + case GQLFieldType.Media: return typeof content === 'object' ? content.url : null; case GQLFieldType.Datetime: return parseDatetimeToReadableStringInCurrentTimeZone(content); diff --git a/client/src/webpages/dashboard/rules/types.ts b/client/src/webpages/dashboard/rules/types.ts index 3a8bea9..73771d5 100644 --- a/client/src/webpages/dashboard/rules/types.ts +++ b/client/src/webpages/dashboard/rules/types.ts @@ -151,6 +151,7 @@ export function isMediaType(it: GQLScalarType): boolean { return ( it === GQLScalarType.Audio || it === GQLScalarType.Video || - it === GQLScalarType.Image + it === GQLScalarType.Image || + it === GQLScalarType.Media ); } diff --git a/server/graphql/generated.ts b/server/graphql/generated.ts index 63f3590..637936b 100644 --- a/server/graphql/generated.ts +++ b/server/graphql/generated.ts @@ -1344,6 +1344,7 @@ export const GQLFieldType = { Image: 'IMAGE', IpAddress: 'IP_ADDRESS', Map: 'MAP', + Media: 'MEDIA', Number: 'NUMBER', PolicyId: 'POLICY_ID', RelatedItem: 'RELATED_ITEM', @@ -4144,6 +4145,7 @@ export const GQLScalarType = { Id: 'ID', Image: 'IMAGE', IpAddress: 'IP_ADDRESS', + Media: 'MEDIA', Number: 'NUMBER', PolicyId: 'POLICY_ID', RelatedItem: 'RELATED_ITEM', @@ -4284,6 +4286,7 @@ export const GQLSignalInputType = { Id: 'ID', Image: 'IMAGE', IpAddress: 'IP_ADDRESS', + Media: 'MEDIA', Number: 'NUMBER', PolicyId: 'POLICY_ID', RelatedItem: 'RELATED_ITEM', diff --git a/server/graphql/schema.ts b/server/graphql/schema.ts index b8ee660..8c8676a 100644 --- a/server/graphql/schema.ts +++ b/server/graphql/schema.ts @@ -59,6 +59,7 @@ const typeDefs = /* GraphQL */ ` AUDIO IMAGE VIDEO + MEDIA DATETIME GEOHASH RELATED_ITEM @@ -77,6 +78,7 @@ const typeDefs = /* GraphQL */ ` AUDIO IMAGE VIDEO + MEDIA DATETIME GEOHASH RELATED_ITEM @@ -101,6 +103,7 @@ const typeDefs = /* GraphQL */ ` AUDIO IMAGE VIDEO + MEDIA DATETIME GEOHASH ARRAY diff --git a/server/package-lock.json b/server/package-lock.json index 25d9fa9..efd1540 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -25,7 +25,7 @@ "@opentelemetry/api": "^1.8.0", "@opentelemetry/semantic-conventions": "^1.22.0", "@roostorg/coop-integration-example": "^2.0.0", - "@roostorg/coop-types": "^2.2.0", + "@roostorg/coop-types": "^2.3.0", "@sendgrid/mail": "^8.1.6", "@stdlib/stats-binomial-test": "^0.0.7", "@total-typescript/ts-reset": "^0.3.7", @@ -3551,9 +3551,9 @@ } }, "node_modules/@roostorg/coop-types": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@roostorg/coop-types/-/coop-types-2.2.0.tgz", - "integrity": "sha512-qq7WmaGIPZo/bzJfhIme5yVU/SRtFeifEzq3NJi/PFjOEh2lgj30vUZVCeZHsi8sEjmj63sFeUyrM+kiHZKljg==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@roostorg/coop-types/-/coop-types-2.3.0.tgz", + "integrity": "sha512-aOHomJIoqNUUoHxewxjOe6x2Y7ALcVgweUX7SxD18DwfylYK1KJZi4Q5nZB0SMm6q7V5H6btJfo4jRQMLxUT+g==", "license": "ISC", "dependencies": { "date-fns": "^2.29.3", diff --git a/server/package.json b/server/package.json index 8098e47..e7afdb1 100644 --- a/server/package.json +++ b/server/package.json @@ -40,7 +40,7 @@ "@opentelemetry/api": "^1.8.0", "@opentelemetry/semantic-conventions": "^1.22.0", "@roostorg/coop-integration-example": "^2.0.0", - "@roostorg/coop-types": "^2.2.0", + "@roostorg/coop-types": "^2.3.0", "@sendgrid/mail": "^8.1.6", "@stdlib/stats-binomial-test": "^0.0.7", "@total-typescript/ts-reset": "^0.3.7", @@ -55,8 +55,8 @@ "dataloader": "^2.1.0", "date-fns": "^2.30.0", "express": "^5.1.0", - "fast-check": "^4.6.0", "express-session": "^1.17.2", + "fast-check": "^4.6.0", "form-data": "^4.0.0", "form-data-encoder": "^4.0.2", "formdata-node": "^6.0.3", diff --git a/server/services/itemProcessingService/fieldTypeHandlers.test.ts b/server/services/itemProcessingService/fieldTypeHandlers.test.ts index 4342997..89d7ae8 100644 --- a/server/services/itemProcessingService/fieldTypeHandlers.test.ts +++ b/server/services/itemProcessingService/fieldTypeHandlers.test.ts @@ -9,7 +9,10 @@ import { import fc from 'fast-check'; import { FieldArbitrary } from '../../test/arbitraries/ContentType.js'; -import { fieldTypeHandlers } from './fieldTypeHandlers.js'; +import { + detectMediaKindFromUrl, + fieldTypeHandlers, +} from './fieldTypeHandlers.js'; describe('Content type schemas', () => { describe('fieldTypeHandlers', () => { @@ -94,4 +97,79 @@ describe('Content type schemas', () => { ).toBeInstanceOf(Error); }); }); + + describe('MEDIA coercion', () => { + const { coerce } = fieldTypeHandlers[ScalarTypes.MEDIA]; + + test.each([ + ['https://example.com/cat.jpg', ScalarTypes.IMAGE], + ['https://example.com/cat.JPG', ScalarTypes.IMAGE], + ['https://example.com/photo.jpeg', ScalarTypes.IMAGE], + ['https://example.com/anim.gif', ScalarTypes.IMAGE], + ['https://example.com/pic.webp', ScalarTypes.IMAGE], + ['https://example.com/clip.mp4', ScalarTypes.VIDEO], + ['https://example.com/clip.mov', ScalarTypes.VIDEO], + ['https://example.com/clip.webm', ScalarTypes.VIDEO], + ['https://example.com/song.mp3', ScalarTypes.AUDIO], + ['https://example.com/song.m4a', ScalarTypes.AUDIO], + ['https://example.com/song.wav', ScalarTypes.AUDIO], + ])('resolves %s to mediaType=%s', (url, expectedKind) => { + expect(coerce(url, [])).toEqual({ url, mediaType: expectedKind }); + }); + + test.each([ + // Ambiguous container — .ogg can be audio or video; stay unresolved. + 'https://example.com/song.ogg', + // No extension. + 'https://example.com/profile/123', + // Unknown extension. + 'https://example.com/file.xyz', + // Trailing dot. + 'https://example.com/file.', + ])('returns mediaType=null for unresolved URL %s', (url) => { + expect(coerce(url, [])).toEqual({ url, mediaType: null }); + }); + + test('treats empty string as missing (returns null)', () => { + expect(coerce('', [])).toBeNull(); + }); + + test.each([ + ['not a url', 'invalid string'], + // eslint-disable-next-line no-script-url -- testing that a javascript: URL is rejected + ['javascript:alert(1)', 'blocked scheme'], + ['ftp://example.com/x.mp3', 'unsupported scheme'], + ])('rejects %s (%s)', (url) => { + expect(coerce(url, [])).toBeInstanceOf(Error); + }); + + test.each([42, true, {}, []])('rejects non-string input %p', (value) => { + expect(coerce(value, [])).toBeInstanceOf(Error); + }); + + test('keeps the original URL casing in the returned object', () => { + expect(coerce('https://example.com/cat.PNG?v=2', [])).toEqual({ + url: 'https://example.com/cat.PNG?v=2', + mediaType: ScalarTypes.IMAGE, + }); + }); + }); + + describe('detectMediaKindFromUrl', () => { + test('returns null for unparseable URL', () => { + expect(detectMediaKindFromUrl('not a url')).toBeNull(); + }); + + test('uses the last dot in the pathname', () => { + expect( + detectMediaKindFromUrl('https://example.com/path.with.dots/file.mp4'), + ).toBe(ScalarTypes.VIDEO); + }); + + test('ignores extensions in the query string', () => { + expect( + detectMediaKindFromUrl('https://example.com/foo?x=y.mp4'), + ).toBeNull(); + }); + }); }); diff --git a/server/services/itemProcessingService/fieldTypeHandlers.ts b/server/services/itemProcessingService/fieldTypeHandlers.ts index fa1ed0d..47a2c89 100644 --- a/server/services/itemProcessingService/fieldTypeHandlers.ts +++ b/server/services/itemProcessingService/fieldTypeHandlers.ts @@ -7,6 +7,7 @@ import { type ContainerType, type ContainerTypeRuntimeType, type ItemIdentifier, + type MediaKind, type RelatedItem, type ScalarType, type ScalarTypeRuntimeType, @@ -183,6 +184,10 @@ export const fieldTypeHandlers: Handlers = { coerce: coerceMediaUrlInput, getValues: scalarGetValues, }, + [ScalarTypes.MEDIA]: { + coerce: coerceMediaInput, + getValues: scalarGetValues, + }, [ScalarTypes.DATETIME]: { getValues: scalarGetValues, coerce(v) { @@ -312,6 +317,69 @@ function coerceMediaUrlInput(value: unknown) { : err; } +// Extension → media kind. Lowercase, no leading dot. Conservative — only +// extensions that map unambiguously to one kind are listed, so e.g. .ogg +// (audio or video container) stays unresolved. +const MEDIA_EXTENSION_TO_KIND: Readonly> = { + // Image + jpg: ScalarTypes.IMAGE, + jpeg: ScalarTypes.IMAGE, + png: ScalarTypes.IMAGE, + gif: ScalarTypes.IMAGE, + webp: ScalarTypes.IMAGE, + bmp: ScalarTypes.IMAGE, + svg: ScalarTypes.IMAGE, + avif: ScalarTypes.IMAGE, + heic: ScalarTypes.IMAGE, + heif: ScalarTypes.IMAGE, + tif: ScalarTypes.IMAGE, + tiff: ScalarTypes.IMAGE, + // Video + mp4: ScalarTypes.VIDEO, + m4v: ScalarTypes.VIDEO, + mov: ScalarTypes.VIDEO, + webm: ScalarTypes.VIDEO, + mkv: ScalarTypes.VIDEO, + avi: ScalarTypes.VIDEO, + flv: ScalarTypes.VIDEO, + // Audio + mp3: ScalarTypes.AUDIO, + m4a: ScalarTypes.AUDIO, + wav: ScalarTypes.AUDIO, + aac: ScalarTypes.AUDIO, + flac: ScalarTypes.AUDIO, + opus: ScalarTypes.AUDIO, + wma: ScalarTypes.AUDIO, +}; + +/** + * Best-effort kind detection from a URL's pathname extension. Returns `null` + * when the URL is unparseable, has no extension, or has an extension we don't + * map. Consumers that need stronger guarantees should probe Content-Type. + */ +export function detectMediaKindFromUrl(url: string): MediaKind | null { + let pathname: string; + try { + pathname = new URL(url).pathname; + } catch { + return null; + } + const dot = pathname.lastIndexOf('.'); + if (dot < 0 || dot === pathname.length - 1) return null; + const ext = pathname.slice(dot + 1).toLowerCase(); + return MEDIA_EXTENSION_TO_KIND[ext] ?? null; +} + +function coerceMediaInput(value: unknown) { + const err = new Error('This field, if given, must hold a valid URL.'); + + if (typeof value !== 'string') return err; + if (value === '') return null; + if (!isValidUrl(value)) return err; + + return { url: value, mediaType: detectMediaKindFromUrl(value) }; +} + function coerceIdLikeInput(value: unknown) { // NB: we don't currently have any restrictions on the string in an `ID` field; // in particular, it's allowed to be empty. But note that _item ids_ (like in diff --git a/server/test/arbitraries/ContentType.ts b/server/test/arbitraries/ContentType.ts index 44885a6..c01bed4 100644 --- a/server/test/arbitraries/ContentType.ts +++ b/server/test/arbitraries/ContentType.ts @@ -56,6 +56,16 @@ export const IdLikeArbitrary = fc.string(); export const MediaUrlArbitrary = fc.record({ url: fc.string() /* todo */ }); +export const MediaArbitrary = fc.record({ + url: fc.string(), + mediaType: fc.constantFrom( + ScalarTypes.AUDIO, + ScalarTypes.IMAGE, + ScalarTypes.VIDEO, + null, + ), +}); + export const RelatedItemArbitrary = fc.record({ id: fc.string(), typeId: fc.string(), @@ -89,6 +99,7 @@ export const ScalarValidValuesArbitraries = { [ScalarTypes.URL]: fc.webUrl({ validSchemes: ['http', 'https'] }), [ScalarTypes.USER_ID]: IdLikeArbitrary, [ScalarTypes.VIDEO]: MediaUrlArbitrary, + [ScalarTypes.MEDIA]: MediaArbitrary, [ScalarTypes.DATETIME]: DateStringArbitrary, [ScalarTypes.RELATED_ITEM]: RelatedItemArbitrary, [ScalarTypes.POLICY_ID]: IdLikeArbitrary,