From 7d69e77ac6348f62acfa3fc2734df6c148efc3d0 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 20 Mar 2026 15:22:53 +0000 Subject: [PATCH] Implement proper play conversion for frontend/backend testing Convert to json-equivalent object, and back, without needing json stringify/parse --- package-lock.json | 11 +++ package.json | 1 + .../tests/component/transformers.test.ts | 2 +- .../tests/musicbrainz/musicbrainz.test.ts | 6 +- src/backend/tests/plays/playParsing.test.ts | 5 +- .../tests/scrobbler/scrobblers.test.ts | 8 +- src/core/Atomic.ts | 17 ++-- src/core/PlayTestUtils.ts | 20 ++--- src/core/tests/utils/fixtures.ts | 81 ++++++++++++++++++- src/stories/storyUtils.ts | 4 +- 10 files changed, 120 insertions(+), 35 deletions(-) diff --git a/package-lock.json b/package-lock.json index 05622870..6da2d3d2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -170,6 +170,7 @@ "mocha": "^10.3.0", "mockdate": "^3.0.5", "msw": "^2.12.10", + "neotraverse": "^0.6.18", "nodemon": "^3.0.3", "playwright": "^1.58.2", "sinon": "^21.0.2", @@ -15127,6 +15128,16 @@ "node": ">= 0.6" } }, + "node_modules/neotraverse": { + "version": "0.6.18", + "resolved": "https://registry.npmjs.org/neotraverse/-/neotraverse-0.6.18.tgz", + "integrity": "sha512-Z4SmBUweYa09+o6pG+eASabEpP6QkQ70yHj351pQoEXIs8uHbaU2DWVmzBANKgflPa47A50PtB2+NgRpQvr7vA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, "node_modules/next-themes": { "version": "0.4.6", "resolved": "https://registry.npmjs.org/next-themes/-/next-themes-0.4.6.tgz", diff --git a/package.json b/package.json index 1d7a3eb2..6819a908 100644 --- a/package.json +++ b/package.json @@ -208,6 +208,7 @@ "mocha": "^10.3.0", "mockdate": "^3.0.5", "msw": "^2.12.10", + "neotraverse": "^0.6.18", "nodemon": "^3.0.3", "playwright": "^1.58.2", "sinon": "^21.0.2", diff --git a/src/backend/tests/component/transformers.test.ts b/src/backend/tests/component/transformers.test.ts index 05aa0810..d35b93c6 100644 --- a/src/backend/tests/component/transformers.test.ts +++ b/src/backend/tests/component/transformers.test.ts @@ -7,7 +7,7 @@ import AbstractComponent, { AbstractComponentConfig } from "../../common/Abstrac import { ConditionalSearchAndReplaceRegExp, STAGE_TYPES, STAGE_TYPES_METADATA, STAGE_TYPES_USER, TRANSFORM_HOOK } from "../../common/infrastructure/Transform.js"; import { isConditionalSearchAndReplace } from "../../utils/PlayTransformUtils.js"; -import { asPlays, generateArtistsStr, generatePlay, normalizePlays } from "../../../core/PlayTestUtils.js"; +import { generateArtistsStr, generatePlay } from "../../../core/PlayTestUtils.js"; import { WebhookPayload } from "../../common/infrastructure/config/health/webhooks.js"; import { findCauseByMessage } from "../../utils/ErrorUtils.js"; import NativeTransformer from "../../common/transforms/NativeTransformer.js"; diff --git a/src/backend/tests/musicbrainz/musicbrainz.test.ts b/src/backend/tests/musicbrainz/musicbrainz.test.ts index fea951f3..a9c396ec 100644 --- a/src/backend/tests/musicbrainz/musicbrainz.test.ts +++ b/src/backend/tests/musicbrainz/musicbrainz.test.ts @@ -460,7 +460,7 @@ describe('#MB Missing Types', function() { it('Finds none missing when all mbids are defined', function() { - const play = withBrainz(generatePlay(), ['album', 'artist', 'track']); + const play = withBrainz(generatePlay(), {include: ['album', 'artist', 'track']}); const missing = missingMbidTypes(play); expect(missing.length).eq(0); }); @@ -474,7 +474,7 @@ describe('#MB Missing Types', function() { it('Finds duration missing', function() { - const play = withBrainz(generatePlay(), ['album', 'artist', 'track']); + const play = withBrainz(generatePlay(), {include: ['album', 'artist', 'track']}); delete play.data.duration; const missing = missingMbidTypes(play); expect(missing.length).eq(1); @@ -491,7 +491,7 @@ describe('#MB Missing Types', function() { it('intersect is not empty when missing any desired types', function() { - const play = withBrainz(generatePlay(), ['album', 'track']); + const play = withBrainz(generatePlay(), {include: ['album', 'track']}); const missing = missingMbidTypes(play); expect(missing).to.have.members(['artists']); expect(intersect(DEFAULT_MISSING_TYPES, missing)).length.is.greaterThan(0); diff --git a/src/backend/tests/plays/playParsing.test.ts b/src/backend/tests/plays/playParsing.test.ts index 9b277408..1b1b53cf 100644 --- a/src/backend/tests/plays/playParsing.test.ts +++ b/src/backend/tests/plays/playParsing.test.ts @@ -1,10 +1,9 @@ -import { loggerTest, loggerDebug, childLogger } from "@foxxmd/logging"; import chai, { assert, expect } from 'chai'; import asPromised from 'chai-as-promised'; import { after, before, describe, it } from 'mocha'; -import { asPlays, generateArtistsStr, generatePlay, normalizePlays } from "../../../core/PlayTestUtils.js"; -import { parseArtistCredits, parseContextAwareStringList, parseCredits, parseTrackCredits, uniqueNormalizedStrArr } from "../../utils/StringUtils.js"; +import { generateArtistsStr, } from "../../../core/PlayTestUtils.js"; +import { parseArtistCredits, parseCredits, parseTrackCredits, uniqueNormalizedStrArr } from "../../utils/StringUtils.js"; import testData from '../utils/playTestData.json' with { type: "json" }; import { intersect } from "../../utils.js"; import { ExpectedResults } from "../utils/interfaces.js"; diff --git a/src/backend/tests/scrobbler/scrobblers.test.ts b/src/backend/tests/scrobbler/scrobblers.test.ts index f88fd7bf..c0c9d820 100644 --- a/src/backend/tests/scrobbler/scrobblers.test.ts +++ b/src/backend/tests/scrobbler/scrobblers.test.ts @@ -12,7 +12,7 @@ import { genGroupIdStr } from '../../../core/PlayUtils.js'; import mixedDuration from '../plays/mixedDuration.json' with { type: 'json' }; import withDuration from '../plays/withDuration.json' with { type: 'json' }; import { MockNetworkError, withRequestInterception } from "../utils/networking.js"; -import { asPlays, generatePlay, generatePlayPlatformId, generatePlays, generateSourcePlayerObj, normalizePlays } from "../../../core/PlayTestUtils.js"; +import { generatePlay, generatePlayPlatformId, generatePlays, generateSourcePlayerObj, normalizePlays } from "../../../core/PlayTestUtils.js"; import MockDate from 'mockdate'; import { NowPlayingScrobbler, TestAuthScrobbler, TestScrobbler } from "./TestScrobbler.js"; @@ -20,14 +20,16 @@ import { PaginatedTimeRangeOptions, PlayPlatformId, REFRESH_STALE_DEFAULT } from import { defaultLifecycle } from '../../utils/PlayTransformUtils.js'; import { shuffleArray } from '../../utils/DataUtils.js'; import { DEFAULT_CONSOLIDATE_DURATION, DEFAULT_GROUP_DURATION, groupPlaysToTimeRanges } from '../../utils/ListenFetchUtils.js'; +import { asPlay } from '../../../core/tests/utils/fixtures.js'; chai.use(asPromised); const firstPlayDate = dayjs().subtract(1, 'hour'); const olderFirstPlayDate = dayjs().subtract(4, 'hour'); -const withDurPlays = asPlays(withDuration); -const mixedDurPlays = asPlays(mixedDuration); +const withDurPlays = withDuration.map(asPlay); +// @ts-expect-error mixed duration is missing meta but not used for tests +const mixedDurPlays = mixedDuration.map(asPlay); const normalizedWithDur = normalizePlays(withDurPlays, {initialDate: firstPlayDate}); const normalizedWithMixedDur = normalizePlays(mixedDurPlays, {initialDate: firstPlayDate}); diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts index aa0f7976..7dd123be 100644 --- a/src/core/Atomic.ts +++ b/src/core/Atomic.ts @@ -84,8 +84,8 @@ export interface TrackStringOptions { } } -export interface PlayProgressAmb { - timestamp: string | Dayjs +export interface PlayProgressAmb { + timestamp: D position?: number positionPercent?: number } @@ -299,7 +299,7 @@ export interface ScrobbleResult { export interface PlayLifecycle { input?: object - original: PlayObjectLifecycleless + original: PlayObjectLifecycleless steps: LifecycleStep[] scrobble?: ScrobbleResult } @@ -321,13 +321,13 @@ export interface ScrobbleActionResult { warnings?: string[] } -export interface PlayMatchResult { +export interface PlayMatchResult { match: boolean score: number breakdowns: string[] reason?: string - closestMatchedPlay?: PlayObjectLifecycleless - transformedPlay?: PlayObjectLifecycleless + closestMatchedPlay?: PlayObjectLifecycleless + transformedPlay?: PlayObjectLifecycleless summary?: String } @@ -575,3 +575,8 @@ export interface numberFormatOptions { indicate?: boolean; }; } +/** Only checks for DateT since we can reasonbly sure if this exists its a date we can parse with dayjs + * + * It needs to be cheap since we mostly use this when walking play objects to transform strings back to dayjs and there may be many strings to check + */ +export const REGEX_ISO8601_LOOSE = new RegExp(/\d{4}-[01]\d-[0-3]\dT/); \ No newline at end of file diff --git a/src/core/PlayTestUtils.ts b/src/core/PlayTestUtils.ts index 896206ef..f2735fbd 100644 --- a/src/core/PlayTestUtils.ts +++ b/src/core/PlayTestUtils.ts @@ -23,20 +23,6 @@ dayjs.extend(relativeTime); dayjs.extend(duration); dayjs.extend(timezone); -export const asPlays = (data: object[]): PlayObject[] => { - return data.map(x => { - const y = x as JsonPlayObject; - return { - ...y, - data: { - ...y.data, - playDate: dayjs(y.data.playDate), - playDateCompleted: y.data.playDateCompleted !== undefined ? dayjs(y.data.playDateCompleted) : undefined - } - } - }); -} - export const normalizePlays = (plays: PlayObject[], options?: { //sortFunc?: (a: PlayObject, b: PlayObject) => 0 | 1 | -1 @@ -194,7 +180,11 @@ export const generateJsonPlays = (...args: Parameters): Js return JSON.parse(JSON.stringify(plays)); } -export const withBrainz = (play: PlayObject, include: ('track' | 'artist' | 'album')[]): PlayObject => { +export interface WithBrainzOptions { + include: ('track' | 'artist' | 'album')[] +} +export const withBrainz = (play: PlayObject, opts: WithBrainzOptions): PlayObject => { + const {include} = opts; for(const i of include) { switch(i) { case 'track': diff --git a/src/core/tests/utils/fixtures.ts b/src/core/tests/utils/fixtures.ts index e575ba52..37e1beb8 100644 --- a/src/core/tests/utils/fixtures.ts +++ b/src/core/tests/utils/fixtures.ts @@ -1,4 +1,81 @@ +import { Traverse, TraverseContext } from 'neotraverse/modern'; +import dayjs from 'dayjs'; +import { AmbPlayObject, JsonPlayObject, PlayObject, PlayProgressAmb, REGEX_ISO8601_LOOSE } from '../../Atomic.js'; +import { ListenRange } from '../../../backend/sources/PlayerState/ListenRange.js'; +import { ListenProgressPositional, ListenProgressTS } from '../../../backend/sources/PlayerState/ListenProgress.js'; +import { clone } from 'jsondiffpatch'; -export const generatePlayWithLifecycle = () => { - +interface BlockPath { key: string, parent: string }; +type BlockPaths = BlockPath[]; + +/** We know some nodes will never have data that needs to be transformed + * and these nodes can have lots of data so we can optimize them away by not (recursively) traversing them + */ +const blockedKeys: PropertyKey[] = ['patch', 'inputs', 'payload', 'response', 'error']; +/** We know some paths/nodes will never have data that needs to be transformed + * and these nodes can have lots of data so we can optimize them away by not (recursively) traversing them + */ +const blockedPaths: BlockPaths = [ + { + parent: 'data', + key: 'meta' + }, + { + parent: 'lifecycle', + key: 'input' + } +] + +const shouldBlock = (ctx: TraverseContext): boolean => { + if (blockedKeys.includes(ctx.key)) { + return true; + } + return blockedPaths.some((x) => { + let blocked = x.key === ctx.key; + if (blocked && x.parent !== undefined) { + blocked = ctx.parent !== undefined && ctx.parent.key === x.parent; + } + return blocked; + }) +} + +export const asJsonPlayObject = (play: AmbPlayObject): JsonPlayObject => { + const cloned = clone(play); + new Traverse(cloned).forEach((ctx, x) => { + if (shouldBlock(ctx)) { + ctx.block(); + return; + } + + if (dayjs.isDayjs(x)) { + ctx.update(x.toISOString()); + } else if (x instanceof ListenRange) { + ctx.update(x.toJSON(), true); + } + }); + return cloned as unknown as JsonPlayObject; +} + +export const asPlay = (data: JsonPlayObject): PlayObject => { + const cloned = clone(data); + new Traverse(cloned).forEach((ctx, x) => { + if (shouldBlock(ctx)) { + ctx.block(); + return; + } + + if (typeof x === 'string' && REGEX_ISO8601_LOOSE.test(x)) { + ctx.update(dayjs(x), true); + } else if (ctx.key === 'listenRanges') { + const ranges = x[0].map((y: PlayProgressAmb) => { + if (y.positionPercent === undefined) { + return new ListenProgressPositional({ timestamp: dayjs(y.timestamp), position: y.position }); + } else { + return new ListenProgressTS({ timestamp: dayjs(y.timestamp), positionPercent: y.positionPercent }); + } + }) + ctx.update(ranges, true); + } + }); + return cloned as unknown as PlayObject; } \ No newline at end of file diff --git a/src/stories/storyUtils.ts b/src/stories/storyUtils.ts index a6c780ae..87d5c53c 100644 --- a/src/stories/storyUtils.ts +++ b/src/stories/storyUtils.ts @@ -1,6 +1,6 @@ import { JsonPlayObject, PlayLifecycle } from "../core/Atomic.js"; -const exampleLifecycle = (): PlayLifecycle => ({ +const exampleLifecycle = (): PlayLifecycle => ({ "input": { "device": { "id": "3cc6dc47a8551498a70e519ee09cd8423ff9d48c", @@ -1101,7 +1101,7 @@ export const examplePlay = (): JsonPlayObject => ({ } }); -const lastfmErrorLifcycle: PlayLifecycle = { +const lastfmErrorLifcycle: PlayLifecycle = { "input": { "track": { "album": { -- 2.51.2