diff --git a/package-lock.json b/package-lock.json index 34976ab9..131432f8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -72,7 +72,7 @@ "icecast-metadata-stats": "^0.1.12", "iso-websocket": "^0.3.0", "iti": "^0.7.0", - "jest-diff": "^30.2.0", + "json-diff-ts": "^5.0.0-alpha.2", "json5": "^2.2.3", "jsondiffpatch": "^0.7.3", "jsondiffpatch-react": "^1.0.8", @@ -2568,36 +2568,6 @@ "axios": "^1.3.4" } }, - "node_modules/@jest/diff-sequences": { - "version": "30.0.1", - "resolved": "https://registry.npmjs.org/@jest/diff-sequences/-/diff-sequences-30.0.1.tgz", - "integrity": "sha512-n5H8QLDJ47QqbCNn5SuFjCRDrOLEZ0h8vAHCK5RL9Ls7Xa8AQLa/YxAc9UjFqoEDM48muwtBGjtMY5cr0PLDCw==", - "license": "MIT", - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/@jest/get-type": { - "version": "30.1.0", - "resolved": "https://registry.npmjs.org/@jest/get-type/-/get-type-30.1.0.tgz", - "integrity": "sha512-eMbZE2hUnx1WV0pmURZY9XoXPkUYjpc55mb0CrhtdWLtzMQPFvu/rZkTLZFTsdaVQa+Tr4eWAteqcUzoawq/uA==", - "license": "MIT", - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/@jest/schemas": { - "version": "30.0.5", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.5.tgz", - "integrity": "sha512-DmdYgtezMkh3cpU8/1uyXakv3tJRcmcXxBOcO0tbaozPwpmh4YMsnWrQm9ZmZMfa5ocbxzbFk6O4bDPEc/iAnA==", - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.34.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, "node_modules/@joshwooding/vite-plugin-react-docgen-typescript": { "version": "0.6.4", "resolved": "https://registry.npmjs.org/@joshwooding/vite-plugin-react-docgen-typescript/-/vite-plugin-react-docgen-typescript-0.6.4.tgz", @@ -3928,12 +3898,6 @@ "integrity": "sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg==", "license": "MIT" }, - "node_modules/@sinclair/typebox": { - "version": "0.34.41", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.34.41.tgz", - "integrity": "sha512-6gS8pZzSXdyRHTIqoqSVknxolr1kzfy4/CeDnrzsVz8TTIWUbOBr6gnzOmTYJ3eXQNh4IYHIGi5aIL7sOZ2G/g==", - "license": "MIT" - }, "node_modules/@sindresorhus/is": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-2.1.1.tgz", @@ -13525,53 +13489,6 @@ } } }, - "node_modules/jest-diff": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.2.0.tgz", - "integrity": "sha512-dQHFo3Pt4/NLlG5z4PxZ/3yZTZ1C7s9hveiOj+GCN+uT109NC2QgsoVZsVOAvbJ3RgKkvyLGXZV9+piDpWbm6A==", - "license": "MIT", - "dependencies": { - "@jest/diff-sequences": "30.0.1", - "@jest/get-type": "30.1.0", - "chalk": "^4.1.2", - "pretty-format": "30.2.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-diff/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/jest-diff/node_modules/pretty-format": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.2.0.tgz", - "integrity": "sha512-9uBdv/B4EefsuAL+pWqueZyZS2Ba+LxfFeQ9DN14HU4bN8bhaxKdkpjpB6fs9+pSjIBu+FXQHImEg8j/Lw0+vA==", - "license": "MIT", - "dependencies": { - "@jest/schemas": "30.0.5", - "ansi-styles": "^5.2.0", - "react-is": "^18.3.1" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-diff/node_modules/react-is": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", - "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", - "license": "MIT" - }, "node_modules/jest-get-type": { "version": "27.5.1", "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-27.5.1.tgz", @@ -13684,6 +13601,12 @@ "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", "license": "MIT" }, + "node_modules/json-diff-ts": { + "version": "5.0.0-alpha.2", + "resolved": "https://registry.npmjs.org/json-diff-ts/-/json-diff-ts-5.0.0-alpha.2.tgz", + "integrity": "sha512-7AI6rZNrvqW3Bkmm9Wsq6985MeC8KkK68mgtW7z3aq39QBGMcoyClrbAXlUr6yQ0VbafVtAM+f5b4bAwlxVSAg==", + "license": "MIT" + }, "node_modules/json-parse-even-better-errors": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", diff --git a/package.json b/package.json index da9e685a..5e7112f5 100644 --- a/package.json +++ b/package.json @@ -110,7 +110,7 @@ "icecast-metadata-stats": "^0.1.12", "iso-websocket": "^0.3.0", "iti": "^0.7.0", - "jest-diff": "^30.2.0", + "json-diff-ts": "^5.0.0-alpha.2", "json5": "^2.2.3", "jsondiffpatch": "^0.7.3", "jsondiffpatch-react": "^1.0.8", diff --git a/src/backend/common/AbstractComponent.ts b/src/backend/common/AbstractComponent.ts index 3e1e4037..8369c856 100644 --- a/src/backend/common/AbstractComponent.ts +++ b/src/backend/common/AbstractComponent.ts @@ -10,28 +10,24 @@ import { CommonClientConfig } from "./infrastructure/config/client/index.js"; import { CommonSourceConfig } from "./infrastructure/config/source/index.js"; import { mergeSimpleError, SkipTransformStageError, StagePrerequisiteError, StageTransformError, TransformRulesError } from "./errors/MSErrors.js"; import { + FLOW_CONTROL_TERM, PlayTransformRules, StageConfig, TRANSFORM_HOOK, TransformHook } from "./infrastructure/Transform.js"; import AbstractInitializable from "./AbstractInitializable.js"; -import play = Simulate.play; import TransformerManager from "./transforms/TransformerManager.js"; import { getRoot } from "../ioc.js"; import { nanoid } from "nanoid"; -import {diffStringsUnified, DiffOptionsColor} from 'jest-diff'; -import chalk from 'chalk'; import { isDebugMode } from "../utils.js"; import { findCauseByReference } from "../utils/ErrorUtils.js"; import { hashObject } from "../utils/StringUtils.js"; import { metaInvariantTransform, playContentInvariantTransform } from "../utils/PlayComparisonUtils.js"; import { MSCache } from "./Cache.js"; -import { jdiff } from "../../core/DataUtils.js"; -import ConsoleFormatter from "jsondiffpatch/formatters/console"; +import { diffObjects, diffObjectsConsoleOutput, patchObject } from "../../core/DataUtils.js"; import clone from "clone"; - -const console = new ConsoleFormatter(); +import { loggerNoop } from "./MaybeLogger.js"; export type AbstractComponentConfig = (CommonClientConfig | CommonSourceConfig) & { transformManager?: TransformerManager }; @@ -169,7 +165,7 @@ export default abstract class AbstractComponent extends AbstractInitializable { return partArr.map(x => this.transformManager.parseTransformerConfig(x)); } - public transformPlay = async (play: PlayObject, hookType: TransformHook, log?: boolean) => { + public transformPlay = async (play: PlayObject, hookType: TransformHook, log?: boolean | 'all') => { const asyncId = nanoid(6); @@ -200,177 +196,123 @@ export default abstract class AbstractComponent extends AbstractInitializable { const shouldLog = log ?? this.config.options?.playTransform?.log ?? isDebugMode(); const transformHash = `playTransform-${hashObject(hook)}-${hashObject(playContentInvariantTransform(play))}`; - const cachedTransformPlay = await this.cache.cacheTransform.get(transformHash); - if(cachedTransformPlay !== undefined) { - logger.trace(`Cache hit ${transformHash}`); - return cachedTransformPlay; + const cachedSteps = await this.cache.cacheTransform.get(transformHash); + if(cachedSteps !== undefined) { + logger.trace(`Cache hit for Steps => ${transformHash}`); + //return cachedTransformPlay; } logger.debug(`Transform start for => ${buildTrackString(play)}`); - let transformedPlay: PlayObject = play; - let transformHistory: TransformResult[] = []; - let shouldBreak = false; + let transformedPlay: PlayObject = clone(play); let cacheOk: boolean = true; - for(const hookItem of hook) { + const steps: LifecycleStep[] = []; - const { - onSuccess = 'continue', - onFailure = 'stop', - onSkip = 'continue', - failureReturnPartial = false - } = hookItem; + const opts = { + logger, + asyncId + } - const existingStepIndex = transformedPlay.meta.lifecycle.steps.findIndex(x => x.name === 'hookType' && x.source === this.getIdentifier()); - const step: LifecycleStep = existingStepIndex !== -1 ? transformedPlay.meta.lifecycle.steps[existingStepIndex] : { - name: hookType, - source: this.getIdentifier(), + if(cachedSteps !== undefined) { + // don't re-cache cached steps + cacheOk = false; + // only patch play if steps didn't end in a failure that we are okay with returning partial from + let shouldTransform = true; + const lastCachedStep = cachedSteps[cachedSteps.length - 1]; + if(lastCachedStep.error !== undefined && lastCachedStep.flowKnownState !== 'skip' && !lastCachedStep.returnPartial) { + shouldTransform = false; } - let newTransformedPlay: PlayObject, - stageName: string = 'Unnamed', - err: Error; - try { - [newTransformedPlay, stageName] = await this.transformManager.handleStage(hookItem, transformedPlay, asyncId); - } catch (e) { - err = e; - if(e instanceof StageTransformError) { - stageName = e.stageName; + for(const s of cachedSteps) { + steps.push({...s, cached: true}); + if(shouldTransform && s.patch !== undefined) { + transformedPlay.data = patchObject(transformedPlay.data, s.patch); // jdiff.patch(clone(transformedPlay.data),s.patch); } } - - if(err !== undefined) { - const merged = mergeSimpleError(err); - step.error = merged; - - const skipError = findCauseByReference(err, SkipTransformStageError); - if(skipError !== undefined) { - let skipMsg = `Stage '${stageName}' was skipped`; - step.flowResult = onSkip; - step.flowKnownState = 'skip'; - - if(onSkip === 'stop') { - skipMsg += ' and will stop transform due to onSkip: stop'; - } - step.flowReason = skipMsg; - - logger.debug(merged, skipMsg); - if(onSkip === 'stop') { - shouldBreak = true; - } - } else { - step.flowResult = onFailure; - let reason: string; - - const reqError = findCauseByReference(err, StagePrerequisiteError); - if(reqError !== undefined) { - reason = 'Transform could not be completed due to prerequisite failure'; - step.flowKnownState = 'prereq'; - } else { - reason = 'Transform encountered an error'; - cacheOk = false; - } - - if(onFailure === 'continue') { - reason += ' but will continue due to onFailure: continue'; - logger.warn(merged, reason); + } else { + for(const hookItem of hook) { + const [step, stepPlay] = await this.generateStepFromStage(transformedPlay, hookItem, hookType, opts); + steps.push(step); + cacheOk = step.error === undefined || (step.error !== undefined && step.flowKnownState !== undefined); + if(step.flowResult === FLOW_CONTROL_TERM.stop) { + if(step.error !== undefined && step.flowKnownState !== 'skip' && !step.returnPartial) { + // revert to original play but keep steps for paper trail + transformedPlay = play; + transformedPlay.meta.lifecycle.steps = steps; } else { - logger[reqError !== undefined ? 'warn' : 'error'](merged, reason); - - if(!failureReturnPartial) { - // rewind to original play so we don't return partial transform - transformedPlay = play; - transformHistory = []; - } else if(transformHistory.length > 0) { - cacheOk = true; - step.flowReason += ' | Preserving play transformations up to this point due to failureReturnPartial=true' - } - shouldBreak = true; + transformedPlay = stepPlay; } - - step.flowReason = reason; - } - } else { - // reset cache state - // in the event it was false due to previous step error but onFailure: continue - cacheOk = true; - step.flowResult = onSuccess; - - if(!deepEqual(play, newTransformedPlay)) { - const o = JSON.parse(JSON.stringify(play)); - const t = JSON.parse(JSON.stringify(newTransformedPlay)); - const patch = jdiff.diff(o,t); - step.patch = patch; - } - - if(newTransformedPlay.meta.lifecycleInputs?.length > 0) { - step.inputs = clone(newTransformedPlay.meta.lifecycleInputs) - } else if(transformedPlay.meta.lifecycleInputs?.length > 0) { - step.inputs = clone(transformedPlay.meta.lifecycleInputs) + break; + } else { + transformedPlay = stepPlay; } - - delete transformedPlay.meta.lifecycleInputs; - - // for logging only - transformHistory.push({ - type: hookItem.type, - name: stageName, - play: newTransformedPlay.data - }); - transformedPlay = newTransformedPlay; } + } - if(err === undefined && onSuccess === 'stop') { - logger.debug(`Stopping transform due to onSuccess: stop`); - shouldBreak = true; + const historyToDiff: {name: string, data?: PlayData}[] = [ + {name: 'Pre Transform', data: play.data} + ]; + if(steps.length > 0 && transformedPlay.meta?.lifecycle?.steps === undefined) { + const { + meta: { + lifecycle: { + steps = [], + ...lifecycleRest + }, + } = {} + } = transformedPlay; + transformedPlay.meta.lifecycle = { + ...lifecycleRest, + steps } - + } + steps.forEach((s, index) => { + const existingStepIndex = transformedPlay.meta.lifecycle.steps.findIndex(x => x.name === 'hookType' && x.source === this.getIdentifier()); if(existingStepIndex !== -1) { - transformedPlay.meta.lifecycle.steps[existingStepIndex] = step; + transformedPlay.meta.lifecycle.steps[existingStepIndex] = s; } else { - transformedPlay.meta.lifecycle.steps.push(step); + transformedPlay.meta.lifecycle.steps.push(s); } - if(shouldBreak) { - break; + if(shouldLog === 'all') { + if(s.patch === undefined) { + historyToDiff.push({name: `${s.source}-${s.name}`}); + } else { + const patched = patchObject(historyToDiff[historyToDiff.length - 1].data, s.patch); + historyToDiff.push({name: `${s.source}-${s.name}${s.cached ? ' (Cached)' : ''}`, data: patched}); + } } - } - + }); if(shouldLog !== false) { - if(transformHistory.length === 0) { + if(steps.filter(x => x.patch !== undefined).length === 0) { logger.debug('Transform Diff: No Change'); } else { - const historyToDiff: {name: string, data: PlayData}[] = [ - {name: 'Original', data: play.data} - ]; - if(shouldLog === true) { - const last = transformHistory[transformHistory.length - 1]; - historyToDiff.push({name: `${last.type}-${last.name}`, data: last.play}); - } else { - for(const t of transformHistory) { - historyToDiff.push({name: `${t.type}-${t.name}`, data: t.play}) - } - } const diffs: string[] = []; + historyToDiff.forEach((curr, index) => { if(index === 0) { return; } const last = historyToDiff[index - 1]; - if(deepEqual(last.data, curr.data)) { + if(curr.data === undefined) { diffs.push(`${last.name} => ${curr.name} -- No Change`); } else { - const unchanged = index - 1 === 0 ? last.data : undefined; - - const formattedDiff = console.format(jdiff.diff(last.data, curr.data), unchanged); + const formattedDiff = diffObjectsConsoleOutput(last.data, curr.data); diffs.push(`${last.name} => ${curr.name}\n${formattedDiff}`); } }); + + if(shouldLog === true || steps.filter(x => x.patch !== undefined).length > 2) { + const finalData = transformedPlay.data; + const formattedDiff = diffObjectsConsoleOutput(play.data, finalData, true); + diffs.push(`Original => Final\n${formattedDiff}`); + } + logger.debug(`Transform Diff\n${diffs.join('\n')}`); } } if(cacheOk) { - await this.cache.cacheTransform.set(transformHash, transformedPlay, '10m'); + await this.cache.cacheTransform.set(transformHash, steps, '10m'); } return transformedPlay; @@ -379,4 +321,103 @@ export default abstract class AbstractComponent extends AbstractInitializable { return play; } } + + protected generateStepFromStage = async (playTruth: PlayObject, hookItem: StageConfig, hookType: TransformHook, opts: { logger?: Logger, asyncId?: string } = {}): Promise<[LifecycleStep, PlayObject]> => { + const { + onSuccess = 'continue', + onFailure = 'stop', + onSkip = 'continue', + failureReturnPartial = false + } = hookItem; + + const { + logger = loggerNoop, + asyncId = nanoid(6) + } = opts; + + const stepName = `${hookType} - ${hookItem.type} - ${hookItem.name}` + const existingStepIndex = playTruth.meta.lifecycle.steps.findIndex(x => x.name === stepName && x.source === this.getIdentifier()); + const step: LifecycleStep = existingStepIndex !== -1 ? playTruth.meta.lifecycle.steps[existingStepIndex] : { + name: stepName, + source: this.getIdentifier(), + } + + let newTransformedPlay: PlayObject, + stageName: string = 'Unnamed', + err: Error; + try { + [newTransformedPlay, stageName] = await this.transformManager.handleStage(hookItem, playTruth, asyncId); + newTransformedPlay = clone(newTransformedPlay); + } catch (e) { + err = e; + if (e instanceof StageTransformError) { + stageName = e.stageName; + } + } + + if (err !== undefined) { + const merged = mergeSimpleError(err); + step.error = merged; + + const skipError = findCauseByReference(err, SkipTransformStageError); + if (skipError !== undefined) { + let skipMsg = `Stage '${stageName}' was skipped`; + step.flowResult = onSkip; + step.flowKnownState = 'skip'; + + if (onSkip === 'stop') { + skipMsg += ' and will stop transform due to onSkip: stop'; + } + step.flowReason = skipMsg; + + logger.debug(merged, skipMsg); + } else { + step.flowResult = onFailure; + let reason: string; + + const reqError = findCauseByReference(err, StagePrerequisiteError); + if (reqError !== undefined) { + reason = 'Transform could not be completed due to prerequisite failure'; + step.flowKnownState = 'prereq'; + } else { + reason = 'Transform encountered an error'; + } + + if (onFailure === 'continue') { + reason += ' but will continue due to onFailure: continue'; + logger.warn(merged, reason); + } else { + if (failureReturnPartial) { + reason += ' | Preserving play transformations up to this point due to failureReturnPartial=true'; + step.returnPartial = true; + } + + logger[reqError !== undefined ? 'warn' : 'error'](merged, reason); + } + step.flowReason = reason; + + return [step, playTruth]; + } + } else { + step.flowResult = onSuccess; + + if (!deepEqual(playTruth.data, newTransformedPlay.data)) { + const o = JSON.parse(JSON.stringify(playTruth.data)); + const t = JSON.parse(JSON.stringify(newTransformedPlay.data)); + const patch = diffObjects(o, t); // jdiff.diff(o, t); + step.patch = patch; + } + + if (newTransformedPlay.meta.lifecycleInputs?.length > 0) { + step.inputs = clone(newTransformedPlay.meta.lifecycleInputs) + } else if (playTruth.meta.lifecycleInputs?.length > 0) { + logger.warn({ label: `${hookItem.type} - ${hookItem.name}` }, `Should only be adding inputs to transformed play!`); + step.inputs = clone(playTruth.meta.lifecycleInputs) + } + + delete newTransformedPlay.meta.lifecycleInputs; + } + + return [step, newTransformedPlay]; + } } diff --git a/src/backend/common/infrastructure/Transform.ts b/src/backend/common/infrastructure/Transform.ts index 4a3d6aef..ef1b0143 100644 --- a/src/backend/common/infrastructure/Transform.ts +++ b/src/backend/common/infrastructure/Transform.ts @@ -50,6 +50,10 @@ export interface Whennable { } export type FlowControlTerm = 'continue' | 'stop' +export const FLOW_CONTROL_TERM = { + continue: 'continue', + stop: 'stop' +} as const satisfies Record; export interface FlowControl { onSuccess: FlowControlTerm diff --git a/src/backend/tests/component/transformers.test.ts b/src/backend/tests/component/transformers.test.ts index d35b93c6..b7766a42 100644 --- a/src/backend/tests/component/transformers.test.ts +++ b/src/backend/tests/component/transformers.test.ts @@ -16,6 +16,8 @@ import { Cacheable } from "cacheable"; import { TransformerCommonConfig } from "../../../core/Atomic.js"; import TransformerManager from "../../common/transforms/TransformerManager.js"; import { transientCache } from "../utils/CacheTestUtils.js"; +import dayjs from "dayjs"; +import clone from "clone"; chai.use(asPromised); @@ -638,6 +640,49 @@ describe('Play Transforms', function () { }); + describe("Stage Caching", function () { + + it('Re-uses steps without modifying other Play properties', async function () { + component.config = { + options: { + playTransform: { + preCompare: [ + { + name: "barChange", + title: [ + { + search: "something", + replace: "bar" + } + ] + }, + { + type: 'native' + } + ] + } + } + } + + const [str, primaries, secondaries] = generateArtistsStr({primary: {max: 3, ambiguousJoinedNames: true, trailingAmpersand: true, finalJoiner: false}}); + + component.buildTransformRules(); + const play = generatePlay({ track: 'My cool something track', artists: [str], playDate: dayjs().subtract(10, 'm') }); + const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare, 'all'); + expect(transformed.data.track).equal('My cool bar track'); + expect(transformed.data.artists).eql(primaries.concat(secondaries)); + + const cachablePlay = clone(play); + const laterDate = dayjs().subtract(5, 'm'); + cachablePlay.data.playDate = laterDate; + const cacheTransformed = await component.transformPlay(cachablePlay, TRANSFORM_HOOK.preCompare, 'all'); + expect(cacheTransformed.data.track).equal('My cool bar track'); + expect(cacheTransformed.data.artists).eql(primaries.concat(secondaries)); + expect(cacheTransformed.data.playDate.isSame(cachablePlay.data.playDate)); + }); + + }); + describe('Transform Manager', function() { it('Uses user transforms in the order supplied within component', async function() { diff --git a/src/client/components/JsonDiff.tsx b/src/client/components/JsonDiff.tsx index e3e7b5cd..dedc7aba 100644 --- a/src/client/components/JsonDiff.tsx +++ b/src/client/components/JsonDiff.tsx @@ -1,10 +1,10 @@ import React, { ComponentProps } from 'react'; import JsonDiffReact from 'jsondiffpatch-react'; import { MSErrorBoundary } from './ErrorBoundary'; -import { Delta } from 'jsondiffpatch'; import { MarkOptional } from 'ts-essentials'; -import { jdiff } from '../../core/DataUtils'; +import { patchObject } from '../../core/DataUtils'; import './JsonDiff.css'; +import { IJsonDelta } from 'json-diff-ts'; type DiffableVal = [] | object | string; @@ -18,7 +18,7 @@ export interface JsonDiffReactProps { } export type JsonDiffPatchProps = MarkOptional & { - diff?: Delta | object + diff?: IJsonDelta } export const JsonDiffPatch = (props: JsonDiffPatchProps) => { @@ -31,9 +31,9 @@ export const JsonDiffPatch = (props: JsonDiffPatchProps) => { const detachedLeft = JSON.parse(JSON.stringify(left)); let realRight: DiffableVal; if (right !== undefined) { - realRight = JSON.parse(JSON.stringify(right)); + realRight = structuredClone(right); } else if (diff !== undefined) { - realRight = jdiff.patch(JSON.parse(JSON.stringify(left)), diff as Delta) as DiffableVal; + realRight = patchObject(left, diff); } else { throw new Error(`must provide either 'right' or 'diff'`); } diff --git a/src/client/components/TransformSteps.tsx b/src/client/components/TransformSteps.tsx index 1958ad3b..8784968b 100644 --- a/src/client/components/TransformSteps.tsx +++ b/src/client/components/TransformSteps.tsx @@ -12,7 +12,7 @@ import { FaStop } from "react-icons/fa"; import { MdMusicNote } from "react-icons/md"; import { ChakraCodeBlockShort, ChakraPlainBlockShort } from "./CodeBlock"; import { JsonDiffPatch } from "./JsonDiff"; -import { jdiff } from "../../core/DataUtils"; +import { jdiff, patchObject } from "../../core/DataUtils"; import { MSCollapsible, MSCollapsibleExternalProps } from "./MSCollapsible"; import { Muted } from "./Typography"; @@ -23,7 +23,7 @@ export interface LifeycleStepsTimelineProps extends MSCollapsibleExternalProps { const diffElements = (original: JsonPlayObject, steps: LifecycleStep[]): [JSX.Element[], JsonPlayObject?] => { - let currentPlay: JsonPlayObject = JSON.parse(JSON.stringify(original)); + let currentPlay: JsonPlayObject = structuredClone(original); // JSON.parse(JSON.stringify(original)); let patchFailed = false; const diffElements: JSX.Element[] | null = []; @@ -49,7 +49,7 @@ const diffElements = (original: JsonPlayObject, steps: LifecycleStep[]): [JSX.El diffElements.push(); continue; } - let left: JsonPlayObject = JSON.parse(JSON.stringify(currentPlay)); + let left: JsonPlayObject = structuredClone(currentPlay); // JSON.parse(JSON.stringify(currentPlay)); left.data.meta = { ...(left.data.meta ?? {}), brainz: { @@ -64,10 +64,10 @@ const diffElements = (original: JsonPlayObject, steps: LifecycleStep[]): [JSX.El } try { - currentPlay = jdiff.patch(currentPlay, patch) as JsonPlayObject; + currentPlay.data = patchObject(currentPlay.data, patch)// jdiff.patch(currentPlay, patch) as JsonPlayObject; diffElements.push( - - + + ) } catch (e) { diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts index b7ec4510..816523cf 100644 --- a/src/core/Atomic.ts +++ b/src/core/Atomic.ts @@ -7,6 +7,7 @@ import { MarkOptional } from "ts-essentials"; import { ErrorObject } from "serialize-error"; import { PlayPlatformIdStr } from "../backend/common/infrastructure/Atomic.js"; import { FlowControlTerm } from "../backend/common/infrastructure/Transform.js"; +import { IJsonDelta } from "json-diff-ts"; export interface SourceStatusData { status: string; @@ -311,11 +312,12 @@ export interface LifecycleStep { name: string source: string cached?: boolean + returnPartial?: boolean flowResult?: FlowControlTerm flowReason?: string flowKnownState?: 'skip' | 'prereq' error?: ErrorLike - patch?: Delta + patch?: IJsonDelta inputs?: LifecycleInput[] } diff --git a/src/core/DataUtils.ts b/src/core/DataUtils.ts index b772ba71..873139f1 100644 --- a/src/core/DataUtils.ts +++ b/src/core/DataUtils.ts @@ -1,6 +1,14 @@ import { create as diffCreate } from "jsondiffpatch"; import { numberFormatOptions } from './Atomic.js'; +import { diffDelta, applyDelta, IJsonDelta, DeltaOptions } from 'json-diff-ts'; +// may want to return to this one day +// but currently the jsondiffpatch formatter is the best console/ansi diff output for humans :( +//import {DiffOptions, DiffOptionsColor, diff as jestDiff} from 'jest-diff'; +import chalk from 'chalk'; +import clone from "clone"; +import ConsoleFormatter from "jsondiffpatch/formatters/console"; +const console = new ConsoleFormatter(); export const jdiff = diffCreate({ propertyFilter(name, context) { @@ -10,6 +18,34 @@ export const jdiff = diffCreate({ //omitRemovedValues: true }); +const diffOptions: DeltaOptions = { + /*arrayIdentityKeys: {artists: '$value'},*/ + reversible: false, + keysToSkip: ['playDate','playDateCompleted','listenRanges'] +}; + +export const diffObjects = (a: object, b: object) => { + return diffDelta(a, b, diffOptions); +} + +export const patchObject = (a: T, b: IJsonDelta): T => { + return applyDelta(clone(a), b); +} + +// const jestDiffOptions: DiffOptions = { +// aAnnotation: 'Old', +// bAnnotation: 'New', +// aColor: chalk.red, +// bColor: chalk.green +// } + +export const diffObjectsConsoleOutput = (a: object, b: object, showUnchanged: boolean = false) => { + //return jestDiff(a, b, jestDiffOptions); + + const left = JSON.parse(JSON.stringify(a)); + return console.format(jdiff.diff(left, JSON.parse(JSON.stringify(b))), showUnchanged ? left : undefined); +} + export const formatNumber = (val: number | string, options?: numberFormatOptions) => { const { toFixed = 2, defaultVal = null, prefix = '', suffix = '', round, diff --git a/src/core/tests/utils/fixtures.ts b/src/core/tests/utils/fixtures.ts index ca1e405c..6ec28152 100644 --- a/src/core/tests/utils/fixtures.ts +++ b/src/core/tests/utils/fixtures.ts @@ -8,7 +8,7 @@ import { MarkOptional } from 'ts-essentials'; import { generateBrainz, generateMbid, generatePlay, GeneratePlayOpts, generatePlays } from '../../PlayTestUtils.js'; import { lifecyclelessInvariantTransform } from '../../PlayUtils.js'; import clone from 'clone'; -import { jdiff } from '../../DataUtils.js'; +import { diffObjects } from '../../DataUtils.js'; import { existingScrobble } from '../../../backend/utils/PlayComparisonUtils.js'; import { UpstreamError } from '../../../backend/common/errors/UpstreamError.js'; import { playToListenPayload } from '../../../backend/common/vendor/listenbrainz/lzUtils.js'; @@ -312,7 +312,7 @@ export const generateLifecycleStep = (play: PlayObject, opts: GenerateLifecycleO }); } - step.patch = jdiff.diff(play, modifiedPlay); + step.patch = diffObjects(play.data, modifiedPlay.data);// jdiff.diff(play, modifiedPlay); return [step, modifiedPlay]; } diff --git a/src/stories/ActivityTimeline.stories.tsx b/src/stories/ActivityTimeline.stories.tsx index 8616f9a9..7f3a0020 100644 --- a/src/stories/ActivityTimeline.stories.tsx +++ b/src/stories/ActivityTimeline.stories.tsx @@ -8,7 +8,7 @@ import {Provider} from "../client/components/Provider"; import { generateJsonPlays } from "../core/PlayTestUtils.js"; import { ErrorLike, JsonPlayObject, PlayLifecycle } from "../core/Atomic.js"; import { examplePlay, lastfmErrorExample } from "./storyUtils.js"; -import { generatePlayWithLifecycle, playWithLifecycleScrobble } from "../core/tests/utils/fixtures.js"; +import { asJsonPlayObject, generatePlayWithLifecycle, playWithLifecycleScrobble } from "../core/tests/utils/fixtures.js"; // More on how to set up stories at: https://storybook.js.org/docs/writing-stories#default-export const meta = preview.meta({ @@ -35,14 +35,14 @@ decorators: [ export const ActivityTimelineStory = meta.story({ loaders: [ async () => { - const scrobbleError = await playWithLifecycleScrobble(generatePlayWithLifecycle( + const scrobbleError = asJsonPlayObject(await playWithLifecycleScrobble(generatePlayWithLifecycle( { lifecycleSteps: { preCompare: 1, postCompare: 1, } } - )) + ))) return {play: scrobbleError}; } ], @@ -51,7 +51,7 @@ export const ActivityTimelineStory = meta.story({ export const ScrobbleError = meta.story({ loaders: [ async () => { - const scrobbleError = await playWithLifecycleScrobble(generatePlayWithLifecycle(), {error: true}); + const scrobbleError = asJsonPlayObject(await playWithLifecycleScrobble(generatePlayWithLifecycle(), {error: true})); return {play: scrobbleError}; } ], @@ -60,12 +60,12 @@ export const ScrobbleError = meta.story({ export const TransformError = meta.story({ loaders: [ async () => { - const scrobbleError = await generatePlayWithLifecycle({ + const scrobbleError = asJsonPlayObject(await generatePlayWithLifecycle({ lifecycleSteps: { preCompare: 2, postCompare: [false], } - }); + })); return {play: scrobbleError}; } ], @@ -74,11 +74,11 @@ export const TransformError = meta.story({ export const TransformSkip = meta.story({ loaders: [ async () => { - const play = await generatePlayWithLifecycle({ + const play = asJsonPlayObject(await generatePlayWithLifecycle({ lifecycleSteps: { preCompare: [true, 'skipped', true], } - }); + })); return {play}; } ], @@ -87,11 +87,11 @@ export const TransformSkip = meta.story({ export const TransformPrereq = meta.story({ loaders: [ async () => { - const play = await generatePlayWithLifecycle({ + const play = asJsonPlayObject(await generatePlayWithLifecycle({ lifecycleSteps: { preCompare: [true, 'prereq'], } - }); + })); return {play}; } ], @@ -100,11 +100,11 @@ export const TransformPrereq = meta.story({ export const TransformStop = meta.story({ loaders: [ async () => { - const play = await generatePlayWithLifecycle({ + const play = asJsonPlayObject(await generatePlayWithLifecycle({ lifecycleSteps: { preCompare: [true, 'stop'], } - }); + })); return {play}; } ], diff --git a/src/stories/List.stories.tsx b/src/stories/List.stories.tsx index 9b14b1d3..0f3e1086 100644 --- a/src/stories/List.stories.tsx +++ b/src/stories/List.stories.tsx @@ -8,7 +8,7 @@ import {Provider} from "../client/components/Provider"; import { generateJsonPlays, normalizePlays } from "../core/PlayTestUtils.js"; import { ErrorLike, JsonPlayObject } from "../core/Atomic.js"; import {examplePlay, lastfmErrorExample} from './storyUtils.js'; -import {playWithLifecycleScrobble, generatePlayWithLifecycle} from '../core/tests/utils/fixtures' +import {playWithLifecycleScrobble, generatePlayWithLifecycle, asJsonPlayObject} from '../core/tests/utils/fixtures' import { generateArray } from "../core/DataUtils.js"; import dayjs from "dayjs"; @@ -55,14 +55,14 @@ decorators: [ export const List = meta.story({ loaders: [ async () => { - const queued = normalizePlays(generateArray(7,() => generatePlayWithLifecycle()), {endDate: dayjs()}).map(x => ({play: x, status: 'queued'})); + const queued = normalizePlays(generateArray(7,() => generatePlayWithLifecycle()), {endDate: dayjs()}).map(x => ({play: asJsonPlayObject(x), status: 'queued'})); - const scrobbled = await playWithLifecycleScrobble(generatePlayWithLifecycle({lifecycleSteps: {preCompare: [true, 'skipped', true]}})); - const scrobbleError = await playWithLifecycleScrobble(generatePlayWithLifecycle(), {error: true}); + const scrobbled = asJsonPlayObject(await playWithLifecycleScrobble(generatePlayWithLifecycle({lifecycleSteps: {preCompare: [true, 'skipped', true]}}))); + const scrobbleError = asJsonPlayObject(await playWithLifecycleScrobble(generatePlayWithLifecycle(), {error: true})); const promisedScrobbled = generateArray(10,() => playWithLifecycleScrobble(generatePlayWithLifecycle({lifecycleSteps: {preCompare: [true, 'skipped', true]}}))); const promised = await Promise.all(promisedScrobbled); - const yesterdayScrobbled = normalizePlays(promised, {endDate: dayjs().subtract(1, 'd').subtract(100, 'm')}).map((x) => ({play: x, status: 'scrobbled'})); + const yesterdayScrobbled = normalizePlays(promised, {endDate: dayjs().subtract(1, 'd').subtract(100, 'm')}).map((x) => ({play: asJsonPlayObject(x), status: 'scrobbled'})); return {data: [ ...queued, {play: scrobbled, status: 'scrobbled'}, diff --git a/src/stories/TransformSteps.stories.tsx b/src/stories/TransformSteps.stories.tsx index 460cf295..b0c989a0 100644 --- a/src/stories/TransformSteps.stories.tsx +++ b/src/stories/TransformSteps.stories.tsx @@ -8,7 +8,7 @@ import {Provider} from "../client/components/Provider"; import { generateJsonPlays } from "../core/PlayTestUtils.js"; import { ErrorLike, JsonPlayObject, PlayLifecycle } from "../core/Atomic.js"; import { examplePlay, lastfmErrorExample } from "./storyUtils.js"; -import {generatePlayWithLifecycle} from '../core/tests/utils/fixtures' +import {asJsonPlayObject, generatePlayWithLifecycle} from '../core/tests/utils/fixtures' // More on how to set up stories at: https://storybook.js.org/docs/writing-stories#default-export const meta = preview.meta({ @@ -27,7 +27,7 @@ decorators: [ // Use `fn` to spy on the onClick arg, which will appear in the actions panel once invoked: https://storybook.js.org/docs/essentials/actions#story-args }); -const multiPlay = generatePlayWithLifecycle({lifecycleSteps: { preCompare: 2}}); +const multiPlay = asJsonPlayObject(generatePlayWithLifecycle({lifecycleSteps: { preCompare: 2}})); // More on writing stories with args: https://storybook.js.org/docs/writing-stories/args export const Multiple = meta.story({ args: {