diff --git a/src/backend/common/AbstractComponent.ts b/src/backend/common/AbstractComponent.ts index 2dd60da6..6e5e2669 100644 --- a/src/backend/common/AbstractComponent.ts +++ b/src/backend/common/AbstractComponent.ts @@ -33,7 +33,7 @@ import { getRetentionCompactAfterFromEnv, getRetentionDeleteAfterFromEnv, isComp import type {DbConcrete} from "./database/drizzle/drizzleUtils.ts"; import type {ComponentSelect} from "./database/drizzle/drizzleTypes.ts"; import { DrizzlePlayRepository } from "./database/drizzle/repositories/PlayRepository.ts"; -import type {ClientType, MonitoringStatus} from "../../core/Atomic.ts"; +import type {ClientType, MonitoringStatus, OptionalCacheUsage} from "../../core/Atomic.ts"; import type {SourceType} from "../../core/Atomic.ts"; import { DrizzleComponentRepository } from "./database/drizzle/repositories/ComponentRepository.ts"; import dayjs, { type Dayjs } from "dayjs"; @@ -248,7 +248,12 @@ export default abstract class AbstractComponent extends AbstractInitializable { return partArr.map(x => this.transformManager.parseTransformerConfig(x)); } - public transformPlay = async (play: PlayObject, hookType: TransformHook, log?: boolean | 'all') => { + public transformPlay = async (play: PlayObject, hookType: TransformHook, transformOpts: {log?: boolean | 'all'} & OptionalCacheUsage = {}) => { + + const { + log, + useCachedResult = true + } = transformOpts; let logger: Logger; @@ -280,7 +285,7 @@ export default abstract class AbstractComponent extends AbstractInitializable { const shouldLog = log ?? this.config.options?.playTransform?.log ?? isDebugMode(); const transformHash = `playTransform-${hashObject(hook)}-${hashObject(playContentInvariantTransform(play))}`; - const cachedSteps = await this.cache.cacheTransform.get(transformHash); + const cachedSteps = useCachedResult ? await this.cache.cacheTransform.get(transformHash) : undefined; if(cachedSteps !== undefined) { logger.trace(`Cache hit for Steps => ${transformHash}`); //return cachedTransformPlay; @@ -294,7 +299,8 @@ export default abstract class AbstractComponent extends AbstractInitializable { const opts = { logger, - asyncId + asyncId, + useCachedResult } if(cachedSteps !== undefined) { @@ -415,17 +421,18 @@ export default abstract class AbstractComponent extends AbstractInitializable { } } - protected generateStepFromStage = async (playTruth: PlayObject, hookItem: StageConfig, hookType: TransformHook, opts: { logger?: Logger, asyncId?: string } = {}): Promise<[LifecycleStep, PlayObject]> => { + protected generateStepFromStage = async (playTruth: PlayObject, hookItem: StageConfig, hookType: TransformHook, opts: { logger?: Logger, asyncId?: string } & OptionalCacheUsage = {}): Promise<[LifecycleStep, PlayObject]> => { const { onSuccess = 'continue', onFailure = 'stop', onSkip = 'continue', - failureReturnPartial = false + failureReturnPartial = false, } = hookItem; const { logger = loggerNoop, - asyncId = nanoid(6) + asyncId = nanoid(6), + useCachedResult } = opts; const { @@ -446,7 +453,7 @@ export default abstract class AbstractComponent extends AbstractInitializable { stageName: string = 'Unnamed', err: Error; try { - [newTransformedPlay, stageName] = await this.transformManager.handleStage(hookItem, playTruth, asyncId); + [newTransformedPlay, stageName] = await this.transformManager.handleStage(hookItem, playTruth, {asyncId, useCachedResult}); newTransformedPlay = clone(newTransformedPlay); } catch (e) { err = e; diff --git a/src/backend/common/transforms/AbstractTransformer.ts b/src/backend/common/transforms/AbstractTransformer.ts index c2bc8c73..0e7f1979 100644 --- a/src/backend/common/transforms/AbstractTransformer.ts +++ b/src/backend/common/transforms/AbstractTransformer.ts @@ -1,5 +1,5 @@ import { childLogger, type Logger } from "@foxxmd/logging"; -import type { PlayObject, TransformerCommon, TransformerCommonConfig } from "../../../core/Atomic.ts"; +import type { OptionalCacheUsage, PlayObject, TransformerCommon, TransformerCommonConfig } from "../../../core/Atomic.ts"; import { isStageTyped, testWhenConditions } from "../../utils/PlayTransformUtils.ts"; import AbstractInitializable from "../AbstractInitializable.ts"; import type { StageConfig } from "../../../core/Transform.ts"; @@ -61,13 +61,16 @@ export default abstract class AbstractTransformer { + public async handle(data: Y, play: PlayObject, opts?: OptionalCacheUsage): Promise { + const { + useCachedResult = true, + } = (opts ?? {}); const cacheKey = `transformResult-${this.configHash}-${hashObject(data)}-${hashObject(playContentInvariantTransform(play))}` try { - const cachedTransformData = await this.cache.get(cacheKey); + const cachedTransformData = useCachedResult ? await this.cache.get(cacheKey) : undefined; if(cachedTransformData !== undefined) { this.logger.debug('Transform cache hit'); - const transformed = await this.doHandle(data, play, cachedTransformData); + const transformed = await this.doHandle(data, play, cachedTransformData, opts); return transformed; } } catch (e) { @@ -82,7 +85,7 @@ export default abstract class AbstractTransformer; + protected abstract doHandle(data: StageConfig, play: PlayObject, transformData: T, opts?: OptionalCacheUsage): Promise; - public async getTransformerData(play: PlayObject, stageConfig: Y): Promise { + public async getTransformerData(play: PlayObject, stageConfig: Y, opts?: OptionalCacheUsage): Promise { return undefined; } - public async handlePostFetch(play: PlayObject, transformData: any, stageConfig: Y): Promise { + public async handlePostFetch(play: PlayObject, transformData: any, stageConfig: Y, opts?: OptionalCacheUsage): Promise { return transformData; } - public async handlePreFetch(play: PlayObject, stageConfig: Y): Promise { + public async handlePreFetch(play: PlayObject, stageConfig: Y, opts?: OptionalCacheUsage): Promise { return } } \ No newline at end of file diff --git a/src/backend/common/transforms/MusicbrainzTransformer.ts b/src/backend/common/transforms/MusicbrainzTransformer.ts index ee47d028..cfc0cc9b 100644 --- a/src/backend/common/transforms/MusicbrainzTransformer.ts +++ b/src/backend/common/transforms/MusicbrainzTransformer.ts @@ -1,4 +1,4 @@ -import { type ArtistCredit, asMBReleasePrimaryGroupType, asMBReleaseSecondaryGroupType, asMBReleaseStatus, DEFAULT_MISSING_TYPES, type LifecycleInput, type MBReleaseGroupPrimaryType, type MBReleaseGroupSecondaryType, type MBReleaseStatus, type MissingMbidType, type PlayObject, type TrackMeta, type TransformerCommon, type TransformOptions } from "../../../core/Atomic.ts"; +import { type ArtistCredit, asMBReleasePrimaryGroupType, asMBReleaseSecondaryGroupType, asMBReleaseStatus, DEFAULT_MISSING_TYPES, type LifecycleInput, type MBReleaseGroupPrimaryType, type MBReleaseGroupSecondaryType, type MBReleaseStatus, type MissingMbidType, type OptionalCacheUsage, type PlayObject, type TrackMeta, type TransformerCommon, type TransformOptions } from "../../../core/Atomic.ts"; import { isWhenCondition, testWhenConditions } from "../../utils/PlayTransformUtils.ts"; import type {WebhookPayload} from "../infrastructure/config/health/webhooks.ts"; import type {ExternalMetadataTerm, PlayTransformMetadataStage} from "../../../core/Transform.ts"; @@ -429,7 +429,7 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer { + public async getTransformerData(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage, opts?: OptionalCacheUsage): Promise { const { // preserve order of search from before searchOrder @@ -443,25 +443,25 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer { + public async searchByBasicFields(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage, opts: OptionalCacheUsage = {}): Promise { this.logger.debug({labels: ['Basic Search']}, 'Searching by artist/album/track'); - return await this.api.searchByRecording(play); + return await this.api.searchByRecording(play, opts); } - public async searchByBasicFieldsOrMBIDs(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage): Promise { + public async searchByBasicFieldsOrMBIDs(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage, opts: OptionalCacheUsage = {}): Promise { const using: UsingTypes[] = []; const { data: { @@ -512,31 +512,31 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer 0 ? 'mbidartist' : 'artist'); this.logger.debug({labels: ['Basic Or MBID Search']}, `Searching using ${using.join(', ')}}`); - return await this.api.searchByRecording(play, {using}); + return await this.api.searchByRecording(play, {using, ...opts}); } - public async searchByIsrc(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage): Promise { + public async searchByIsrc(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage, opts: OptionalCacheUsage = {}): Promise { if(play.data.isrc !== undefined) { this.logger.debug({labels: ['ISRC Search']},'Searching with ISRC'); - return await this.api.searchByRecording(play, {using: ['isrc']}); + return await this.api.searchByRecording(play, {using: ['isrc'], ...opts}); } throw new SearchPrerequisiteError('Play does not have ISRC'); } - public async searchByRecordingMbid(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage): Promise { + public async searchByRecordingMbid(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage, opts: OptionalCacheUsage = {}): Promise { if(play.data.meta?.brainz?.recording !== undefined) { this.logger.debug({labels: ['MBID Search']},'Searching with Recording MBID'); - return await this.api.searchByRecording(play, {using: ['mbidrecording']}); + return await this.api.searchByRecording(play, {using: ['mbidrecording'], ...opts}); } throw new SearchPrerequisiteError('Play does not have recording MBID'); } - public async searchByAlbum(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage): Promise { + public async searchByAlbum(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage, opts: OptionalCacheUsage = {}): Promise { // possibly the artist is incorrect (may be combined as one string) // if we have an album we can likely still get a decent hit w/o using artist if(play.data.album !== undefined && play.data.artists !== undefined && play.data.artists.length > 0) { this.logger.debug({labels: ['Album Search']},'Searching with only track+album'); - return await this.api.searchByRecording(play, {using: ['title','album']}); + return await this.api.searchByRecording(play, {using: ['title','album'], ...opts}); } if(play.data.album === undefined) { throw new SearchPrerequisiteError('Play does not have an album'); @@ -549,7 +549,7 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer { + public async searchByArtist(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage, opts: OptionalCacheUsage = {}): Promise { const { searchArtistMethod = this.defaults.searchArtistMethod, } = stageConfig; @@ -582,7 +582,7 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer { + public async searchByFreeText(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage, opts: OptionalCacheUsage = {}): Promise { this.logger.debug({labels: ['Freetext Search']},'Trying freetext search'); - const results = await this.api.searchByRecording(play, {freetext: true}) as IRecordingMSList; + const results = await this.api.searchByRecording(play, {freetext: true, ...opts}) as IRecordingMSList; results.freeText = true; return results; } diff --git a/src/backend/common/transforms/TransformerManager.ts b/src/backend/common/transforms/TransformerManager.ts index 0d269d2f..c92707d1 100644 --- a/src/backend/common/transforms/TransformerManager.ts +++ b/src/backend/common/transforms/TransformerManager.ts @@ -1,6 +1,6 @@ import { childLogger, type Logger } from "@foxxmd/logging"; import type AbstractTransformer from "./AbstractTransformer.ts"; -import type {TransformerCommonConfig} from "../../../core/Atomic.ts"; +import type {OptionalCacheUsage, TransformerCommonConfig} from "../../../core/Atomic.ts"; import UserTransformer from "./UserTransformer.ts"; import type {StageConfig} from "../../../core/Transform.ts"; import type {PlayObject} from "../../../core/Atomic.ts"; @@ -146,10 +146,10 @@ export default class TransformerManager { return config; } - public async handleStage(data: StageConfig, play: PlayObject, asyncId: string = nanoid(6)): Promise<[PlayObject, string]> { + public async handleStage(data: StageConfig, play: PlayObject, opts: {asyncId?: string} & OptionalCacheUsage): Promise<[PlayObject, string]> { const t: AbstractTransformer = this.getTransformerByStage(data); try { - const transformedPlay = await this.asyncStore.run(asyncId, async () => { + const transformedPlay = await this.asyncStore.run(opts.asyncId ?? nanoid(6), async () => { return await t.handle(data, play); }); return [transformedPlay, t.name]; diff --git a/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts b/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts index 57510c9d..0aaa7af8 100644 --- a/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts +++ b/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts @@ -1,5 +1,5 @@ import type { Response } from 'superagent'; -import type {ArtistCredit, PlayObject, PlayObjectMinimal, URLData} from "../../../../core/Atomic.ts"; +import type {ArtistCredit, OptionalCacheUsage, PlayObject, PlayObjectMinimal, URLData} from "../../../../core/Atomic.ts"; import { UpstreamError } from "../../errors/UpstreamError.ts"; import { type AbstractApiOptions, type FormatPlayObjectOptions, MUSICBRAINZ_URL, type MusicbrainzApiConfigData } from "../../infrastructure/Atomic.ts"; import AbstractApiClient from "../AbstractApiClient.ts"; @@ -114,17 +114,18 @@ export class MusicbrainzApiClient extends AbstractApiClient { return 'API'; } - callApi = async (func: (mb: MusicBrainzApi) => Promise, options?: { timeout?: number, cacheKey?: string }): Promise => { + callApi = async (func: (mb: MusicBrainzApi) => Promise, options?: { timeout?: number, cacheKey?: string } & OptionalCacheUsage): Promise => { let apiConfig = this.rrApis.next().value; const { timeout = 30000, - cacheKey + cacheKey, + useCachedResult = true } = options || {}; try { - const cachedTransform = await this.cache.get(cacheKey); + const cachedTransform = useCachedResult ? await this.cache.get(cacheKey) : undefined; if(cachedTransform !== undefined) { const cacheUrl = await this.cache.get(`${cacheKey}-url`); const cacheQs = await this.cache.get(`${cacheKey}-qs`); @@ -218,13 +219,14 @@ export class MusicbrainzApiClient extends AbstractApiClient { } } - searchByRecording = async(play: PlayObject, options?: SearchOptions): Promise => { + searchByRecording = async(play: PlayObject, options?: SearchOptions & OptionalCacheUsage): Promise => { const { escapeCharacters = true, removeCharacters = false, using = ['album','artist','title'], - freetext + freetext, + useCachedResult } = options || {}; const cacheKey = `mb-recSearch-${hashObject({...playContentInvariantTransform(play), using})}`; @@ -347,7 +349,8 @@ export class MusicbrainzApiClient extends AbstractApiClient { query: q }); }, { - cacheKey + cacheKey, + useCachedResult }); if(res === undefined) { diff --git a/src/backend/tests/component/transformers.test.ts b/src/backend/tests/component/transformers.test.ts index 0a99dbb1..05566718 100644 --- a/src/backend/tests/component/transformers.test.ts +++ b/src/backend/tests/component/transformers.test.ts @@ -700,14 +700,14 @@ describe('Play Transforms', function () { component.buildTransformRules(); const play = generatePlay({ track: 'My cool something track', artists: artistNamesToCredits([str]), playDate: dayjs().subtract(10, 'm') }); - const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare, 'all'); + const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare, {log: 'all'}); expect(transformed.data.track).equal('My cool bar track'); expect(artistCreditsToNames(transformed.data.artists)).eql(primaries.concat(secondaries)); const cachablePlay = clone(play); const laterDate = dayjs().subtract(5, 'm'); cachablePlay.data.playDate = laterDate; - const cacheTransformed = await component.transformPlay(cachablePlay, TRANSFORM_HOOK.preCompare, 'all'); + const cacheTransformed = await component.transformPlay(cachablePlay, TRANSFORM_HOOK.preCompare, {log: 'all'}); expect(cacheTransformed.data.track).equal('My cool bar track'); expect(artistCreditsToNames(cacheTransformed.data.artists)).eql(primaries.concat(secondaries)); expect(cacheTransformed.data.playDate.isSame(cachablePlay.data.playDate)); diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts index b8c9e3c6..914c9057 100644 --- a/src/core/Atomic.ts +++ b/src/core/Atomic.ts @@ -806,3 +806,6 @@ export const NO_DEVICE = 'NoDevice';export const NO_USER = 'SingleUser'; export const SINGLE_USER_PLATFORM_ID: PlayPlatformId = [NO_DEVICE, NO_USER]; export const SINGLE_USER_PLATFORM_ID_STR = `${NO_DEVICE}-${NO_USER}`; +export interface OptionalCacheUsage { + useCachedResult?: boolean +} \ No newline at end of file