From 46389fd81d552d1e47554068d5f0bef0b91ec25c Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Sat, 25 Apr 2026 00:03:11 +0000 Subject: [PATCH] feat(database): Add compacting config --- src/backend/common/AbstractComponent.ts | 20 ++- src/backend/common/database/Database.ts | 138 ++++++++++++++---- .../common/infrastructure/config/aioConfig.ts | 4 +- .../infrastructure/config/client/index.ts | 4 +- .../common/infrastructure/config/database.ts | 25 +++- .../infrastructure/config/source/index.ts | 4 +- src/backend/utils.ts | 5 +- 7 files changed, 151 insertions(+), 49 deletions(-) diff --git a/src/backend/common/AbstractComponent.ts b/src/backend/common/AbstractComponent.ts index eff45e8f..62706649 100644 --- a/src/backend/common/AbstractComponent.ts +++ b/src/backend/common/AbstractComponent.ts @@ -6,7 +6,7 @@ import { LifecycleStep, PlayData, PlayObject, TransformResult } from "../../core import { buildPlayHumanDiffable, buildTrackString } from "../../core/StringUtils.js"; import { CommonClientConfig } from "./infrastructure/config/client/index.js"; import { CommonSourceConfig } from "./infrastructure/config/source/index.js"; -import { mergeSimpleError, SkipTransformStageError, StagePrerequisiteError, StageTransformError, TransformRulesError } from "./errors/MSErrors.js"; +import { mergeSimpleError, SimpleError, SkipTransformStageError, StagePrerequisiteError, StageTransformError, TransformRulesError } from "./errors/MSErrors.js"; import { FLOW_CONTROL_TERM, PlayTransformRules, @@ -20,15 +20,15 @@ import { getRoot } from "../ioc.js"; import { nanoid } from "nanoid"; import { isDebugMode } from "../utils.js"; import { findCauseByReference } from "../utils/ErrorUtils.js"; -import { hashObject } from "../utils/StringUtils.js"; +import { hashObject, parseArrayFromMaybeString } from "../utils/StringUtils.js"; import { metaInvariantTransform, playContentInvariantTransform } from "../utils/PlayComparisonUtils.js"; import { MSCache } from "./Cache.js"; import { diffObjects, diffObjectsConsoleOutput, patchObject } from "../../core/DataUtils.js"; import clone from "clone"; import { loggerNoop } from "./MaybeLogger.js"; import { objectsEqual } from "../utils/DataUtils.js"; -import { RetentionOptionsFull } from "./infrastructure/config/database.js"; -import { parseRetentionOptions } from "./database/Database.js"; +import { RetentionOptions } from "./infrastructure/config/database.js"; +import { getRetentionCompactAfterFromEnv, getRetentionDeleteAfterFromEnv, isCompactableProperty, parseRetentionOptions, parseRetentionOptionsDurations } from "./database/Database.js"; export type AbstractComponentConfig = (CommonClientConfig | CommonSourceConfig) & { transformManager?: TransformerManager }; @@ -40,13 +40,21 @@ export default abstract class AbstractComponent extends AbstractInitializable { regexCache!: ReturnType; protected transformManager: TransformerManager; protected cache: MSCache; - protected retentionOpts: RetentionOptionsFull; + protected retentionOpts: RetentionOptions; protected constructor(config: AbstractComponentConfig) { super(config); this.transformManager = config.transformManager ?? getRoot().items.transformerManager; this.cache = getRoot().items.cache(); - this.retentionOpts = parseRetentionOptions(config.options.retention); + const cProps = config.options?.retention?.compact ?? parseArrayFromMaybeString(process.env.COMPACT_PROPERTIES, {lower: true}); + if(!cProps.every(isCompactableProperty)) { + throw new SimpleError(`Compactable properties must be one of 'transform' or 'input'. Given: ${cProps.join(',')}`); + } + this.retentionOpts = { + deleteAfter: parseRetentionOptionsDurations(config.options?.retention?.deleteAfter, getRetentionDeleteAfterFromEnv()), + compactAfter: parseRetentionOptions(config.options?.retention?.compactAfter, getRetentionCompactAfterFromEnv()), + compact: cProps + }; } protected postCache(): Promise { diff --git a/src/backend/common/database/Database.ts b/src/backend/common/database/Database.ts index 2da1cca0..05e1f14e 100644 --- a/src/backend/common/database/Database.ts +++ b/src/backend/common/database/Database.ts @@ -4,11 +4,14 @@ import { promises as fs } from 'fs' import { childLogger, Logger } from '@foxxmd/logging'; import { loggerNoop } from '../MaybeLogger.js'; import { fileExists, fileOrDirectoryIsWriteable } from '../../utils/FSUtils.js'; -import { DEFAULT_RETENTION_DELETE_AFTER, RententionGranular, RetentionOptions, RetentionOptionsFull } from '../infrastructure/config/database.js'; +import { COMPACTABLE, compactableProperties, CompactableProperty, DEFAULT_RETENTION_DELETE_AFTER, RententionGranular, RetentionConfig, RetentionConfigValue, RetentionOption, RetentionValue, RetentionValueUnparsed } from '../infrastructure/config/database.js'; import { DurationValue } from '../infrastructure/Atomic.js'; import { Duration } from 'dayjs/plugin/duration.js'; import dayjs from 'dayjs'; import { parseDurationFromDurationValue } from '../../utils/TimeUtils.js'; +import assert, { AssertionError } from 'node:assert'; +import { parseBoolStrict } from '../../utils.js'; +import { SimpleError } from '../errors/MSErrors.js'; export const MEMORY_DB_NAME = ':memory:'; export const isMemoryDb = (name: string): boolean => name === MEMORY_DB_NAME; @@ -52,59 +55,132 @@ export const backupDb = async (dbName: string, opts: { logger?: Logger, workingD } } -const parseRetentionFromEnv = (): Required> => { +const parseRetentionValue = (val: RetentionValueUnparsed): RetentionValue => { + if(typeof val === 'string' || typeof val === 'boolean') { + try { + const boolVal = parseBoolStrict(val); + assert(boolVal === false, 'retention value cannot be true'); + return boolVal; + } catch (e) { + // swallow + } + } else if(dayjs.isDuration(val)) { + return val; + } else { + return parseDurationFromDurationValue(val); + } + throw new SimpleError('retention value be of one: false, number, or string'); +} + +const parseRetentionFromEnv = (): RetentionOption => { const deleteAfterEnv = process.env.RETENTION_DELETE_AFTER ?? DEFAULT_RETENTION_DELETE_AFTER, deleteCompletedEnv = process.env.RETENTION_DELETE_COMPLETED_AFTER ?? deleteAfterEnv, deleteFailedEnv = process.env.RETENTION_DELETE_FAILED_AFTER ?? deleteAfterEnv, deleteDupedEnv = process.env.RETENTION_DELETE_DUPED_AFTER ?? deleteAfterEnv; return { - completed: parseDurationFromDurationValue(deleteCompletedEnv), - failed: parseDurationFromDurationValue(deleteFailedEnv), - duped: parseDurationFromDurationValue(deleteDupedEnv) + completed: parseRetentionValue(deleteCompletedEnv), + failed: parseRetentionValue(deleteFailedEnv), + duped: parseRetentionValue(deleteDupedEnv) } } -let retentionFromEnv: Required>; -const getRetentionFromEnv = () => { - if (retentionFromEnv === undefined) { - retentionFromEnv = parseRetentionFromEnv(); +const isRetentionOptionDurations = (val: RetentionOption): val is RetentionOption => { + return dayjs.isDuration(val.completed) + && dayjs.isDuration(val.duped) + && dayjs.isDuration(val.failed); +} + +let retentionDeleteAfterFromEnv: RetentionOption, +retentionCompactAfterFromEnv: RetentionOption; + +export const getRetentionDeleteAfterFromEnv = () => { + if (retentionDeleteAfterFromEnv === undefined) { + const deleteEnv = parseRetentionFromEnv(); + if(isRetentionOptionDurations(deleteEnv)) { + retentionDeleteAfterFromEnv = deleteEnv; + } else { + throw new SimpleError('retention deleteAfter values from env must all be one of: number or string'); + } + } + return retentionDeleteAfterFromEnv; +} +export const getRetentionCompactAfterFromEnv = () => { + if (retentionCompactAfterFromEnv === undefined) { + const compactEnv = parseRetentionFromEnv(); + retentionCompactAfterFromEnv = compactEnv; } - return retentionFromEnv; + return retentionCompactAfterFromEnv; } -export const parseRetentionOptions = (opts: RetentionOptions = {}): RetentionOptionsFull => { - if (typeof opts.deleteAfter === 'number' || typeof opts.deleteAfter === 'string') { - const dur = parseDurationFromDurationValue(opts.deleteAfter); +export const parseRetentionOptions = (opts: RetentionConfigValue = {}, defaults: RetentionOption): RetentionOption => { + if (typeof opts === 'number' || typeof opts === 'string') { + const dur = parseDurationFromDurationValue(opts); return { - deleteAfter: { - completed: dur, - duped: dur, - failed: dur - } + completed: dur, + duped: dur, + failed: dur } } - const fromEnv = getRetentionFromEnv(); - if (opts.deleteAfter === undefined) { + if (opts === undefined) { + return defaults; + } + + if(dayjs.isDuration(opts)) { return { - deleteAfter: fromEnv + completed: opts, + failed: opts, + duped: opts } } const { - deleteAfter: { - completed = fromEnv.completed, - failed = fromEnv.failed, - duped = fromEnv.duped - } = {} + completed = defaults.completed, + failed = defaults.failed, + duped = defaults.duped } = opts; return { - deleteAfter: { - completed: dayjs.isDuration(completed) ? completed : parseDurationFromDurationValue(completed), - failed: dayjs.isDuration(failed) ? failed : parseDurationFromDurationValue(failed), - duped: dayjs.isDuration(duped) ? duped : parseDurationFromDurationValue(duped), + completed: parseRetentionValue(completed), + failed: parseRetentionValue(failed), + duped: parseRetentionValue(duped), + } +} + +export const parseRetentionOptionsDurations = (opts: RetentionConfigValue> = {}, defaults: RetentionOption): RetentionOption => { + if (typeof opts === 'number' || typeof opts === 'string') { + const dur = parseDurationFromDurationValue(opts); + return { + completed: dur, + duped: dur, + failed: dur + } + } + + if(dayjs.isDuration(opts)) { + return { + completed: opts, + failed: opts, + duped: opts } } -} \ No newline at end of file + + if (opts === undefined) { + return defaults; + } + + const { + completed = defaults.completed, + failed = defaults.failed, + duped = defaults.duped + } = opts; + + return { + completed: dayjs.isDuration(completed) ? completed : parseDurationFromDurationValue(completed), + failed: dayjs.isDuration(failed) ? failed : parseDurationFromDurationValue(failed), + duped: dayjs.isDuration(duped) ? duped : parseDurationFromDurationValue(duped), + } +} + +export const isCompactableProperty = (val: string): val is CompactableProperty => val === COMPACTABLE.input || val === COMPACTABLE.transform; \ No newline at end of file diff --git a/src/backend/common/infrastructure/config/aioConfig.ts b/src/backend/common/infrastructure/config/aioConfig.ts index ef3dd3ee..51c11778 100644 --- a/src/backend/common/infrastructure/config/aioConfig.ts +++ b/src/backend/common/infrastructure/config/aioConfig.ts @@ -7,7 +7,7 @@ import { CommonSourceOptions, SourceRetryOptions } from "./source/index.js"; import { SourceAIOConfig } from "./source/sources.js"; import { CacheConfigOptions, DurationValue } from "../Atomic.js"; import { TransformerCommonConfig } from "../../../../core/Atomic.js"; -import { RetentionOptions } from "./database.js"; +import { RetentionConfig } from "./database.js"; export interface SourceDefaults extends CommonSourceOptions { @@ -72,7 +72,7 @@ export interface AIOConfig { transformers?: TransformerCommonConfig[] database?: { - retention?: RetentionOptions + retention?: RetentionConfig } } diff --git a/src/backend/common/infrastructure/config/client/index.ts b/src/backend/common/infrastructure/config/client/index.ts index a4d80cec..83b13a59 100644 --- a/src/backend/common/infrastructure/config/client/index.ts +++ b/src/backend/common/infrastructure/config/client/index.ts @@ -1,7 +1,7 @@ import { DurationValue } from "../../Atomic.js"; import { PlayTransformConfig, PlayTransformOptions } from "../../Transform.js"; import { CommonConfig, CommonData, RequestRetryOptions } from "../common.js"; -import { RetentionOptions } from "../database.js"; +import { RetentionConfig } from "../database.js"; /** * Scrobble matching (between new source track and existing client scrobbles) logging options. Used for debugging. @@ -108,7 +108,7 @@ export interface CommonClientOptions extends RequestRetryOptions, UpstreamRefres playTransform?: PlayTransformOptions - retention?: RetentionOptions + retention?: RetentionConfig } export interface CommonClientConfig extends CommonConfig { diff --git a/src/backend/common/infrastructure/config/database.ts b/src/backend/common/infrastructure/config/database.ts index 66379ae2..aa05d1a9 100644 --- a/src/backend/common/infrastructure/config/database.ts +++ b/src/backend/common/infrastructure/config/database.ts @@ -1,18 +1,33 @@ import { Duration } from "dayjs/plugin/duration.js"; import { DurationValue } from "../Atomic.js"; -export interface RententionGranular { +export type RetentionValueUnparsed = DurationValue | Duration | false; +export type RetentionValue = Duration | false; +export interface RententionGranular { failed?: T completed?: T duped?: T } -export interface RetentionOptions { - deleteAfter?: T | RententionGranular +export type RetentionConfigValue = T | RententionGranular; +export type RetentionOption = Required>; + +export type CompactableProperty = 'transform' | 'input'; +export const COMPACTABLE = { + transform: 'transform', + input: 'input' +} as const satisfies Record; +export const compactableProperties: CompactableProperty[] = [COMPACTABLE.transform, COMPACTABLE.input]; +export interface RetentionConfig { + deleteAfter?: RetentionConfigValue + compactAfter?: RetentionConfigValue + compact?: CompactableProperty[] } -export interface RetentionOptionsFull { - deleteAfter: RententionGranular +export interface RetentionOptions { + deleteAfter: RetentionOption + compactAfter: RetentionOption + compact: CompactableProperty[] } export const DEFAULT_RETENTION_DELETE_AFTER = 604800; // 7 days \ No newline at end of file diff --git a/src/backend/common/infrastructure/config/source/index.ts b/src/backend/common/infrastructure/config/source/index.ts index aa6d1196..8d87c72f 100644 --- a/src/backend/common/infrastructure/config/source/index.ts +++ b/src/backend/common/infrastructure/config/source/index.ts @@ -2,7 +2,7 @@ import { FileLogOptions, LogLevel } from "@foxxmd/logging"; import { PlayTransformConfig, PlayTransformOptions } from "../../Transform.js"; import { CommonConfig, CommonData, RequestRetryOptions } from "../common.js"; -import { RetentionOptions } from "../database.js"; +import { RetentionConfig } from "../database.js"; import { DurationValue } from "../../Atomic.js"; export interface SourceRetryOptions extends RequestRetryOptions { @@ -111,7 +111,7 @@ export interface CommonSourceOptions extends SourceRetryOptions { playTransform?: PlayTransformOptions - retention?: RetentionOptions + retention?: RetentionConfig } export interface ManualListeningOptions { diff --git a/src/backend/utils.ts b/src/backend/utils.ts index 861da8da..7ed9cb33 100644 --- a/src/backend/utils.ts +++ b/src/backend/utils.ts @@ -293,7 +293,10 @@ export function parseBool(value: any, prev: any = false): boolean { throw new Error(`'${value.toString()}' is not a boolean value.`); } -export function parseBoolStrict(value: string): boolean { +export function parseBoolStrict(value: string | boolean): boolean { + if(typeof value === 'boolean') { + return value; + } const strTrue = ['1', 'true', 'yes'].includes(value.toLocaleLowerCase().trim()); if (strTrue) { return strTrue; -- 2.51.2