diff --git a/src/backend/tests/utils/strings.test.ts b/src/backend/tests/utils/strings.test.ts index 73f5c358..44598053 100644 --- a/src/backend/tests/utils/strings.test.ts +++ b/src/backend/tests/utils/strings.test.ts @@ -3,8 +3,10 @@ import { describe, it } from 'mocha'; import { compareNormalizedStrings, normalizeStr, + replaceInterpolatedValues, } from "../../utils/StringUtils.js"; import { splitByFirstFound } from '../../../core/StringUtils.js'; +import { noCasePropObj } from '../../utils/DataUtils.js'; describe('String Comparisons', function () { @@ -122,3 +124,39 @@ describe('String Splitting', function() { }); }); +it('Proxy object has case-insensitive keys', function() { + const myObj: Record = {FOO: 'bar'}; + + const iObj = noCasePropObj(myObj); + + expect(iObj.foO).is.not.undefined; + expect(iObj.foo).eq('bar'); +}); + +describe('Interpolation', function() { + + it('interpolates values', function() { + const replaced = replaceInterpolatedValues('My cool string has [[asecret]] in it', {asecret: 'foo'}); + expect(replaced).not.includes('['); + expect(replaced).not.includes(']'); + expect(replaced).includes('foo'); + }); + + it('interpolates values case-insensitive', function() { + const replaced = replaceInterpolatedValues('My cool string has [[ASECREt]] in it', {aSeCrEt: 'foo'}); + expect(replaced).includes('foo'); + }); + + it('interpolates values after trimming', function() { + const replaced = replaceInterpolatedValues('My cool string has [[ ASECREt ]] in it', {aSeCrEt: 'foo'}); + expect(replaced).includes('foo'); + }); + + it('leaves match in place if not interpolated', function() { + const replaced = replaceInterpolatedValues('My cool string has [[asecret]] and [[coolsecret]] in it', {bar: 'foo', coolsecret: 'xxx'}); + expect(replaced).includes('[[asecret]]'); + expect(replaced).includes('xxx'); + expect(replaced).not.includes('[[coolsecret]]'); + }); +}); + diff --git a/src/backend/utils/DataUtils.ts b/src/backend/utils/DataUtils.ts index 7722466c..e6e76130 100644 --- a/src/backend/utils/DataUtils.ts +++ b/src/backend/utils/DataUtils.ts @@ -1,6 +1,85 @@ export const asArray = (data: T | T[]): T[] => { - if(Array.isArray(data)) { + if (Array.isArray(data)) { return data; } return [data]; +} + +const handler: ProxyHandler = +{ + get: function (target, key) { + //console.log("key: " + key.toString()); + if (typeof key == "string") { + const uKey = key.toUpperCase(); + + if ((key != uKey) && (key in target)) + return target[key]; + return target[uKey]; + } + return target[key]; + }, + set: function (target, key, value) { + if (typeof key == "string") { + const uKey = key.toUpperCase(); + + if ((key != uKey) && (key in target)) + target[key] = value; + target[uKey] = value; + return true; + } + else { + target[key] = value; + return true; + } + + }, + deleteProperty: function (target, key) { + if (typeof key == "string") { + const uKey = key.toUpperCase(); + + if ((key != uKey) && (key in target)) + delete target[key]; + if (uKey in target) + delete target[uKey]; + return true; + } + else { + delete target[key]; + return true; + } + + }, +}; + +const checkAtomic = (value) => { + if (typeof value == "object") + return noCasePropObj(value); // recursive call only for Objects + return value; +} + +/** Return a Proxy of an object where keys can be accessed case-insensitive + * + * https://stackoverflow.com/a/50102779/1469797 + */ +export const noCasePropObj = (obj: T): T => { + let newObj; + + if (typeof obj == "object") { + newObj = new Proxy({}, handler); + // traverse the Original object converting string keys to upper case + for (var key in obj) { + if (typeof key == "string") { + var objKey = key.toUpperCase(); + if (!(key in newObj)) + newObj[objKey] = checkAtomic(obj[key]); + } + } + } + else if (Array.isArray(obj)) { + // in an array of objects convert to upper case string keys within each row + newObj = new Array(); + for (var i = 0; i < obj.length; i++) + newObj[i] = checkAtomic(obj[i]); + } + return newObj; // object with upper cased keys } \ No newline at end of file diff --git a/src/backend/utils/StringUtils.ts b/src/backend/utils/StringUtils.ts index 0e3783f3..1d93916c 100644 --- a/src/backend/utils/StringUtils.ts +++ b/src/backend/utils/StringUtils.ts @@ -4,6 +4,8 @@ import { PlayObject } from "../../core/Atomic.js"; import { asPlayerStateData, DELIMITERS, DELIMITERS_NO_AMP, PlayerStateDataMaybePlay } from "../common/infrastructure/Atomic.js"; import { genGroupIdStr, getPlatformIdFromData, intersect, parseRegexSingleOrFail } from "../utils.js"; import { buildTrackString } from "../../core/StringUtils.js"; +import { MaybeLogger } from "../common/logging.js"; +import { noCasePropObj } from "./DataUtils.js"; const {levenStrategy, diceStrategy} = strategies; @@ -430,4 +432,26 @@ export const hashObject = (obj: object, h: HashFunction = defaultHashFunc): stri const NON_ALPHANUMWHITESPACE_CHARS: RegExp = new RegExp(/[^a-zA-Z\d\s]/); export const hasNonAlphanumericChars = (str: string): boolean => { return NON_ALPHANUMWHITESPACE_CHARS.test(str); +} + +export const INTERPOLATION_WRAPPED_REGEX: RegExp = new RegExp(/\[\[([^\r\n\[\]]+?)\]\]/g); +export const replaceInterpolatedValues = (str: string, fromVals: Record, logger: MaybeLogger = new MaybeLogger()): string => { + const cleanFromValKeys = noCasePropObj(fromVals); + + const matched = new Set(), + unmatched = new Set(); + const replaced = str.replaceAll(INTERPOLATION_WRAPPED_REGEX, (match, p1) => { + //const fv = cleanFromValKeys[p1.toLocaleLowerCase().trim()]; + const fv = cleanFromValKeys[p1.trim()]; + if(fv !== undefined) { + matched.add(p1); + return fv; + } + unmatched.add(p1); + return match; + }); + if(matched.size !== 0 || unmatched.size !== 0) { + logger.debug(`Matched: ${matched.size === 0 ? 'None' : Array.from(matched.values()).join(', ')} | Unmatched: ${unmatched.size === 0 ? 'None' : Array.from(unmatched.values()).join(', ')}`); + } + return replaced; } \ No newline at end of file -- 2.51.2 From 36ed40f628ee714f126ae47b8ea992345c495eec Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Tue, 16 Dec 2025 20:53:00 +0000 Subject: [PATCH 2/6] feat: Implement secret interpolation from ENV #420 --- src/backend/common/vendor/LastfmApiClient.ts | 5 +- src/backend/index.ts | 5 +- src/backend/scrobblers/ScrobbleClients.ts | 7 ++- src/backend/sources/DeezerInternalSource.ts | 2 +- src/backend/sources/DeezerSource.ts | 5 +- src/backend/sources/ScrobbleSources.ts | 7 ++- src/backend/sources/SpotifySource.ts | 4 +- src/backend/tests/utils/strings.test.ts | 2 +- src/backend/utils.ts | 19 ------- src/backend/utils/DataUtils.ts | 58 +++++++++++++++++++- src/backend/utils/StringUtils.ts | 24 +------- 11 files changed, 79 insertions(+), 59 deletions(-) diff --git a/src/backend/common/vendor/LastfmApiClient.ts b/src/backend/common/vendor/LastfmApiClient.ts index 7f7aefe9..16dc38cc 100644 --- a/src/backend/common/vendor/LastfmApiClient.ts +++ b/src/backend/common/vendor/LastfmApiClient.ts @@ -7,7 +7,8 @@ import LastFm, { } from "lastfm-node-client"; import { PlayObject } from "../../../core/Atomic.js"; import { nonEmptyStringOrDefault, splitByFirstFound } from "../../../core/StringUtils.js"; -import { readJson, removeUndefinedKeys, sleep, writeFile } from "../../utils.js"; +import { removeUndefinedKeys, sleep, writeFile } from "../../utils.js"; +import { readJson } from '../../utils/DataUtils.js'; import { joinedUrl } from "../../utils/NetworkUtils.js"; import { getScrobbleTsSOCDate } from "../../utils/TimeUtils.js"; import { getNodeNetworkException, isNodeNetworkException } from "../errors/NodeErrors.js"; @@ -169,7 +170,7 @@ export default class LastfmApiClient extends AbstractApiClient { initialize = async (): Promise => { try { - const creds = await readJson(this.workingCredsPath, {throwOnNotFound: false}); + const creds = await readJson(this.workingCredsPath, {throwOnNotFound: false, interpolateEnvs: false}); const {sessionKey} = creds || {}; if (this.client.sessionKey === undefined && sessionKey !== undefined) { this.client.sessionKey = sessionKey; diff --git a/src/backend/index.ts b/src/backend/index.ts index 765ae5b1..9bd1b81d 100644 --- a/src/backend/index.ts +++ b/src/backend/index.ts @@ -16,7 +16,8 @@ import { getRoot, parseVersion } from "./ioc.js"; import { initServer } from "./server/index.js"; import { createHeartbeatClientsTask } from "./tasks/heartbeatClients.js"; import { createHeartbeatSourcesTask } from "./tasks/heartbeatSources.js"; -import { isDebugMode, parseBool, readJson, retry, sleep } from "./utils.js"; +import { isDebugMode, parseBool, retry, sleep } from "./utils.js"; +import { readJson } from './utils/DataUtils.js'; import { createVegaGenerator } from './utils/SchemaUtils.js'; import ScrobbleClients from './scrobblers/ScrobbleClients.js'; import ScrobbleSources from './sources/ScrobbleSources.js'; @@ -62,7 +63,7 @@ const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`) let appConfigFail: Error | undefined = undefined; let config = {}; try { - config = await readJson(`${configDir}/config.json`, {throwOnNotFound: false}); + config = await readJson(`${configDir}/config.json`, {throwOnNotFound: false, logger: childLogger(initLogger, 'Secrets')}); } catch (e) { appConfigFail = e; } diff --git a/src/backend/scrobblers/ScrobbleClients.ts b/src/backend/scrobblers/ScrobbleClients.ts index b2fb598c..d2722339 100644 --- a/src/backend/scrobblers/ScrobbleClients.ts +++ b/src/backend/scrobblers/ScrobbleClients.ts @@ -10,7 +10,8 @@ import { ListenBrainzClientConfig } from "../common/infrastructure/config/client import { MalojaClientConfig } from "../common/infrastructure/config/client/maloja.js"; import { WildcardEmitter } from "../common/WildcardEmitter.js"; import { Notifiers } from "../notifier/Notifiers.js"; -import { isDebugMode, readJson } from "../utils.js"; +import { isDebugMode } from "../utils.js"; +import { readJson } from '../utils/DataUtils.js'; import { joinedUrl } from "../utils/NetworkUtils.js"; import { getTypeSchemaFromConfigGenerator } from "../utils/SchemaUtils.js"; import { validateJson } from "../utils/ValidationUtils.js"; @@ -128,7 +129,7 @@ export default class ScrobbleClients { let configFile; try { - configFile = await readJson(`${this.configDir}/config.json`, {throwOnNotFound: false}); + configFile = await readJson(`${this.configDir}/config.json`, {throwOnNotFound: false, logger: childLogger(this.logger, `Secrets`)}); } catch (e) { // think this should stay as show-stopper since config could include important defaults (delay, retries) we don't want to ignore throw new Error('config.json could not be parsed'); @@ -294,7 +295,7 @@ export default class ScrobbleClients { } let rawClientConfigs; try { - rawClientConfigs = await readJson(`${this.configDir}/${clientType}.json`, {throwOnNotFound: false}); + rawClientConfigs = await readJson(`${this.configDir}/${clientType}.json`, {throwOnNotFound: false, logger: childLogger(this.logger, `${clientType} Secrets`)}); } catch (e) { const errMsg = `${clientType}.json config file could not be parsed`; this.emitter.emit('error', errMsg); diff --git a/src/backend/sources/DeezerInternalSource.ts b/src/backend/sources/DeezerInternalSource.ts index 4451b50b..8215aaaa 100644 --- a/src/backend/sources/DeezerInternalSource.ts +++ b/src/backend/sources/DeezerInternalSource.ts @@ -5,7 +5,7 @@ import { PlayObject, SOURCE_SOT, TA_CLOSE, TA_DURING, TA_EXACT, TA_FUZZY, Tempor import { DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions, InternalConfig } from "../common/infrastructure/Atomic.js"; import { DeezerInternalSourceConfig, DeezerInternalTrackData, DeezerSourceConfig } from "../common/infrastructure/config/source/deezer.js"; import { TRANSFORM_HOOK } from "../common/infrastructure/Transform.js"; -import { parseRetryAfterSecsFromObj, playObjDataMatch, readJson, sleep, sortByOldestPlayDate, writeFile, } from "../utils.js"; +import { parseRetryAfterSecsFromObj, playObjDataMatch, sleep, sortByOldestPlayDate} from "../utils.js"; import AbstractSource, { RecentlyPlayedOptions } from "./AbstractSource.js"; import { CookieJar, Cookie } from 'tough-cookie'; import { MixedCookieAgent } from 'http-cookie-agent/http'; diff --git a/src/backend/sources/DeezerSource.ts b/src/backend/sources/DeezerSource.ts index 14210540..c9197072 100644 --- a/src/backend/sources/DeezerSource.ts +++ b/src/backend/sources/DeezerSource.ts @@ -6,7 +6,8 @@ import request from 'superagent'; import { PlayObject } from "../../core/Atomic.js"; import { DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions, InternalConfig } from "../common/infrastructure/Atomic.js"; import { DeezerSourceConfig } from "../common/infrastructure/config/source/deezer.js"; -import { parseRetryAfterSecsFromObj, readJson, sleep, sortByOldestPlayDate, writeFile, } from "../utils.js"; +import { parseRetryAfterSecsFromObj, sleep, sortByOldestPlayDate, writeFile, } from "../utils.js"; +import { readJson } from '../utils/DataUtils.js'; import { joinedUrl } from "../utils/NetworkUtils.js"; import AbstractSource, { RecentlyPlayedOptions } from "./AbstractSource.js"; @@ -95,7 +96,7 @@ export default class DeezerSource extends AbstractSource { this.logger.warn('This Source is DEPRECATED! Deezer has dropped support official API support. New apps cannot be created and existing apps are not guaranteed to continue working. Refer to the MS documentation for a new Deezer Source implementation.'); try { - const credFile = await readJson(this.workingCredsPath, {throwOnNotFound: false}); + const credFile = await readJson(this.workingCredsPath, {throwOnNotFound: false, interpolateEnvs: false}); if(credFile !== undefined) { this.config.data.accessToken = credFile.accessToken; } else { diff --git a/src/backend/sources/ScrobbleSources.ts b/src/backend/sources/ScrobbleSources.ts index 02602eeb..e6fbf8ce 100644 --- a/src/backend/sources/ScrobbleSources.ts +++ b/src/backend/sources/ScrobbleSources.ts @@ -34,7 +34,8 @@ import { VLCData, VLCSourceConfig } from "../common/infrastructure/config/source import { WebScrobblerSourceConfig } from "../common/infrastructure/config/source/webscrobbler.js"; import { YTMusicData, YTMusicSourceConfig } from "../common/infrastructure/config/source/ytmusic.js"; import { WildcardEmitter } from "../common/WildcardEmitter.js"; -import { parseBool, readJson } from "../utils.js"; +import { parseBool } from "../utils.js"; +import { readJson } from '../utils/DataUtils.js'; import { validateJson } from "../utils/ValidationUtils.js"; import AbstractSource from "./AbstractSource.js"; import { AzuracastSource } from "./AzuracastSource.js"; @@ -270,7 +271,7 @@ export default class ScrobbleSources { let configFile; try { - configFile = await readJson(`${this.internalConfig.configDir}/config.json`, {throwOnNotFound: false}); + configFile = await readJson(`${this.internalConfig.configDir}/config.json`, {throwOnNotFound: false, logger: childLogger(this.logger, `Secrets`)}); } catch (e) { throw new Error('config.json could not be parsed'); } @@ -711,7 +712,7 @@ export default class ScrobbleSources { } let rawSourceConfigs; try { - rawSourceConfigs = await readJson(`${this.internalConfig.configDir}/${sourceType}.json`, {throwOnNotFound: false}); + rawSourceConfigs = await readJson(`${this.internalConfig.configDir}/${sourceType}.json`, {throwOnNotFound: false, logger: childLogger(this.logger, `${sourceType} Secrets`)}); } catch (e) { const errMsg = `${sourceType}.json config file could not be parsed`; this.emitter.emit('error', errMsg); diff --git a/src/backend/sources/SpotifySource.ts b/src/backend/sources/SpotifySource.ts index ec91aeb1..f30187ec 100644 --- a/src/backend/sources/SpotifySource.ts +++ b/src/backend/sources/SpotifySource.ts @@ -19,11 +19,11 @@ import { import { SpotifySourceConfig } from "../common/infrastructure/config/source/spotify.js"; import { parseRetryAfterSecsFromObj, - readJson, sleep, sortByOldestPlayDate, writeFile, } from "../utils.js"; +import { readJson } from '../utils/DataUtils.js'; import { findCauseByFunc } from "../utils/ErrorUtils.js"; import { joinedUrl } from "../utils/NetworkUtils.js"; import { RecentlyPlayedOptions } from "./AbstractSource.js"; @@ -219,7 +219,7 @@ export default class SpotifySource extends MemoryPositionalSource { let spotifyCreds = {}; try { - spotifyCreds = await readJson(this.workingCredsPath, {throwOnNotFound: false}) as any; + spotifyCreds = await readJson(this.workingCredsPath, {throwOnNotFound: false, interpolateEnvs: false}) as any; } catch (e) { this.logger.warn('Current spotify credentials file exists but could not be parsed', { path: this.workingCredsPath }); } diff --git a/src/backend/tests/utils/strings.test.ts b/src/backend/tests/utils/strings.test.ts index 44598053..caec15c1 100644 --- a/src/backend/tests/utils/strings.test.ts +++ b/src/backend/tests/utils/strings.test.ts @@ -3,8 +3,8 @@ import { describe, it } from 'mocha'; import { compareNormalizedStrings, normalizeStr, - replaceInterpolatedValues, } from "../../utils/StringUtils.js"; +import { replaceInterpolatedValues } from "../../utils/DataUtils.js"; import { splitByFirstFound } from '../../../core/StringUtils.js'; import { noCasePropObj } from '../../utils/DataUtils.js'; diff --git a/src/backend/utils.ts b/src/backend/utils.ts index a7b884a9..0d78a5b4 100644 --- a/src/backend/utils.ts +++ b/src/backend/utils.ts @@ -5,7 +5,6 @@ import { Duration } from "dayjs/plugin/duration.js"; import utc from 'dayjs/plugin/utc.js'; import { Request } from "express"; import { accessSync, constants, promises } from "fs"; -import JSON5 from 'json5'; // https://github.com/jfromaniello/url-join#in-nodejs import pathUtil from "path"; import { TimeoutError, WebapiError } from "spotify-web-api-node/src/response-error.js"; @@ -26,24 +25,6 @@ import { //const { default: Ajv } = AjvNS; dayjs.extend(utc); -export async function readJson(this: any, path: any, {throwOnNotFound = true} = {}) { - try { - await promises.access(path, constants.R_OK); - const data = await promises.readFile(path); - return JSON5.parse(data as unknown as string); - } catch (e) { - const {code} = e; - if (code === 'ENOENT') { - if (throwOnNotFound) { - throw new Error(`No file found at given path: ${path}`, {cause: e}); - } else { - return; - } - } - throw new Error(`Encountered error while parsing file: ${path}`, {cause: e}) - } -} - export async function readText(path: any) { await promises.access(path, constants.R_OK); const data = await promises.readFile(path); diff --git a/src/backend/utils/DataUtils.ts b/src/backend/utils/DataUtils.ts index e6e76130..2a949fd4 100644 --- a/src/backend/utils/DataUtils.ts +++ b/src/backend/utils/DataUtils.ts @@ -1,3 +1,7 @@ +import JSON5 from "json5"; +import { constants, promises } from "fs"; +import { MaybeLogger } from "../common/logging.js"; + export const asArray = (data: T | T[]): T[] => { if (Array.isArray(data)) { return data; @@ -82,4 +86,56 @@ export const noCasePropObj = (obj: T): T => { newObj[i] = checkAtomic(obj[i]); } return newObj; // object with upper cased keys -} \ No newline at end of file +} +export async function readJson(this: any, path: any, options: ReadJsonOptions = {}) { + const { + throwOnNotFound = true, + interpolateEnvs = true, + logger + } = options; + + try { + await promises.access(path, constants.R_OK); + const data = (await promises.readFile(path)).toString(); + if(interpolateEnvs) { + const replaced = replaceInterpolatedValues(data, process.env, logger); + return JSON5.parse(replaced); + } + return data; + } catch (e) { + const { code } = e; + if (code === 'ENOENT') { + if (throwOnNotFound) { + throw new Error(`No file found at given path: ${path}`, { cause: e }); + } else { + return; + } + } + throw new Error(`Encountered error while parsing file: ${path}`, { cause: e }); + } +}export interface ReadJsonOptions { + throwOnNotFound?: boolean; + interpolateEnvs?: boolean; + logger?: MaybeLogger +} +export const replaceInterpolatedValues = (str: string, fromVals: Record, logger: MaybeLogger = new MaybeLogger()): string => { + const cleanFromValKeys = noCasePropObj(fromVals); + + const matched = new Set(), unmatched = new Set(); + const replaced = str.replaceAll(INTERPOLATION_WRAPPED_REGEX, (match, p1) => { + //const fv = cleanFromValKeys[p1.toLocaleLowerCase().trim()]; + const fv = cleanFromValKeys[p1.trim()]; + if (fv !== undefined) { + matched.add(p1); + return fv; + } + unmatched.add(p1); + return match; + }); + if (matched.size !== 0 || unmatched.size !== 0) { + logger.debug(`Matched: ${matched.size === 0 ? 'None' : Array.from(matched.values()).join(', ')} | Unmatched: ${unmatched.size === 0 ? 'None' : Array.from(unmatched.values()).join(', ')}`); + } + return replaced; +}; +export const INTERPOLATION_WRAPPED_REGEX: RegExp = new RegExp(/\[\[([^\r\n\[\]]+?)\]\]/g); + diff --git a/src/backend/utils/StringUtils.ts b/src/backend/utils/StringUtils.ts index 1d93916c..39ce01f7 100644 --- a/src/backend/utils/StringUtils.ts +++ b/src/backend/utils/StringUtils.ts @@ -4,8 +4,6 @@ import { PlayObject } from "../../core/Atomic.js"; import { asPlayerStateData, DELIMITERS, DELIMITERS_NO_AMP, PlayerStateDataMaybePlay } from "../common/infrastructure/Atomic.js"; import { genGroupIdStr, getPlatformIdFromData, intersect, parseRegexSingleOrFail } from "../utils.js"; import { buildTrackString } from "../../core/StringUtils.js"; -import { MaybeLogger } from "../common/logging.js"; -import { noCasePropObj } from "./DataUtils.js"; const {levenStrategy, diceStrategy} = strategies; @@ -434,24 +432,4 @@ export const hasNonAlphanumericChars = (str: string): boolean => { return NON_ALPHANUMWHITESPACE_CHARS.test(str); } -export const INTERPOLATION_WRAPPED_REGEX: RegExp = new RegExp(/\[\[([^\r\n\[\]]+?)\]\]/g); -export const replaceInterpolatedValues = (str: string, fromVals: Record, logger: MaybeLogger = new MaybeLogger()): string => { - const cleanFromValKeys = noCasePropObj(fromVals); - - const matched = new Set(), - unmatched = new Set(); - const replaced = str.replaceAll(INTERPOLATION_WRAPPED_REGEX, (match, p1) => { - //const fv = cleanFromValKeys[p1.toLocaleLowerCase().trim()]; - const fv = cleanFromValKeys[p1.trim()]; - if(fv !== undefined) { - matched.add(p1); - return fv; - } - unmatched.add(p1); - return match; - }); - if(matched.size !== 0 || unmatched.size !== 0) { - logger.debug(`Matched: ${matched.size === 0 ? 'None' : Array.from(matched.values()).join(', ')} | Unmatched: ${unmatched.size === 0 ? 'None' : Array.from(unmatched.values()).join(', ')}`); - } - return replaced; -} \ No newline at end of file + -- 2.51.2 From f0c5686b4acbc15cf8500d0d8ded699f14517225 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Tue, 16 Dec 2025 23:53:38 +0000 Subject: [PATCH 3/6] fix: Parse non-interpolated json data as data --- src/backend/utils/DataUtils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/utils/DataUtils.ts b/src/backend/utils/DataUtils.ts index 2a949fd4..4a83de40 100644 --- a/src/backend/utils/DataUtils.ts +++ b/src/backend/utils/DataUtils.ts @@ -101,7 +101,7 @@ export async function readJson(this: any, path: any, options: ReadJsonOptions = const replaced = replaceInterpolatedValues(data, process.env, logger); return JSON5.parse(replaced); } - return data; + return JSON5.parse(data); } catch (e) { const { code } = e; if (code === 'ENOENT') { -- 2.51.2 From 0cd36e57d60d4bfd6a75be0efbac7c96ceb49796 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Wed, 17 Dec 2025 01:30:52 +0000 Subject: [PATCH 4/6] feat: Log as WARN if there are unmatched interpolation keys #420 --- src/backend/utils/DataUtils.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/backend/utils/DataUtils.ts b/src/backend/utils/DataUtils.ts index 4a83de40..f3149ba8 100644 --- a/src/backend/utils/DataUtils.ts +++ b/src/backend/utils/DataUtils.ts @@ -133,7 +133,13 @@ export const replaceInterpolatedValues = (str: string, fromVals: Record 0) { + logger.warn(logMsg); + } else { + logger.debug(logMsg); + } + } return replaced; }; -- 2.51.2 From 939ae6f8925e5487e30bad698d5c2784102f9479 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Wed, 17 Dec 2025 01:52:07 +0000 Subject: [PATCH 5/6] docs: Add config interpolation docs --- docsite/docs/configuration/configuration.mdx | 71 ++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/docsite/docs/configuration/configuration.mdx b/docsite/docs/configuration/configuration.mdx index 94a34d33..09b98d7d 100644 --- a/docsite/docs/configuration/configuration.mdx +++ b/docsite/docs/configuration/configuration.mdx @@ -166,6 +166,77 @@ See the [Configuration Types](#configuration-types) above for your options for c Each entry for a Source/Client includes a **Configuration** section that describes how to configure it using a [configuration type](#configuration-types). +## Secrets Interpolation + +When using [File](./?configType=file#configuration-types) or [AIO](./?configType=aio#configuration-types) Configuration, Multi-Scrobbler can interpolate Environmental Variables into your json files. This can be used, for example, to keep sensitive data (like [Last.fm Client/Secret](/configurations/clients/lastfm#configuration)) out of your configuration files so that they can be committed to git. + +Multi-scrobbler will look for patterns in text fields within _all_ of your json files: + +* Some part of the text field matches: `[[MY_ENV]]` +* Is replaced by the value of the Environmental Variable named `MY_ENV` + +
+ +Example + +Given this [Last.fm](/configurations/clients/lastfm#configuration) [File](./?configType=file#configuration-types) config: + +```json title="lastfm.json" +[ + { + "name": "myLastFmClient", + "configureAs": "client", + "data": { + "apiKey": "[[MY_APIKEY]]", + "secret": "[[MY_SECRET]]", + "redirectUri": "http://localhost:9078/lastfm/callback" + } + } +] +``` + +And these environmental variables, in this scenario set through `environment` in the [docker compose installation](/quickstart#create-docker-compose-file): + +``` +MY_APIKEY=a89cba1569901a0671d5a9875fed4be1 +MY_SECRET=ec42e09d5ae0ee0f0816ca151008412a +``` + +The resulting json multi-scrobbler would use: + +```json title="lastfm.json" +[ + { + "name": "myLastFmClient", + "configureAs": "client", + "data": { + "apiKey": "a89cba1569901a0671d5a9875fed4be1", + "secret": "ec42e09d5ae0ee0f0816ca151008412a", + "redirectUri": "http://localhost:9078/lastfm/callback" + } + } +] +``` + +
+ +:::note[Caveats] + +* ENV variable names/interpolation keys are case-insensitive +* Interpolation only works for **string** values within json. This cannot be used for numbers, booleans, objects, etc... + +::: + +:::warning + +Multi-scrobbler **will not** throw an error if the environmental value is not found. It will instead leave the string as-is and log a warning (`WARN` level) with the names of the missing environmental variable names like so: + +``` +[2025-12-16 15:49:03.532 -0500] DEBUG : [App] [Sources] [spotify Secrets] Matched: None | Unmatched: SPOTIFY_SECRET +``` + +::: + ## Application Options These options affect multi-scrobbler's behavior and are not specific to any source/client. -- 2.51.2 From 15d667f6530c652560d0e85be44f5e9a768411f9 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Wed, 17 Dec 2025 01:59:05 +0000 Subject: [PATCH 6/6] docs: Better warnings for ENV interpolation --- docsite/docs/configuration/configuration.mdx | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/docsite/docs/configuration/configuration.mdx b/docsite/docs/configuration/configuration.mdx index 09b98d7d..cfc23329 100644 --- a/docsite/docs/configuration/configuration.mdx +++ b/docsite/docs/configuration/configuration.mdx @@ -227,16 +227,22 @@ The resulting json multi-scrobbler would use: ::: -:::warning +:::warning[Missing ENVS] -Multi-scrobbler **will not** throw an error if the environmental value is not found. It will instead leave the string as-is and log a warning (`WARN` level) with the names of the missing environmental variable names like so: +Multi-scrobbler **will not** throw an error if the environmental value is not found. Instead, it will leave the string as-is and log a warning (`WARN` level) with the names of the missing environmental variable names like so: ``` -[2025-12-16 15:49:03.532 -0500] DEBUG : [App] [Sources] [spotify Secrets] Matched: None | Unmatched: SPOTIFY_SECRET +WARN : [App] [Sources] [spotify Secrets] Matched: None | Unmatched: SPOTIFY_SECRET ``` ::: +:::warning[ENV Name Collisions] + +**Verify that interpolation keys/environmental variable names you will use do not collide with existing ENV names used by multi-scrobbler.** Use the docs search to verify the name you want to use is not already used elsewhere by multi-scrobbler. + +::: + ## Application Options These options affect multi-scrobbler's behavior and are not specific to any source/client.