diff --git a/package-lock.json b/package-lock.json index efbaac85..f46fdf39 100644 --- a/package-lock.json +++ b/package-lock.json @@ -162,6 +162,7 @@ "@types/sinon": "^21.0.0", "@types/spotify-web-api-node": "^5.0.7", "@types/superagent": "^8.1.9", + "@types/supertest": "^7.2.1", "@types/xml2js": "^0.4.11", "@vitejs/plugin-react": "^6.0.2", "@welldone-software/why-did-you-render": "^10.0.1", @@ -176,7 +177,7 @@ "eslint-import-resolver-typescript": "^4.4.5", "eslint-plugin-boundaries": "^7.0.2", "eslint-plugin-import-x": "^4.17.1", - "eslint-plugin-mocha": "^11.3.0", + "eslint-plugin-mocha": "^12.0.2", "eslint-plugin-prefer-arrow-functions": "^3.2.4", "eslint-plugin-react-hooks": "^7.1.1", "eslint-plugin-storybook": "^10.3.6", @@ -205,6 +206,7 @@ "sinon": "^21.0.2", "storybook": "^10.4.3", "storybook-addon-remix-react-router": "^6.1.0", + "supertest": "^7.2.2", "tailwindcss": "^4.2.2", "tinyexec": "^1.1.1", "ts-essentials": "^10.2.0", @@ -2274,9 +2276,9 @@ } }, "node_modules/@eslint-community/eslint-utils": { - "version": "4.9.1", - "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.1.tgz", - "integrity": "sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ==", + "version": "4.10.1", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.10.1.tgz", + "integrity": "sha512-cuadcxVFE8sDK6iWJbs8Sn0av2Nrh2QSGQhVlBW9AaAHqHwjWsZHT8LJ4hFGPh7ASBV2deFdM7H/DPjulmh8rg==", "dev": true, "license": "MIT", "dependencies": { @@ -7214,6 +7216,17 @@ "form-data": "^4.0.0" } }, + "node_modules/@types/supertest": { + "version": "7.2.1", + "resolved": "https://registry.npmjs.org/@types/supertest/-/supertest-7.2.1.tgz", + "integrity": "sha512-4CbBvoYVLHL7+yhbYrZET0vsvuyXTC05aRe7dNQkwMzm56auceoy6Yu3K50uZmwfHna1os3CMSgM/3QVkUtPTw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/methods": "^1.1.4", + "@types/superagent": "^8.1.0" + } + }, "node_modules/@types/unist": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", @@ -11855,27 +11868,37 @@ } }, "node_modules/eslint-plugin-mocha": { - "version": "11.3.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-mocha/-/eslint-plugin-mocha-11.3.0.tgz", - "integrity": "sha512-anENwrIwmdvunmmssjMn5a4nTd+mYMkqBlwjksxOECcIThLNhefWJIiTWY7pY/arMQFjNwHQjVOZb6pQ9PrLjg==", + "version": "12.0.2", + "resolved": "https://registry.npmjs.org/eslint-plugin-mocha/-/eslint-plugin-mocha-12.0.2.tgz", + "integrity": "sha512-RzjnzVBoidGFjVcvzQShLbfWug5pL/6d6LZFbBZk1b5duZ0ooq6dasmpu6qLyNowUZBqaWvQbnR5U1hsKX/+5g==", "dev": true, "license": "MIT", "dependencies": { - "@eslint-community/eslint-utils": "^4.4.1", - "globals": "^15.14.0" + "@eslint-community/eslint-utils": "4.10.1", + "@eslint/core": "1.2.1", + "@types/estree": "1.0.9", + "globals": "17.8.0", + "json-schema-to-ts": "3.1.1", + "type-fest": "5.8.0" + }, + "engines": { + "node": ">=22.0.0" }, "peerDependencies": { - "eslint": ">=9.0.0" + "eslint": ">=10.2.0" } }, - "node_modules/eslint-plugin-mocha/node_modules/globals": { - "version": "15.15.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-15.15.0.tgz", - "integrity": "sha512-7ACyT3wmyp3I61S4fG682L0VA2RGD9otkqGJIwNUMF1SWUombIIk+af1unuDYgMm082aHYwD+mzJvv9Iu8dsgg==", + "node_modules/eslint-plugin-mocha/node_modules/type-fest": { + "version": "5.8.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-5.8.0.tgz", + "integrity": "sha512-YGYEVz3Fm5iy/AybuA0oyNFq7H4CgQNfRp/qfe8nurE1kuCeNm3/vfm9X4Mtl+qLyaKJUh5xrFZwogr41SMjYA==", "dev": true, - "license": "MIT", + "license": "(MIT OR CC0-1.0)", + "dependencies": { + "tagged-tag": "^1.0.0" + }, "engines": { - "node": ">=18" + "node": ">=20" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -13566,9 +13589,9 @@ } }, "node_modules/globals": { - "version": "17.7.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-17.7.0.tgz", - "integrity": "sha512-Czmyns5dUsq4seFBR/Kdydhmo8y9kC79hiSkPn0YcGtNnYWnrgt0vjrSjx9tspoDGWm2CMarffRuLjM4xUz8xg==", + "version": "17.8.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-17.8.0.tgz", + "integrity": "sha512-Zz/LMDZScFmkakeL2cTHzf+PbWKdpU3uclqkZT7TjDG58j5WPt0PpA+n9uPI24fZtlw07q0OtEi84K+umsRzqQ==", "dev": true, "license": "MIT", "engines": { @@ -14845,6 +14868,20 @@ "dev": true, "license": "MIT" }, + "node_modules/json-schema-to-ts": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz", + "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.3", + "ts-algebra": "^2.0.0" + }, + "engines": { + "node": ">=16" + } + }, "node_modules/json-stable-stringify": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.3.0.tgz", @@ -19393,6 +19430,42 @@ "url": "https://ko-fi.com/tunnckoCore/commissions" } }, + "node_modules/supertest": { + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/supertest/-/supertest-7.2.2.tgz", + "integrity": "sha512-oK8WG9diS3DlhdUkcFn4tkNIiIbBx9lI2ClF8K+b2/m8Eyv47LSawxUzZQSNKUrVb2KsqeTDCcjAAVPYaSLVTA==", + "dev": true, + "license": "MIT", + "dependencies": { + "cookie-signature": "^1.2.2", + "methods": "^1.1.2", + "superagent": "^10.3.0" + }, + "engines": { + "node": ">=14.18.0" + } + }, + "node_modules/supertest/node_modules/superagent": { + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/superagent/-/superagent-10.3.0.tgz", + "integrity": "sha512-B+4Ik7ROgVKrQsXTV0Jwp2u+PXYLSlqtDAhYnkkD+zn3yg8s/zjA2MeGayPoY/KICrbitwneDHrjSotxKL+0XQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "component-emitter": "^1.3.1", + "cookiejar": "^2.1.4", + "debug": "^4.3.7", + "fast-safe-stringify": "^2.1.1", + "form-data": "^4.0.5", + "formidable": "^3.5.4", + "methods": "^1.1.2", + "mime": "2.6.0", + "qs": "^6.14.1" + }, + "engines": { + "node": ">=14.18.0" + } + }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -19665,6 +19738,13 @@ "integrity": "sha512-0s7O4OFi0f8DpD03spao04Osrt2b4p1v2uags7JBMLsAe36pUI0mQU6XMgQuQSv5jOTqTFS0W50wTPrlzST80w==", "license": "MIT" }, + "node_modules/ts-algebra": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", + "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==", + "dev": true, + "license": "MIT" + }, "node_modules/ts-dedent": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/ts-dedent/-/ts-dedent-2.2.0.tgz", diff --git a/package.json b/package.json index 12ef83dd..83608814 100644 --- a/package.json +++ b/package.json @@ -202,6 +202,7 @@ "@types/sinon": "^21.0.0", "@types/spotify-web-api-node": "^5.0.7", "@types/superagent": "^8.1.9", + "@types/supertest": "^7.2.1", "@types/xml2js": "^0.4.11", "@vitejs/plugin-react": "^6.0.2", "@welldone-software/why-did-you-render": "^10.0.1", @@ -216,7 +217,7 @@ "eslint-import-resolver-typescript": "^4.4.5", "eslint-plugin-boundaries": "^7.0.2", "eslint-plugin-import-x": "^4.17.1", - "eslint-plugin-mocha": "^11.3.0", + "eslint-plugin-mocha": "^12.0.2", "eslint-plugin-prefer-arrow-functions": "^3.2.4", "eslint-plugin-react-hooks": "^7.1.1", "eslint-plugin-storybook": "^10.3.6", @@ -245,6 +246,7 @@ "sinon": "^21.0.2", "storybook": "^10.4.3", "storybook-addon-remix-react-router": "^6.1.0", + "supertest": "^7.2.2", "tailwindcss": "^4.2.2", "tinyexec": "^1.1.1", "ts-essentials": "^10.2.0", diff --git a/src/backend/common/AbstractComponent.ts b/src/backend/common/AbstractComponent.ts index ec55ee01..646bd534 100644 --- a/src/backend/common/AbstractComponent.ts +++ b/src/backend/common/AbstractComponent.ts @@ -67,7 +67,7 @@ export default abstract class AbstractComponent extends AbstractInitializable { type: ClientType | SourceType; name: string; - lastActiveAt?: Dayjs + lastActiveAt?: Dayjs; lastReadyAt?: Dayjs; protected lastUpdatedComponentDatesAt?: Dayjs @@ -116,8 +116,8 @@ export default abstract class AbstractComponent extends AbstractInitializable { name: this.config?.name ?? this.name }); this.componentId = this.dbComponent.id; - this.lastActiveAt = this.dbComponent.lastActiveAt; - this.lastReadyAt = this.dbComponent.lastReadyAt; + this.lastActiveAt = this.dbComponent.lastActiveAt ?? undefined; + this.lastReadyAt = this.dbComponent.lastReadyAt ?? undefined; return true; } @@ -572,8 +572,8 @@ export default abstract class AbstractComponent extends AbstractInitializable { monitoringStatus: this.getMonitoringStatus(), countNonLive: this.dbComponent.countNonLive, createdAt: this.dbComponent.createdAt?.toISOString(), - lastReadyAt: this.lastReadyAt?.toISOString(), - lastActiveAt: this.lastActiveAt?.toISOString(), + lastReadyAt: this.lastActiveAt !== undefined ? this.lastReadyAt.toISOString() : undefined, + lastActiveAt: this.lastActiveAt !== undefined ? this.lastActiveAt?.toISOString() : undefined, errors: this.errors.map(x => x instanceof Error ? serializeError(x) : x), warnings: this.warnings.map(x => x instanceof Error ? serializeError(x) : x), ...this.additionalApiData() diff --git a/src/backend/common/vendor/webscrobbler/interfaces.ts b/src/backend/common/vendor/webscrobbler/interfaces.ts index d94d7609..fb4b0846 100644 --- a/src/backend/common/vendor/webscrobbler/interfaces.ts +++ b/src/backend/common/vendor/webscrobbler/interfaces.ts @@ -9,7 +9,7 @@ const processedSongDataSchema = z.object({ album: z.string().nullish(), albumArtist: z.string().nullish(), track: z.string().nullish(), - duration: z.number().nullish(), + duration: z.number().positive().nullish(), }); export type ProcessedSongData = z.output; @@ -44,7 +44,7 @@ export type Flags = z.infer; export const metadataSchema = z.looseObject({ label: z.string(), - startTimestamp: z.number(), + startTimestamp: z.int().positive(), albumMbId: z.string().optional(), albumUrl: z.string().optional(), artistUrl: z.string().optional(), @@ -77,7 +77,7 @@ export type WebScrobblerSong = z.infer; export const webScrobblePayloadSchema = z.object({ eventName: webScrobblerHookEventSchema, - time: z.number().optional(), + time: z.int().positive().optional(), data: z.object({ song: webScrobblerSongSchema, songs: z.array(webScrobblerSongSchema).optional(), diff --git a/src/backend/index.ts b/src/backend/index.ts index 88fb1936..8d4d6fce 100644 --- a/src/backend/index.ts +++ b/src/backend/index.ts @@ -173,7 +173,7 @@ const dataDir = getDataDir(); await root.items.cache().init(true); - initServer(logger, appLoggerStream, output, scrobbleSources, scrobbleClients); + initServer({sources: scrobbleSources, clients: scrobbleClients},{logger, appLoggerStream, initialOutput: output}); if(process.env.IS_LOCAL === 'true') { logger.info('multi-scrobbler can be run as a background service! See: https://docs.multi-scrobbler.app/installation/service'); diff --git a/src/backend/server/api.ts b/src/backend/server/api.ts index 5a0660a3..001a3952 100644 --- a/src/backend/server/api.ts +++ b/src/backend/server/api.ts @@ -1,17 +1,15 @@ -import type {LogDataPretty, Logger, LogLevel} from "@foxxmd/logging"; +import {loggerTest, type LogDataPretty, type Logger, type LogLevel} from "@foxxmd/logging"; import type {Express} from 'express'; import bsseDef from 'better-sse'; import bodyParser from "body-parser"; import { FixedSizeList } from 'fixed-size-list'; -import type { PassThrough } from "node:stream"; +import { PassThrough } from "node:stream"; import { Transform } from "stream"; import { DEAD_QUEUE, type ClientStatusData, type DeadLetterScrobble, type LogOutputConfig, - PLAY_CLIENT_STATE, - PLAY_SOURCE_STATE, type PlayObject, SOURCE_SOT, type SOURCE_SOT_TYPES, @@ -49,6 +47,7 @@ import { serializeError } from "serialize-error"; import { z } from 'zod'; import type { createTypedRouter } from "@minisylar/express-typed-router"; import pEvent from "p-event"; +import { hasMetricRepositories, registerMetrics, setMetricRepositories } from "./promMetrics.ts"; const maxBufferSize = 300; const output: Record> = {}; @@ -72,7 +71,34 @@ const getLogs = (minLevel: number, limit: number = maxBufferSize, sort: 'asc' | return allLogs.flat(1).sort((a, b) => a.time - b.time).slice(0, limit); } -export const setupApi = (app: Express, router: ReturnType, logger: Logger, appLoggerStream: PassThrough, initialLogOutput: LogDataPretty[] = [], scrobbleSources: ScrobbleSources, scrobbleClients: ScrobbleClients) => { +export interface ApiArgs { + app: Express + router: ReturnType + scrobbleSources: ScrobbleSources + scrobbleClients: ScrobbleClients +} + +export interface ApiOptions { + logger?: Logger + appLoggerStream?: PassThrough + initialLogOutput?: LogDataPretty[] + testMode?: boolean +} + +export const setupApi = (args: ApiArgs, opts: ApiOptions = {}) => { + const { + app, + router, + scrobbleSources, + scrobbleClients + } = args; + const { + logger = loggerTest, + appLoggerStream = new PassThrough(), + initialLogOutput = [], + testMode + } = opts; + for(const level of Object.keys(logger.levels.labels)) { output[level] = new FixedSizeList(maxBufferSize); } @@ -949,134 +975,18 @@ export const setupApi = (app: Express, router: ReturnType x['componentId'] === source.componentId); - for(const s of PLAY_SOURCE_STATE) { - const rel = relevant.find(x => x['state'] === s); - const count = rel === undefined ? 0 : rel['count(*)']; - this.labels({name: source.getSafeExternalName(), type: source.type, state: s}).set(count); - } - } - } - }); - const sourceRetention = new prom.Gauge({ - name: 'multiscrobbler_source_plays_compacted', - help: 'Count of compacted, stored plays by compaction type for Sources', - labelNames: ['name', 'type', 'compactionType'], - async collect() { - const res = await playRepo.getCompactedPlayCountByComponent(); - for(const source of scrobbleSources.sources) { - const relevant = res.filter(x => x['componentId'] === source.componentId); - for(const s of ['input','transform','input-transform']) { - const rel = relevant.find(x => x['compacted'] === s); - const count = rel === undefined ? 0 : rel['count(*)']; - this.labels({name: source.getSafeExternalName(), type: source.type, compactionType: s}).set(count); - } - } - } - }); - const clientPlays = new prom.Gauge({ - name: 'multiscrobbler_client_plays', - help: 'Count of stored plays by state for Clients', - labelNames: ['name', 'type', 'state'], - async collect() { - const res = await playRepo.getPlayCountByState(); - for(const client of scrobbleClients.clients) { - const relevant = res.filter(x => x['componentId'] === client.componentId); - for(const s of PLAY_CLIENT_STATE) { - const rel = relevant.find(x => x['state'] === s); - const count = rel === undefined ? 0 : rel['count(*)']; - this.labels({name: client.getSafeExternalName(), type: client.type, state: s}).set(count); - } - } - } - }); - const clientRetention = new prom.Gauge({ - name: 'multiscrobbler_client_plays_compacted', - help: 'Count of compacted, stored plays by compaction type for Clients', - labelNames: ['name', 'type', 'compactionType'], - async collect() { - const res = await playRepo.getCompactedPlayCountByComponent(); - for(const client of scrobbleClients.clients) { - const relevant = res.filter(x => x['componentId'] === client.componentId); - for(const s of ['input','transform','input-transform']) { - const rel = relevant.find(x => x['compacted'] === s); - const count = rel === undefined ? 0 : rel['count(*)']; - this.labels({name: client.getSafeExternalName(), type: client.type, compactionType: s}).set(count); - } - } - } - }); - const clientHistoricalPlays = new prom.Gauge({ - name: 'multiscrobbler_client_historical_plays', - help: 'Count of stored historical plays for Clients', - labelNames: ['name', 'type'], - async collect() { - const res = await playHistoricalRepo.getPlayCountByComponent(); - for(const client of scrobbleClients.clients) { - if(client instanceof AbstractHistoricalScrobbleClient) { - const relevant = res.filter(x => x['componentId'] === client.componentId); - for(const rel of relevant) { - this.labels({name: client.getSafeExternalName(), type: client.type}).set(rel['count(*)']); - } - } - } - } - }); - - if(process.env.PROMETHEUS_FULL === 'true') { - prom.collectDefaultMetrics(); + if(testMode !== true) { + registerMetrics(scrobbleSources, scrobbleClients); + if(process.env.PROMETHEUS_FULL === 'true') { + prom.collectDefaultMetrics(); + } } router.get('/api/metrics', {tags: ['App Meta']}, async (req, res) => { - if(playRepo === undefined) { + if(!hasMetricRepositories()) { const db = await getRoot().items.db(); - playRepo = new DrizzlePlayRepository(db); - playHistoricalRepo = new DrizzlePlayHistoricalRepository(db); + setMetricRepositories(new DrizzlePlayRepository(db),new DrizzlePlayHistoricalRepository(db)) } const metricsString = await prom.register.metrics(); diff --git a/src/backend/server/endpointListenbrainzRoutes.ts b/src/backend/server/endpointListenbrainzRoutes.ts index 63b94e5a..e93665e6 100644 --- a/src/backend/server/endpointListenbrainzRoutes.ts +++ b/src/backend/server/endpointListenbrainzRoutes.ts @@ -82,7 +82,7 @@ export const setupLZEndpointRoutes = (app: Express, router: ReturnType]> => { + + const app = express(); -const router = createTypedRouter(); + const router = createTypedRouter(); -//const router = Router(); -app.set('query parser', (str: string) => qs.parse(str, qsOptions)); + app.set('query parser', (str: string) => qs.parse(str, qsOptions)); -export const initServer = async (parentLogger: Logger, appLoggerStream: PassThrough, initialOutput: LogDataPretty[] = [], sources: ScrobbleSources, clients: ScrobbleClients) => { + const { + sources, + clients + } = args; + const { + logger: parentLogger = loggerTest, + } = opts; - const logger = childLogger(parentLogger, 'API'); // parentLogger.child({labels: ['API']}, mergeArr); + const logger = childLogger(parentLogger, 'API'); try { app.use(router.getRouter()); @@ -51,13 +70,17 @@ export const initServer = async (parentLogger: Logger, appLoggerStream: PassThro return; } + setupApi({app, router, scrobbleSources: sources, scrobbleClients: clients}, {...opts, logger}); + + if(opts.testMode === true) { + return [app, router]; + } + const isProd = root.get('isProd'); const port = root.get('port'); const local = root.get('localUrl'); const localDefined = root.get('hasDefinedBaseUrl'); - setupApi(app, router, logger, appLoggerStream, initialOutput, sources, clients); - const addy = getAddress(); const addresses: string[] = []; let dockerHint = ''; @@ -123,6 +146,7 @@ export const initServer = async (parentLogger: Logger, appLoggerStream: PassThro }).on('error', (err) => { throw new Error('Server encountered unrecoverable error', {cause: err}); }); + return [app, router]; } catch (e) { throw new Error('Server encountered unrecoverable error', {cause: e}); } diff --git a/src/backend/server/promMetrics.ts b/src/backend/server/promMetrics.ts new file mode 100644 index 00000000..dff2c10f --- /dev/null +++ b/src/backend/server/promMetrics.ts @@ -0,0 +1,145 @@ +import type { DrizzlePlayHistoricalRepository } from "../common/database/drizzle/repositories/PlayHistoricalRepository.ts"; +import type { DrizzlePlayRepository } from "../common/database/drizzle/repositories/PlayRepository.ts"; +import prom from 'prom-client'; +import type ScrobbleSources from "../sources/ScrobbleSources.ts"; +import type ScrobbleClients from "../scrobblers/ScrobbleClients.ts"; +import { PLAY_CLIENT_STATE, PLAY_SOURCE_STATE } from "../../core/Atomic.ts"; +import AbstractHistoricalScrobbleClient from "../scrobblers/AbstractHistoricalScrobbleClient.ts"; + +let playRepo: DrizzlePlayRepository, + playHistoricalRepo: DrizzlePlayHistoricalRepository; + +export const setMetricRepositories = (play: DrizzlePlayRepository, playHistorical: DrizzlePlayHistoricalRepository) => { + playRepo = play; + playHistoricalRepo = playHistorical; +} +export const hasMetricRepositories = () => playRepo !== undefined; + +export const registerMetrics = (scrobbleSources: ScrobbleSources, scrobbleClients: ScrobbleClients) => { + if (prom.register.getSingleMetric('multiscrobbler_client_issues') === undefined) { + new prom.Gauge({ + name: 'multiscrobbler_client_issues', + help: 'Number of errors/issues with Client', + labelNames: ['name', 'type'], + async collect() { + for (const client of scrobbleClients.clients) { + let issues = 0; + if (!(await client.isReady())) { + issues++; + } + this.labels({ name: client.getSafeExternalName(), type: client.type }).set(issues); + } + } + }); + } + if (prom.register.getSingleMetric('multiscrobbler_source_issues') === undefined) { + new prom.Gauge({ + name: 'multiscrobbler_source_issues', + help: 'Number of errors/issues with Source', + labelNames: ['name', 'type'], + async collect() { + for (const source of scrobbleSources.sources) { + let issues = 0; + if (source.requiresAuth && !source.authed) { + issues++; + } + if (source.canPoll && !source.polling) { + issues++; + } + this.labels({ name: source.getSafeExternalName(), type: source.type }).set(issues); + } + } + }); + } + if (prom.register.getSingleMetric('multiscrobbler_source_plays') === undefined) { + new prom.Gauge({ + name: 'multiscrobbler_source_plays', + help: 'Count of stored plays by state for Sources', + labelNames: ['name', 'type', 'state'], + async collect() { + const res = await playRepo.getPlayCountByState(); + for (const source of scrobbleSources.sources) { + const relevant = res.filter(x => x['componentId'] === source.componentId); + for (const s of PLAY_SOURCE_STATE) { + const rel = relevant.find(x => x['state'] === s); + const count = rel === undefined ? 0 : rel['count(*)']; + this.labels({ name: source.getSafeExternalName(), type: source.type, state: s }).set(count); + } + } + } + }); + } + + if (prom.register.getSingleMetric('multiscrobbler_source_plays_compacted') === undefined) { + new prom.Gauge({ + name: 'multiscrobbler_source_plays_compacted', + help: 'Count of compacted, stored plays by compaction type for Sources', + labelNames: ['name', 'type', 'compactionType'], + async collect() { + const res = await playRepo.getCompactedPlayCountByComponent(); + for (const source of scrobbleSources.sources) { + const relevant = res.filter(x => x['componentId'] === source.componentId); + for (const s of ['input', 'transform', 'input-transform']) { + const rel = relevant.find(x => x['compacted'] === s); + const count = rel === undefined ? 0 : rel['count(*)']; + this.labels({ name: source.getSafeExternalName(), type: source.type, compactionType: s }).set(count); + } + } + } + }); + } + if (prom.register.getSingleMetric('multiscrobbler_client_plays') === undefined) { + new prom.Gauge({ + name: 'multiscrobbler_client_plays', + help: 'Count of stored plays by state for Clients', + labelNames: ['name', 'type', 'state'], + async collect() { + const res = await playRepo.getPlayCountByState(); + for (const client of scrobbleClients.clients) { + const relevant = res.filter(x => x['componentId'] === client.componentId); + for (const s of PLAY_CLIENT_STATE) { + const rel = relevant.find(x => x['state'] === s); + const count = rel === undefined ? 0 : rel['count(*)']; + this.labels({ name: client.getSafeExternalName(), type: client.type, state: s }).set(count); + } + } + } + }); + } + if (prom.register.getSingleMetric('multiscrobbler_client_plays_compacted') === undefined) { + new prom.Gauge({ + name: 'multiscrobbler_client_plays_compacted', + help: 'Count of compacted, stored plays by compaction type for Clients', + labelNames: ['name', 'type', 'compactionType'], + async collect() { + const res = await playRepo.getCompactedPlayCountByComponent(); + for (const client of scrobbleClients.clients) { + const relevant = res.filter(x => x['componentId'] === client.componentId); + for (const s of ['input', 'transform', 'input-transform']) { + const rel = relevant.find(x => x['compacted'] === s); + const count = rel === undefined ? 0 : rel['count(*)']; + this.labels({ name: client.getSafeExternalName(), type: client.type, compactionType: s }).set(count); + } + } + } + }); + } + if (prom.register.getSingleMetric('multiscrobbler_client_historical_plays') === undefined) { + new prom.Gauge({ + name: 'multiscrobbler_client_historical_plays', + help: 'Count of stored historical plays for Clients', + labelNames: ['name', 'type'], + async collect() { + const res = await playHistoricalRepo.getPlayCountByComponent(); + for (const client of scrobbleClients.clients) { + if (client instanceof AbstractHistoricalScrobbleClient) { + const relevant = res.filter(x => x['componentId'] === client.componentId); + for (const rel of relevant) { + this.labels({ name: client.getSafeExternalName(), type: client.type }).set(rel['count(*)']); + } + } + } + } + }); + } +} \ No newline at end of file diff --git a/src/backend/sources/AbstractSource.ts b/src/backend/sources/AbstractSource.ts index e29ccf14..21a2cbee 100644 --- a/src/backend/sources/AbstractSource.ts +++ b/src/backend/sources/AbstractSource.ts @@ -54,7 +54,6 @@ import { PLAY_EVENT_TYPE, type PlayEvent } from '../../core/PlayEvent.ts'; import { dupeCheckToPlayEvent, entityIsPlayEntity, queueStateToPlayEvent, stateChangeToPlayEvent, transformToPlayEvent } from '../common/database/drizzle/entityUtils.ts'; import type { PlayProcessingResult } from '../common/infrastructure/PlayProcessing.ts'; import { PlayProcessingError } from '../common/errors/PlayProcessingError.ts'; -import { runTransaction } from '../common/database/drizzle/drizzleUtils.ts'; export interface RecentlyPlayedOptions { limit?: number diff --git a/src/backend/sources/EndpointListenbrainzSource.ts b/src/backend/sources/EndpointListenbrainzSource.ts index fabd62b7..8694a7c0 100644 --- a/src/backend/sources/EndpointListenbrainzSource.ts +++ b/src/backend/sources/EndpointListenbrainzSource.ts @@ -20,7 +20,7 @@ import type {Logger} from "@foxxmd/logging"; import type {PlayerStateOptions} from "./PlayerState/AbstractPlayerState.ts"; import { AUTH_HEADER_DEFAULT_REGEX, parseSlugFromRequest, parseTokenFromRequest, type RequestIdentifierRegexes } from "../utils/RequestUtils.ts"; -const noSlugMatch = new RegExp(/(?:\/api\/listenbrainz\/?)$|(?:\/1\/?|\/1\/submit-listens\/?\/1\/validate-token\/|)$/i); +const noSlugMatch = new RegExp(/\/api\/listenbrainz\/?$|(\/1\/?|\/1\/.+)$/i); const slugMatch = new RegExp(/\/api\/listenbrainz\/([^\/]+)$/i); export const requestMatchers: RequestIdentifierRegexes = { diff --git a/src/backend/sources/PlayerState/AbstractPlayerState.ts b/src/backend/sources/PlayerState/AbstractPlayerState.ts index 973fb866..e79454e3 100644 --- a/src/backend/sources/PlayerState/AbstractPlayerState.ts +++ b/src/backend/sources/PlayerState/AbstractPlayerState.ts @@ -84,7 +84,7 @@ export abstract class AbstractPlayerState { currentListenRange?: ListenRange listenRanges: ListenRange[] = []; createdAt: Dayjs = dayjs(); - stateLastUpdatedAt: Dayjs = dayjs(0); + stateLastUpdatedAt: Dayjs = dayjs(1); lastPlay?: PlayObject lastPlayUpdatedAt?: Dayjs @@ -173,6 +173,9 @@ export abstract class AbstractPlayerState { update(state: PlayerStateDataMaybePlay, reportedTS?: Dayjs) { this.stateLastUpdatedAt = state.stateUpdatedAt ?? dayjs(); + if(!this.stateLastUpdatedAt.isValid()) { + this.stateLastUpdatedAt = dayjs(); + } const {play, status} = state; diff --git a/src/backend/sources/WebScrobblerSource.ts b/src/backend/sources/WebScrobblerSource.ts index 7cad7c3f..4edddb1c 100644 --- a/src/backend/sources/WebScrobblerSource.ts +++ b/src/backend/sources/WebScrobblerSource.ts @@ -121,18 +121,18 @@ export class WebScrobblerSource extends MemorySource { } } = obj; - const track = processed.track ?? parsed.track; - const artist = processed.artist ?? parsed.artist; - const album = processed.album ?? parsed.album; - const albumArtist = processed.albumArtist ?? parsed.albumArtist; - const duration = parsed.duration ?? processed.duration; + const track = processed.track ?? parsed.track ?? undefined; + const artist = processed.artist ?? parsed.artist ?? undefined; + const album = processed.album ?? parsed.album ?? undefined; + const albumArtist = processed.albumArtist ?? parsed.albumArtist ?? undefined; + const duration = parsed.duration ?? processed.duration ?? undefined; const play: PlayObjectMinimal = { data: { track, - artists: [artistNameToCredit(artist)], + artists: artist !== undefined ? [artistNameToCredit(artist)] : [], album: album === null ? undefined : album, - albumArtists: albumArtist === null ? undefined : [artistNameToCredit(albumArtist)], + albumArtists: albumArtist === null ? undefined : albumArtist === undefined ? undefined : [artistNameToCredit(albumArtist)], playDate: dayjs.unix(startTimestamp), duration: duration === null ? undefined : duration, meta: { diff --git a/src/backend/tests/ingress/ingress.test.ts b/src/backend/tests/ingress/ingress.test.ts new file mode 100644 index 00000000..4e8d42d6 --- /dev/null +++ b/src/backend/tests/ingress/ingress.test.ts @@ -0,0 +1,176 @@ +import chai, { expect } from 'chai'; +import asPromised from 'chai-as-promised'; +import { describe, it } from 'mocha'; +import request from 'supertest'; +import ScrobbleSources from '../../sources/ScrobbleSources.ts'; +import { WildcardEmitter } from '../../common/WildcardEmitter.ts'; +import { loggerTest } from '@foxxmd/logging'; +import type { ListenbrainzEndpointSourceConfig } from '../../common/infrastructure/config/source/endpointlz.ts'; +import { initServer } from '../../server/index.ts'; +import ScrobbleClients from '../../scrobblers/ScrobbleClients.ts'; +import { zocker } from "zocker"; +import { webScrobblePayloadSchema } from '../../common/vendor/webscrobbler/interfaces.ts'; +import type { WebScrobblerSourceConfig } from '../../common/infrastructure/config/source/webscrobbler.ts'; +import { sleep } from '../../utils.ts'; +import { playToSubmitPayload } from '../../common/vendor/listenbrainz/lzUtils.ts'; +import { generatePlay } from '../../../core/tests/utils/PlayTestUtils.ts'; + +chai.use(asPromised); + +const internalConfig = { localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test' }; + +const defaultLzConfig: ListenbrainzEndpointSourceConfig & {source: string} = { + id: 'test', + enable: true, + data: {}, + source: 'file' +}; +const defaultWebscrobblerConfig: WebScrobblerSourceConfig & {source: string} = { + id: 'test', + enable: true, + data: {}, + source: 'file' +}; + +const generateSources = () => new ScrobbleSources(new WildcardEmitter(), internalConfig, loggerTest); +describe('Listenbrainz Endpoint', function() { + + let clients: ScrobbleClients; + before(function() { + clients = new ScrobbleClients(new WildcardEmitter(), new WildcardEmitter(), internalConfig, loggerTest); + }); + + describe('Accepts requests on standard endpoints', function() { + + it('accepts requests to /1/validate-token with a config without token', async function() { + const sources = generateSources(); + await sources.addSource('endpointlz', [defaultLzConfig]); + const [app] = await initServer({sources, clients}, {testMode: true}); + //const source = sources.sources[0] as EndpointListenbrainzSource; + const response = await request(app).get('/1/validate-token') + + expect(response.status).eq(200); + expect(response.body.user_name).eq(defaultLzConfig.id); + }); + + it('accepts requests to /1/validate-token with a config with token', async function() { + const sources = generateSources(); + await sources.addSource('endpointlz', [defaultLzConfig]); + await sources.addSource('endpointlz', [{...defaultLzConfig, data: {token: 'foo'}, id: 'tokenTest'}]); + const [app] = await initServer({sources, clients}, {testMode: true}); + //const source = sources.sources[1] as EndpointListenbrainzSource; + //source.queueIdleMs = 2; + //await source.initialize(); + const response = await request(app) + .get('/1/validate-token') + .set('Authorization', 'Token foo'); + + expect(response.status).eq(200); + expect(response.body.user_name).eq('tokenTest'); + }); + + it('accepts requests to /1/submit-listens', async function() { + const sources = generateSources(); + await sources.addSource('endpointlz', [defaultLzConfig]); + const [app] = await initServer({sources, clients}, {testMode: true}); + const source = sources.sources[0]; + source.queueIdleMs = 2; + await source.initialize(); + + const response = await request(app).post('/1/submit-listens') + .set('Content-Type', 'application/json') + .send(JSON.stringify(playToSubmitPayload(generatePlay()))); + + expect(response.status).eq(200); + await sleep(4); + expect( source.getApiData().queued).eq(1); + }); + + it('accepts requests to /1/playing-now', async function() { + const sources = generateSources(); + await sources.addSource('endpointlz', [defaultLzConfig]); + const [app] = await initServer({sources, clients}, {testMode: true}); + const source = sources.sources[0]; + source.queueIdleMs = 2; + await source.initialize(); + + await request(app).post('/1/submit-listens') + .set('Content-Type', 'application/json') + .send(JSON.stringify({...playToSubmitPayload(generatePlay()), listen_type: 'playing_now'})); + await sleep(4); + + const response = await request(app).get('/1/user/test/playing-now'); + expect(response.body.payload.listens).to.exist; + }); + + }); + + describe('Accepts requests on slug endpoints', function() { + + it('accepts requests for submit-listens payload on /api/listenbrainz with no slug', async function() { + const sources = generateSources(); + await sources.addSource('endpointlz', [defaultLzConfig]); + const [app] = await initServer({sources, clients}, {testMode: true}); + const source = sources.sources[0]; + source.queueIdleMs = 2; + await source.initialize(); + + const response = await request(app).post('/api/listenbrainz') + .set('Content-Type', 'application/json') + .send(JSON.stringify(playToSubmitPayload(generatePlay()))); + + expect(response.status).eq(200); + await sleep(4); + expect( source.getApiData().queued).eq(1); + }); + + it('accepts requests for submit-listens payload on /api/listenbrainz with slug', async function() { + const sources = generateSources(); + await sources.addSource('endpointlz', [defaultLzConfig]); + await sources.addSource('endpointlz', [{...defaultLzConfig, id: 'foo', data: {slug: 'foobar'}}]); + const [app] = await initServer({sources, clients}, {testMode: true}); + const source = sources.sources[1]; + source.queueIdleMs = 2; + await source.initialize(); + + const response = await request(app).post('/api/listenbrainz/foobar') + .set('Content-Type', 'application/json') + .send(JSON.stringify(playToSubmitPayload(generatePlay()))); + + expect(response.status).eq(200); + await sleep(4); + expect( source.getApiData().queued).eq(1); + }); + + }); + +}); + +describe('Webscrobbler Endpoint', function() { + + let clients: ScrobbleClients; + before(function() { + clients = new ScrobbleClients(new WildcardEmitter(), new WildcardEmitter(), internalConfig, loggerTest); + }); + + it('accepts request to /api/webscrobbler', async function() { + const sources = generateSources(); + await sources.addSource('webscrobbler', [defaultWebscrobblerConfig]); + const source = sources.sources[0]; + source.queueIdleMs = 2; + await source.initialize(); + const [app] = await initServer({sources, clients}, {testMode: true}); + + const payload = zocker(webScrobblePayloadSchema) + .supply(webScrobblePayloadSchema.shape.data.shape.currentlyPlaying, true) + .supply(webScrobblePayloadSchema.shape.eventName, 'scrobble').generate() + + const response = await request(app).post('/api/webscrobbler') + .set('Content-Type', 'application/json') + .send(JSON.stringify(payload)); + + expect(response.status).eq(200); + await sleep(4); + expect( source.getApiData().queued).eq(1); + }); +}); \ No newline at end of file diff --git a/src/backend/tests/setup.ts b/src/backend/tests/setup.ts index 37e4d7b5..0a5c3edd 100644 --- a/src/backend/tests/setup.ts +++ b/src/backend/tests/setup.ts @@ -1,6 +1,22 @@ import { loggerTest } from '@foxxmd/logging'; import { getRoot } from "../ioc.ts"; import { transientCache, transientDb } from './utils/TransientTestUtils.ts'; +import dayjs from 'dayjs'; +import duration from 'dayjs/plugin/duration.js'; +import isBetween from 'dayjs/plugin/isBetween.js'; +import relativeTime from 'dayjs/plugin/relativeTime.js'; +import isToday from 'dayjs/plugin/isToday.js'; +import timezone from 'dayjs/plugin/timezone.js'; +import week from 'dayjs/plugin/weekOfYear.js'; +import utc from 'dayjs/plugin/utc.js'; + +dayjs.extend(utc) +dayjs.extend(isBetween); +dayjs.extend(relativeTime); +dayjs.extend(duration); +dayjs.extend(timezone); +dayjs.extend(isToday); +dayjs.extend(week); const root = getRoot({cache: transientCache, logger: loggerTest, db: transientDb}); root.items.cache().init(); \ No newline at end of file diff --git a/src/backend/utils/RequestUtils.ts b/src/backend/utils/RequestUtils.ts index 0987d9b4..1994af51 100644 --- a/src/backend/utils/RequestUtils.ts +++ b/src/backend/utils/RequestUtils.ts @@ -241,9 +241,9 @@ export const parseSlugFromString = (path: string, expData: Pick, expData: Pick): string | false | undefined => parseSlugFromString(req.baseUrl, expData); +export const parseSlugFromRequest = (req: Pick, expData: Pick): string | false | undefined => parseSlugFromString(req.originalUrl, expData); -export const parseIdentifiersFromRequest = (req: Pick, matchers: MarkOptional): [string | false | undefined, false | string | undefined] => { +export const parseIdentifiersFromRequest = (req: Pick, matchers: MarkOptional): [string | false | undefined, false | string | undefined] => { const slug = parseSlugFromRequest(req, matchers); let token: string | false | undefined; if(matchers.token !== undefined) { @@ -253,7 +253,7 @@ export const parseIdentifiersFromRequest = (req: Pick, matchers: MarkOptional): [string, string] => { +export const parseDisplayIdentifiersFromRequest = (req: Pick, matchers: MarkOptional): [string, string] => { const [slug, token] = parseIdentifiersFromRequest(req, matchers); let slugStr = '(no slug)'; if (slug === false) {