From d3cd07c5c3d2c381a33821a43039b09f9f2e278f Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 13 Mar 2025 16:20:37 +0000 Subject: [PATCH] feat(listenbrainz): Implement MVP LZ endpoint source --- config/endpointlz.json.example | 9 + src/backend/common/infrastructure/Atomic.ts | 2 + .../config/source/endpointlz.ts | 32 +++ .../infrastructure/config/source/sources.ts | 3 + .../common/vendor/ListenbrainzApiClient.ts | 39 +++- src/backend/server/api.ts | 2 + .../server/endpointListenbrainzRoutes.ts | 54 +++++ .../sources/EndpointListenbrainzSource.ts | 191 ++++++++++++++++++ src/backend/sources/ScrobbleSources.ts | 25 +++ .../ingressNotifiers/LZEndpointNotifier.ts | 46 +++++ 10 files changed, 402 insertions(+), 1 deletion(-) create mode 100644 config/endpointlz.json.example create mode 100644 src/backend/common/infrastructure/config/source/endpointlz.ts create mode 100644 src/backend/server/endpointListenbrainzRoutes.ts create mode 100644 src/backend/sources/EndpointListenbrainzSource.ts create mode 100644 src/backend/sources/ingressNotifiers/LZEndpointNotifier.ts diff --git a/config/endpointlz.json.example b/config/endpointlz.json.example new file mode 100644 index 00000000..0c8139b1 --- /dev/null +++ b/config/endpointlz.json.example @@ -0,0 +1,9 @@ +[ + { + "name": "myLz", + "enable": true, + "data": { + "token": "myToken" + } + } +] diff --git a/src/backend/common/infrastructure/Atomic.ts b/src/backend/common/infrastructure/Atomic.ts index e87d6882..dab935f1 100644 --- a/src/backend/common/infrastructure/Atomic.ts +++ b/src/backend/common/infrastructure/Atomic.ts @@ -15,6 +15,7 @@ export type SourceType = | 'jellyfin' | 'lastfm' | 'deezer' + | 'endpointlz' | 'ytmusic' | 'mpris' | 'mopidy' @@ -35,6 +36,7 @@ export const sourceTypes: SourceType[] = [ 'jellyfin', 'lastfm', 'deezer', + 'endpointlz', 'ytmusic', 'mpris', 'mopidy', diff --git a/src/backend/common/infrastructure/config/source/endpointlz.ts b/src/backend/common/infrastructure/config/source/endpointlz.ts new file mode 100644 index 00000000..db8d5459 --- /dev/null +++ b/src/backend/common/infrastructure/config/source/endpointlz.ts @@ -0,0 +1,32 @@ +import { CommonSourceConfig, CommonSourceData } from "./index.js"; + +export interface ListenbrainzEndpointData extends CommonSourceData { + /** + * The URL ending that should be used to identify scrobbles for this source + * + * If you are using multiple Listenbrainz endpoint sources (scrobbles for many users) you can use a slug to match Sources with individual users/origins + * + * Example: + * + * * slug: 'usera' => API URL: http://localhost:9078/api/listenbrainz/usera + * * slug: 'originb' => API URL: http://localhost:9078/api/listenbrainz/originb + * + * If no slug is found from an extension's incoming webhook event the first Listenbrainz source without a slug will be used + * */ + slug?: string | null + + /** + * If an LZ submission request contains this token in the Authorization Header it will be used to match the submission with this Source + * + * See: https://listenbrainz.readthedocs.io/en/latest/users/api/index.html#add-the-user-token-to-your-requests + * */ + token?: string | null +} + +export interface ListenbrainzEndpointSourceConfig extends CommonSourceConfig { + data?: ListenbrainzEndpointData +} + +export interface ListenbrainzEndpointSourceAIOConfig extends ListenbrainzEndpointSourceConfig { + type: 'endpointlz' +} diff --git a/src/backend/common/infrastructure/config/source/sources.ts b/src/backend/common/infrastructure/config/source/sources.ts index 77cfbbd9..8adbfd44 100644 --- a/src/backend/common/infrastructure/config/source/sources.ts +++ b/src/backend/common/infrastructure/config/source/sources.ts @@ -1,4 +1,5 @@ import { ChromecastSourceAIOConfig, ChromecastSourceConfig } from "./chromecast.js"; +import { ListenbrainzEndpointSourceAIOConfig, ListenbrainzEndpointSourceConfig } from "./endpointlz.js"; import { DeezerSourceAIOConfig, DeezerSourceConfig } from "./deezer.js"; import { JellyApiSourceAIOConfig, JellyApiSourceConfig, JellySourceAIOConfig, JellySourceConfig } from "./jellyfin.js"; import { JRiverSourceAIOConfig, JRiverSourceConfig } from "./jriver.js"; @@ -24,6 +25,7 @@ export type SourceConfig = | PlexApiSourceConfig | TautulliSourceConfig | DeezerSourceConfig + | ListenbrainzEndpointSourceConfig | SubSonicSourceConfig | JellySourceConfig | JellyApiSourceConfig @@ -46,6 +48,7 @@ export type SourceAIOConfig = | PlexApiSourceAIOConfig | TautulliSourceAIOConfig | DeezerSourceAIOConfig + | ListenbrainzEndpointSourceAIOConfig | SubsonicSourceAIOConfig | JellySourceAIOConfig | JellyApiSourceAIOConfig diff --git a/src/backend/common/vendor/ListenbrainzApiClient.ts b/src/backend/common/vendor/ListenbrainzApiClient.ts index 99d7eec3..5540f238 100644 --- a/src/backend/common/vendor/ListenbrainzApiClient.ts +++ b/src/backend/common/vendor/ListenbrainzApiClient.ts @@ -81,7 +81,7 @@ export interface ListenPayload { } export interface SubmitPayload { - listen_type: 'single', + listen_type: 'single' | 'playing_now', payload: [ListenPayload] } @@ -300,6 +300,43 @@ export class ListenbrainzApiClient extends AbstractApiClient { } } + static listenPayloadToPlay(payload: ListenPayload, nowPlaying: boolean = false): PlayObject { + const { + listened_at = dayjs().unix(), + track_metadata: { + artist_name, + track_name, + additional_info: { + duration, + track_mbid, + artist_mbids, + release_mbid, + release_group_mbid + } = {} + } = {}, + } = payload; + + return { + data: { + playDate: typeof listened_at === 'number' ? dayjs.unix(listened_at) : dayjs(listened_at), + track: track_name, + artists: [artist_name], + duration, + meta: { + brainz: { + artist: artist_mbids !== undefined ? artist_mbids : undefined, + album: release_mbid, + albumArtist: release_group_mbid, + track: track_mbid + } + } + }, + meta: { + nowPlaying, + } + } + } + static listenResponseToPlay(listen: ListenResponse): PlayObject { const { listened_at, diff --git a/src/backend/server/api.ts b/src/backend/server/api.ts index 8596dafb..3bae9b49 100644 --- a/src/backend/server/api.ts +++ b/src/backend/server/api.ts @@ -24,6 +24,7 @@ import { parseBool, sortByNewestPlayDate } from "../utils.js"; import { setupAuthRoutes } from "./auth.js"; import { setupDeezerRoutes } from "./deezerRoutes.js"; import { setupJellyfinRoutes } from "./jellyfinRoutes.js"; +import {setupLZEndpointRoutes} from "./endpointListenbrainzRoutes.js"; import { makeClientCheckMiddle, makeSourceCheckMiddle } from "./middleware.js"; import { setupPlexRoutes } from "./plexRoutes.js"; import { setupTautulliRoutes } from "./tautulliRoutes.js"; @@ -152,6 +153,7 @@ export const setupApi = (app: ExpressWithAsync, logger: Logger, appLoggerStream: setupJellyfinRoutes(app, logger, scrobbleSources); setupDeezerRoutes(app, logger, scrobbleSources); setupWebscrobblerRoutes(app, logger, scrobbleSources); + setupLZEndpointRoutes(app, logger, scrobbleSources); setupAuthRoutes(app, logger, sourceRequiredMiddle, clientRequiredMiddle, scrobbleSources, scrobbleClients); app.putAsync('/api/webscrobbler', bodyParser.json({type: ['text/*', 'application/json']}), async (req, res) => { diff --git a/src/backend/server/endpointListenbrainzRoutes.ts b/src/backend/server/endpointListenbrainzRoutes.ts new file mode 100644 index 00000000..f82513a6 --- /dev/null +++ b/src/backend/server/endpointListenbrainzRoutes.ts @@ -0,0 +1,54 @@ +/* eslint-disable prefer-arrow-functions/prefer-arrow-functions */ +import { ExpressWithAsync } from "@awaitjs/express"; +import { childLogger, Logger } from "@foxxmd/logging"; +import bodyParser from "body-parser"; +import { EndpointListenbrainzSource, playStateFromRequest } from "../sources/EndpointListenbrainzSource.js"; +import { LZEndpointNotifier } from "../sources/ingressNotifiers/LZEndpointNotifier.js"; +import ScrobbleSources from "../sources/ScrobbleSources.js"; +import { nonEmptyBody } from "./middleware.js"; + +export const setupLZEndpointRoutes = (app: ExpressWithAsync, parentLogger: Logger, scrobbleSources: ScrobbleSources) => { + + const logger = childLogger(parentLogger, ['Ingress', 'Listenbrainz']); + + const lzJsonParser = bodyParser.json({ + type: ['text/*', 'application/json'], + }); + const nonEmptyCheck = nonEmptyBody(logger, 'LZ Endpoint'); + + const webhookIngress = new LZEndpointNotifier(logger); + app.useAsync(/\/api\/listenbrainz.*/, + async function (req, res, next) { + // track request before parsing body to ensure we at least log that something is happening + // (in the event body parsing does not work or request is not POST/PATCH) + webhookIngress.trackIngress(req, true); + if (req.method !== 'POST') { + return res.sendStatus(405); + } + next(); + }, + lzJsonParser, nonEmptyCheck, async function (req, res) { + webhookIngress.trackIngress(req, false); + + res.sendStatus(200); + + + const sources = scrobbleSources.getByType('endpointlz') as EndpointListenbrainzSource[]; + if (sources.length === 0) { + logger.warn('Received Listenbrainz endpoint payload but no Listenbrainz endpoint sources are configured'); + } + + const validSources = sources.filter(x => x.matchRequest(req)); + if (validSources.length === 0) { + const [slug, token] = EndpointListenbrainzSource.parseDisplayIdentifiersFromRequest(req); + logger.warn(`No Listenbrainz endpoint config matched => Slug: ${slug} | Token: ${token}`); + } + + const playerState = playStateFromRequest(req.body); + + for (const source of validSources) { + await source.handle(playerState); + } + }); +} + diff --git a/src/backend/sources/EndpointListenbrainzSource.ts b/src/backend/sources/EndpointListenbrainzSource.ts new file mode 100644 index 00000000..9acf42d1 --- /dev/null +++ b/src/backend/sources/EndpointListenbrainzSource.ts @@ -0,0 +1,191 @@ +import dayjs from "dayjs"; +import EventEmitter from "events"; +import { PlayObject, SOURCE_SOT } from "../../core/Atomic.js"; +import { + ExpressRequest, + FormatPlayObjectOptions, + InternalConfig, + NO_USER, + PlayerStateData, + REPORTED_PLAYER_STATUSES, + ReportedPlayerStatus +} from "../common/infrastructure/Atomic.js"; +import { ListenbrainzEndpointSourceConfig } from "../common/infrastructure/config/source/endpointlz.js"; +import { ListenbrainzApiClient, ListenPayload, SubmitPayload } from "../common/vendor/ListenbrainzApiClient.js"; +import { parseRegexSingleOrFail } from "../utils.js"; +import MemorySource from "./MemorySource.js"; + +const noSlugMatch = new RegExp(/\/api\/listenbrainz(?:\/?|\/1\/?|\/1\/submit-listens\/?)$/i); +const slugMatch = new RegExp(/\/api\/listenbrainz\/([^\/]+)(?:\/?|\/1\/?|\/1\/submit-listens\/?)$/i); + +export const authHeaderRegex = new RegExp(/Token (.+)$/i); + +export class EndpointListenbrainzSource extends MemorySource { + + declare config: ListenbrainzEndpointSourceConfig; + + constructor(name: any, config: ListenbrainzEndpointSourceConfig, internal: InternalConfig, emitter: EventEmitter) { + super('endpointlz', name, config, internal, emitter); + this.multiPlatform = true; + this.playerSourceOfTruth = SOURCE_SOT.HISTORY; + + const { + data = {}, + data: { + slug, + } = {} + } = this.config; + this.config.data = { + token: undefined, + ...data, + slug: slug === null ? undefined : slug, + }; + } + + static parseSlugFromString(path: string): string | false | undefined { + const noSlug = parseRegexSingleOrFail(noSlugMatch, path); + if (noSlug !== undefined) { + return undefined; + } + const slugResult = parseRegexSingleOrFail(slugMatch, path); + if (slugResult !== undefined) { + return slugResult.groups[0]; + } + return false; + } + + static parseSlugFromRequest(req: ExpressRequest): string | false | undefined { + return EndpointListenbrainzSource.parseSlugFromString(req.baseUrl); + } + + static parseTokenFromString(str: string): string | undefined { + const tokenMatch = parseRegexSingleOrFail(authHeaderRegex, str); + if(tokenMatch !== undefined) { + return tokenMatch.groups[0]; + } + return undefined; + } + + static parseTokenFromRequest(req: ExpressRequest): string | false | undefined { + const auth = req.header('Authorization'); + if(typeof auth === 'string' && auth !== '') { + const matchedToken = EndpointListenbrainzSource.parseTokenFromString(auth); + if(matchedToken === undefined) { + return false; + } + return matchedToken; + } + return undefined; + } + + static parseIdentifiersFromRequest(req: ExpressRequest): [string | false | undefined, false | string | undefined] { + const slug = EndpointListenbrainzSource.parseSlugFromRequest(req); + const token = EndpointListenbrainzSource.parseTokenFromRequest(req); + + return [slug, token]; + } + + static parseDisplayIdentifiersFromRequest(req: ExpressRequest): [string, string] { + const [slug, token] = EndpointListenbrainzSource.parseIdentifiersFromRequest(req); + let slugStr = '(no slug)'; + if (slug === false) { + slugStr = '(invalid slug)'; + } else if (slug !== undefined) { + slugStr = slug; + } + + let tokenStr = '(no token)'; + if (token === false) { + tokenStr = '(invalid token)'; + } else if (token !== undefined) { + tokenStr = `${token.substring(0,3)}****` + } + return [slugStr, tokenStr]; + } + + matchRequest(req: ExpressRequest): boolean { + let matchesToken = this.config.data.token === undefined; + const reqToken = EndpointListenbrainzSource.parseTokenFromRequest(req); + if (reqToken === false) { + return false; + } + matchesToken = this.config.data.token === undefined && reqToken === undefined || + (reqToken !== undefined && this.config.data.token !== undefined + && this.config.data.token.toLowerCase().trim() === reqToken.toLowerCase().trim()); + + if (!matchesToken) { + return false; + } + + let matchesPath = false; + const slug = EndpointListenbrainzSource.parseSlugFromRequest(req); + if (slug === false) { + return false; + } else { + matchesPath = (this.config.data.slug === undefined && slug === undefined) || (slug !== undefined && this.config.data.slug !== undefined && this.config.data.slug.toLowerCase().trim() === slug.toLocaleLowerCase().trim()); + } + + return matchesToken && matchesPath; + } + + static listenTypeAsPlayerStatus(event: string): ReportedPlayerStatus { + switch (event) { + case 'single': + case 'playing_now': + return REPORTED_PLAYER_STATUSES.playing; + default: + return REPORTED_PLAYER_STATUSES.unknown; + } + } + + static formatPlayObj(obj: ListenPayload, options: FormatPlayObjectOptions & { + nowPlaying?: boolean + } = {}): PlayObject { + return ListenbrainzApiClient.listenPayloadToPlay(obj, options.nowPlaying); + } + + getRecentlyPlayed = async (options = {}) => { + return this.getFlatRecentlyDiscoveredPlays(); + } + + isValidScrobble = (playObj: PlayObject) => { + return true; + } + + handle = async (stateData: PlayerStateData) => { + + this.processRecentPlays([stateData]); + + if (stateData.play.meta.nowPlaying === false && this.isValidScrobble(stateData.play)) { + const discovered = this.discover([stateData.play]); + if (discovered.length > 0) { + this.scrobble(discovered); + } + } + } +} + +export const playStateFromRequest = (obj: SubmitPayload): PlayerStateData => { + const { + listen_type, + payload, + } = obj; + + const play = ListenbrainzApiClient.listenPayloadToPlay(payload[0], listen_type === 'playing_now'); + return { + platformId: [play.meta.deviceId, NO_USER], + play, + status: listenTypeAsPlayerStatus(listen_type), + timestamp: dayjs() + } +} + +export const listenTypeAsPlayerStatus = (event: string): ReportedPlayerStatus => { + switch (event) { + case 'single': + case 'playing_now': + return REPORTED_PLAYER_STATUSES.playing; + default: + return REPORTED_PLAYER_STATUSES.unknown; + } +} \ No newline at end of file diff --git a/src/backend/sources/ScrobbleSources.ts b/src/backend/sources/ScrobbleSources.ts index 3bc2a44c..58affd01 100644 --- a/src/backend/sources/ScrobbleSources.ts +++ b/src/backend/sources/ScrobbleSources.ts @@ -5,6 +5,7 @@ import { ConfigMeta, InternalConfig, isSourceType, SourceType, sourceTypes } fro import { AIOConfig, SourceDefaults } from "../common/infrastructure/config/aioConfig.js"; import { ChromecastSourceConfig } from "../common/infrastructure/config/source/chromecast.js"; import { DeezerData, DeezerSourceConfig } from "../common/infrastructure/config/source/deezer.js"; +import { ListenbrainzEndpointSourceConfig, ListenbrainzEndpointData } from "../common/infrastructure/config/source/endpointlz.js"; import { JellyApiData, JellyApiSourceConfig, @@ -33,6 +34,7 @@ import { validateJson } from "../utils/ValidationUtils.js"; import AbstractSource from "./AbstractSource.js"; import { ChromecastSource } from "./ChromecastSource.js"; import DeezerSource from "./DeezerSource.js"; +import { EndpointListenbrainzSource } from "./EndpointListenbrainzSource.js"; import JellyfinApiSource from "./JellyfinApiSource.js"; import JellyfinSource from "./JellyfinSource.js"; import { JRiverSource } from "./JRiverSource.js"; @@ -130,6 +132,9 @@ export default class ScrobbleSources { case 'deezer': this.schemaDefinitions[type] = getTypeSchemaFromConfigGenerator("DeezerSourceConfig"); break; + case 'endpointlz': + this.schemaDefinitions[type] = getTypeSchemaFromConfigGenerator("ListenbrainzEndpointSourceConfig"); + break; case 'subsonic': this.schemaDefinitions[type] = getTypeSchemaFromConfigGenerator("SubSonicSourceConfig"); break; @@ -377,6 +382,23 @@ export default class ScrobbleSources { // sane default for lastfm is that user want to scrobble TO it, not FROM it -- this is also existing behavior defaultConfigureAs = 'client'; break; + case 'endpointlz': + const lzShouldUse = parseBool(process.env.LZENDPOINT_ENABLE); + const lze = { + slug: process.env.LZE_SLUG, + token: process.env.LZE_TOKEN + } + if (!Object.values(lze).every(x => x === undefined) || lzShouldUse) { + configs.push({ + type: 'endpointlz', + name: 'unnamed', + source: 'ENV', + mode: 'single', + configureAs: defaultConfigureAs, + data: lze as ListenbrainzEndpointData + }); + } + break; case 'jriver': const jr = { url: process.env.JRIVER_URL, @@ -663,6 +685,9 @@ export default class ScrobbleSources { case 'listenbrainz': newSource = await new ListenbrainzSource(name, compositeConfig as ListenBrainzSourceConfig, this.internalConfig, this.emitter); break; + case 'endpointlz': + newSource = await new EndpointListenbrainzSource(name, compositeConfig as ListenbrainzEndpointSourceConfig, this.internalConfig, this.emitter); + break; case 'jriver': newSource = await new JRiverSource(name, compositeConfig as JRiverSourceConfig, this.internalConfig, this.emitter); break; diff --git a/src/backend/sources/ingressNotifiers/LZEndpointNotifier.ts b/src/backend/sources/ingressNotifiers/LZEndpointNotifier.ts new file mode 100644 index 00000000..b99bfec3 --- /dev/null +++ b/src/backend/sources/ingressNotifiers/LZEndpointNotifier.ts @@ -0,0 +1,46 @@ +import { Logger } from "@foxxmd/logging"; +import { Request } from "express"; +import { EndpointListenbrainzSource } from "../EndpointListenbrainzSource.js"; +import { IngressNotifier } from "./IngressNotifier.js"; + +export class LZEndpointNotifier extends IngressNotifier { + + constructor(logger: Logger) { + super('Listenbrainz Endpoint', logger); + } + + seenSlugs: Record = {}; + notifyBySource(req: Request, isRaw: boolean): [boolean, (string | undefined)] { + + if(!isRaw) { + + const [slug, token] = EndpointListenbrainzSource.parseIdentifiersFromRequest(req); + if(slug === false) { + return [false, `Request URL was not a valid: ${req.baseUrl}`]; + } + const slugStr = slug ?? '(no slug)'; + + if(token === false) { + return [false, `Request URL was valid and 'Authorization' header was present but invalid. Authorization header should be 'Token tokenValue' but was '${req.header('Authorization')}'`]; + } + const tokenStr = token ?? '(no token)'; + const redactedToken = token !== undefined ? `${(token as string).substring(0,3)}****` : '(no token)'; + + const identifier = `${slugStr}-${tokenStr}`; + + if(this.seenSlugs[identifier] === undefined) { + this.seenSlugs[identifier] = true; + return [true, `Received a well formed request to endpoint with -- Slug: ${slugStr} | Token: ${redactedToken} -- for the first time.`]; + } + } + + return [true, undefined]; + } + + notifyByRequest(req: Request, isRaw: boolean): string | undefined { + if(req.method !== 'POST') { + return `Expected POST request (submit-listen payload) but received ${req.method}`; + } + return; + } +} -- 2.51.2