From 3680ac8af1693d4afab7f26b73ad04c0b81541cc Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 13 Mar 2025 18:44:08 +0000 Subject: [PATCH] feat(lastm): Implement MVP Lastfm endpoint source --- config/endpointlfm.json.example | 9 ++ src/backend/common/infrastructure/Atomic.ts | 2 + .../config/source/endpointlfm.ts | 25 ++++ .../infrastructure/config/source/sources.ts | 3 + .../typings/lastfm-node-client.d.ts | 8 ++ src/backend/common/vendor/LastfmApiClient.ts | 62 ++++++++- src/backend/server/api.ts | 2 + src/backend/server/endpointLastfmRoutes.ts | 60 +++++++++ src/backend/sources/EndpointLastfmSource.ts | 122 ++++++++++++++++++ .../sources/EndpointListenbrainzSource.ts | 2 +- src/backend/sources/ScrobbleSources.ts | 24 ++++ .../ingressNotifiers/LFMEndpointNotifier.ts | 50 +++++++ 12 files changed, 367 insertions(+), 2 deletions(-) create mode 100644 config/endpointlfm.json.example create mode 100644 src/backend/common/infrastructure/config/source/endpointlfm.ts create mode 100644 src/backend/server/endpointLastfmRoutes.ts create mode 100644 src/backend/sources/EndpointLastfmSource.ts create mode 100644 src/backend/sources/ingressNotifiers/LFMEndpointNotifier.ts diff --git a/config/endpointlfm.json.example b/config/endpointlfm.json.example new file mode 100644 index 00000000..bdb20351 --- /dev/null +++ b/config/endpointlfm.json.example @@ -0,0 +1,9 @@ +[ + { + "name": "myLfm", + "enable": true, + "data": { + "slug": "mySlug" + } + } +] diff --git a/src/backend/common/infrastructure/Atomic.ts b/src/backend/common/infrastructure/Atomic.ts index dab935f1..a18f38c5 100644 --- a/src/backend/common/infrastructure/Atomic.ts +++ b/src/backend/common/infrastructure/Atomic.ts @@ -16,6 +16,7 @@ export type SourceType = | 'lastfm' | 'deezer' | 'endpointlz' + | 'endpointlfm' | 'ytmusic' | 'mpris' | 'mopidy' @@ -37,6 +38,7 @@ export const sourceTypes: SourceType[] = [ 'lastfm', 'deezer', 'endpointlz', + 'endpointlfm', 'ytmusic', 'mpris', 'mopidy', diff --git a/src/backend/common/infrastructure/config/source/endpointlfm.ts b/src/backend/common/infrastructure/config/source/endpointlfm.ts new file mode 100644 index 00000000..80abc281 --- /dev/null +++ b/src/backend/common/infrastructure/config/source/endpointlfm.ts @@ -0,0 +1,25 @@ +import { CommonSourceConfig, CommonSourceData } from "./index.js"; + +export interface LastFMEndpointData extends CommonSourceData { + /** + * The URL ending that should be used to identify scrobbles for this source + * + * If you are using multiple Last.fm endpoint sources (scrobbles for many users) you can use a slug to match Sources with individual users/origins + * + * Example: + * + * * slug: 'usera' => API URL: http://localhost:9078/api/lastfm/usera + * * slug: 'originb' => API URL: http://localhost:9078/api/lastfm/originb + * + * If no slug is found from an extension's incoming webhook event the first Last.fm source without a slug will be used + * */ + slug?: string | null +} + +export interface LastFMEndpointSourceConfig extends CommonSourceConfig { + data?: LastFMEndpointData +} + +export interface LastFMEndpointSourceAIOConfig extends LastFMEndpointSourceConfig { + type: 'endpointlfm' +} diff --git a/src/backend/common/infrastructure/config/source/sources.ts b/src/backend/common/infrastructure/config/source/sources.ts index 8adbfd44..d4212b07 100644 --- a/src/backend/common/infrastructure/config/source/sources.ts +++ b/src/backend/common/infrastructure/config/source/sources.ts @@ -1,5 +1,6 @@ import { ChromecastSourceAIOConfig, ChromecastSourceConfig } from "./chromecast.js"; import { ListenbrainzEndpointSourceAIOConfig, ListenbrainzEndpointSourceConfig } from "./endpointlz.js"; +import { LastFMEndpointSourceAIOConfig, LastFMEndpointSourceConfig } from "./endpointlfm.js"; import { DeezerSourceAIOConfig, DeezerSourceConfig } from "./deezer.js"; import { JellyApiSourceAIOConfig, JellyApiSourceConfig, JellySourceAIOConfig, JellySourceConfig } from "./jellyfin.js"; import { JRiverSourceAIOConfig, JRiverSourceConfig } from "./jriver.js"; @@ -26,6 +27,7 @@ export type SourceConfig = | TautulliSourceConfig | DeezerSourceConfig | ListenbrainzEndpointSourceConfig + | LastFMEndpointSourceConfig | SubSonicSourceConfig | JellySourceConfig | JellyApiSourceConfig @@ -49,6 +51,7 @@ export type SourceAIOConfig = | TautulliSourceAIOConfig | DeezerSourceAIOConfig | ListenbrainzEndpointSourceAIOConfig + | LastFMEndpointSourceAIOConfig | SubsonicSourceAIOConfig | JellySourceAIOConfig | JellyApiSourceAIOConfig diff --git a/src/backend/common/infrastructure/typings/lastfm-node-client.d.ts b/src/backend/common/infrastructure/typings/lastfm-node-client.d.ts index 8ee66f5c..8026e1eb 100644 --- a/src/backend/common/infrastructure/typings/lastfm-node-client.d.ts +++ b/src/backend/common/infrastructure/typings/lastfm-node-client.d.ts @@ -30,6 +30,10 @@ declare module 'lastfm-node-client' { mbid?: string } + export interface TrackScrobbleRequest extends TrackScrobblePayload { + method: 'track.scrobble' + } + export interface TrackScrobbleResponse { scrobbles: { '@attr': { @@ -52,6 +56,10 @@ declare module 'lastfm-node-client' { export type NowPlayingPayload = Omit + export type NowPlayingRequest = NowPlayingPayload & { method: 'track.updateNowPlaying' }; + + export type LastfmTrackUpdateRequest = TrackScrobbleRequest | NowPlayingRequest; + export interface NowPlayingResponse { nowplaying: { ignoredMessage: { diff --git a/src/backend/common/vendor/LastfmApiClient.ts b/src/backend/common/vendor/LastfmApiClient.ts index 1fca76f5..445db4c8 100644 --- a/src/backend/common/vendor/LastfmApiClient.ts +++ b/src/backend/common/vendor/LastfmApiClient.ts @@ -1,6 +1,8 @@ -import dayjs from "dayjs"; +import dayjs, { Dayjs } from "dayjs"; import LastFm, { AuthGetSessionResponse, + LastfmTrackUpdateRequest, + NowPlayingPayload, NowPlayingResponse, TrackObject, TrackScrobblePayload, @@ -16,6 +18,7 @@ import { UpstreamError } from "../errors/UpstreamError.js"; import { AbstractApiOptions, DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions } from "../infrastructure/Atomic.js"; import { LastfmData } from "../infrastructure/config/client/lastfm.js"; import AbstractApiClient from "./AbstractApiClient.js"; +import { parseArtistCredits } from "../../utils/StringUtils.js"; const badErrors = [ 'api key suspended', @@ -275,3 +278,60 @@ export default class LastfmApiClient extends AbstractApiClient { } } } + +export const scrobblePayloadToPlay = (obj: LastfmTrackUpdateRequest): PlayObject => { + const { + artist, + track, + duration, + album, + albumArtist, + } = obj; + + let ts: Dayjs | undefined; + if('timestamp' in obj) { + ts = dayjs.unix(obj.timestamp); + } + const mbid = 'mbid' in obj ? obj.mbid : undefined; + + + let artists: string[] = []; + + const credits = parseArtistCredits(artist); + if(credits !== undefined) { + artists.push(credits.primary); + if(credits.secondary !== undefined) { + const nonEmptyArtists = credits.secondary.filter(x => x !== undefined && x !== null && x.trim() !== ''); + if(nonEmptyArtists.length > 0) { + artists = [...artists, ...nonEmptyArtists]; + } + } + } else { + artists = [artist]; + } + + const play: PlayObject = { + data: { + track, + album: nonEmptyStringOrDefault(album), + albumArtists: nonEmptyStringOrDefault(albumArtist) !== undefined ? [albumArtist] : undefined, + duration: typeof duration === 'string' ? parseInt(duration, 10) : duration, + playDate: ts, + artists + }, + meta: { + source: 'lastfm', + nowPlaying: obj.method === 'track.updateNowPlaying' + } + }; + + if(nonEmptyStringOrDefault(mbid) !== undefined) { + play.data.meta = { + brainz: { + track: mbid + } + }; + } + + return play; +} \ No newline at end of file diff --git a/src/backend/server/api.ts b/src/backend/server/api.ts index 3bae9b49..625f7c7a 100644 --- a/src/backend/server/api.ts +++ b/src/backend/server/api.ts @@ -25,6 +25,7 @@ import { setupAuthRoutes } from "./auth.js"; import { setupDeezerRoutes } from "./deezerRoutes.js"; import { setupJellyfinRoutes } from "./jellyfinRoutes.js"; import {setupLZEndpointRoutes} from "./endpointListenbrainzRoutes.js"; +import {setupLastfmEndpointRoutes} from "./endpointLastfmRoutes.js"; import { makeClientCheckMiddle, makeSourceCheckMiddle } from "./middleware.js"; import { setupPlexRoutes } from "./plexRoutes.js"; import { setupTautulliRoutes } from "./tautulliRoutes.js"; @@ -154,6 +155,7 @@ export const setupApi = (app: ExpressWithAsync, logger: Logger, appLoggerStream: setupDeezerRoutes(app, logger, scrobbleSources); setupWebscrobblerRoutes(app, logger, scrobbleSources); setupLZEndpointRoutes(app, logger, scrobbleSources); + setupLastfmEndpointRoutes(app, logger, scrobbleSources); setupAuthRoutes(app, logger, sourceRequiredMiddle, clientRequiredMiddle, scrobbleSources, scrobbleClients); app.putAsync('/api/webscrobbler', bodyParser.json({type: ['text/*', 'application/json']}), async (req, res) => { diff --git a/src/backend/server/endpointLastfmRoutes.ts b/src/backend/server/endpointLastfmRoutes.ts new file mode 100644 index 00000000..23a3bd79 --- /dev/null +++ b/src/backend/server/endpointLastfmRoutes.ts @@ -0,0 +1,60 @@ +/* eslint-disable prefer-arrow-functions/prefer-arrow-functions */ +import { ExpressWithAsync } from "@awaitjs/express"; +import { childLogger, Logger } from "@foxxmd/logging"; +import bodyParser from "body-parser"; +import ScrobbleSources from "../sources/ScrobbleSources.js"; +import { nonEmptyBody } from "./middleware.js"; +import { LFMEndpointNotifier } from "../sources/ingressNotifiers/LFMEndpointNotifier.js"; +import { EndpointLastfmSource, playStateFromRequest, parseDisplayIdentifiersFromRequest } from "../sources/EndpointLastfmSource.js"; +import { LastfmTrackUpdateRequest } from "lastfm-node-client"; + +export const setupLastfmEndpointRoutes = (app: ExpressWithAsync, parentLogger: Logger, scrobbleSources: ScrobbleSources) => { + + const logger = childLogger(parentLogger, ['Ingress', 'LFM']); + + const nonEmptyCheck = nonEmptyBody(logger, 'LFM Endpoint'); + + const webhookIngress = new LFMEndpointNotifier(logger); + app.useAsync(/(\/api\/lastfm.*)|(\/2.0\/?)$/, + async function (req, res, next) { + // track request before parsing body to ensure we at least log that something is happening + // (in the event body parsing does not work or request is not POST/PATCH) + webhookIngress.trackIngress(req, true); + if (req.method !== 'POST') { + return res.sendStatus(405); + } + next(); + }, + bodyParser.urlencoded({ extended: true }), + nonEmptyCheck, async function (req, res) { + webhookIngress.trackIngress(req, false); + + const sources = scrobbleSources.getByType('endpointlfm') as EndpointLastfmSource[]; + if (sources.length === 0) { + logger.warn('Received Lastfm endpoint payload but no Lastfm endpoint sources are configured'); + } + + const validSources = sources.filter(x => x.matchRequest(req)); + if (validSources.length === 0) { + const [slug] = parseDisplayIdentifiersFromRequest(req); + logger.warn(`No Lastfm endpoint config matched => Slug: ${slug}`); + } + + if(!('method' in req.body)) { + return res.status(400).json({error: `Missing 'method' param`}); + } + const method = (req.body as LastfmTrackUpdateRequest).method; + if(!['track.updateNowPlaying','track.scrobble'].includes(method)) { + return res.status(400).json({error: `Unexpected 'method' param value '${method}', expected either 'track.updateNowPlaying' or 'track.scrobble'`}); + } + + res.sendStatus(200); + + const playerState = playStateFromRequest(req.body); + + for (const source of validSources) { + await source.handle(playerState); + } + }); +} + diff --git a/src/backend/sources/EndpointLastfmSource.ts b/src/backend/sources/EndpointLastfmSource.ts new file mode 100644 index 00000000..72444e3d --- /dev/null +++ b/src/backend/sources/EndpointLastfmSource.ts @@ -0,0 +1,122 @@ +import dayjs from "dayjs"; +import EventEmitter from "events"; +import { PlayObject, SOURCE_SOT } from "../../core/Atomic.js"; +import { + ExpressRequest, + FormatPlayObjectOptions, + InternalConfig, + NO_USER, + PlayerStateData, + REPORTED_PLAYER_STATUSES, + ReportedPlayerStatus +} from "../common/infrastructure/Atomic.js"; +import { parseRegexSingleOrFail } from "../utils.js"; +import MemorySource from "./MemorySource.js"; +import { LastFMEndpointSourceConfig } from "../common/infrastructure/config/source/endpointlfm.js"; +import { LastfmTrackUpdateRequest, NowPlayingPayload, TrackScrobblePayload } from "lastfm-node-client"; +import { scrobblePayloadToPlay } from "../common/vendor/LastfmApiClient.js"; + +const noSlugMatch = new RegExp(/(?:\/api\/lastfm\/?)$|(?:\/1\/?|\/2.0\/?)$/i); +const slugMatch = new RegExp(/\/api\/lastfm\/([^\/]+)$/i); + +export const authHeaderRegex = new RegExp(/Token (.+)$/i); + +export class EndpointLastfmSource extends MemorySource { + + declare config: LastFMEndpointSourceConfig; + + constructor(name: any, config: LastFMEndpointSourceConfig, internal: InternalConfig, emitter: EventEmitter) { + super('endpointlfm', name, config, internal, emitter); + this.multiPlatform = false; + this.playerSourceOfTruth = SOURCE_SOT.HISTORY; + + const { + data = {}, + data: { + slug, + } = {} + } = this.config; + this.config.data = { + ...data, + slug: slug === null ? undefined : slug, + }; + } + + matchRequest(req: ExpressRequest): boolean { + let matchesPath = false; + const slug = parseSlugFromRequest(req); + if (slug === false) { + return false; + } else { + matchesPath = (this.config.data.slug === undefined && slug === undefined) || (slug !== undefined && this.config.data.slug !== undefined && this.config.data.slug.toLowerCase().trim() === slug.toLocaleLowerCase().trim()); + } + + return matchesPath; + } + + static formatPlayObj(obj: LastfmTrackUpdateRequest, options: FormatPlayObjectOptions = {}): PlayObject { + return scrobblePayloadToPlay(obj); + } + + getRecentlyPlayed = async (options = {}) => { + return this.getFlatRecentlyDiscoveredPlays(); + } + + isValidScrobble = (playObj: PlayObject) => { + return true; + } + + handle = async (stateData: PlayerStateData) => { + + this.processRecentPlays([stateData]); + + if (stateData.play.meta.nowPlaying === false && this.isValidScrobble(stateData.play)) { + const discovered = this.discover([stateData.play]); + if (discovered.length > 0) { + this.scrobble(discovered); + } + } + } +} + +export const playStateFromRequest = (obj: LastfmTrackUpdateRequest): PlayerStateData => { + + const play = scrobblePayloadToPlay(obj); + return { + platformId: [play.meta.deviceId, NO_USER], + play, + status: obj.method === 'track.updateNowPlaying' ? REPORTED_PLAYER_STATUSES.playing : REPORTED_PLAYER_STATUSES.unknown, + timestamp: dayjs() + } +} + +export const parseSlugFromString = (path: string): string | false | undefined => { + const noSlug = parseRegexSingleOrFail(noSlugMatch, path); + if (noSlug !== undefined) { + return undefined; + } + const slugResult = parseRegexSingleOrFail(slugMatch, path); + if (slugResult !== undefined) { + return slugResult.groups[0]; + } + return false; +} + +export const parseSlugFromRequest = (req: ExpressRequest): string | false | undefined => parseSlugFromString(req.baseUrl); + +export const parseIdentifiersFromRequest = (req: ExpressRequest): [string | false | undefined] => { + const slug = parseSlugFromRequest(req); + + return [slug]; +} + +export const parseDisplayIdentifiersFromRequest = (req: ExpressRequest): [string] => { + const [slug] = parseIdentifiersFromRequest(req); + let slugStr = '(no slug)'; + if (slug === false) { + slugStr = '(invalid slug)'; + } else if (slug !== undefined) { + slugStr = slug; + } + return [slugStr]; +} \ No newline at end of file diff --git a/src/backend/sources/EndpointListenbrainzSource.ts b/src/backend/sources/EndpointListenbrainzSource.ts index b2a20d78..9bf8c47a 100644 --- a/src/backend/sources/EndpointListenbrainzSource.ts +++ b/src/backend/sources/EndpointListenbrainzSource.ts @@ -26,7 +26,7 @@ export class EndpointListenbrainzSource extends MemorySource { constructor(name: any, config: ListenbrainzEndpointSourceConfig, internal: InternalConfig, emitter: EventEmitter) { super('endpointlz', name, config, internal, emitter); - this.multiPlatform = true; + this.multiPlatform = false; this.playerSourceOfTruth = SOURCE_SOT.HISTORY; const { diff --git a/src/backend/sources/ScrobbleSources.ts b/src/backend/sources/ScrobbleSources.ts index 58affd01..76e3751c 100644 --- a/src/backend/sources/ScrobbleSources.ts +++ b/src/backend/sources/ScrobbleSources.ts @@ -6,6 +6,7 @@ import { AIOConfig, SourceDefaults } from "../common/infrastructure/config/aioCo import { ChromecastSourceConfig } from "../common/infrastructure/config/source/chromecast.js"; import { DeezerData, DeezerSourceConfig } from "../common/infrastructure/config/source/deezer.js"; import { ListenbrainzEndpointSourceConfig, ListenbrainzEndpointData } from "../common/infrastructure/config/source/endpointlz.js"; +import { LastFMEndpointSourceConfig, LastFMEndpointData } from "../common/infrastructure/config/source/endpointlfm.js"; import { JellyApiData, JellyApiSourceConfig, @@ -35,6 +36,7 @@ import AbstractSource from "./AbstractSource.js"; import { ChromecastSource } from "./ChromecastSource.js"; import DeezerSource from "./DeezerSource.js"; import { EndpointListenbrainzSource } from "./EndpointListenbrainzSource.js"; +import { EndpointLastfmSource } from "./EndpointLastfmSource.js"; import JellyfinApiSource from "./JellyfinApiSource.js"; import JellyfinSource from "./JellyfinSource.js"; import { JRiverSource } from "./JRiverSource.js"; @@ -135,6 +137,9 @@ export default class ScrobbleSources { case 'endpointlz': this.schemaDefinitions[type] = getTypeSchemaFromConfigGenerator("ListenbrainzEndpointSourceConfig"); break; + case 'endpointlfm': + this.schemaDefinitions[type] = getTypeSchemaFromConfigGenerator("LastFMEndpointSourceConfig"); + break; case 'subsonic': this.schemaDefinitions[type] = getTypeSchemaFromConfigGenerator("SubSonicSourceConfig"); break; @@ -399,6 +404,22 @@ export default class ScrobbleSources { }); } break; + case 'endpointlfm': + const lfmShouldUse = parseBool(process.env.LFMENDPOINT_ENABLE); + const lfme = { + slug: process.env.LFM_SLUG, + } + if (!Object.values(lfme).every(x => x === undefined) || lfmShouldUse) { + configs.push({ + type: 'endpointlfm', + name: 'unnamed', + source: 'ENV', + mode: 'single', + configureAs: defaultConfigureAs, + data: lfme as LastFMEndpointData + }); + } + break; case 'jriver': const jr = { url: process.env.JRIVER_URL, @@ -688,6 +709,9 @@ export default class ScrobbleSources { case 'endpointlz': newSource = await new EndpointListenbrainzSource(name, compositeConfig as ListenbrainzEndpointSourceConfig, this.internalConfig, this.emitter); break; + case 'endpointlfm': + newSource = await new EndpointLastfmSource(name, compositeConfig as LastFMEndpointSourceConfig, this.internalConfig, this.emitter); + break; case 'jriver': newSource = await new JRiverSource(name, compositeConfig as JRiverSourceConfig, this.internalConfig, this.emitter); break; diff --git a/src/backend/sources/ingressNotifiers/LFMEndpointNotifier.ts b/src/backend/sources/ingressNotifiers/LFMEndpointNotifier.ts new file mode 100644 index 00000000..c1e3269b --- /dev/null +++ b/src/backend/sources/ingressNotifiers/LFMEndpointNotifier.ts @@ -0,0 +1,50 @@ +import { Logger } from "@foxxmd/logging"; +import { Request } from "express"; +import { parseIdentifiersFromRequest } from "../EndpointLastfmSource.js"; +import { IngressNotifier } from "./IngressNotifier.js"; +import { LastfmTrackUpdateRequest } from "lastfm-node-client"; + +export class LFMEndpointNotifier extends IngressNotifier { + + constructor(logger: Logger) { + super('Lastfm Endpoint', logger); + } + + seenSlugs: Record = {}; + notifyBySource(req: Request, isRaw: boolean): [boolean, (string | undefined)] { + + if(!isRaw) { + + const [slug] = parseIdentifiersFromRequest(req); + if(slug === false) { + return [false, `Request URL was not valid: ${req.baseUrl}`]; + } + const slugStr = slug ?? '(no slug)'; + + const identifier = `${slugStr}`; + + if(this.seenSlugs[identifier] === undefined) { + this.seenSlugs[identifier] = true; + return [true, `Received a request to endpoint with -- Slug: ${slugStr} -- for the first time.`]; + } + } + + return [true, undefined]; + } + + notifyByRequest(req: Request, isRaw: boolean): string | undefined { + if(req.method !== 'POST') { + return `Expected POST request (track.scrobble payload) but received ${req.method}`; + } + if(!isRaw) { + if(!('method' in req.body)) { + return `Body is missing 'method' param` + } + const method = (req.body as LastfmTrackUpdateRequest).method; + if(!['track.updateNowPlaying','track.scrobble'].includes(method)) { + return `Unexpected 'method' param value '${method}', expected either 'track.updateNowPlaying' or 'track.scrobble'` + } + } + return; + } +} -- 2.51.2