From be9b16e5c1545beda8d00a972e52b5dd04b37620 Mon Sep 17 00:00:00 2001 From: sudbrack Date: Mon, 28 Sep 2026 20:09:10 -0500 Subject: [PATCH] feat: periodically backfill songs from spotify history api for missed live playback state --- .../sources/_env_configs/_spotify.md | 17 +-- .../docs/configuration/sources/spotify.mdx | 14 +++ .../infrastructure/config/source/spotify.ts | 7 ++ src/backend/sources/AbstractSource.ts | 15 +-- src/backend/sources/SpotifySource.ts | 29 +++++ src/backend/tests/spotify/spotify.test.ts | 102 ++++++++++++++++-- 6 files changed, 163 insertions(+), 21 deletions(-) diff --git a/docsite/docs/configuration/sources/_env_configs/_spotify.md b/docsite/docs/configuration/sources/_env_configs/_spotify.md index 56a423f3..41e966a4 100644 --- a/docsite/docs/configuration/sources/_env_configs/_spotify.md +++ b/docsite/docs/configuration/sources/_env_configs/_spotify.md @@ -1,8 +1,9 @@ -| Environmental Variable | Type | Default | Description | -| ----------------------------- | ------- | ------------------------------ | -------------------------------------------------------------------- | -| _**`SPOTIFY_ID`**_ | string | | A globally unique ID EX `myComponentId` | -| `SPOTIFY_NAME` | string | Value of `SPOTIFY_ID` | A vanity name EX `My Cool Component` | -| `SPOTIFY_ENABLE` | boolean | true | Should this component be used? | -| _**`SPOTIFY_CLIENT_ID`**_ | string | | spotify client id | -| _**`SPOTIFY_CLIENT_SECRET`**_ | string | | spotify client secret | -| `SPOTIFY_REDIRECT_URI` | string | http://localhost:9078/callback | spotify redirect URI -- required only if not the default shown here. | \ No newline at end of file +| Environmental Variable | Type | Default | Description | +| ----------------------------- | ------- | ------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| _**`SPOTIFY_ID`**_ | string | | A globally unique ID EX `myComponentId` | +| `SPOTIFY_NAME` | string | Value of `SPOTIFY_ID` | A vanity name EX `My Cool Component` | +| `SPOTIFY_ENABLE` | boolean | true | Should this component be used? | +| _**`SPOTIFY_CLIENT_ID`**_ | string | | spotify client id | +| _**`SPOTIFY_CLIENT_SECRET`**_ | string | | spotify client secret | +| `SPOTIFY_REDIRECT_URI` | string | http://localhost:9078/callback | spotify redirect URI -- required only if not the default shown here. | +| `SPOTIFY_SCROBBLE_BACKLOG` | boolean | true | Fetch recent listening history on startup and periodically refetch from Spotify's API to reconcile plays missed during live tracking. Note: Backfilled tracks scrobble under Spotify's ~30s history rule rather than normal listen thresholds (see [Scrobble Threshold Trade-off](#scrobbling-backlog-and-reconciling-history)). | \ No newline at end of file diff --git a/docsite/docs/configuration/sources/spotify.mdx b/docsite/docs/configuration/sources/spotify.mdx index 441bfc8e..f7b1b0a5 100644 --- a/docsite/docs/configuration/sources/spotify.mdx +++ b/docsite/docs/configuration/sources/spotify.mdx @@ -117,6 +117,20 @@ To work around this, multi-scrobbler makes one extra call to the `tracks/{id}` e If you would rather not make this extra API call set `enrichIsrc` to `false` in your Spotify source config. +#### Scrobbling Backlog and Reconciling History + +When `scrobbleBacklog` is enabled (default `true`), multi-scrobbler will: +* On startup, check Spotify's recent listening history to scrobble any backlogged tracks played while MS was not running. +* Periodically (every 15 minutes), check Spotify's recent listening history to reconcile and backfill any plays that may have been missed by real-time tracking (for example, during Spotify Connect device switches or transient API dropouts). Any tracks already tracked or scrobbled in real-time are automatically deduplicated. + +:::caution Scrobble Threshold Trade-off +Under normal live playback tracking, multi-scrobbler enforces your configured [scrobble thresholds](/configuration/sources#scrobble-thresholds) (by default, listening to at least **50% of the track** or 4 minutes) before submitting a scrobble. + +Due to Spotify API limitations, the recent history endpoint does not report playback duration or whether a track was skipped. Spotify automatically records any listen lasting at least ~30 seconds into your recent history. Consequently, **any track queued from backlog (at startup or via periodic reconcile) will be scrobbled if Spotify logged it (~30+ seconds)**, bypassing the standard 50% or 4-minute listen threshold. + +If you prefer to strictly enforce duration thresholds and avoid scrobbling short/skipped listens that Spotify logged to history, set `scrobbleBacklog` to `false` in your Spotify source options (or via ENV `SPOTIFY_SCROBBLE_BACKLOG=false`). +::: + ## Configuration diff --git a/src/backend/common/infrastructure/config/source/spotify.ts b/src/backend/common/infrastructure/config/source/spotify.ts index b66be132..b5ab01a4 100644 --- a/src/backend/common/infrastructure/config/source/spotify.ts +++ b/src/backend/common/infrastructure/config/source/spotify.ts @@ -82,6 +82,10 @@ const envDataSchema = z.object({ SPOTIFY_CLIENT_ID: spotifySourceDataSchema.shape.clientId, SPOTIFY_CLIENT_SECRET: spotifySourceDataSchema.shape.clientSecret, SPOTIFY_REDIRECT_URI: spotifySourceDataSchema.shape.redirectUri, + SPOTIFY_SCROBBLE_BACKLOG: z.stringbool().optional().meta({ + description: "Fetch recent listening history on startup and periodically refetch from Spotify's API to reconcile plays missed during live tracking. Note: Backfilled tracks scrobble under Spotify's ~30s history rule rather than normal listen thresholds (see [Scrobble Threshold Trade-off](#scrobbling-backlog-and-reconciling-history)).", + default: true + }), }); export const envSchemas: EnvSourceSchema = { @@ -92,6 +96,9 @@ export const envSchemas: EnvSourceSchema { + protected processBacklog = async (signal: AbortSignal, label?: string) => { if (this.canBacklog) { + const logger = label !== undefined ? childLogger(this.logger, label) : this.logger; const { options: { @@ -548,12 +549,12 @@ export default abstract class AbstractSource extends AbstractComponent implement } = this.config; if(scrobbleBacklog === false) { - this.logger.info('Source is able to scrobble backlog but was it disabled by user.'); + logger.info('Source is able to scrobble backlog but was it disabled by user.'); this.setStatus('Not scrobbling backlog because it was disabled by user'); return; } - this.logger.info('Discovering backlogged tracks from recently played API...'); + logger.info('Discovering backlogged tracks from recently played API...'); this.setStatus('Discovering backlogged tracks from recently played API...'); let backlogPlays: PlayObject[]; const { @@ -561,18 +562,18 @@ export default abstract class AbstractSource extends AbstractComponent implement } = this.config.options || {}; let backlogLimit = scrobbleBacklogCount; if(backlogLimit > this.SCROBBLE_BACKLOG_COUNT) { - this.logger.warn(`scrobbleBacklogCount (${scrobbleBacklogCount}) cannot be greater than max API limit (${this.SCROBBLE_BACKLOG_COUNT}), reverting to max...`); + logger.warn(`scrobbleBacklogCount (${scrobbleBacklogCount}) cannot be greater than max API limit (${this.SCROBBLE_BACKLOG_COUNT}), reverting to max...`); backlogLimit = this.SCROBBLE_BACKLOG_COUNT; } try { - this.logger.verbose(`Fetching the last ${backlogLimit}${backlogLimit === this.SCROBBLE_BACKLOG_COUNT ? ' (max) ' : ''} listens to check for backlogging...`); + logger.verbose(`Fetching the last ${backlogLimit}${backlogLimit === this.SCROBBLE_BACKLOG_COUNT ? ' (max) ' : ''} listens to check for backlogging...`); backlogPlays = (await this.getBackloggedPlays({limit: backlogLimit})).map((x) => ({...x, meta: {...x.meta, parsedFrom: PARSED_FROM.backlog}})); signal.throwIfAborted(); } catch (e) { throw new Error('Error occurred while fetching backlogged plays', {cause: e}); } await this.queuePlay(backlogPlays); - this.logger.info('Backlog Plays added to discovery queue.'); + logger.info('Backlog Plays added to discovery queue.'); } return; } @@ -630,7 +631,7 @@ export default abstract class AbstractSource extends AbstractComponent implement fork(async (fSignal) => { try { - await this.processBacklog(fSignal); + await this.processBacklog(fSignal, 'Startup'); } catch (e) { throwIfAborted(fSignal); await this.notify({ diff --git a/src/backend/sources/SpotifySource.ts b/src/backend/sources/SpotifySource.ts index 6090ff63..4e3f76a3 100644 --- a/src/backend/sources/SpotifySource.ts +++ b/src/backend/sources/SpotifySource.ts @@ -40,6 +40,7 @@ import { MemoryPositionalSource } from "./MemoryPositionalSource.ts"; import { baseFormatPlayObj } from "../utils/PlayTransformUtils.ts"; import { createGetScrobblesForTimeRangeFunc } from "../utils/ListenFetchUtils.ts"; import { AuthError, SimpleError } from "../common/errors/MSErrors.ts"; +import { AsyncTask, SimpleIntervalJob } from "toad-scheduler"; const scopes = ['user-read-recently-played', 'user-read-currently-playing', 'user-read-playback-state', 'user-read-playback-position']; const state = 'random'; @@ -677,6 +678,34 @@ export default class SpotifySource extends MemoryPositionalSource implements Pag return true; } + public override initTasks(opts: {deadDelay?: number} = {}) { + super.initTasks(opts); + const { scrobbleBacklog = true } = this.config.options ?? {}; + if(!this.scheduler.existsById('reconcile') && scrobbleBacklog) { + this.logger.info('Adding Backlog Reconcile Task'); + this.scheduler.addSimpleIntervalJob(new SimpleIntervalJob({ + minutes: 15, + runImmediately: false + }, new AsyncTask( + 'Reconcile', + (): Promise => { + if(this.isReady()) { + return this.processBacklog(new AbortController().signal, 'Reconcile').then(() => null).catch((err) => { + this.errors.push(err); + this.logger.error(err); + }); + } + this.logger.info('Not running reconcile job because Source is not ready.'); + return Promise.resolve(); + }, + (err: Error) => { + this.logger.error(err); + this.errors.push(err); + } + ), {id: 'reconcile'})); + } + } + protected getBackloggedPlays = async (options: RecentlyPlayedOptions = {}) => await this.getPlayHistory({formatted: true, ...options}) } diff --git a/src/backend/tests/spotify/spotify.test.ts b/src/backend/tests/spotify/spotify.test.ts index bc5a5e4d..deec6174 100644 --- a/src/backend/tests/spotify/spotify.test.ts +++ b/src/backend/tests/spotify/spotify.test.ts @@ -4,21 +4,20 @@ import { expect } from 'chai'; import EventEmitter from "events"; import sinon from 'sinon'; import clone from 'clone'; +import { SimpleIntervalJob } from 'toad-scheduler'; import SpotifySource from "../../sources/SpotifySource.ts"; -import type { SpotifySourceConfig } from "../../common/infrastructure/config/source/spotify.ts"; +import { envSchemas, type SpotifySourceConfig } from "../../common/infrastructure/config/source/spotify.ts"; import currentlyPlayingNoIsrcPayload from '../plays/spotifyCurrentlyPlayingNoIsrc.json' with { type: "json" }; import playbackState from '../plays/spotifyCurrentPlaybackState.json' with { type: "json" }; -const createSpotifySource = (enrichIsrc?: boolean): SpotifySource => { +const createSpotifySource = (options: SpotifySourceConfig['options'] = {}): SpotifySource => { const config = { id: `test-${Date.now()}-${Math.random()}`, data: { clientId: 'test-client', clientSecret: 'test-secret', }, - options: { - enrichIsrc - } + options } as unknown as SpotifySourceConfig; return new SpotifySource('test', config, { localUrl: new URL('http://test'), configDir: 'test', logger: loggerTest, version: 'test' }, new EventEmitter()); @@ -70,7 +69,7 @@ describe('Spotify - ISRC Enrichment', function () { const payload = clone(currentlyPlayingNoIsrcPayload); payload.item.id = 'track-disabled'; - const source = createSpotifySource(false); + const source = createSpotifySource({ enrichIsrc: false }); const getTrackStub = sinon.stub(); (source as any).spotifyApi = { getMyCurrentPlayingTrack: sinon.stub().resolves({ body: payload }), @@ -117,3 +116,94 @@ describe('Spotify - ISRC Enrichment', function () { expect(play?.data.track).to.equal('The Sandpits Of Zonhoven'); }); }); + +describe('Spotify - Backlog Reconcile Task', function () { + + afterEach(function () { + sinon.restore(); + }); + + it('adds reconcile task to scheduler by default with 15 minute interval', function () { + const source = createSpotifySource(); + source.initTasks(); + + expect(source.scheduler.existsById('reconcile')).to.be.true; + const job = source.scheduler.getById('reconcile') as SimpleIntervalJob; + expect((job as any).schedule).to.deep.equal({ minutes: 15, runImmediately: false }); + }); + + it('adds reconcile task when scrobbleBacklog is explicitly true', function () { + const source = createSpotifySource({ scrobbleBacklog: true }); + source.initTasks(); + + expect(source.scheduler.existsById('reconcile')).to.be.true; + }); + + it('does not add reconcile task when scrobbleBacklog is false', function () { + const source = createSpotifySource({ scrobbleBacklog: false }); + source.initTasks(); + + expect(source.scheduler.existsById('reconcile')).to.be.false; + }); + + it('does not add duplicate task if initTasks is called again', function () { + const source = createSpotifySource(); + source.initTasks(); + source.initTasks(); + + expect(source.scheduler.getAllJobs().filter(j => j.id === 'reconcile')).to.have.lengthOf(1); + }); + + it('reconcile task executes processBacklog with Reconcile label when source is ready', async function () { + const source = createSpotifySource(); + const processStub = sinon.stub(source as any, 'processBacklog').resolves(); + sinon.stub(source, 'isReady').returns(true); + + source.initTasks(); + const job = source.scheduler.getById('reconcile') as SimpleIntervalJob; + await (job as any).task.executeAsync(); + + expect(processStub.calledOnce).to.be.true; + expect(processStub.firstCall.args[1]).to.equal('Reconcile'); + }); + + it('reconcile task does not execute processBacklog when source is not ready', async function () { + const source = createSpotifySource(); + const processStub = sinon.stub(source as any, 'processBacklog').resolves(); + sinon.stub(source, 'isReady').returns(false); + + source.initTasks(); + const job = source.scheduler.getById('reconcile') as SimpleIntervalJob; + await (job as any).task.executeAsync(); + + expect(processStub.called).to.be.false; + }); + + it('reconcile task pushes to errors when processBacklog fails', async function () { + const source = createSpotifySource(); + const error = new Error('Spotify API error'); + sinon.stub(source as any, 'processBacklog').rejects(error); + sinon.stub(source, 'isReady').returns(true); + + source.initTasks(); + const job = source.scheduler.getById('reconcile') as SimpleIntervalJob; + await (job as any).task.executeAsync(); + + expect(source.errors).to.include(error); + }); +}); + +describe('Spotify - Configuration', function () { + + it('Parses SPOTIFY_SCROBBLE_BACKLOG from ENV', function () { + const parsed = envSchemas.toConfig({ + SPOTIFY_CLIENT_ID: 'cid', + SPOTIFY_CLIENT_SECRET: 'csec', + SPOTIFY_REDIRECT_URI: 'http://localhost/callback', + SPOTIFY_SCROBBLE_BACKLOG: false, + }); + + expect(parsed.options?.scrobbleBacklog).to.be.false; + }); +}); + -- 2.51.2