diff --git a/docsite/docs/configuration/clients/clients.mdx b/docsite/docs/configuration/clients/clients.mdx
index e7468a2d..0a040a70 100644
--- a/docsite/docs/configuration/clients/clients.mdx
+++ b/docsite/docs/configuration/clients/clients.mdx
@@ -23,6 +23,71 @@ A **Client** is an application that stores the historical information about what
## Features
+### Duplicate Detection
+
+Before a Play is scrobbled to a client MS checks existing scrobbles from the Client's API to see if the Play has already been scrobbled.
+
+For each Play, MS fetches (cached) scrobbles from the Client in a time range inclusive of the Play's listening timestamp and then scores existing scrobbles against the Play based on:
+
+* Similarity of Title, Artists, and Album
+* Temporal closeness of the Play's timestamp to the existing scrobble's timestamp
+* Whether MS detected the Play as a repeat (applicable to Sources that report realtime Player data)
+
+Detailed scoring breakdowns against each existing scrobble are logged at the `TRACE` level.
+
+
+
+Detailed Explanation
+
+A **candidate** (to be scrobbled) Play is first transformed using the configured [`compare.candidate` Hook](/configuration/transforms/#hook), if any exists.
+
+Next, MS checks (up to) the last 100 scrobbles *in-memory* scrobbles that it has made. These are *not* from the Client but the actual scrobbles MS made while it has been running. The data from these scrobbles is much richer than what is usually parsed from the Client which makes it easier to detect duplicates from.
+
+If no in-memory scrobble matches then MS starts comparing the candidate against historical scrobbles fetched from an inclusive time range of the candidate's timestamp.
+
+Matching Title/Artist/Album
+
+For these string-based values MS uses an [token-order-invariant](https://github.com/FoxxMD/multi-scrobbler/blob/0dfa4c7aad6df98e13aee6d395827665c2414adb/src/backend/utils/StringUtils.ts#L299) method that scores string similarity based on a (token count) weighted average of two [similarity](https://foxxmd.github.io/string-sameness/#md:strategies) algorithms, [Levenshtien Distance](https://en.wikipedia.org/wiki/Levenshtein_distance) and [Dice's Coefficient](https://en.wikipedia.org/wiki/S%C3%B8rensen%E2%80%93Dice_coefficient). The weighted average ensures that very long strings (Track titles) are scored with a confidence proportional to their length.
+
+Matching Timestamp
+
+Timestamps are scored on how temporally close they are. There are four possible scores with decreasing value:
+
+* **Exact** - Timestamps are within 1 second of each other
+* **Close** - Timestamps are within `threshold` seconds of each other
+ * This is determined by the smallest update interval of Source
+ * Some Sources (subsonic) only update every 60 seconds so this is the smallest "close" value possible. Most are 10 seconds. Signified by `(Needed <10s)` in the breakdown example below.
+* **Fuzzy** - One timestamp is within `threshold` seconds of the *end* of the other timestamp
+ * Sources can set the scrobble timestamp at different times. Some do it when the track is started listening to, some when it ends, some when the *player* stops.
+ * Where possible, MS knows and keeps track of when this timestamp *should* be, for each Source. If it's not possible then Fuzzy may be allowed.
+* **None** - There is no correlation between timestamps
+
+Scoring and Breakdows
+
+A candidate Play must score >= 1 to be detected as a duplicate of an existing scrobble.
+
+Each score and a breakdown of the scores for its individual components can be see at the `TRACE` logging level or in the [debug data](/help#copy-play-debug-data) for a scrobble. An example:
+
+```
+* Artist: 0.06 * 0.3 = 0.02
+* Title: 0.04 * 0.4 = 0.02
+* Time: (Exact) 1 * 0.5 = 0.50
+ * Existing: 19:13:33-04:00 - Candidate: 19:13:33-04:00
+ * Temporal Sameness: Exact
+ * Play Diff: 0s (Needed <10s)
+ * Range Comparison N/A
+Score 0.54 => No Match
+```
+
+In each component equation, the first number is the similarity (or temporal closeness).
+
+* 0 = no correlation
+* 1 = exactly the same
+
+The second number is the *weight* of that component in the final score.
+
+
+
### Dead Scrobbles
If multi-scrobbler is unable to submit a scrobble to a Client then it places the scrobble into a queue which is retried every 5 minutes for a number of times before it gives up.
--
2.51.2
From 358355c612a01c2c841e51c9ef2fc6d8189107cb Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Wed, 25 Mar 2026 00:41:10 +0000
Subject: [PATCH 2/3] refactor: Move stagger config responsibility to
transformer
---
.../common/transforms/AbstractTransformer.ts | 3 ++
.../transforms/MusicbrainzTransformer.ts | 4 ++
.../common/transforms/TransformerManager.ts | 28 +++--------
src/backend/ioc.ts | 5 +-
.../scrobblers/AbstractScrobbleClient.ts | 47 +++++++++++++++++--
src/backend/sources/AbstractSource.ts | 46 +++++++++++++++---
src/backend/tests/setup.ts | 2 +-
src/backend/utils/AsyncUtils.ts | 4 +-
8 files changed, 99 insertions(+), 40 deletions(-)
diff --git a/src/backend/common/transforms/AbstractTransformer.ts b/src/backend/common/transforms/AbstractTransformer.ts
index 99916a93..65343f7d 100644
--- a/src/backend/common/transforms/AbstractTransformer.ts
+++ b/src/backend/common/transforms/AbstractTransformer.ts
@@ -9,6 +9,7 @@ import { hashObject } from "../../utils/StringUtils.js";
import { playContentInvariantTransform } from "../../utils/PlayComparisonUtils.js";
import { isSimpleError, SkipTransformStageError, StagePrerequisiteError } from "../errors/MSErrors.js";
import { capitalize } from "../../../core/StringUtils.js";
+import { StaggerOptions } from "../../utils/AsyncUtils.js";
export interface TransformerOptions {
logger: Logger
@@ -34,6 +35,8 @@ export default abstract class AbstractTransformer = { initialInterval: 0, maxRandomStagger: 0};
+
public constructor(config: TransformerCommon, options: TransformerOptions) {
super(config);
this.name = config.name;
diff --git a/src/backend/common/transforms/MusicbrainzTransformer.ts b/src/backend/common/transforms/MusicbrainzTransformer.ts
index 92e8f288..4eb841a0 100644
--- a/src/backend/common/transforms/MusicbrainzTransformer.ts
+++ b/src/backend/common/transforms/MusicbrainzTransformer.ts
@@ -355,6 +355,10 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer {
diff --git a/src/backend/common/transforms/TransformerManager.ts b/src/backend/common/transforms/TransformerManager.ts
index 319eb0e4..47bf1155 100644
--- a/src/backend/common/transforms/TransformerManager.ts
+++ b/src/backend/common/transforms/TransformerManager.ts
@@ -110,7 +110,11 @@ export default class TransformerManager {
return this.transformers.has(type);
}
- protected getTransformerByStage(data: StageConfig): AbstractTransformer {
+ public getTransformerType(type: string): AbstractTransformer[] | undefined {
+ return this.transformers.get(type);
+ }
+
+ public getTransformerByStage(data: StageConfig): AbstractTransformer {
const list = this.transformers.get(data.type);
if (list === undefined || list.length === 0) {
throw new Error(`No transformer of type '${data.type}' is registered.`);
@@ -143,27 +147,7 @@ export default class TransformerManager {
}
public async handleStage(data: StageConfig, play: PlayObject, asyncId: string = nanoid(6)): Promise<[PlayObject, string]> {
- const list = this.transformers.get(data.type);
- if (list === undefined || list.length === 0) {
- throw new Error(`No transformer of type '${data.type}' is registered.`);
- }
-
- let t: AbstractTransformer;
- if (list.length > 1) {
- if(data.name === undefined) {
- this.logger.warn(`More than one '${data.type}' transformer but name was not specified, using first registered`);
- t = list[0];
- } else {
- const named = list.find(x => x.name === data.name);
- if(named === undefined) {
- throw new Error(`No ${data.type} transformer with name '${data.name}'`)
- }
- t = named;
- }
- } else {
- t = list[0];
- }
-
+ const t: AbstractTransformer = this.getTransformerByStage(data);
try {
const transformedPlay = await this.asyncStore.run(asyncId, async () => {
return await t.handle(data, play);
diff --git a/src/backend/ioc.ts b/src/backend/ioc.ts
index 2df8c8aa..9bd2da09 100644
--- a/src/backend/ioc.ts
+++ b/src/backend/ioc.ts
@@ -27,7 +27,6 @@ export interface RootOptions {
cache?: CacheConfigOptions | MSCache | (() => MSCache)
mbMap?: MusicBrainzSingletonMap | (() => MusicBrainzSingletonMap)
transformers?: TransformerCommonConfig[]
- staggerOptions?: Partial
}
const discovered = new prom.Counter({
@@ -61,8 +60,7 @@ const createRoot = (options: RootOptions = {logger: loggerDebug}) => {
logger,
cache,
mbMap,
- transformers = [],
- staggerOptions,
+ transformers = []
} = options || {};
const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`);
let disableWeb = dw;
@@ -150,7 +148,6 @@ const createRoot = (options: RootOptions = {logger: loggerDebug}) => {
cache: () => maybeSingletonCache !== undefined ? () => maybeSingletonCache : cacheFunc,
mbMap: () => maybeSingletonMb !== undefined ? () => maybeSingletonMb : mbFunc,
coverArtApi,
- staggerOptions: staggerOptions ?? {},
}).add((items) => {
const localUrl = generateBaseURL(baseUrl, items.port)
return {
diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts
index 542ae800..48f6e19e 100644
--- a/src/backend/scrobblers/AbstractScrobbleClient.ts
+++ b/src/backend/scrobblers/AbstractScrobbleClient.ts
@@ -125,6 +125,10 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i
protected problemGauge: Gauge;
protected staggerOpts: Partial;
+ protected staggerMappers = {
+ preCompare: staggerMapper({concurrency: 2}),
+ existing: staggerMapper({concurrency: 2})
+ }
constructor(type: any, name: any, config: CommonClientConfig, notifier: Notifiers, emitter: EventEmitter, logger: Logger) {
super(config);
@@ -190,7 +194,42 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i
existingSubmitted: this.findExistingSubmittedPlayObj
}
this.existingScrobble = (playObjPre: PlayObject, existingScrobbles: PlayObject[], log?: boolean) => existingScrobble(playObjPre, existingScrobbles, existingScrobbleOpts, log);
- this.staggerOpts = getRoot().items.staggerOptions;
+ }
+
+ protected async postCache(): Promise {
+ await super.postCache();
+ this.generateStaggerMappers();
+ }
+
+ protected generateStaggerMappers() {
+ const {
+ preCompare = [],
+ compare: {
+ existing = []
+ } = {}
+ } = this.transformRules;
+
+ if(preCompare.length > 0) {
+ let pcInits: number[] = [0],
+ pcMaxStagger: number[] = [];
+ for(const hook of this.transformRules.preCompare) {
+ const t = this.transformManager.getTransformerByStage({type: hook.type, name: hook.name});
+ pcInits.push(t.staggerOpts?.initialInterval ?? 0);
+ pcMaxStagger.push(t.staggerOpts?.maxRandomStagger ?? 0)
+ }
+ this.staggerMappers.preCompare = staggerMapper({initialInterval: Math.max(...pcInits), maxRandomStagger: Math.max(...pcMaxStagger), concurrency: 2});
+ }
+
+ if(existing.length > 0) {
+ let eInits: number[] = [0],
+ eMaxStagger: number[] = [];
+ for(const hook of this.transformRules.postCompare) {
+ const t = this.transformManager.getTransformerByStage({type: hook.type, name: hook.name});
+ eInits.push(t.staggerOpts?.initialInterval ?? 0);
+ eMaxStagger.push(t.staggerOpts?.maxRandomStagger ?? 0)
+ }
+ this.staggerMappers.existing = staggerMapper({initialInterval: Math.max(...eInits), maxRandomStagger: Math.max(...eMaxStagger), concurrency: 2});
+ }
}
protected getIdentifier() {
@@ -449,8 +488,7 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i
const playObj = await this.transformPlay(playObjPre, TRANSFORM_HOOK.candidate);
- const sm = staggerMapper({...this.staggerOpts, concurrency: 2});
- const dtInvariantMatches = (await pMap(this.scrobbledPlayObjs.data, sm(async x => ({...x, play: await this.transformPlay(x.play, TRANSFORM_HOOK.existing)})), {concurrency: 2}))
+ const dtInvariantMatches = (await pMap(this.scrobbledPlayObjs.data, this.staggerMappers.existing(async x => ({...x, play: await this.transformPlay(x.play, TRANSFORM_HOOK.existing)})), {concurrency: 2}))
.filter(x => playObjDataMatch(playObj, x.play));
if (dtInvariantMatches.length === 0) {
@@ -851,8 +889,7 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i
queueScrobble = async (data: PlayObject | PlayObject[], source: string) => {
const plays = (Array.isArray(data) ? data : [data]).map(x => ({...x, meta: {...x.meta, seenAt: dayjs()}}));
- const sm = staggerMapper({...this.staggerOpts, concurrency: 2});
- for await(const play of pMapIterable(plays, sm(async x => await this.transformPlay(x, TRANSFORM_HOOK.preCompare)), {concurrency: 2})) {
+ for await(const play of pMapIterable(plays, this.staggerMappers.preCompare(async x => await this.transformPlay(x, TRANSFORM_HOOK.preCompare)), {concurrency: 2})) {
try {
const existingQueued = await this.existingScrobble(play, this.queuedScrobbles.map(x => x.play), false);
// want to be very confident of this
diff --git a/src/backend/sources/AbstractSource.ts b/src/backend/sources/AbstractSource.ts
index d2ca2218..6837aec6 100644
--- a/src/backend/sources/AbstractSource.ts
+++ b/src/backend/sources/AbstractSource.ts
@@ -94,7 +94,10 @@ export default abstract class AbstractSource extends AbstractComponent implement
protected discoveredCounter: Counter;
- protected staggerOpts: Partial;
+ protected staggerMappers = {
+ preCompare: staggerMapper({concurrency: 2}),
+ postCompare: staggerMapper({concurrency: 2})
+ }
constructor(type: SourceType, name: string, config: SourceConfig, internal: InternalConfig, emitter: EventEmitter) {
super(config);
@@ -112,7 +115,40 @@ export default abstract class AbstractSource extends AbstractComponent implement
this.emitter = emitter;
this.discoveredCounter = getRoot().items.sourceMetics.discovered;
- this.staggerOpts = getRoot().items.staggerOptions;
+ }
+
+ protected async postCache(): Promise {
+ await super.postCache();
+ this.generateStaggerMappers();
+ }
+
+ protected generateStaggerMappers() {
+ const {
+ preCompare = [],
+ postCompare = [],
+ } = this.transformRules;
+
+ if (preCompare.length > 0) {
+ let pcInits: number[] = [0],
+ pcMaxStagger: number[] = [0];
+ for (const hook of this.transformRules.preCompare) {
+ const t = this.transformManager.getTransformerByStage({ type: hook.type, name: hook.name });
+ pcInits.push(t.staggerOpts?.initialInterval ?? 0);
+ pcMaxStagger.push(t.staggerOpts?.maxRandomStagger ?? 0)
+ }
+ this.staggerMappers.preCompare = staggerMapper({ initialInterval: Math.max(...pcInits), maxRandomStagger: Math.max(...pcMaxStagger), concurrency: 2 });
+ }
+
+ if (postCompare.length > 0) {
+ let postInits: number[] = [0],
+ postMaxStagger: number[] = [0];
+ for (const hook of this.transformRules.postCompare) {
+ const t = this.transformManager.getTransformerByStage({ type: hook.type, name: hook.name });
+ postInits.push(t.staggerOpts?.initialInterval ?? 0);
+ postMaxStagger.push(t.staggerOpts?.maxRandomStagger ?? 0)
+ }
+ this.staggerMappers.postCompare = staggerMapper({ initialInterval: Math.max(...postInits), maxRandomStagger: Math.max(...postMaxStagger), concurrency: 2 });
+ }
}
protected getIdentifier() {
@@ -222,8 +258,7 @@ export default abstract class AbstractSource extends AbstractComponent implement
discover = async (plays: PlayObject[], options: { checkAll?: boolean, [key: string]: any } = {}): Promise => {
const newDiscoveredPlays: PlayObject[] = [];
- const sm = staggerMapper({...this.staggerOpts, concurrency: 2});
- for await(const play of pMapIterable(plays, sm(async x => await this.transformPlay(x, TRANSFORM_HOOK.preCompare)), {concurrency: 2})) {
+ for await(const play of pMapIterable(plays, this.staggerMappers.preCompare(async x => await this.transformPlay(x, TRANSFORM_HOOK.preCompare)), {concurrency: 2})) {
if(!(await this.alreadyDiscovered(play, options))) {
this.addPlayToDiscovered(play);
newDiscoveredPlays.push(play);
@@ -254,9 +289,8 @@ export default abstract class AbstractSource extends AbstractComponent implement
return;
}
newDiscoveredPlays.sort(sortByOldestPlayDate);
- const sm = staggerMapper({...this.staggerOpts, concurrency: 2});
this.emitter.emit('discoveredToScrobble', {
- data: await pMap(newDiscoveredPlays, sm(async (x) => await this.transformPlay(x, TRANSFORM_HOOK.postCompare)), {concurrency: 2}),
+ data: await pMap(newDiscoveredPlays, this.staggerMappers.postCompare(async (x) => await this.transformPlay(x, TRANSFORM_HOOK.postCompare)), {concurrency: 2}),
options: {
...options,
checkTime: newDiscoveredPlays[newDiscoveredPlays.length-1].data.playDate.add(2, 'second'),
diff --git a/src/backend/tests/setup.ts b/src/backend/tests/setup.ts
index 4f143853..ecc3656a 100644
--- a/src/backend/tests/setup.ts
+++ b/src/backend/tests/setup.ts
@@ -2,5 +2,5 @@ import { loggerTest } from '@foxxmd/logging';
import { getRoot } from "../ioc.js";
import { transientCache } from './utils/CacheTestUtils.js';
-const root = getRoot({cache: transientCache, logger: loggerTest, staggerOptions: {initialInterval: 1, maxRandomStagger: 1}});
+const root = getRoot({cache: transientCache, logger: loggerTest});
root.items.cache().init();
\ No newline at end of file
diff --git a/src/backend/utils/AsyncUtils.ts b/src/backend/utils/AsyncUtils.ts
index 9073b87a..949ad901 100644
--- a/src/backend/utils/AsyncUtils.ts
+++ b/src/backend/utils/AsyncUtils.ts
@@ -54,8 +54,8 @@ export interface StaggerOptions {
}
export function staggerMapper(options: StaggerOptions) {
const {
- initialInterval = 300,
- maxRandomStagger = 300,
+ initialInterval = 0,
+ maxRandomStagger = 0,
concurrency
} = options;
let initialStagger = 0;
--
2.51.2
From 22cf95e58cba0fa24f491d5afe4758f0769f12c9 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Wed, 25 Mar 2026 01:37:54 +0000
Subject: [PATCH 3/3] feat(listenbrainz): Use upstream for backlogged
---
src/backend/sources/ListenbrainzSource.ts | 4 ++--
src/backend/utils/ListenFetchUtils.ts | 1 -
2 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/src/backend/sources/ListenbrainzSource.ts b/src/backend/sources/ListenbrainzSource.ts
index 0fba021d..57f0df33 100644
--- a/src/backend/sources/ListenbrainzSource.ts
+++ b/src/backend/sources/ListenbrainzSource.ts
@@ -88,7 +88,7 @@ export default class ListenbrainzSource extends MemorySource {
getUpstreamRecentlyPlayed = async (options: RecentlyPlayedOptions = {}): Promise => {
try {
- return await this.getScrobblesForTimeRange({limit: 20});
+ return await this.getScrobblesForTimeRange({limit: 20, ...options});
} catch (e) {
throw e;
}
@@ -98,7 +98,7 @@ export default class ListenbrainzSource extends MemorySource {
return 'second';
}
- protected getBackloggedPlays = async (options: RecentlyPlayedOptions = {}) => await this.getRecentlyPlayed({formatted: true, ...options})
+ protected getBackloggedPlays = async (options: RecentlyPlayedOptions = {}) => await this.getUpstreamRecentlyPlayed({formatted: true, ...options})
getNewPlayer = (logger: Logger, id: PlayPlatformId, opts: PlayerStateOptions) => new NowPlayingPlayerState(logger, id, opts);
}
diff --git a/src/backend/utils/ListenFetchUtils.ts b/src/backend/utils/ListenFetchUtils.ts
index 435c3f3f..66d1a359 100644
--- a/src/backend/utils/ListenFetchUtils.ts
+++ b/src/backend/utils/ListenFetchUtils.ts
@@ -32,7 +32,6 @@ export const createGetScrobblesForTimeRangeFunc =