diff --git a/src/backend/common/infrastructure/Atomic.ts b/src/backend/common/infrastructure/Atomic.ts index 0f3f3c8d..338be622 100644 --- a/src/backend/common/infrastructure/Atomic.ts +++ b/src/backend/common/infrastructure/Atomic.ts @@ -4,7 +4,7 @@ import { Dayjs, ManipulateType } from "dayjs"; import { Request, Response } from "express"; import { NextFunction, ParamsDictionary, Query } from "express-serve-static-core"; import { FixedSizeList } from 'fixed-size-list'; -import { isPlayObject, PlayMeta, PlayMetaLifecycleless, PlayObject, PlayObjectLifecycleless } from "../../../core/Atomic.js"; +import { isPlayObject, PlayMeta, PlayMetaLifecycleless, PlayObject, PlayObjectLifecycleless, UnixTimestamp } from "../../../core/Atomic.js"; import TupleMap from "../TupleMap.js"; import { MusicBrainzApi } from 'musicbrainz-api'; @@ -407,6 +407,8 @@ export const MBID_VARIOUS_ARTISTS = "89ad4ac3-39f7-470e-963a-56509c546377"; export type MusicBrainzSingletonMap = Map; + + /* https://websocket.org/reference/close-codes/ */ export const WEBSOCKET_CLOSE_CODE_REASONS = { 1000: 'Closed gracefully', @@ -436,9 +438,9 @@ export const WEBSOCKET_CLOSE_CODES_RETRY = [1006,1011,1001];export interface Pag export interface PaginatedTimeRangeOptions { /** Unix timestamp */ - from: number + from: UnixTimestamp /** Unix timestamp */ - to: number + to: UnixTimestamp } export interface PaginatedListensOptions extends PaginatedLimit { @@ -453,6 +455,7 @@ export interface PaginatedListensTimeRangeOptions extends Partial { return 'getPaginatedListens' in obj; } +export interface PaginatedTimeRangeListensResult { + data: PlayObject[]; + meta: PaginatedListensTimeRangeOptions & PaginatedResults; +} export interface PaginatedTimeRangeListens { - getPaginatedTimeRangeListens(params: PaginatedListensTimeRangeOptions): Promise<{data: PlayObject[], meta: PaginatedListensTimeRangeOptions & PaginatedResults}> + getPaginatedTimeRangeListens(params: PaginatedListensTimeRangeOptions): Promise getPaginatedUnitOfTime(): ManipulateType; } @@ -472,12 +479,17 @@ export const hasPaginatedTimeRangeListens = (obj: Object): obj is PaginatedTimeR return 'getPaginatedTimeRangeListens' in obj; } +export interface PagelessTimeRangeListensResult { + data: PlayObject[] + meta: PagelessListensTimeRangeOptions & PaginatedResults +} export interface PagelessTimeRangeListens { - getPagelessTimeRangeListens(params: PagelessListensTimeRangeOptions): Promise<{data: PlayObject[], meta: PagelessListensTimeRangeOptions & PaginatedResults}> + getPagelessTimeRangeListens(params: PagelessListensTimeRangeOptions): Promise + getPaginatedUnitOfTime(): ManipulateType; } export const hasPagelessTimeRangeListens = (obj: Object): obj is PagelessTimeRangeListens => { - return 'getPaginatedTimeRangeListens' in obj; + return 'getPagelessTimeRangeListens' in obj; } export type PaginatedSource = PaginatedListens | PaginatedTimeRangeListens | PagelessTimeRangeListens; \ No newline at end of file diff --git a/src/backend/common/infrastructure/config/client/koito.ts b/src/backend/common/infrastructure/config/client/koito.ts index e9a85230..12244f05 100644 --- a/src/backend/common/infrastructure/config/client/koito.ts +++ b/src/backend/common/infrastructure/config/client/koito.ts @@ -1,3 +1,4 @@ +import { UnixTimestamp } from "../../../../../core/Atomic.js" import { RequestRetryOptions } from "../common.js" import { CommonClientConfig, CommonClientData } from "./index.js" @@ -15,6 +16,18 @@ export interface ListenObjectResponse { track: TrackResponse } +export interface GetListensOptions { + limit?: number + page?: number + week?: number + month?: number + year?: number + + // new in 0.1.0 + to?: UnixTimestamp + from?: UnixTimestamp +} + export interface TrackResponse { id: number title: string diff --git a/src/backend/common/vendor/LastfmApiClient.ts b/src/backend/common/vendor/LastfmApiClient.ts index 6a2ed989..9cbf923c 100644 --- a/src/backend/common/vendor/LastfmApiClient.ts +++ b/src/backend/common/vendor/LastfmApiClient.ts @@ -1,5 +1,5 @@ -import dayjs, { Dayjs } from "dayjs"; -import { BrainzMeta, PlayObject, PlayObjectLifecycleless, ScrobbleActionResult, URLData } from "../../../core/Atomic.js"; +import dayjs, { Dayjs, ManipulateType } from "dayjs"; +import { BrainzMeta, PlayObject, PlayObjectLifecycleless, ScrobbleActionResult, UnixTimestamp, URLData, Writeable } from "../../../core/Atomic.js"; import { nonEmptyStringOrDefault, splitByFirstFound } from "../../../core/StringUtils.js"; import { removeUndefinedKeys, sleep, writeFile } from "../../utils.js"; import { objectIsEmpty, readJson } from '../../utils/DataUtils.js'; @@ -7,11 +7,11 @@ import { isPortReachableConnect, joinedUrl, normalizeWebAddress } from "../../ut import { getScrobbleTsSOCDate } from "../../utils/TimeUtils.js"; import { getNodeNetworkException, isNodeNetworkException } from "../errors/NodeErrors.js"; import { UpstreamError } from "../errors/UpstreamError.js"; -import { AbstractApiOptions, DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions, InternalConfigOptional } from "../infrastructure/Atomic.js"; +import { AbstractApiOptions, DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions, InternalConfigOptional, PaginatedListensTimeRangeOptions, PaginatedTimeRangeListens, PaginatedTimeRangeListensResult } from "../infrastructure/Atomic.js"; import { LastfmData } from "../infrastructure/config/client/lastfm.js"; import AbstractApiClient from "./AbstractApiClient.js"; import { normalizeStr, parseArtistCredits } from "../../utils/StringUtils.js"; -import { LastFMUser, LastFMAuth, LastFMTrack, LastFMUserGetRecentTracksResponse, LastFMBooleanNumber, LastFMUpdateNowPlayingResponse, LastFMUserGetInfoResponse } from 'lastfm-ts-api'; +import { LastFMUser, LastFMAuth, LastFMTrack, LastFMUserGetRecentTracksResponse, LastFMBooleanNumber, LastFMUpdateNowPlayingResponse, LastFMUserGetInfoResponse, LastFMUserGetRecentTracksParams } from 'lastfm-ts-api'; import clone from 'clone'; import { IncomingMessage } from "http"; import { baseFormatPlayObj } from "../../utils/PlayTransformUtils.js"; @@ -37,7 +37,7 @@ export const LIBREFM_PATH = '/2.0/'; export const LASTFM_HOST = 'ws.audioscrobbler.com'; export const LASTFM_PATH = '/2.0'; -export default class LastfmApiClient extends AbstractApiClient { +export default class LastfmApiClient extends AbstractApiClient implements PaginatedTimeRangeListens { user?: string; declare config: LastfmData; @@ -249,24 +249,28 @@ export default class LastfmApiClient extends AbstractApiClient { return await this.getRecentTracks({limit, from, to, page, extended: 1}); } - getRecentTracks = async (options: TracksFetchOptions = {}): Promise => { + getPaginatedUnitOfTime(): ManipulateType { + return 'second'; + } - let resp: LastFMUserGetRecentTracksResponse; - try { - resp = await this.callApi(() => this.userApi.getRecentTracks({ - ...options, - user: this.user, - // sk is required if user has "Hide recent listening" enabled on their profile - sk: this.sessionKey, - extended: 1 - })); + getPaginatedTimeRangeListens = async (fetchOptions: PaginatedListensTimeRangeOptions, options: {includeNowPlaying?: boolean} = {}): Promise => { - const { - recenttracks: { - track: list = [], - } = {} - } = resp; - return list.reduce((acc: any, x: any) => { + const resp = await this.getRecentTracksWithPagination(fetchOptions); + + const {includeNowPlaying = false} = options; + + const { + recenttracks: { + track: list = [], + '@attr': { + total, + totalPages, + page + } + } = {}, + } = resp; + + const plays = list.reduce((acc: any, x: any) => { try { const formatted = formatPlayObj(x); const { @@ -279,7 +283,7 @@ export default class LastfmApiClient extends AbstractApiClient { nowPlaying, } } = formatted; - if (nowPlaying === true) { + if (nowPlaying === true && !includeNowPlaying) { // if the track is "now playing" it doesn't get a timestamp so we can't determine when it started playing // and don't want to accidentally count the same track at different timestamps by artificially assigning it 'now' as a timestamp // so we'll just ignore it in the context of recent tracks since really we only want "tracks that have already finished being played" anyway @@ -292,11 +296,37 @@ export default class LastfmApiClient extends AbstractApiClient { return acc.concat(formatted); } catch (e) { this.logger.warn(new Error(`Failed to format ${this.upstreamName} recently scrobbled track, omitting from time frame check`, { cause: e })); - this.logger.debug('Full api response object:'); - this.logger.debug(x); + this.logger.debug({data: x}, 'Full api response object:'); return acc; } }, []); + + return {data: plays, meta: {...fetchOptions, total: parseInt(total, 10), more: fetchOptions.page < parseInt(totalPages, 10)}}; + + } + + getRecentTracks = async (options: TracksFetchOptions = {}): Promise => { + + const { to, from, extended = 1, ...rest} = options; + + const requestOpts: Writeable = { + ...rest, + // sk is required if user has "Hide recent listening" enabled on their profile + sk: this.sessionKey, + user: this.user, + extended + }; + + if(to !== undefined) { + requestOpts.to = to.toString(); + } + if(from !== undefined) { + requestOpts.from = from.toString(); + } + + let resp: LastFMUserGetRecentTracksResponse; + try { + return await this.callApi(() => this.userApi.getRecentTracks(requestOpts)); } catch (e) { if(e.message.includes('Invalid resource specified')) { // likely the user does not have any scrobbles on their profile yet @@ -304,7 +334,18 @@ export default class LastfmApiClient extends AbstractApiClient { // https://github.com/libre-fm/libre-fm/discussions/91#discussioncomment-15456070 // so we log as a warning and return empty array instead this.logger.warn(new Error('This error occurs when a librefm (and lastfm?) account has no existing scrobbles yet. If you are seeing this warning and this is not the case, please create an issue', {cause: e})); - return []; + return { + recenttracks: { + track: [], + '@attr': { + user: this.user, + totalPages: '0', + total: '0', + page: '1', + perPage: '50' + } + } + } } this.logger.debug(resp); throw e; @@ -406,10 +447,11 @@ export default class LastfmApiClient extends AbstractApiClient { } export interface TracksFetchOptions { - to?: string - from?: string + to?: UnixTimestamp + from?: UnixTimestamp extended?: 0 | 1 page?: number + /** Max 200. Default is 50 */ limit?: number } diff --git a/src/backend/common/vendor/ListenbrainzApiClient.ts b/src/backend/common/vendor/ListenbrainzApiClient.ts index 1cf41bee..504a21be 100644 --- a/src/backend/common/vendor/ListenbrainzApiClient.ts +++ b/src/backend/common/vendor/ListenbrainzApiClient.ts @@ -1,7 +1,7 @@ import { stringSameness } from '@foxxmd/string-sameness'; import dayjs from "dayjs"; import request, { Request, Response } from 'superagent'; -import { BrainzMeta, PlayObject, PlayObjectLifecycleless, ScrobbleActionResult, URLData } from "../../../core/Atomic.js"; +import { BrainzMeta, PlayObject, PlayObjectLifecycleless, ScrobbleActionResult, UnixTimestamp, URLData } from "../../../core/Atomic.js"; import { combinePartsToString, slice } from "../../../core/StringUtils.js"; import { findDelimiters, @@ -14,7 +14,7 @@ import { } from "../../utils/StringUtils.js"; import { getScrobbleTsSOCDate } from "../../utils/TimeUtils.js"; import { UpstreamError } from "../errors/UpstreamError.js"; -import { AbstractApiOptions, DEFAULT_RETRY_MULTIPLIER, DELIMITERS, FormatPlayObjectOptions } from "../infrastructure/Atomic.js"; +import { AbstractApiOptions, DEFAULT_RETRY_MULTIPLIER, DELIMITERS, FormatPlayObjectOptions, PagelessListensTimeRangeOptions, PagelessTimeRangeListens, PagelessTimeRangeListensResult } from "../infrastructure/Atomic.js"; import { ListenBrainzClientData } from "../infrastructure/config/client/listenbrainz.js"; import AbstractApiClient from "./AbstractApiClient.js"; import { getBaseFromUrl, isPortReachableConnect, joinedUrl, normalizeWebAddress } from '../../utils/NetworkUtils.js'; @@ -34,9 +34,29 @@ interface SubmitOptions { export interface ListensResponse { count: number; listens: ListenResponse[]; + latest_listen_ts: number; + oldest_listen_ts: number; } -export class ListenbrainzApiClient extends AbstractApiClient { +export interface UserListensOptions { + /** unix epoch timestamp, listens with listened_at less than (but not including) this value will be returned. */ + max_ts?: UnixTimestamp + /** unix epoch timestamp, listens with listened_at greater than (but not including) this value will be returned. */ + min_ts?: UnixTimestamp + /** number of listens to return. Max is `MAX_ITEMS_PER_GET` + * + * @default 25 + */ + count?: number +} + +/** https://github.com/metabrainz/listenbrainz-server/pull/2572 + * https://github.com/metabrainz/listenbrainz-server/blob/master/listenbrainz/webserver/views/api_tools.py#L48 + */ +const MAX_ITEMS_PER_GET = 1000; +const DEFAULT_ITEMS_PER_GET = 25; + +export class ListenbrainzApiClient extends AbstractApiClient implements PagelessTimeRangeListens { declare config: ListenBrainzClientData; url: URLData; @@ -262,38 +282,67 @@ export class ListenbrainzApiClient extends AbstractApiClient { return playObj; } - getUserListensWithPagination = async (options: { - count?: number; - minTs?: number; - maxTs?: number; - user?: string; - } = {}): Promise => { - const { count = 100, minTs, maxTs, user } = options; + getUserListensWithPagination = async (options: UserListensOptions & {user?: string} = {}): Promise => { + const { count = 100, user } = options; + + try { + /** https://rain0r.github.io/listenbrainz-openapi/#/lbCore/listensForUser + * https://listenbrainz.readthedocs.io/en/latest/users/api/core.html#get--1-user-(mb_username-user_name)-listens + */ + const resp = await this.callApi(request + .get(`${joinedUrl(this.url.url,'1/user', user ?? this.config.username, 'listens')}`) + .timeout({ + response: 15000, + deadline: 30000 + }) + .query({...options, count: Math.min(count, MAX_ITEMS_PER_GET)})); + + const {body: {payload}} = resp as any; + return payload as ListensResponse; + } catch (e) { + throw e; + } + } + + getPagelessTimeRangeListens = async (options: PagelessListensTimeRangeOptions & {user?: string} = {}): Promise => { + const { limit = 100, to, from, user } = options; + + const lzListensOptions: UserListensOptions = { + count: Math.min(limit, MAX_ITEMS_PER_GET), + }; try { - const query: any = { count }; - if (minTs !== undefined) { - query.min_ts = minTs; + if (from !== undefined) { + lzListensOptions.min_ts = from; } - if (maxTs !== undefined) { - query.max_ts = maxTs; + if (to !== undefined) { + lzListensOptions.max_ts = to; } + /** https://rain0r.github.io/listenbrainz-openapi/#/lbCore/listensForUser + * https://listenbrainz.readthedocs.io/en/latest/users/api/core.html#get--1-user-(mb_username-user_name)-listens + */ const resp = await this.callApi(request .get(`${joinedUrl(this.url.url,'1/user', user ?? this.config.username, 'listens')}`) .timeout({ response: 15000, deadline: 30000 }) - .query(query)); + .query(lzListensOptions)); const {body: {payload}} = resp as any; - return payload as ListensResponse; + const lr = payload as ListensResponse; + const more = to > lr.latest_listen_ts; + return {data: lr.listens.map(x => listenResponseToPlay(x)), meta: {...options, total: lr.count, more}}; } catch (e) { throw e; } } + getPaginatedUnitOfTime(): dayjs.ManipulateType { + return 'second'; + } + static formatPlayObj(obj: any, options: FormatPlayObjectOptions): PlayObject { return listenResponseToPlay(obj); diff --git a/src/backend/common/vendor/koito/KoitoApiClient.ts b/src/backend/common/vendor/koito/KoitoApiClient.ts index c6fc9006..efc43309 100644 --- a/src/backend/common/vendor/koito/KoitoApiClient.ts +++ b/src/backend/common/vendor/koito/KoitoApiClient.ts @@ -1,7 +1,7 @@ import dayjs from "dayjs"; import { PlayObject, PlayObjectLifecycleless, ScrobbleActionResult, URLData } from "../../../../core/Atomic.js"; -import { AbstractApiOptions, DEFAULT_RETRY_MULTIPLIER, PaginatedListensTimeRangeOptions, PaginatedTimeRangeListens } from "../../infrastructure/Atomic.js"; -import { KoitoData, ListenObjectResponse, ListensResponse } from "../../infrastructure/config/client/koito.js"; +import { AbstractApiOptions, DEFAULT_RETRY_MULTIPLIER, PaginatedListensTimeRangeOptions, PaginatedTimeRangeListens, PaginatedTimeRangeListensResult } from "../../infrastructure/Atomic.js"; +import { GetListensOptions, KoitoData, ListenObjectResponse, ListensResponse } from "../../infrastructure/config/client/koito.js"; import AbstractApiClient from "../AbstractApiClient.js"; import { getBaseFromUrl, isPortReachableConnect, joinedUrl, normalizeWebAddress } from "../../../utils/NetworkUtils.js"; import request, { Request, Response } from 'superagent'; @@ -128,15 +128,16 @@ export class KoitoApiClient extends AbstractApiClient implements PaginatedTimeRa } } - getUserListens = async (maxTracks: number): Promise => { + getUserListens = async (options: GetListensOptions): Promise => { + const {page = 0} = options; try { const resp = await this.callApi(request .get(`${joinedUrl(this.url.url, '/apis/web/v1/listens')}`) .query({ period: 'all_time', - page: 0, - limit: maxTracks + page, + ...options }) ); const { body } = resp as any; @@ -146,92 +147,39 @@ export class KoitoApiClient extends AbstractApiClient implements PaginatedTimeRa } } - getUserListensWithPagination = async (options: { - count?: number; - minTs?: number; - maxTs?: number; - } = {}): Promise => { - const { count = 100, minTs, maxTs } = options; + getPaginatedTimeRangeListens = async (params: PaginatedListensTimeRangeOptions): Promise => { + let resp: ListensResponse; try { - const query: any = { count }; - if (minTs !== undefined) { - query.min_ts = minTs; - } - if (maxTs !== undefined) { - query.max_ts = maxTs; - } - - const resp = await this.callApi(request - .get(`${joinedUrl(this.url.url, '/apis/listenbrainz/1/user', this.config.username, 'listens')}`) - .timeout({ - response: 15000, - deadline: 30000 - }) - .query(query)); - - const { body: { payload } } = resp as any; - return payload as ListensResponse; - } catch (e) { - throw e; - } + resp = await this.getUserListens(params); + } catch (e) { + throw new Error('Error occurred while getting Koito paginated listens', { cause: e }); } - getPaginatedTimeRangeListens = async (params: PaginatedListensTimeRangeOptions) => { - let dateData: {week?: number, month?: number, year?: number} = {}; - if(params.from !== undefined && params.to !== undefined) { - const from = dayjs.unix(params.from); - const to = dayjs.unix(params.to); - if(from.week() === to.week()) { - from.subtract - dateData = { - year: from.year(), - month: from.month(), - week: from.week() - } - } else if(from.month() === to.month()) { - dateData = { - year: from.year(), - month: from.month(), - } - } else { - dateData = { - year: from.year() - } - } - } - const resp = await this.callApi(request - .get(`${joinedUrl(this.url.url, '/apis/web/v1/listens')}`) - .query({ - page: params.page, - limit: params.limit, - ...dateData - })); - - const r = resp.body as ListensResponse; - return { - data: r.items.map((x => listenObjectResponseToPlay(x))), + data: resp.items.map((x => listenObjectResponseToPlay(x, { url: this.url.url }))), meta: { ...params, - total: r.total_record_count + total: resp.total_record_count, + page: resp.current_page, + more: resp.has_next_page } } } getPaginatedUnitOfTime(): dayjs.ManipulateType { - return 'week'; + return 'second'; } - getRecentlyPlayed = async (maxTracks: number): Promise => { - try { - const resp = await this.getUserListens(maxTracks); - return resp.items.map(x => listenObjectResponseToPlay(x, { url: this.url.url })); - } catch (e) { - this.logger.error(`Error encountered while getting User listens | Error => ${e.message}`); - return []; - } - } + // getRecentlyPlayed = async (maxTracks: number): Promise => { + // try { + // const resp = await this.getUserListens({limit: maxTracks}); + // return resp.items.map(x => listenObjectResponseToPlay(x, { url: this.url.url })); + // } catch (e) { + // this.logger.error(`Error encountered while getting User listens | Error => ${e.message}`); + // return []; + // } + // } submitListen = async (play: PlayObject, options: SubmitOptions = {}): Promise => { const { log = false, listenType = 'single' } = options; diff --git a/src/backend/common/vendor/maloja/MalojaApiClient.ts b/src/backend/common/vendor/maloja/MalojaApiClient.ts index 8c96330d..a782db34 100644 --- a/src/backend/common/vendor/maloja/MalojaApiClient.ts +++ b/src/backend/common/vendor/maloja/MalojaApiClient.ts @@ -5,7 +5,7 @@ import AbstractApiClient from "../AbstractApiClient.js"; import { getBaseFromUrl, isPortReachableConnect, joinedUrl, normalizeWebAddress } from "../../../utils/NetworkUtils.js"; import { MalojaData } from "../../infrastructure/config/client/maloja.js"; import { PlayObject, PlayObjectLifecycleless, ScrobbleActionResult, URLData } from "../../../../core/Atomic.js"; -import { AbstractApiOptions, DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions, PaginatedListensTimeRangeOptions, PaginatedTimeRangeListens } from "../../infrastructure/Atomic.js"; +import { AbstractApiOptions, DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions, PaginatedListensTimeRangeOptions, PaginatedTimeRangeListens, PaginatedTimeRangeListensResult } from "../../infrastructure/Atomic.js"; import { isNodeNetworkException } from "../../errors/NodeErrors.js"; import { isSuperAgentResponseError } from "../../errors/ErrorUtils.js"; import { getNonEmptyVal, parseRetryAfterSecsFromObj, removeUndefinedKeys, sleep } from "../../../utils.js"; @@ -179,30 +179,42 @@ export class MalojaApiClient extends AbstractApiClient implements PaginatedTimeR } } - getRecentScrobbles = async (limit: number) => { - const resp = await this.callApi(request.get(`${this.url.url}/apis/mlj_1/scrobbles?perpage=${limit}`)); - const { - body: { - list = [], - } = {}, - } = resp; - return list.map(formatPlayObj); - } + // getRecentScrobbles = async (limit: number) => { + // const resp = await this.callApi(request.get(`${this.url.url}/apis/mlj_1/scrobbles?perpage=${limit}`)); + // const { + // body: { + // list = [], + // } = {}, + // } = resp; + // return list.map(formatPlayObj); + // } - getPaginatedTimeRangeListens = async (params: PaginatedListensTimeRangeOptions) => { - const resp = await this.callApi(request.get(`${this.url.url}/apis/mlj_1/scrobbles`).query({ + getPaginatedTimeRangeListens = async (params: PaginatedListensTimeRangeOptions): Promise => { + + const opts: RecentlyPlayedRequestOptions = { perpage: params.limit, page: params.page, from: params.from !== undefined ? dayjs.unix(params.from).format('YYYY/MM/DD') : undefined, - to: params.to !== undefined ? dayjs.unix(params.from).format('YYYY/MM/DD') : undefined - })); + until: params.to !== undefined ? dayjs.unix(params.from).format('YYYY/MM/DD') : undefined + }; + + const resp = await this.getScrobbles(opts); return { - data: resp.body.list.map(formatPlayObj), - meta: params + data: resp.list.map(x => formatPlayObj(x)), + meta: {...params, more: resp.pagination.next_page !== null} } } + getScrobbles = async (options: RecentlyPlayedRequestOptions = {}): Promise => { + const resp = await this.callApi(request.get(`${this.url.url}/apis/mlj_1/scrobbles`).query(removeUndefinedKeys(options))); + const { + body + } = resp; + + return body as RecentlyPlayedResponse; + } + getPaginatedUnitOfTime(): ManipulateType { return 'day'; } @@ -456,4 +468,24 @@ export const playToScrobblePayload = (playObj: PlayObject, apiKey?: string): Mal // https://github.com/FoxxMD/multi-scrobbler/issues/454#issuecomment-3806367420 return scrobbleData; +} + +export interface RecentlyPlayedRequestOptions { + page?: number; + perpage?: number; + /** start of timerange, smallest granularity like '2026/02/10 */ + from?: string; + /** end of timerange, smallest granularity like '2026/02/10 */ + until?: string; +} + +export interface RecentlyPlayedResponse { + list: MalojaScrobbleData[]; + pagination: { + next_page: null |string + prev_page: null | string + page: number + perpage: number + } + status: string; } \ No newline at end of file diff --git a/src/backend/scrobblers/KoitoScrobbler.ts b/src/backend/scrobblers/KoitoScrobbler.ts index e32ffdc6..82d1d9de 100644 --- a/src/backend/scrobblers/KoitoScrobbler.ts +++ b/src/backend/scrobblers/KoitoScrobbler.ts @@ -67,7 +67,8 @@ export default class KoitoScrobbler extends AbstractScrobbleClient { } getScrobblesForRefresh = async (limit: number) => { - return await this.api.getRecentlyPlayed(limit); + const resp = await this.api.getPaginatedTimeRangeListens({limit, page: 0 }); + return resp.data; } doScrobble = async (playObj: PlayObject) => { diff --git a/src/backend/scrobblers/LastfmScrobbler.ts b/src/backend/scrobblers/LastfmScrobbler.ts index ba09ff6e..1caabd92 100644 --- a/src/backend/scrobblers/LastfmScrobbler.ts +++ b/src/backend/scrobblers/LastfmScrobbler.ts @@ -59,51 +59,52 @@ export default class LastfmScrobbler extends AbstractScrobbleClient { } getScrobblesForRefresh = async (limit: number) => { - return await this.api.getRecentTracks({limit}); + const {data: plays} = await this.api.getPaginatedTimeRangeListens({limit, page: 1}); + return plays; } - getScrobblesForTimeRange = async (fromDate?: Dayjs, toDate?: Dayjs, limit: number = 1000): Promise => { - const allPlays: PlayObject[] = []; - let currentPage = 1; - const perPage = 200; - - while (allPlays.length < limit) { - const resp = await this.api.getRecentTracksWithPagination({ - page: currentPage, - limit: perPage, - from: fromDate?.unix().toString(), - to: toDate?.unix().toString(), - }); - - const { - recenttracks: { - track: rawTracks = [], - '@attr': pageInfo - } = {} - } = resp; - - if (rawTracks.length === 0) { - break; - } + // getScrobblesForTimeRange = async (fromDate?: Dayjs, toDate?: Dayjs, limit: number = 1000): Promise => { + // const allPlays: PlayObject[] = []; + // let currentPage = 1; + // const perPage = 200; - const plays = resp - .filter(p => p.data.playDate !== undefined && p.data.playDate.isValid()); // Filter out plays with invalid dates + // while (allPlays.length < limit) { + // const resp = await this.api.getRecentTracksWithPagination({ + // page: currentPage, + // limit: perPage, + // from: fromDate?.unix(), + // to: toDate?.unix(), + // }); - allPlays.push(...plays); + // const { + // recenttracks: { + // track: rawTracks = [], + // '@attr': pageInfo + // } = {} + // } = resp; - if (allPlays.length >= limit) { - break; - } + // if (rawTracks.length === 0) { + // break; + // } - if (pageInfo && currentPage >= parseInt(pageInfo.totalPages, 10)) { - break; - } + // const plays = resp + // .filter(p => p.data.playDate !== undefined && p.data.playDate.isValid()); // Filter out plays with invalid dates - currentPage++; - } + // allPlays.push(...plays); - return allPlays.slice(0, limit); - } + // if (allPlays.length >= limit) { + // break; + // } + + // if (pageInfo && currentPage >= parseInt(pageInfo.totalPages, 10)) { + // break; + // } + + // currentPage++; + // } + + // return allPlays.slice(0, limit); + // } cleanSourceSearchTitle = (playObj: PlayObject) => { const { diff --git a/src/backend/scrobblers/ListenbrainzScrobbler.ts b/src/backend/scrobblers/ListenbrainzScrobbler.ts index 9e651c50..3d42d8c2 100644 --- a/src/backend/scrobblers/ListenbrainzScrobbler.ts +++ b/src/backend/scrobblers/ListenbrainzScrobbler.ts @@ -70,43 +70,43 @@ export default class ListenbrainzScrobbler extends AbstractScrobbleClient { return await this.api.getRecentlyPlayed(limit); } - getScrobblesForTimeRange = async (fromDate?: Dayjs, toDate?: Dayjs, limit: number = 1000): Promise => { - const allPlays: PlayObject[] = []; - let maxTs = toDate?.unix(); - const minTs = fromDate?.unix(); - - while (allPlays.length < limit) { - const batchSize = Math.min(100, limit - allPlays.length); - const resp = await this.api.getUserListensWithPagination({ - count: batchSize, - minTs, - maxTs, - }); - - if (!resp.listens || resp.listens.length === 0) { - break; - } - - const plays = resp.listens - .map(l => ListenbrainzApiClient.formatPlayObj(l, {})) - .filter(p => p.data.playDate && p.data.playDate.isValid()); // Filter out plays with invalid dates - - allPlays.push(...plays); - - if (plays.length < batchSize) { - break; - } - - const oldestPlay = plays[plays.length - 1]; - if (oldestPlay.data.playDate) { - maxTs = oldestPlay.data.playDate.unix() - 1; - } else { - break; - } - } - - return allPlays; - } + // getScrobblesForTimeRange = async (fromDate?: Dayjs, toDate?: Dayjs, limit: number = 1000): Promise => { + // const allPlays: PlayObject[] = []; + // let maxTs = toDate?.unix(); + // const minTs = fromDate?.unix(); + + // while (allPlays.length < limit) { + // const batchSize = Math.min(100, limit - allPlays.length); + // const resp = await this.api.getUserListensWithPagination({ + // count: batchSize, + // minTs, + // maxTs, + // }); + + // if (!resp.listens || resp.listens.length === 0) { + // break; + // } + + // const plays = resp.listens + // .map(l => ListenbrainzApiClient.formatPlayObj(l, {})) + // .filter(p => p.data.playDate && p.data.playDate.isValid()); // Filter out plays with invalid dates + + // allPlays.push(...plays); + + // if (plays.length < batchSize) { + // break; + // } + + // const oldestPlay = plays[plays.length - 1]; + // if (oldestPlay.data.playDate) { + // maxTs = oldestPlay.data.playDate.unix() - 1; + // } else { + // break; + // } + // } + + // return allPlays; + // } public playToClientPayload(playObj: PlayObject): ListenPayload { return playToListenPayload(playObj); diff --git a/src/backend/scrobblers/MalojaScrobbler.ts b/src/backend/scrobblers/MalojaScrobbler.ts index 73bb55b4..49d4dced 100644 --- a/src/backend/scrobblers/MalojaScrobbler.ts +++ b/src/backend/scrobblers/MalojaScrobbler.ts @@ -77,7 +77,8 @@ export default class MalojaScrobbler extends AbstractScrobbleClient { } getScrobblesForRefresh = async (limit: number) => { - return await this.api.getRecentScrobbles(limit); + const resp = await this.api.getPaginatedTimeRangeListens({limit, page: 0}); + return resp.data; } public playToClientPayload(playObj: PlayObject): MalojaScrobbleRequestData { diff --git a/src/backend/sources/KoitoSource.ts b/src/backend/sources/KoitoSource.ts index 9f2327c5..12da3548 100644 --- a/src/backend/sources/KoitoSource.ts +++ b/src/backend/sources/KoitoSource.ts @@ -58,12 +58,14 @@ export default class KoitoSource extends MemorySource implements PaginatedTimeRa getRecentlyPlayed = async(options: RecentlyPlayedOptions = {}) => { const {limit = 20} = options; await this.processRecentPlays([]); - return await this.api.getRecentlyPlayed(limit); + const resp = await this.api.getPaginatedTimeRangeListens({limit, page: 0 }); + return resp.data; } getUpstreamRecentlyPlayed = async (options: RecentlyPlayedOptions = {}): Promise => { try { - return await this.api.getRecentlyPlayed(20); + const resp = await this.api.getPaginatedTimeRangeListens({limit: 20, page: 0 }); + return resp.data; } catch (e) { throw e; } diff --git a/src/backend/sources/LastfmSource.ts b/src/backend/sources/LastfmSource.ts index 540b25a5..c8ae792f 100644 --- a/src/backend/sources/LastfmSource.ts +++ b/src/backend/sources/LastfmSource.ts @@ -71,7 +71,7 @@ export default class LastfmSource extends MemorySource implements PaginatedTimeR getLastfmRecentTrack = async(options: RecentlyPlayedOptions = {}): Promise<[PlayObject[], PlayObject[]]> => { const {limit = 20} = options; try { - const plays = await this.api.getRecentTracks({limit}); + const {data: plays} = await this.api.getPaginatedTimeRangeListens({limit, page: 1}); plays.sort(sortByOldestPlayDate); // if the track is "now playing" it doesn't get a timestamp so we can't determine when it started playing // and don't want to accidentally count the same track at different timestamps by artificially assigning it 'now' as a timestamp diff --git a/src/backend/sources/MalojaSource.ts b/src/backend/sources/MalojaSource.ts index fb711fc4..a565c847 100644 --- a/src/backend/sources/MalojaSource.ts +++ b/src/backend/sources/MalojaSource.ts @@ -56,12 +56,14 @@ export default class MalojaSource extends MemorySource implements PaginatedTimeR getRecentlyPlayed = async (options: RecentlyPlayedOptions = {}) => { const { limit = 20 } = options; await this.processRecentPlays([]); - return await this.api.getRecentScrobbles(limit); + const resp = await this.api.getPaginatedTimeRangeListens({limit, page: 0}); + return resp.data; } getUpstreamRecentlyPlayed = async (options: RecentlyPlayedOptions = {}): Promise => { try { - return await this.api.getRecentScrobbles(20); + const resp = await this.api.getPaginatedTimeRangeListens({limit: 20, page: 0}); + return resp.data; } catch (e) { throw e; } diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts index 763bf566..1f2f4f11 100644 --- a/src/core/Atomic.ts +++ b/src/core/Atomic.ts @@ -562,4 +562,9 @@ export const KNOWN_MEDIA_PROVIDER_URLS = [ 'play.google.com', 'listenbrainz.org', 'musicbrainz.org' -]; \ No newline at end of file +]; + +/** Number of SECONDS since 1970 */ +export type UnixTimestamp = number; + +export type Writeable = { -readonly [P in keyof T]: T[P] }; \ No newline at end of file