diff --git a/package-lock.json b/package-lock.json index de9e0322..f5c2f9c8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "dependencies": { "@astronautlabs/mdns": "^1.0.7", "@atcute/identity-resolver-node": "^1.0.3", + "@atcute/tid": "^1.1.1", "@atproto/api": "^0.18.0", "@atproto/oauth-client-node": "^0.3.10", "@awaitjs/express": "^0.6.3", @@ -236,6 +237,26 @@ "esm-env": "^1.2.2" } }, + "node_modules/@atcute/tid": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@atcute/tid/-/tid-1.1.1.tgz", + "integrity": "sha512-djJ8UGhLkTU5V51yCnBEruMg35qETjWzWy5sJG/2gEOl2Gd7rQWHSaf+yrO6vMS5EFA38U2xOWE3EDUPzvc2ZQ==", + "license": "0BSD", + "dependencies": { + "@atcute/time-ms": "^1.0.0" + } + }, + "node_modules/@atcute/time-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@atcute/time-ms/-/time-ms-1.2.1.tgz", + "integrity": "sha512-7tzcYMLz1IQCyneupNP5AMkgfLywzYnsWLTW/n1Ku3vM/LgicGgBVbHKhqP10H8LUz3gEf8DUMrKbTkM/VLUgA==", + "hasInstallScript": true, + "license": "0BSD", + "dependencies": { + "@types/bun": "^1.3.8", + "node-gyp-build": "^4.8.4" + } + }, "node_modules/@atcute/util-fetch": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/@atcute/util-fetch/-/util-fetch-1.0.3.tgz", @@ -3727,6 +3748,15 @@ "@types/node": "*" } }, + "node_modules/@types/bun": { + "version": "1.3.9", + "resolved": "https://registry.npmjs.org/@types/bun/-/bun-1.3.9.tgz", + "integrity": "sha512-KQ571yULOdWJiMH+RIWIOZ7B2RXQGpL1YQrBtLIV3FqDcCu6FsbFUBwhdKUlCKUpS3PJDsHlJ1QKlpxoVR+xtw==", + "license": "MIT", + "dependencies": { + "bun-types": "1.3.9" + } + }, "node_modules/@types/cacheable-request": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.3.tgz", @@ -4988,6 +5018,15 @@ "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" }, + "node_modules/bun-types": { + "version": "1.3.9", + "resolved": "https://registry.npmjs.org/bun-types/-/bun-types-1.3.9.tgz", + "integrity": "sha512-+UBWWOakIP4Tswh0Bt0QD0alpTY8cb5hvgiYeWCMet9YukHbzuruIEeXC2D7nMJPB12kbh8C7XJykSexEqGKJg==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -9893,6 +9932,17 @@ } } }, + "node_modules/node-gyp-build": { + "version": "4.8.4", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz", + "integrity": "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==", + "license": "MIT", + "bin": { + "node-gyp-build": "bin.js", + "node-gyp-build-optional": "optional.js", + "node-gyp-build-test": "build-test.js" + } + }, "node_modules/node-object-hash": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/node-object-hash/-/node-object-hash-3.1.1.tgz", diff --git a/package.json b/package.json index 52fd1aba..9a78586e 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "dependencies": { "@astronautlabs/mdns": "^1.0.7", "@atcute/identity-resolver-node": "^1.0.3", + "@atcute/tid": "^1.1.1", "@atproto/api": "^0.18.0", "@atproto/oauth-client-node": "^0.3.10", "@awaitjs/express": "^0.6.3", diff --git a/src/backend/common/vendor/bluesky/AbstractBlueSkyApiClient.ts b/src/backend/common/vendor/bluesky/AbstractBlueSkyApiClient.ts index 591902b0..4ce9a4b7 100644 --- a/src/backend/common/vendor/bluesky/AbstractBlueSkyApiClient.ts +++ b/src/backend/common/vendor/bluesky/AbstractBlueSkyApiClient.ts @@ -1,22 +1,23 @@ import { getRoot } from "../../../ioc.js"; -import { AbstractApiOptions } from "../../infrastructure/Atomic.js"; +import { AbstractApiOptions, PagelessListensTimeRangeOptions, PagelessTimeRangeListens, PagelessTimeRangeListensResult } from "../../infrastructure/Atomic.js"; import { ListRecord, ScrobbleRecord, TealClientData } from "../../infrastructure/config/client/tealfm.js"; import AbstractApiClient from "../AbstractApiClient.js"; -import { Agent, ComAtprotoRepoCreateRecord } from "@atproto/api"; +import { Agent, ComAtprotoRepoCreateRecord, ComAtprotoRepoListRecords } from "@atproto/api"; import { MSCache } from "../../Cache.js"; import { BrainzMeta, PlayObject, PlayObjectLifecycleless, ScrobbleActionResult } from "../../../../core/Atomic.js"; import { musicServiceToCononical } from "../ListenbrainzApiClient.js"; import { parseRegexSingle } from "@foxxmd/regex-buddy-core"; import { RecordOptions } from "../../infrastructure/config/client/tealfm.js"; -import dayjs from "dayjs"; +import dayjs, { ManipulateType } from "dayjs"; import { getScrobbleTsSOCDateWithContext } from "../../../utils/TimeUtils.js"; import { removeUndefinedKeys } from "../../../utils.js"; import { baseFormatPlayObj } from "../../../utils/PlayTransformUtils.js"; import { ScrobbleSubmitError } from "../../errors/MSErrors.js"; import { UpstreamError } from "../../errors/UpstreamError.js"; +import * as TID from '@atcute/tid'; +import { randomInt } from "node:crypto"; - -export abstract class AbstractBlueSkyApiClient extends AbstractApiClient { +export abstract class AbstractBlueSkyApiClient extends AbstractApiClient implements PagelessTimeRangeListens { declare config: TealClientData; @@ -48,18 +49,41 @@ export abstract class AbstractBlueSkyApiClient extends AbstractApiClient { } } - async listScrobbleRecord(limit: number = 20): Promise[]> { + async listScrobbleRecord(options: {limit?: number, cursor?: string} = {}): Promise { + const {limit = 20, cursor} = options; try { + // records are returned newest to oldest const response = await this.agent.com.atproto.repo.listRecords({ repo: this.agent.sessionManager.did, collection: "fm.teal.alpha.feed.play", - limit + limit, + cursor // cursor TID is EXCLUSIVE IE first record returned will be the first older than cursor }); - return response.data.records as unknown as ListRecord[]; + return response; } catch (e) { throw new UpstreamError(`Failed to list scrobble record`, { cause: e, response: 'response' in e ? e.response : undefined }); } } + + async getPagelessTimeRangeListens(params: PagelessListensTimeRangeOptions): Promise { + const {to, limit} = params; + + let cursor: string; + if(to !== undefined) { + cursor = TID.create(to, randomInt(1023)); + } + + const resp = await this.listScrobbleRecord({cursor, limit}); + const fromTS = TID.parse(resp.data.cursor); + + const plays = (resp.data.records as unknown as ListRecord[]).map(x => listRecordToPlay(x)); + + return {data: plays, meta: {to, from: fromTS.timestamp, limit}}; + } + + getPaginatedUnitOfTime(): ManipulateType { + return 'second'; + } } export const playToRecord = (play: PlayObject): ScrobbleRecord => { diff --git a/src/backend/scrobblers/TealfmScrobbler.ts b/src/backend/scrobblers/TealfmScrobbler.ts index 7fdb700d..0c7613e2 100644 --- a/src/backend/scrobblers/TealfmScrobbler.ts +++ b/src/backend/scrobblers/TealfmScrobbler.ts @@ -3,13 +3,12 @@ import EventEmitter from "events"; import { PlayObject } from "../../core/Atomic.js"; import { buildTrackString, capitalize } from "../../core/StringUtils.js"; import { isNodeNetworkException } from "../common/errors/NodeErrors.js"; -import { hasUpstreamError, UpstreamError } from "../common/errors/UpstreamError.js"; import { FormatPlayObjectOptions } from "../common/infrastructure/Atomic.js"; import { playToListenPayload } from "../common/vendor/ListenbrainzApiClient.js"; import { Notifiers } from "../notifier/Notifiers.js"; import AbstractScrobbleClient from "./AbstractScrobbleClient.js"; -import { ListRecord, ScrobbleRecord, TealClientConfig } from "../common/infrastructure/config/client/tealfm.js"; +import { TealClientConfig } from "../common/infrastructure/config/client/tealfm.js"; import { BlueSkyAppApiClient } from "../common/vendor/bluesky/BlueSkyAppApiClient.js"; import { BlueSkyOauthApiClient } from "../common/vendor/bluesky/BlueSkyOauthApiClient.js"; import { AbstractBlueSkyApiClient, listRecordToPlay, playToRecord, recordToPlay } from "../common/vendor/bluesky/AbstractBlueSkyApiClient.js"; @@ -93,13 +92,12 @@ export default class TealScrobbler extends AbstractScrobbleClient { } getScrobblesForRefresh = async (limit: number) => { - let list: ListRecord[]; try { - list = await this.client.listScrobbleRecord(limit) + const {data} = await this.client.getPagelessTimeRangeListens({limit}) + return data; } catch (e) { throw new Error('Error occurred while trying to fetch records', {cause: e}); } - return list.map(x => listRecordToPlay(x)); } doScrobble = async (playObj: PlayObject) => { diff --git a/src/backend/sources/ListenbrainzSource.ts b/src/backend/sources/ListenbrainzSource.ts index 04a2ef2f..4f1531a6 100644 --- a/src/backend/sources/ListenbrainzSource.ts +++ b/src/backend/sources/ListenbrainzSource.ts @@ -13,7 +13,7 @@ import { PlayerStateOptions } from "./PlayerState/AbstractPlayerState.js"; import { NowPlayingPlayerState } from "./PlayerState/NowPlayingPlayerState.js"; import { ManipulateType } from "dayjs"; -export default class ListenbrainzSource extends MemorySource implements PagelessTimeRangeListens { +export default class ListenbrainzSource extends MemorySource { api: ListenbrainzApiClient; requiresAuth = true; @@ -92,15 +92,6 @@ export default class ListenbrainzSource extends MemorySource implements Pageless } } - getPagelessTimeRangeListens = async (options: PagelessListensTimeRangeOptions) => { - const resp = await this.api.getUserListensWithPagination({ - count: options.limit, - minTs: options.from, - maxTs: options.to - }); - return {data: resp.listens.map(x => ListenbrainzSource.formatPlayObj(x)), meta: {...options, total: resp.count}}; - } - getPaginatedUnitOfTime(): ManipulateType { return 'second'; } diff --git a/src/backend/sources/TealfmSource.ts b/src/backend/sources/TealfmSource.ts index a8f46f44..21bf09b7 100644 --- a/src/backend/sources/TealfmSource.ts +++ b/src/backend/sources/TealfmSource.ts @@ -8,7 +8,6 @@ import { AbstractBlueSkyApiClient, listRecordToPlay, recordToPlay } from "../com import { TealSourceConfig } from "../common/infrastructure/config/source/tealfm.js"; import { BlueSkyAppApiClient } from "../common/vendor/bluesky/BlueSkyAppApiClient.js"; import { BlueSkyOauthApiClient } from "../common/vendor/bluesky/BlueSkyOauthApiClient.js"; -import { ListRecord, ScrobbleRecord } from "../common/infrastructure/config/client/tealfm.js"; import { parseArrayFromMaybeString } from "../utils/StringUtils.js"; export default class TealfmSource extends MemorySource { @@ -107,14 +106,14 @@ export default class TealfmSource extends MemorySource { getRecentlyPlayed = async(options: RecentlyPlayedOptions = {}) => { const {limit = 20} = options; - let list: ListRecord[]; + let plays: PlayObject[]; try { - list = await this.client.listScrobbleRecord(limit) + const {data} = await this.client.getPagelessTimeRangeListens({limit}) + plays = data; } catch (e) { throw new Error('Error occurred while trying to fetch records', {cause: e}); } await this.processRecentPlays([]); - let plays = list.map(x => listRecordToPlay(x)); if(this.serviceAllow.length > 0) { plays = plays.filter(x => (x.meta.musicService !== undefined && this.serviceAllow.some(y => x.meta.musicService.toLocaleLowerCase().includes(y)))