diff --git a/config/mixcloud.json.example b/config/mixcloud.json.example index 90e290c8..fedc2347 100644 --- a/config/mixcloud.json.example +++ b/config/mixcloud.json.example @@ -5,8 +5,17 @@ "enable": true, "clients": [], "data": { + // Option 1: monitor a user's public listening history (no authentication needed) "username": "MyUsername", + // Option 2: authenticate with a Mixcloud application to monitor your own + // listening history, including private (pro account) history. + // Create an application at https://www.mixcloud.com/developers/ to get + // these, then use the Authorize button on the source to complete sign-in. + // When used, `username` is optional. + // "clientId": "...", + // "clientSecret": "...", + // optional: polling interval in seconds (default 60) "interval": 60 } diff --git a/docsite/docs/configuration/sources/_env_configs/_mixcloud.md b/docsite/docs/configuration/sources/_env_configs/_mixcloud.md index 59346d35..7c249af6 100644 --- a/docsite/docs/configuration/sources/_env_configs/_mixcloud.md +++ b/docsite/docs/configuration/sources/_env_configs/_mixcloud.md @@ -1,6 +1,8 @@ -| Environmental Variable | Type | Default | Description | -| ------------------------- | ------- | ---------------------- | ------------------------------------------------------------- | -| _**`MIXCLOUD_ID`**_ | string | | A globally unique ID EX `myComponentId` | -| `MIXCLOUD_NAME` | string | Value of `MIXCLOUD_ID` | A vanity name EX `My Cool Component` | -| `MIXCLOUD_ENABLE` | boolean | true | Should this component be used? | -| _**`MIXCLOUD_USERNAME`**_ | string | | Mixcloud username whose listening history should be monitored | \ No newline at end of file +| Environmental Variable | Type | Default | Description | +| ------------------------ | ------- | ---------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| _**`MIXCLOUD_ID`**_ | string | | A globally unique ID EX `myComponentId` | +| `MIXCLOUD_NAME` | string | Value of `MIXCLOUD_ID` | A vanity name EX `My Cool Component` | +| `MIXCLOUD_ENABLE` | boolean | true | Should this component be used? | +| `MIXCLOUD_USERNAME` | string | | Mixcloud username whose listening history should be monitored. Only required when clientId/clientSecret are not defined. | +| `MIXCLOUD_CLIENT_ID` | string | | Client ID of a Mixcloud application, from https://www.mixcloud.com/developers/. When defined (with clientSecret) the source authenticates with OAuth and can monitor private (pro account) listening histories. | +| `MIXCLOUD_CLIENT_SECRET` | string | | Client secret of a Mixcloud application, from https://www.mixcloud.com/developers/. | \ No newline at end of file diff --git a/docsite/docs/configuration/sources/mixcloud.mdx b/docsite/docs/configuration/sources/mixcloud.mdx index da536901..5ecb9e67 100644 --- a/docsite/docs/configuration/sources/mixcloud.mdx +++ b/docsite/docs/configuration/sources/mixcloud.mdx @@ -13,7 +13,7 @@ import ENVConfig from "@site/docs/configuration/sources/_env_configs/_mixcloud.m [**Mixcloud**](https://www.mixcloud.com/) is a platform for DJ mixes, radio shows, and podcasts. -This Source monitors the public [listening history](https://www.mixcloud.com/developers/#listens) of a Mixcloud user and scrobbles newly discovered listens to configured clients. +This Source monitors the [listening history](https://www.mixcloud.com/developers/#listens) of a Mixcloud user and scrobbles newly discovered listens to configured clients. :::info[Mixes, not Tracks] @@ -27,10 +27,24 @@ Mixcloud's public API does not expose tracklists for mixes so each listen is scr ## Setup +There are two ways to configure this Source: + +### Public history with a username + No authentication is required. Your Mixcloud listening history must be **public** (the default), which you can verify by visiting `https://www.mixcloud.com/YourUsername/listens/` while logged out. Your **username** is the last part of your profile URL IE `https://www.mixcloud.com/MyUsername/` => `MyUsername` +### Your own history with an application (including pro accounts) + +If your listening history is private, or you have a pro account, authenticate with a Mixcloud application instead: + +1. [Create an application](https://www.mixcloud.com/developers/) to get a **clientId** and **clientSecret** +2. Set them in the source configuration -- `username` is not needed, the authorized user's listens are monitored +3. Start multi-scrobbler and use the **Authorize** button/link for this Source to complete sign-in at Mixcloud + +The authorize flow redirects to `/api/mixcloud/callback` on the address the multi-scrobbler web UI is served from (default `http://localhost:9078`) -- if multi-scrobbler is behind a reverse proxy, complete sign-in from that address. The access token is stored on your machine and used for future restarts, sign-in is only needed once. + Mixes are long, so this Source polls less frequently than others by default: every 60 seconds, backing off to at most 300 seconds when there is no activity. ## Configuration diff --git a/src/backend/common/infrastructure/config/source/mixcloud.ts b/src/backend/common/infrastructure/config/source/mixcloud.ts index 47cb0128..898217ad 100644 --- a/src/backend/common/infrastructure/config/source/mixcloud.ts +++ b/src/backend/common/infrastructure/config/source/mixcloud.ts @@ -8,14 +8,32 @@ export const mixcloudDataSchema = z.object({ /** * Mixcloud username whose listening history should be monitored * + * Only required when clientId/clientSecret are not defined. When authenticating + * with an application the username is discovered from the authorized user's profile. + * * This is the last part of your profile URL IE https://www.mixcloud.com/MyUsername/ => MyUsername * * @examples ["MyUsername"] * */ - username: z.string().meta({ - description: "Mixcloud username whose listening history should be monitored", + username: z.string().optional().meta({ + description: "Mixcloud username whose listening history should be monitored. Only required when clientId/clientSecret are not defined.", examples: ["MyUsername"] }), + /** + * Client ID of a Mixcloud application, from https://www.mixcloud.com/developers/ + * + * When defined (with clientSecret) the source authenticates with OAuth and can monitor + * private (pro account) listening histories. The username is then optional. + * */ + clientId: z.string().optional().meta({ + description: "Client ID of a Mixcloud application, from https://www.mixcloud.com/developers/. When defined (with clientSecret) the source authenticates with OAuth and can monitor private (pro account) listening histories." + }), + /** + * Client secret of a Mixcloud application, from https://www.mixcloud.com/developers/ + * */ + clientSecret: z.string().optional().meta({ + description: "Client secret of a Mixcloud application, from https://www.mixcloud.com/developers/." + }), }); export type MixcloudData = z.infer; @@ -30,6 +48,8 @@ export type MixcloudSourceConfig = z.infer; const envDataSchema = z.object({ MIXCLOUD_USERNAME: mixcloudDataSchema.shape.username, + MIXCLOUD_CLIENT_ID: mixcloudDataSchema.shape.clientId, + MIXCLOUD_CLIENT_SECRET: mixcloudDataSchema.shape.clientSecret, }); export const envSchemas: EnvSourceSchema = { @@ -37,7 +57,9 @@ export const envSchemas: EnvSourceSchema ({ data: { - username: partial.MIXCLOUD_USERNAME + username: partial.MIXCLOUD_USERNAME, + clientId: partial.MIXCLOUD_CLIENT_ID, + clientSecret: partial.MIXCLOUD_CLIENT_SECRET } }) }; diff --git a/src/backend/server/auth.ts b/src/backend/server/auth.ts index 26524d06..c4b621ca 100644 --- a/src/backend/server/auth.ts +++ b/src/backend/server/auth.ts @@ -3,6 +3,7 @@ import type {Logger} from "@foxxmd/logging"; import type LastfmScrobbler from "../scrobblers/LastfmScrobbler.ts"; import type ScrobbleClients from "../scrobblers/ScrobbleClients.ts"; import type LastfmSource from "../sources/LastfmSource.ts"; +import type MixcloudSource from "../sources/MixcloudSource.ts"; import type ScrobbleSources from "../sources/ScrobbleSources.ts"; import type SpotifySource from "../sources/SpotifySource.ts"; import type YTMusicSource from "../sources/YTMusicSource.ts"; @@ -32,6 +33,15 @@ export const setupAuthRoutes = (app: Express, router: ReturnType findAuthIssue(x) !== undefined}); + source.poll().catch((e) => logger.error(e)); + } else { + if (tokenResult instanceof Error) { + source.replaceErrors(tokenResult, {predicate: (x) => x.message === tokenResult.message}); + source.logger.error(tokenResult); + } else if (typeof tokenResult === 'string') { + const e = new SimpleError(`Token result was unexpected: ${tokenResult}`); + source.replaceErrors(e, {predicate: (x) => x.message === e.message}); + source.logger.error(e); + } + } + } catch (e) { + const err = new SimpleError('Unexpected error while trying to authorize code, or save file', { cause: e }); + source.replaceErrors(err, {predicate: (x) => err.message === x.message}); + source.logger.error(err); + } + res.redirect('/'); + return; } else { // TODO right now all sources requiring source interaction are covered by logic branches (deezer above and spotify here) // but eventually should update all source callbacks to url specific URLS to avoid ambiguity... diff --git a/src/backend/sources/MixcloudSource.ts b/src/backend/sources/MixcloudSource.ts index 85524440..41870fe1 100644 --- a/src/backend/sources/MixcloudSource.ts +++ b/src/backend/sources/MixcloudSource.ts @@ -1,10 +1,15 @@ import dayjs from "dayjs"; import type EventEmitter from "events"; import request from 'superagent'; -import { PARSED_FROM, type PlayObject, type PlayObjectMinimal } from "../../core/Atomic.ts"; +import { COMPONENT_AUTH_TYPE, PARSED_FROM, type PlayObject, type PlayObjectMinimal } from "../../core/Atomic.ts"; +import type { ComponentAuthType } from "../../core/Atomic.ts"; import { artistNamesToCredits } from "../../core/StringUtils.ts"; +import { AuthError, SimpleError } from "../common/errors/MSErrors.ts"; import type { FormatPlayObjectOptions, InternalConfig } from "../common/infrastructure/Atomic.ts"; import type { MixcloudListen, MixcloudSourceConfig } from "../common/infrastructure/config/source/mixcloud.ts"; +import { writeFile } from "../utils/FSUtils.ts"; +import { readJson } from "../utils/DataUtils.ts"; +import { joinedUrl } from "../utils/NetworkUtils.ts"; import { sortByOldestPlayDate } from "../utils.ts"; import { baseFormatPlayObj } from "../utils/PlayTransformUtils.ts"; import AbstractSource, { type RecentlyPlayedOptions } from "./AbstractSource.ts"; @@ -12,9 +17,27 @@ import AbstractSource, { type RecentlyPlayedOptions } from "./AbstractSource.ts" export default class MixcloudSource extends AbstractSource { baseUrl = 'https://api.mixcloud.com'; + oauthBaseUrl = 'https://www.mixcloud.com/oauth'; declare config: MixcloudSourceConfig; + override authType: ComponentAuthType; + override requiresAuth: boolean = false; + override requiresAuthInteraction: boolean = false; + + /** + * Access token received from the OAuth authorization flow, when the source is + * configured with app credentials instead of a username + * */ + accessToken?: string; + /** + * Username of the OAuth authorized user, resolved from /me/ — used for API + * paths in place of a configured username + * */ + authUsername?: string; + redirectUri?: string; + workingCredsPath?: string; + constructor(name: any, config: MixcloudSourceConfig, internal: InternalConfig, emitter: EventEmitter) { const { data: { @@ -29,6 +52,22 @@ export default class MixcloudSource extends AbstractSource { this.supportsUpstreamRecentlyPlayed = true; // https://www.mixcloud.com/developers/#pagination this.SCROBBLE_BACKLOG_COUNT = 50; + + // app credentials mean an OAuth flow against the authorized user's own + // listens (/me/), which works for both public and pro accounts -- + // otherwise the public history of a configured username is monitored and no auth is needed + const {clientId, clientSecret} = this.config.data ?? {}; + if (clientId !== undefined && clientSecret !== undefined) { + this.authType = COMPONENT_AUTH_TYPE.interactive; + this.requiresAuth = true; + this.requiresAuthInteraction = true; + this.workingCredsPath = `${this.configDir}/currentCreds-${name}.json`; + const u = joinedUrl(this.localUrl, 'api/mixcloud/callback'); + u.searchParams.append('name', this.name); + this.redirectUri = u.toString(); + } else { + this.authType = COMPONENT_AUTH_TYPE.none; + } } static formatPlayObj(obj: MixcloudListen, options: FormatPlayObjectOptions = {}): PlayObject { @@ -72,24 +111,116 @@ export default class MixcloudSource extends AbstractSource { } protected async doBuildInitData(): Promise { - if (this.config.data.username === undefined || this.config.data.username.trim() === '') { - throw new Error('username must be defined'); + const {username, clientId, clientSecret} = this.config.data ?? {}; + if (clientId !== undefined || clientSecret !== undefined) { + if (clientId === undefined || clientSecret === undefined) { + throw new Error('clientId and clientSecret must both be defined'); + } + } else if (username === undefined || username.trim() === '') { + throw new Error('username must be defined when clientId/clientSecret are not'); + } + + if (this.workingCredsPath !== undefined) { + const creds = await readJson(this.workingCredsPath, {throwOnNotFound: false, interpolateEnvs: false}) as any; + this.accessToken = creds?.token; + if (this.accessToken === undefined) { + this.logger.info('No access token is present. User interaction for authentication is required.'); + } + this.logger.info(`Redirect URL that will be used on auth callback: '${this.redirectUri}'`); } return true; } protected async doCheckConnection(): Promise { - // also verifies the configured user exists - await this.callApi(`/${encodeURIComponent(this.config.data.username)}/`); + if (this.accessToken !== undefined) { + // also verifies the access token is accepted + await this.callApi('/me/'); + } else { + // also verifies the configured user exists (validated at init) + await this.callApi(`/${encodeURIComponent(this.config.data.username!)}/`); + } return true; } + doAuthentication = async () => { + try { + if (this.accessToken === undefined) { + throw new AuthError('Cannot use API until an access token has been received from the authorization flow.', {unrecoverable: false}); + } + await this.resolveAuthUser(); + return true; + } catch (e: any) { + if(e instanceof AuthError) { + throw e; + } + // callApi wraps superagent errors, where the status code lives + const status: number | undefined = e?.cause?.status ?? e?.status; + throw new AuthError('Failed to authenticate', {cause: e, unrecoverable: status !== undefined && [401, 403].includes(status)}); + } + } + + createAuthUrl = () => { + const u = new URL(`${this.oauthBaseUrl}/authorize`); + u.searchParams.set('client_id', this.config.data.clientId!); + u.searchParams.set('redirect_uri', this.redirectUri!); + return u.toString(); + } + + handleAuthCodeCallback = async ({ + error, + code + }: any) => { + try { + if (error !== undefined) { + this.logger.warn('Callback contained an error! User may have denied access?') + this.logger.error(error); + return error; + } + // https://www.mixcloud.com/developers/#authorization + const resp = await request.get(`${this.oauthBaseUrl}/access_token`) + .query({ + client_id: this.config.data.clientId, + client_secret: this.config.data.clientSecret, + redirect_uri: this.redirectUri, + code + }); + const token = resp.body?.access_token; + if (typeof token !== 'string' || token === '') { + throw new SimpleError('Access token response from Mixcloud did not include a token', {cause: new Error(`Unexpected response body: ${JSON.stringify(resp.body)}`)}); + } + this.accessToken = token; + await writeFile(this.workingCredsPath!, JSON.stringify({ + token + })); + this.logger.info('Got token from code grant authorization!'); + await this.resolveAuthUser(); + return true; + } catch (e) { + throw e; + } + } + + /** + * Verify the current access token and remember the username it belongs to + * */ + protected resolveAuthUser = async () => { + const me = await this.callApi('/me/'); + const {username} = me ?? {}; + if (typeof username !== 'string' || username === '') { + throw new AuthError('Authorized user response from Mixcloud did not include a username', {unrecoverable: false}); + } + this.authUsername = username; + } + getUpstreamRecentlyPlayed = async (options: RecentlyPlayedOptions = {}): Promise => this.getRecentlyPlayed(options) getRecentlyPlayed = async (options: RecentlyPlayedOptions = {}): Promise => { const {limit = 20} = options; this.setStatus('Checking for new Plays'); - const body = await this.callApi(`/${encodeURIComponent(this.config.data.username)}/listens/`, {limit}); + // /me/listens/ includes private (pro account) history while a username path only exposes public history + // https://www.mixcloud.com/developers/#the-me-shortcut + const path = this.accessToken !== undefined ? '/me/listens/' : `/${encodeURIComponent(this.config.data.username!)}/listens/`; + const body = await this.callApi(path, {limit}); const listens: MixcloudListen[] = body.data ?? []; return listens .filter(x => x.listen_time !== undefined) @@ -105,6 +236,9 @@ export default class MixcloudSource extends AbstractSource { callApi = async (path: string, query: Record = {}): Promise => { try { + if (this.accessToken !== undefined) { + query = {access_token: this.accessToken, ...query}; + } // Mixcloud serves JSON as text/javascript so superagent will not parse it by default const resp = await request.get(`${this.baseUrl}${path}`) .query(query) diff --git a/src/backend/tests/mixcloud/mixcloud.test.ts b/src/backend/tests/mixcloud/mixcloud.test.ts index 6a7e29e9..a45c56c6 100644 --- a/src/backend/tests/mixcloud/mixcloud.test.ts +++ b/src/backend/tests/mixcloud/mixcloud.test.ts @@ -1,9 +1,12 @@ import { loggerTest } from "@foxxmd/logging"; import { assert, expect } from 'chai'; import EventEmitter from "events"; +import { mkdtempSync, readFileSync, rmSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; import { describe, it } from 'mocha'; import { http, HttpResponse } from "msw"; -import { PARSED_FROM } from "../../../core/Atomic.ts"; +import { COMPONENT_AUTH_TYPE, PARSED_FROM } from "../../../core/Atomic.ts"; import { artistCreditsToNames } from "../../../core/StringUtils.ts"; import type { MixcloudData, MixcloudListen } from "../../common/infrastructure/config/source/mixcloud.ts"; import MixcloudSource from "../../sources/MixcloudSource.ts"; @@ -18,12 +21,25 @@ const mixcloudJson = (body: object, init: ResponseInit = {}) => new HttpResponse const listens = listensResponse.data as MixcloudListen[]; -const createSource = (data: MixcloudData = { username: 'TestUser' }): MixcloudSource => new MixcloudSource('Test', { +const appData: MixcloudData = { clientId: 'cid', clientSecret: 'secret' }; + +const createSource = (data: MixcloudData = { username: 'TestUser' }, name: string = 'Test'): MixcloudSource => new MixcloudSource(name, { id: `test-${Date.now()}`, data, options: {} }, { localUrl: new URL('http://test'), configDir: 'test', logger: loggerTest, version: 'test' }, new EventEmitter()); +/** A source with app credentials whose credentials file lives in a temp dir */ +const createAuthSource = (): { source: MixcloudSource, cleanup: () => void } => { + const configDir = mkdtempSync(join(tmpdir(), 'ms-mixcloud-test-')); + const source = new MixcloudSource('Test', { + id: `test-${Date.now()}`, + data: appData, + options: {} + }, { localUrl: new URL('http://test'), configDir, logger: loggerTest, version: 'test' }, new EventEmitter()); + return { source, cleanup: () => rmSync(configDir, { recursive: true, force: true }) }; +}; + describe('Mixcloud Source', function () { describe('Play Parsing', function () { @@ -64,6 +80,35 @@ describe('Mixcloud Source', function () { } }); + it('Does not require auth with only a username', async function () { + const source = createSource(); + await source.buildInitData(); + expect(source.authType).to.equal(COMPONENT_AUTH_TYPE.none); + expect(source.requiresAuth).to.be.false; + }); + + it('Fails init when neither username nor app credentials are defined', async function () { + const source = createSource({}); + try { + await source.buildInitData(); + assert.fail('Expected init to throw'); + } catch (e) { + expect(source.buildOK).to.be.false; + expect(((e as Error).cause as Error).message).to.include('username must be defined'); + } + }); + + it('Fails init when clientId is defined without clientSecret', async function () { + const source = createSource({ clientId: 'cid' }); + try { + await source.buildInitData(); + assert.fail('Expected init to throw'); + } catch (e) { + expect(source.buildOK).to.be.false; + expect(((e as Error).cause as Error).message).to.include('clientId and clientSecret must both be defined'); + } + }); + it('Connection check succeeds when user exists', withRequestInterception( [ http.get('https://api.mixcloud.com/TestUser/', () => mixcloudJson({ @@ -104,6 +149,98 @@ describe('Mixcloud Source', function () { )); }); + describe('Authentication', function () { + + it('Requires interactive auth when app credentials are defined', function () { + const { source, cleanup } = createAuthSource(); + try { + expect(source.authType).to.equal(COMPONENT_AUTH_TYPE.interactive); + expect(source.requiresAuth).to.be.true; + expect(source.requiresAuthInteraction).to.be.true; + } finally { + cleanup(); + } + }); + + it('Builds an authorize url with client id and redirect uri', function () { + const { source, cleanup } = createAuthSource(); + try { + const url = new URL(source.createAuthUrl()!); + expect(url.origin + url.pathname).to.equal('https://www.mixcloud.com/oauth/authorize'); + expect(url.searchParams.get('client_id')).to.equal('cid'); + expect(url.searchParams.get('redirect_uri')).to.equal('http://test/api/mixcloud/callback?name=Test'); + } finally { + cleanup(); + } + }); + + it('Authentication fails when no access token has been received', async function () { + const { source, cleanup } = createAuthSource(); + try { + await source.buildInitData(); + try { + await source.testAuth(true); + assert.fail('Expected auth to throw'); + } catch (e) { + expect(((e as Error).cause as Error).message).to.include('access token'); + } + } finally { + cleanup(); + } + }); + + it('Exchanges an oauth code for a token, persists it, and resolves the user', withRequestInterception( + [ + http.get('https://www.mixcloud.com/oauth/access_token', ({ request }) => { + const url = new URL(request.url); + const expected = { + client_id: 'cid', + client_secret: 'secret', + redirect_uri: 'http://test/api/mixcloud/callback?name=Test', + code: 'thecode' + }; + for (const [key, value] of Object.entries(expected)) { + if (url.searchParams.get(key) !== value) { + return HttpResponse.json({ error: `unexpected ${key}` }, { status: 400 }); + } + } + return HttpResponse.json({ access_token: 'tok123' }); + }), + http.get('https://api.mixcloud.com/me/', ({ request }) => { + const url = new URL(request.url); + if (url.searchParams.get('access_token') !== 'tok123') { + return mixcloudJson({ error: { type: 'Test', message: 'unexpected access token' } }, { status: 400 }); + } + return mixcloudJson({ username: 'ProUser', name: 'Pro User' }); + }) + ], + async function () { + const { source, cleanup } = createAuthSource(); + try { + await source.buildInitData(); + const result = await source.handleAuthCodeCallback({ code: 'thecode' }); + expect(result).to.be.true; + expect(source.accessToken).to.equal('tok123'); + expect(source.authUsername).to.equal('ProUser'); + const persisted = JSON.parse(readFileSync(source.workingCredsPath!, 'utf8')); + expect(persisted).to.deep.equal({ token: 'tok123' }); + } finally { + cleanup(); + } + } + )); + + it('Returns the oauth error when the user denies access', async function () { + const { source, cleanup } = createAuthSource(); + try { + const result = await source.handleAuthCodeCallback({ error: 'access_denied' }); + expect(result).to.equal('access_denied'); + } finally { + cleanup(); + } + }); + }); + describe('Recently Played', function () { it('Returns listens as history plays sorted oldest first', withRequestInterception( @@ -141,5 +278,32 @@ describe('Mixcloud Source', function () { expect(plays).to.have.length(0); } )); + + it('Reads listens from the authorized user with an access token', withRequestInterception( + [ + http.get('https://api.mixcloud.com/me/listens/', ({ request }) => { + const url = new URL(request.url); + if (url.searchParams.get('access_token') !== 'tok123') { + return mixcloudJson({ error: { type: 'Test', message: 'unexpected access token' } }, { status: 400 }); + } + if (url.searchParams.get('limit') !== '20') { + return mixcloudJson({ error: { type: 'Test', message: 'unexpected limit' } }, { status: 400 }); + } + return mixcloudJson(listensResponse); + }) + ], + async function () { + const { source, cleanup } = createAuthSource(); + try { + await source.buildInitData(); + source.accessToken = 'tok123'; + const plays = await source.getRecentlyPlayed(); + expect(plays).to.have.length(2); + assert.equal(plays[0].data.track, 'Deep House Mix 2026'); + } finally { + cleanup(); + } + } + )); }); });