diff --git a/flatpak/io.github.foxxmd.multiscrobbler.yml b/flatpak/io.github.foxxmd.multiscrobbler.yml index 865fcdcc..f6b78ee5 100644 --- a/flatpak/io.github.foxxmd.multiscrobbler.yml +++ b/flatpak/io.github.foxxmd.multiscrobbler.yml @@ -53,10 +53,10 @@ modules: sources: - type: git url: https://github.com/FoxxMD/multi-scrobbler - branch: develop + #branch: develop # in official builds use tag/commit instead of branch -# tag: 0.6.0 -# commit: 7037d6f8332f323312ffc5712f48a1ce96026d97 + tag: 0.6.0 + commit: 7037d6f8332f323312ffc5712f48a1ce96026d97 dest: main # Wrapper to launch the app - type: script diff --git a/package-lock.json b/package-lock.json index 4f5055ea..38757dc7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "multi-scrobbler", - "version": "0.5.2", + "version": "0.6.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "multi-scrobbler", - "version": "0.5.2", + "version": "0.6.0", "license": "MIT", "dependencies": { "@awaitjs/express": "^0.6.3", -- 2.51.2 From eb449eeca68554f2389243064188c50e4899bf77 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 5 Oct 2023 13:58:20 -0400 Subject: [PATCH 02/16] feat: Implement WebScrobbler source --- README.md | 1 + config/subsonic.json.example | 2 +- config/webscrobbler.json.example | 10 + docsite/docs/configuration/configuration.md | 60 ++++++ docsite/src/pages/index.mdx | 1 + src/backend/common/infrastructure/Atomic.ts | 4 +- .../infrastructure/config/source/sources.ts | 6 +- .../config/source/webscrobbler.ts | 68 +++++++ src/backend/common/schema/aio-source.json | 169 +++++++++++++++++ src/backend/common/schema/aio.json | 169 +++++++++++++++++ src/backend/common/schema/source.json | 159 ++++++++++++++++ .../common/vendor/webscrobbler/interfaces.ts | 72 ++++++++ src/backend/server/api.ts | 13 ++ src/backend/server/middleware.ts | 19 ++ src/backend/server/webscrobblerRoutes.ts | 71 ++++++++ src/backend/sources/ScrobbleSources.ts | 25 +++ src/backend/sources/WebScrobblerSource.ts | 172 ++++++++++++++++++ .../ingressNotifiers/WebhookNotifier.ts | 39 ++++ src/backend/utils/StringUtils.ts | 4 + src/core/Atomic.ts | 3 +- 20 files changed, 1061 insertions(+), 6 deletions(-) create mode 100644 config/webscrobbler.json.example create mode 100644 src/backend/common/infrastructure/config/source/webscrobbler.ts create mode 100644 src/backend/common/vendor/webscrobbler/interfaces.ts create mode 100644 src/backend/server/webscrobblerRoutes.ts create mode 100644 src/backend/sources/WebScrobblerSource.ts create mode 100644 src/backend/sources/ingressNotifiers/WebhookNotifier.ts diff --git a/README.md b/README.md index 24b57580..e17fb83a 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ A javascript app to scrobble music you listened to, to [Maloja](https://github.c * [Plex](/docsite/docs/configuration/configuration.md#plex) or [Tautulli](/docsite/docs/configuration/configuration.md#tautulli) * [Subsonic-compatible APIs](/docsite/docs/configuration/configuration.md#subsonic) (like [Airsonic](https://airsonic.github.io/) and [Navidrome](https://www.navidrome.org/)) * [Jellyfin](/docsite/docs/configuration/configuration.md#jellyfin) + * [WebScrobbler](/docsite/docs/configuration/configuration.md#webscrobbler) * [Youtube Music](/docsite/docs/configuration/configuration.md#youtube-music) * [Last.fm](/docsite/docs/configuration/configuration.md#lastfm-source) * [ListenBrainz](/docsite/docs/configuration/configuration.md#listenbrainz--source-) diff --git a/config/subsonic.json.example b/config/subsonic.json.example index 4a01fe1d..e26ebfd9 100644 --- a/config/subsonic.json.example +++ b/config/subsonic.json.example @@ -5,7 +5,7 @@ "data": { "url": "http://localhost:4040/airsonic", "user": "yourUser", - "password": "yourPassword", + "password": "yourPassword" } } ] diff --git a/config/webscrobbler.json.example b/config/webscrobbler.json.example new file mode 100644 index 00000000..1d27837b --- /dev/null +++ b/config/webscrobbler.json.example @@ -0,0 +1,10 @@ +[ + { + "name": "MyWebScrobbler", + "slug": null, + "data": { + "whitelist": [], + "blacklist": [] + } + } +] diff --git a/docsite/docs/configuration/configuration.md b/docsite/docs/configuration/configuration.md index ace31df8..cc1a2289 100644 --- a/docsite/docs/configuration/configuration.md +++ b/docsite/docs/configuration/configuration.md @@ -24,6 +24,7 @@ title: Overview * [Mopidy](#mopidy) * [JRiver](#jriver) * [Kodi](#kodi) + * [WebScrobbler](#webscrobbler) * [Client Configurations](#client-configurations) * [Maloja](#maloja) * [Last.fm](#lastfm) @@ -624,6 +625,65 @@ MS transforms this to: `http://mydomain.com:80/kodiReverse/jsonrpc` See [`kodi.json.example`](https://github.com/FoxxMD/multi-scrobbler/blob/master/config/kodi.json.example) or [explore the schema with an example and live editor/validator](https://json-schema.app/view/%23%2Fdefinitions%2FKodiSourceConfig/%23%2Fdefinitions%2FKodiData?url=https%3A%2F%2Fraw.githubusercontent.com%2FFoxxMD%2Fmulti-scrobbler%2Fdevelop%2Fsrc%2Fcommon%2Fschema%2Fsource.json) +## [WebScrobbler](https://web-scrobbler.com/) + +After installing the extension open the preferences/settings for it: + +* Under **Accounts** + * **Add Webhook** + * API URL: `http://localhost:9078/api/webscrobbler` + * Application name: `(whatever you want)` + +Reload the extension after adding the webhook. + +#### Multiple Users + +If you would like use multiple WebScrobbler sources they can be matched using a **slug** at the end of the **API URL.** This requires using [a file-based config.](#file-based-configuration) + +Example: + +In `webscrobbler.json` + +```json +[ + { + "name": "aUserWS", + "clients": [ + "client1Maloja" + ], + "data": { + "slug": "usera" + } + }, + { + "name": "bUserWS", + "clients": [ + "client2Maloja" + ], + "data": { + "slug": "userb" + } + } +] +``` + +* To use `aUserWS` source set **API URL** to `http://localhost:9078/api/webscrobbler/usera` +* To use `bUserWS` source set **API URL** to `http://localhost:9078/api/webscrobbler/userb` + +Note: `http://localhost:9078/api/webscrobbler` is matched with the first source that _that does not have a slug defined._ + +### ENV-Based + +| Environmental Variable | Required? | Default | Description | +|------------------------|-----------|---------|--------------------------------------------------------------------------| +| WS_ENABLE | No | | Set to 'true' to enable WS without needing to define other ENVs | +| WS_WHITELIST | No | | Only scrobble from these WebScrobbler Connectors. Comma-delimited list | +| WS_BLACKLIST | No | | Do not scrobble from these WebScrobbler Connectors. Comma-delimited list | + +### File-Based + +See [`webscrobbler.json.example`](https://github.com/FoxxMD/multi-scrobbler/blob/master/config/webscrobbler.json.example) or [explore the schema with an example and live editor/validator](https://json-schema.app/view/%23%2Fdefinitions%2FWebScrobblerSourceConfig/%23%2Fdefinitions%2FWebScrobblerData?url=https%3A%2F%2Fraw.githubusercontent.com%2FFoxxMD%2Fmulti-scrobbler%2Fdevelop%2Fsrc%2Fcommon%2Fschema%2Fsource.json) + # Client Configurations ## [Maloja](https://github.com/krateng/maloja) diff --git a/docsite/src/pages/index.mdx b/docsite/src/pages/index.mdx index f82d08a5..ba2e3c19 100644 --- a/docsite/src/pages/index.mdx +++ b/docsite/src/pages/index.mdx @@ -17,6 +17,7 @@ A javascript app to scrobble music you listened to, to [Maloja](https://github.c * [Plex](docs/configuration#plex) or [Tautulli](docs/configuration#tautulli) * [Subsonic-compatible APIs](docs/configuration#subsonic) (like [Airsonic](https://airsonic.github.io/) and [Navidrome](https://www.navidrome.org/)) * [Jellyfin](docs/configuration#jellyfin) + * [WebScrobbler](docs/configuration#webscrobbler) * [Youtube Music](docs/configuration#youtube-music) * [Last.fm](docs/configuration#lastfm-source) * [ListenBrainz](docs/configuration#listenbrainz--source-) diff --git a/src/backend/common/infrastructure/Atomic.ts b/src/backend/common/infrastructure/Atomic.ts index 032900c1..53a2d87a 100644 --- a/src/backend/common/infrastructure/Atomic.ts +++ b/src/backend/common/infrastructure/Atomic.ts @@ -6,8 +6,8 @@ import {Request, Response} from "express"; import {NextFunction, ParamsDictionary, Query} from "express-serve-static-core"; import { LogLevel, logLevels, PlayMeta, PlayObject } from "../../../core/Atomic"; -export type SourceType = 'spotify' | 'plex' | 'tautulli' | 'subsonic' | 'jellyfin' | 'lastfm' | 'deezer' | 'ytmusic' | 'mpris' | 'mopidy' | 'listenbrainz' | 'jriver' | 'kodi'; -export const sourceTypes: SourceType[] = ['spotify', 'plex', 'tautulli', 'subsonic', 'jellyfin', 'lastfm', 'deezer', 'ytmusic', 'mpris', 'mopidy', 'listenbrainz', 'jriver', 'kodi']; +export type SourceType = 'spotify' | 'plex' | 'tautulli' | 'subsonic' | 'jellyfin' | 'lastfm' | 'deezer' | 'ytmusic' | 'mpris' | 'mopidy' | 'listenbrainz' | 'jriver' | 'kodi' | 'webscrobbler'; +export const sourceTypes: SourceType[] = ['spotify', 'plex', 'tautulli', 'subsonic', 'jellyfin', 'lastfm', 'deezer', 'ytmusic', 'mpris', 'mopidy', 'listenbrainz', 'jriver', 'kodi', 'webscrobbler']; export const lowGranularitySources: SourceType[] = ['subsonic','ytmusic']; diff --git a/src/backend/common/infrastructure/config/source/sources.ts b/src/backend/common/infrastructure/config/source/sources.ts index 2bfd3fc7..e75d05cc 100644 --- a/src/backend/common/infrastructure/config/source/sources.ts +++ b/src/backend/common/infrastructure/config/source/sources.ts @@ -11,7 +11,9 @@ import { MopidySourceAIOConfig, MopidySourceConfig } from "./mopidy"; import { ListenBrainzSourceAIOConfig, ListenBrainzSourceConfig } from "./listenbrainz"; import { JRiverSourceAIOConfig, JRiverSourceConfig } from "./jriver"; import { KodiSourceAIOConfig, KodiSourceConfig } from "./kodi"; +import { WebScrobblerSourceAIOConfig, WebScrobblerSourceConfig } from "./webscrobbler"; -export type SourceConfig = SpotifySourceConfig | PlexSourceConfig | TautulliSourceConfig | DeezerSourceConfig | SubSonicSourceConfig | JellySourceConfig | LastfmSourceConfig | YTMusicSourceConfig | MPRISSourceConfig | MopidySourceConfig | ListenBrainzSourceConfig | JRiverSourceConfig | KodiSourceConfig; -export type SourceAIOConfig = SpotifySourceAIOConfig | PlexSourceAIOConfig | TautulliSourceAIOConfig | DeezerSourceAIOConfig | SubsonicSourceAIOConfig | JellySourceAIOConfig | LastFmSouceAIOConfig | YTMusicSourceAIOConfig | MPRISSourceAIOConfig | MopidySourceAIOConfig | ListenBrainzSourceAIOConfig | JRiverSourceAIOConfig | KodiSourceAIOConfig; +export type SourceConfig = SpotifySourceConfig | PlexSourceConfig | TautulliSourceConfig | DeezerSourceConfig | SubSonicSourceConfig | JellySourceConfig | LastfmSourceConfig | YTMusicSourceConfig | MPRISSourceConfig | MopidySourceConfig | ListenBrainzSourceConfig | JRiverSourceConfig | KodiSourceConfig | WebScrobblerSourceConfig; + +export type SourceAIOConfig = SpotifySourceAIOConfig | PlexSourceAIOConfig | TautulliSourceAIOConfig | DeezerSourceAIOConfig | SubsonicSourceAIOConfig | JellySourceAIOConfig | LastFmSouceAIOConfig | YTMusicSourceAIOConfig | MPRISSourceAIOConfig | MopidySourceAIOConfig | ListenBrainzSourceAIOConfig | JRiverSourceAIOConfig | KodiSourceAIOConfig | WebScrobblerSourceAIOConfig; diff --git a/src/backend/common/infrastructure/config/source/webscrobbler.ts b/src/backend/common/infrastructure/config/source/webscrobbler.ts new file mode 100644 index 00000000..894506fa --- /dev/null +++ b/src/backend/common/infrastructure/config/source/webscrobbler.ts @@ -0,0 +1,68 @@ +import { CommonSourceConfig, CommonSourceData } from "./index"; + +export interface WebScrobblerData extends CommonSourceData { + /** + * The URL ending that should be used to identify scrobbles for this source + * + * In WebScrobbler's Webhook you must set an 'API URL'. All MS WebScrobbler sources must start like: + * + * http://localhost:9078/api/webscrobbler + * + * If you are using multiple WebScrobbler sources (scrobbles for many users) you must use a slug to match Sources with each users extension. + * + * Example: + * + * * slug: 'usera' => API URL: http://localhost:9078/api/webscrobbler/usera + * * slug: 'userb' => API URL: http://localhost:9078/api/webscrobbler/userb + * + * If no slug is found from an extension's incoming webhook event the first WebScrobbler source without a slug will be used + * */ + slug?: string | null + + /** + * Block scrobbling from specific WebScrobbler Connectors + * + * @examples [["youtube"]] + * */ + blacklist?: string | string[] + + /** + * Only allow scrobbling from specific WebScrobbler Connectors + * + * @examples [["mixcloud","soundcloud","bandcamp"]] + * */ + whitelist?: string | string[] + + /** + * Additional options for WebScrobbler logging and tuning + * */ + options?: { + /** + * Log raw WebScrobbler webhook payload to debug + * + * @default false + * @examples [false] + * */ + logPayload?: boolean + + /** + * How MS should log when a WebScrobbler event fails a defined filter + * + * * `false` => do not log + * * `debug` => log to DEBUG level + * * `warn` => log to WARN level (default) + * + * @default warn + * @examples ["warn"] + * */ + logFilterFailure?: false | 'debug' | 'warn' + } +} + +export interface WebScrobblerSourceConfig extends CommonSourceConfig { + data?: WebScrobblerData +} + +export interface WebScrobblerSourceAIOConfig extends WebScrobblerSourceConfig { + type: 'webscrobbler' +} diff --git a/src/backend/common/schema/aio-source.json b/src/backend/common/schema/aio-source.json index 3ea6e293..1d146a85 100644 --- a/src/backend/common/schema/aio-source.json +++ b/src/backend/common/schema/aio-source.json @@ -1458,6 +1458,9 @@ }, { "$ref": "#/definitions/KodiSourceAIOConfig" + }, + { + "$ref": "#/definitions/WebScrobblerSourceAIOConfig" } ], "title": "SourceAIOConfig" @@ -1830,6 +1833,172 @@ "title": "TautulliSourceAIOConfig", "type": "object" }, + "WebScrobblerData": { + "properties": { + "blacklist": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "string" + } + ], + "description": "Block scrobbling from specific WebScrobbler Connectors", + "examples": [ + [ + "youtube" + ] + ], + "title": "blacklist" + }, + "maxPollRetries": { + "default": 5, + "description": "default # of automatic polling restarts on error", + "examples": [ + 5 + ], + "title": "maxPollRetries", + "type": "number" + }, + "maxRequestRetries": { + "default": 1, + "description": "default # of http request retries a source can make before error is thrown", + "examples": [ + 1 + ], + "title": "maxRequestRetries", + "type": "number" + }, + "options": { + "description": "Additional options for WebScrobbler logging and tuning", + "properties": { + "logFilterFailure": { + "default": "warn", + "description": "How MS should log when a WebScrobbler event fails a defined filter\n\n* `false` => do not log\n* `debug` => log to DEBUG level\n* `warn` => log to WARN level (default)", + "enum": [ + "debug", + false, + "warn" + ], + "examples": [ + "warn" + ], + "title": "logFilterFailure" + }, + "logPayload": { + "default": false, + "description": "Log raw WebScrobbler webhook payload to debug", + "examples": [ + false + ], + "title": "logPayload", + "type": "boolean" + } + }, + "title": "options", + "type": "object" + }, + "retryMultiplier": { + "default": 1.5, + "description": "default retry delay multiplier (retry attempt * multiplier = # of seconds to wait before retrying)", + "examples": [ + 1.5 + ], + "title": "retryMultiplier", + "type": "number" + }, + "scrobbleThresholds": { + "$ref": "#/definitions/ScrobbleThresholds", + "description": "Set thresholds for when multi-scrobbler should consider a tracked play to be \"scrobbable\". If both duration and percent are defined then if either condition is met the track is scrobbled.", + "title": "scrobbleThresholds" + }, + "slug": { + "description": "The URL ending that should be used to identify scrobbles for this source\n\nIn WebScrobbler's Webhook you must set an 'API URL'. All MS WebScrobbler sources must start like:\n\nhttp://localhost:9078/api/webscrobbler\n\nIf you are using multiple WebScrobbler sources (scrobbles for many users) you must use a slug to match Sources with each users extension.\n\nExample:\n\n* slug: 'usera' => API URL: http://localhost:9078/api/webscrobbler/usera\n* slug: 'userb' => API URL: http://localhost:9078/api/webscrobbler/userb\n\nIf no slug is found from an extension's incoming webhook event the first WebScrobbler source without a slug will be used", + "title": "slug", + "type": "string" + }, + "whitelist": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "string" + } + ], + "description": "Only allow scrobbling from specific WebScrobbler Connectors", + "examples": [ + [ + "mixcloud", + "soundcloud", + "bandcamp" + ] + ], + "title": "whitelist" + } + }, + "title": "WebScrobblerData", + "type": "object" + }, + "WebScrobblerSourceAIOConfig": { + "properties": { + "clients": { + "description": "Restrict scrobbling tracks played from this source to Clients with names from this list. If list is empty is not present Source scrobbles to all configured Clients.", + "examples": [ + [ + "MyMalojaConfigName", + "MyLastFMConfigName" + ] + ], + "items": { + "type": "string" + }, + "title": "clients", + "type": "array" + }, + "data": { + "$ref": "#/definitions/WebScrobblerData", + "title": "data" + }, + "enable": { + "default": true, + "description": "Should MS use this client/source? Defaults to true", + "examples": [ + true + ], + "title": "enable", + "type": "boolean" + }, + "name": { + "description": "Unique identifier for this source.", + "title": "name", + "type": "string" + }, + "options": { + "$ref": "#/definitions/CommonSourceOptions", + "title": "options" + }, + "type": { + "enum": [ + "webscrobbler" + ], + "title": "type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "WebScrobblerSourceAIOConfig", + "type": "object" + }, "YTMusicData": { "properties": { "authUser": { diff --git a/src/backend/common/schema/aio.json b/src/backend/common/schema/aio.json index f38bed05..04dba18f 100644 --- a/src/backend/common/schema/aio.json +++ b/src/backend/common/schema/aio.json @@ -2173,6 +2173,9 @@ }, { "$ref": "#/definitions/KodiSourceAIOConfig" + }, + { + "$ref": "#/definitions/WebScrobblerSourceAIOConfig" } ], "title": "SourceAIOConfig" @@ -2554,6 +2557,172 @@ "title": "TautulliSourceAIOConfig", "type": "object" }, + "WebScrobblerData": { + "properties": { + "blacklist": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "string" + } + ], + "description": "Block scrobbling from specific WebScrobbler Connectors", + "examples": [ + [ + "youtube" + ] + ], + "title": "blacklist" + }, + "maxPollRetries": { + "default": 5, + "description": "default # of automatic polling restarts on error", + "examples": [ + 5 + ], + "title": "maxPollRetries", + "type": "number" + }, + "maxRequestRetries": { + "default": 1, + "description": "default # of http request retries a source can make before error is thrown", + "examples": [ + 1 + ], + "title": "maxRequestRetries", + "type": "number" + }, + "options": { + "description": "Additional options for WebScrobbler logging and tuning", + "properties": { + "logFilterFailure": { + "default": "warn", + "description": "How MS should log when a WebScrobbler event fails a defined filter\n\n* `false` => do not log\n* `debug` => log to DEBUG level\n* `warn` => log to WARN level (default)", + "enum": [ + "debug", + false, + "warn" + ], + "examples": [ + "warn" + ], + "title": "logFilterFailure" + }, + "logPayload": { + "default": false, + "description": "Log raw WebScrobbler webhook payload to debug", + "examples": [ + false + ], + "title": "logPayload", + "type": "boolean" + } + }, + "title": "options", + "type": "object" + }, + "retryMultiplier": { + "default": 1.5, + "description": "default retry delay multiplier (retry attempt * multiplier = # of seconds to wait before retrying)", + "examples": [ + 1.5 + ], + "title": "retryMultiplier", + "type": "number" + }, + "scrobbleThresholds": { + "$ref": "#/definitions/ScrobbleThresholds", + "description": "Set thresholds for when multi-scrobbler should consider a tracked play to be \"scrobbable\". If both duration and percent are defined then if either condition is met the track is scrobbled.", + "title": "scrobbleThresholds" + }, + "slug": { + "description": "The URL ending that should be used to identify scrobbles for this source\n\nIn WebScrobbler's Webhook you must set an 'API URL'. All MS WebScrobbler sources must start like:\n\nhttp://localhost:9078/api/webscrobbler\n\nIf you are using multiple WebScrobbler sources (scrobbles for many users) you must use a slug to match Sources with each users extension.\n\nExample:\n\n* slug: 'usera' => API URL: http://localhost:9078/api/webscrobbler/usera\n* slug: 'userb' => API URL: http://localhost:9078/api/webscrobbler/userb\n\nIf no slug is found from an extension's incoming webhook event the first WebScrobbler source without a slug will be used", + "title": "slug", + "type": "string" + }, + "whitelist": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "string" + } + ], + "description": "Only allow scrobbling from specific WebScrobbler Connectors", + "examples": [ + [ + "mixcloud", + "soundcloud", + "bandcamp" + ] + ], + "title": "whitelist" + } + }, + "title": "WebScrobblerData", + "type": "object" + }, + "WebScrobblerSourceAIOConfig": { + "properties": { + "clients": { + "description": "Restrict scrobbling tracks played from this source to Clients with names from this list. If list is empty is not present Source scrobbles to all configured Clients.", + "examples": [ + [ + "MyMalojaConfigName", + "MyLastFMConfigName" + ] + ], + "items": { + "type": "string" + }, + "title": "clients", + "type": "array" + }, + "data": { + "$ref": "#/definitions/WebScrobblerData", + "title": "data" + }, + "enable": { + "default": true, + "description": "Should MS use this client/source? Defaults to true", + "examples": [ + true + ], + "title": "enable", + "type": "boolean" + }, + "name": { + "description": "Unique identifier for this source.", + "title": "name", + "type": "string" + }, + "options": { + "$ref": "#/definitions/CommonSourceOptions", + "title": "options" + }, + "type": { + "enum": [ + "webscrobbler" + ], + "title": "type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "WebScrobblerSourceAIOConfig", + "type": "object" + }, "WebhookConfig": { "anyOf": [ { diff --git a/src/backend/common/schema/source.json b/src/backend/common/schema/source.json index 770e69b4..fa1b4a7c 100644 --- a/src/backend/common/schema/source.json +++ b/src/backend/common/schema/source.json @@ -39,6 +39,9 @@ }, { "$ref": "#/definitions/KodiSourceConfig" + }, + { + "$ref": "#/definitions/WebScrobblerSourceConfig" } ], "definitions": { @@ -1698,6 +1701,162 @@ "title": "TautulliSourceConfig", "type": "object" }, + "WebScrobblerData": { + "properties": { + "blacklist": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "string" + } + ], + "description": "Block scrobbling from specific WebScrobbler Connectors", + "examples": [ + [ + "youtube" + ] + ], + "title": "blacklist" + }, + "maxPollRetries": { + "default": 5, + "description": "default # of automatic polling restarts on error", + "examples": [ + 5 + ], + "title": "maxPollRetries", + "type": "number" + }, + "maxRequestRetries": { + "default": 1, + "description": "default # of http request retries a source can make before error is thrown", + "examples": [ + 1 + ], + "title": "maxRequestRetries", + "type": "number" + }, + "options": { + "description": "Additional options for WebScrobbler logging and tuning", + "properties": { + "logFilterFailure": { + "default": "warn", + "description": "How MS should log when a WebScrobbler event fails a defined filter\n\n* `false` => do not log\n* `debug` => log to DEBUG level\n* `warn` => log to WARN level (default)", + "enum": [ + "debug", + false, + "warn" + ], + "examples": [ + "warn" + ], + "title": "logFilterFailure" + }, + "logPayload": { + "default": false, + "description": "Log raw WebScrobbler webhook payload to debug", + "examples": [ + false + ], + "title": "logPayload", + "type": "boolean" + } + }, + "title": "options", + "type": "object" + }, + "retryMultiplier": { + "default": 1.5, + "description": "default retry delay multiplier (retry attempt * multiplier = # of seconds to wait before retrying)", + "examples": [ + 1.5 + ], + "title": "retryMultiplier", + "type": "number" + }, + "scrobbleThresholds": { + "$ref": "#/definitions/ScrobbleThresholds", + "description": "Set thresholds for when multi-scrobbler should consider a tracked play to be \"scrobbable\". If both duration and percent are defined then if either condition is met the track is scrobbled.", + "title": "scrobbleThresholds" + }, + "slug": { + "description": "The URL ending that should be used to identify scrobbles for this source\n\nIn WebScrobbler's Webhook you must set an 'API URL'. All MS WebScrobbler sources must start like:\n\nhttp://localhost:9078/api/webscrobbler\n\nIf you are using multiple WebScrobbler sources (scrobbles for many users) you must use a slug to match Sources with each users extension.\n\nExample:\n\n* slug: 'usera' => API URL: http://localhost:9078/api/webscrobbler/usera\n* slug: 'userb' => API URL: http://localhost:9078/api/webscrobbler/userb\n\nIf no slug is found from an extension's incoming webhook event the first WebScrobbler source without a slug will be used", + "title": "slug", + "type": "string" + }, + "whitelist": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "string" + } + ], + "description": "Only allow scrobbling from specific WebScrobbler Connectors", + "examples": [ + [ + "mixcloud", + "soundcloud", + "bandcamp" + ] + ], + "title": "whitelist" + } + }, + "title": "WebScrobblerData", + "type": "object" + }, + "WebScrobblerSourceConfig": { + "properties": { + "clients": { + "description": "Restrict scrobbling tracks played from this source to Clients with names from this list. If list is empty is not present Source scrobbles to all configured Clients.", + "examples": [ + [ + "MyMalojaConfigName", + "MyLastFMConfigName" + ] + ], + "items": { + "type": "string" + }, + "title": "clients", + "type": "array" + }, + "data": { + "$ref": "#/definitions/WebScrobblerData", + "title": "data" + }, + "enable": { + "default": true, + "description": "Should MS use this client/source? Defaults to true", + "examples": [ + true + ], + "title": "enable", + "type": "boolean" + }, + "name": { + "description": "Unique identifier for this source.", + "title": "name", + "type": "string" + }, + "options": { + "$ref": "#/definitions/CommonSourceOptions", + "title": "options" + } + }, + "title": "WebScrobblerSourceConfig", + "type": "object" + }, "YTMusicData": { "properties": { "authUser": { diff --git a/src/backend/common/vendor/webscrobbler/interfaces.ts b/src/backend/common/vendor/webscrobbler/interfaces.ts new file mode 100644 index 00000000..8953e845 --- /dev/null +++ b/src/backend/common/vendor/webscrobbler/interfaces.ts @@ -0,0 +1,72 @@ +export type WebScrobblerHookEvent = 'scrobble' | 'paused' | 'resumedplaying' | 'nowplaying' | string; +export interface WebScrobblerPayload { + eventName: WebScrobblerHookEvent + time?: number + data: { + song: WebScrobblerSong + songs?: WebScrobblerSong[] + currentlyPlaying?: boolean + } +} + +interface ProcessedSongData { + artist?: string | null; + album?: string | null; + albumArtist?: string | null; + track?: string | null; + duration?: number | null; +} + +interface ParsedSongData extends ProcessedSongData { + trackArt?: string | null; + uniqueID?: string | null; + originUrl?: string | null; + isPodcast?: boolean | null; + isPlaying?: boolean | null; + currentTime?: number | null; + isScrobblingAllowed?: boolean | null; +} + +export type Flags = + | { + isScrobbled: boolean; + isCorrectedByUser: boolean; + isRegexEditedByUser: { + track: boolean; + artist: boolean; + album: boolean; + albumArtist: boolean; + }; + isAlbumFetched: boolean; + isValid: boolean; + isMarkedAsPlaying: boolean; + isSkipped: boolean; + isReplaying: boolean; +} + | Record; + +export type Metadata = + | { + label: string; + startTimestamp: number; + + albumMbId?: string; + albumUrl?: string; + artistUrl?: string; + notificationId?: string; + trackArtUrl?: string; + trackUrl?: string; + userPlayCount?: number; + userloved?: boolean; +} + | Record; + +export interface WebScrobblerSong { + controllerTabId: string | number; + parsed: ParsedSongData; + processed: ProcessedSongData; + noRegex: ProcessedSongData; + flags: Flags; + metadata: Metadata; + connectorLabel: string; +} diff --git a/src/backend/server/api.ts b/src/backend/server/api.ts index 381192f1..1ed92bd1 100644 --- a/src/backend/server/api.ts +++ b/src/backend/server/api.ts @@ -26,6 +26,8 @@ import {capitalize} from "../../core/StringUtils"; import {source} from "common-tags"; import AbstractScrobbleClient from "../scrobblers/AbstractScrobbleClient"; import {sortByNewestPlayDate} from "../utils"; +import bodyParser from "body-parser"; +import {setupWebscrobblerRoutes} from "./webscrobblerRoutes"; let output: LogInfo[] = [] @@ -122,8 +124,19 @@ export const setupApi = (app: ExpressWithAsync, logger: Logger, initialLogOutput setupPlexRoutes(app, logger, scrobbleSources); setupJellyfinRoutes(app, logger, scrobbleSources); setupDeezerRoutes(app, logger, scrobbleSources); + setupWebscrobblerRoutes(app, logger, scrobbleSources); setupAuthRoutes(app, logger, sourceRequiredMiddle, clientRequiredMiddle, scrobbleSources, scrobbleClients); + app.putAsync('/api/webscrobbler', bodyParser.json({type: ['text/*', 'application/json']}), async (req, res) => { + logger.info(req.body); + res.sendStatus(200); + }); + + app.getAsync('/api/webscrobbler', bodyParser.json({type: ['text/*', 'application/json']}), async (req, res) => { + logger.info(req.body); + res.sendStatus(200); + }); + app.getAsync('/api/status', async (req, res, next) => { const ss = root.get('sources'); diff --git a/src/backend/server/middleware.ts b/src/backend/server/middleware.ts index cd39d08d..73a0d099 100644 --- a/src/backend/server/middleware.ts +++ b/src/backend/server/middleware.ts @@ -1,4 +1,5 @@ import { ExpressHandler } from "../common/infrastructure/Atomic"; +import {Logger} from "@foxxmd/winston"; export const makeSourceCheckMiddle = (sources: any) => (required: boolean ): ExpressHandler => (req: any, res: any, next: any) => { const { @@ -48,3 +49,21 @@ export const makeClientCheckMiddle = (clients: any) => (required: boolean): Expr next(); } + +export const nonEmptyBody = (logger: Logger, origin: string = 'Origin'): ExpressHandler => async (req, res, next) => { + const bodyEmpty = req.body === undefined || req.body === null || (typeof req.body === 'object' && Object.keys(req.body).length === 0); + if (bodyEmpty) { + const length = req.header('content-length') !== undefined ? Number.parseInt(req.header('content-length')) : undefined; + // can't think of a way a user would send an empty body for a payload but if they meant to do it don't spam them with errors... + if (length === 0) { + return; + } + if (length === undefined) { + logger.warn(`${origin} is not sending a well-formatted request. It does not have valid headers (application/json - text/*) OR it is missing content-length header: Content-Type => '${req.header('content-type')}' | Length => ${length}`); + } else { + logger.warn(`${origin} is not sending a request with valid headers. Content-Type must be either application/json or a text/* wildcard (like text/plain) -- given: Content-Type => '${req.header('content-type')}'`); + } + res.status(400).send('Invalid Content-Type. Must be either application/json or a text wildcard (like text/plain)'); + return; + } +} diff --git a/src/backend/server/webscrobblerRoutes.ts b/src/backend/server/webscrobblerRoutes.ts new file mode 100644 index 00000000..4a18dcb3 --- /dev/null +++ b/src/backend/server/webscrobblerRoutes.ts @@ -0,0 +1,71 @@ +import {mergeArr, parseBool, remoteHostIdentifiers} from "../utils"; +import {ExpressWithAsync} from "@awaitjs/express"; +import {Logger} from "@foxxmd/winston"; +import ScrobbleSources from "../sources/ScrobbleSources"; +import bodyParser from "body-parser"; +import {WebScrobblerPayload} from "../common/vendor/webscrobbler/interfaces"; +import {WebhookNotifier} from "../sources/ingressNotifiers/WebhookNotifier"; +import {nonEmptyBody} from "./middleware"; +import {WebScrobblerSource} from "../sources/WebScrobblerSource"; +import path from "path"; + +export const setupWebscrobblerRoutes = (app: ExpressWithAsync, parentLogger: Logger, scrobbleSources: ScrobbleSources) => { + + const logger = parentLogger.child({labels: ['Ingress', 'WebScrobbler']}, mergeArr); + + const webScrobblerJsonParser = bodyParser.json({ + type: ['text/*', 'application/json'], + // verify: function(req, res, buf, encoding) { + // // get rawBody + // // @ts-ignore + // req.rawBody = buf.toString(); + // } + }); + const webhookIngress = new WebhookNotifier(); + app.postAsync('/api/webscrobbler', + async function (req, res, next) { + // track request before parsing body to ensure we at least log that something is happening + // (in the event body parsing does not work or request is not POST/PATCH) + webhookIngress.trackIngress(req, true); + next(); + }, + webScrobblerJsonParser, nonEmptyBody(logger, 'WebScrobbler Extension'), async function (req, res) { + webhookIngress.trackIngress(req, false); + + res.sendStatus(200); + + const parts = path.parse(req.path); + const slug = parts.name === 'webscrobbler' ? undefined : parts.name; + + // let cleanPath = req.path; + // if (cleanPath.charAt(cleanPath.length - 1) === '/') { + // cleanPath = cleanPath.slice(0, -1); + // } + // const splitPath = cleanPath.split('/'); + // const slug = splitPath[splitPath.length - 1]; + + const playerState = WebScrobblerSource.playStateFromRequest(req.body); + + const sources = scrobbleSources.getByType('webscrobbler') as WebScrobblerSource[]; + if (sources.length === 0) { + logger.warn('Received WebScrobbler payload but no WebScrobbler sources are configured'); + } + + let slugMatched = false; + for (const source of sources) { + if (source.matchSlug(slug)) { + await source.handle(playerState); + slugMatched = true; + } + } + + if (!slugMatched) { + if (slug === undefined) { + logger.warn(`Request URL did not have a slug and no WebScrobbler source was configured without a slug.`); + } else { + logger.warn(`No WebScrobbler souce had the given slug '${slug}'`); + } + } + }); +} + diff --git a/src/backend/sources/ScrobbleSources.ts b/src/backend/sources/ScrobbleSources.ts index 25d2fd5a..2877474f 100644 --- a/src/backend/sources/ScrobbleSources.ts +++ b/src/backend/sources/ScrobbleSources.ts @@ -36,6 +36,8 @@ import { JRiverData, JRiverSourceConfig } from "../common/infrastructure/config/ import { KodiSource } from "./KodiSource"; import { KodiData, KodiSourceConfig } from "../common/infrastructure/config/source/kodi"; import { WildcardEmitter } from "../common/WildcardEmitter"; +import {WebScrobblerSource} from "./WebScrobblerSource"; +import {WebScrobblerSourceConfig} from "../common/infrastructure/config/source/webscrobbler"; type groupedNamedConfigs = {[key: string]: ParsedConfig[]}; @@ -301,6 +303,26 @@ export default class ScrobbleSources { }); } break; + case 'webscrobbler': + const wsShouldUse = parseBool(process.env.WEBSCROBBLER_ENABLE); + const ws = { + blacklist: process.env.WEBSCROBBLER_BLACKLIST, + whitelist: process.env.WEBSCROBBLER_WHITELIST + } + if (!Object.values(mp).every(x => x === undefined) || wsShouldUse) { + configs.push({ + type: 'webscrobbler', + name: 'unnamed', + source: 'ENV', + mode: 'single', + configureAs: defaultConfigureAs, + data: { + blacklist: ws.blacklist !== undefined ? ws.blacklist.split(',') : [], + whitelist: ws.whitelist !== undefined ? ws.whitelist.split(',') : [], + } + }); + } + break; default: break; } @@ -476,6 +498,9 @@ export default class ScrobbleSources { case 'kodi': newSource = await new KodiSource(name, compositeConfig as KodiSourceConfig, internal, this.emitter); break; + case 'webscrobbler': + newSource = await new WebScrobblerSource(name, compositeConfig as WebScrobblerSourceConfig, internal, this.emitter); + break; default: break; } diff --git a/src/backend/sources/WebScrobblerSource.ts b/src/backend/sources/WebScrobblerSource.ts new file mode 100644 index 00000000..d1f189ad --- /dev/null +++ b/src/backend/sources/WebScrobblerSource.ts @@ -0,0 +1,172 @@ +import MemorySource from "./MemorySource"; +import { + FormatPlayObjectOptions, + InternalConfig, + NO_USER, + PlayerStateData, REPORTED_PLAYER_STATUSES, + ReportedPlayerStatus +} from "../common/infrastructure/Atomic"; +import EventEmitter from "events"; +import {PlayObject} from "../../core/Atomic"; +import {WebScrobblerHookEvent, WebScrobblerPayload, WebScrobblerSong} from "../common/vendor/webscrobbler/interfaces"; +import dayjs from "dayjs"; +import {WebScrobblerSourceConfig} from "../common/infrastructure/config/source/webscrobbler"; + +export class WebScrobblerSource extends MemorySource { + + declare config: WebScrobblerSourceConfig; + + constructor(name: any, config: WebScrobblerSourceConfig, internal: InternalConfig, emitter: EventEmitter) { + super('webscrobbler', name, config, internal, emitter); + this.multiPlatform = true; + this.playerSourceOfTruth = false; + + const { + data = {}, + data: { + whitelist = [], + blacklist = [], + slug, + } = {} + } = this.config; + const wl = typeof whitelist === 'string' ? [whitelist] : whitelist; + const bl = typeof blacklist === 'string' ? [blacklist] : blacklist; + this.config.data = { + ...data, + slug: slug === null ? undefined : slug, + blacklist: bl.map(x => x.toLocaleLowerCase().trim()), + whitelist: wl.map(x => x.toLocaleLowerCase().trim()) + }; + } + + matchSlug(slug: string | undefined) { + if (this.config.data.slug === undefined || this.config.data.slug === null) { + return slug === undefined; + } + + return slug.toLowerCase() === this.config.data.slug.toLowerCase().trim(); + } + + static webhookEventAsPlayerStatus(event: WebScrobblerHookEvent): ReportedPlayerStatus { + switch (event) { + case 'nowplaying': + case 'scrobble': + case 'resumedplaying': + return REPORTED_PLAYER_STATUSES.playing; + case 'paused': + return REPORTED_PLAYER_STATUSES.paused; + default: + return REPORTED_PLAYER_STATUSES.unknown; + } + } + + static playStateFromRequest(obj: WebScrobblerPayload,): PlayerStateData { + const { + eventName, + time = dayjs().unix(), + } = obj; + + const play = WebScrobblerSource.formatPlayObj(obj.data.song, {nowPlaying: eventName !== 'scrobble'}); + return { + platformId: [play.meta.deviceId, NO_USER], + play, + status: WebScrobblerSource.webhookEventAsPlayerStatus(eventName), + timestamp: dayjs.unix(time) + } + } + + static formatPlayObj(obj: WebScrobblerSong, options: FormatPlayObjectOptions & { + nowPlaying?: boolean + } = {}): PlayObject { + const { + connectorLabel, + controllerTabId = 'UNK', + processed, + parsed: { + isScrobblingAllowed = true, + originUrl, + uniqueID + }, + parsed, + metadata: { + startTimestamp, + trackUrl, + albumMbId, + label, + } + } = obj; + + const track = processed.track ?? parsed.track; + const artist = processed.artist ?? parsed.artist; + const album = processed.album ?? parsed.album; + const albumArtist = processed.albumArtist ?? parsed.albumArtist; + const duration = parsed.duration ?? processed.duration; + + return { + data: { + track, + artists: [artist], + album: album === null ? undefined : album, + playDate: dayjs.unix(startTimestamp), + duration: duration === null ? undefined : duration, + meta: { + brainz: { + album: albumMbId + } + } + }, + meta: { + trackId: uniqueID, + parsedFrom: connectorLabel, + url: { + web: trackUrl, + origin: originUrl + }, + deviceId: `${connectorLabel}-${controllerTabId}`, + scrobbleAllowed: isScrobblingAllowed, + nowPlaying: options.nowPlaying ?? false + } + } + } + + getRecentlyPlayed = async (options = {}) => { + return this.getFlatRecentlyDiscoveredPlays(); + } + + isValidScrobble = (playObj: PlayObject) => { + if (playObj.meta?.scrobbleAllowed === false) { + this.logger.debug(`Will not scrobble play because it was marked as 'Do Not Scrobble' by extension`); + return false; + } + + if (playObj.meta.parsedFrom !== undefined) { + const lowerSource = playObj.meta.parsedFrom.toLowerCase(); + if (Array.isArray(this.config.data.blacklist) && this.config.data.blacklist.length > 0) { + if (this.config.data.blacklist.some(x => x === lowerSource)) { + this.logger.debug(`Will not scrobble play because it is from a blacklisted connector '${lowerSource}'`); + return false; + } + } + if (Array.isArray(this.config.data.whitelist) && this.config.data.whitelist.length > 0) { + if (!this.config.data.whitelist.some(x => x === lowerSource)) { + this.logger.debug(`Will not scrobble play because it is not from a whitelisted connector '${lowerSource}'`); + return false; + } + } + } + + return true; + } + + handle = async (stateData: PlayerStateData) => { + + this.processRecentPlays([stateData]); + + if (stateData.play.meta.nowPlaying === false && this.isValidScrobble(stateData.play)) { + const discovered = this.discover([stateData.play]); + if (discovered.length > 0) { + this.scrobble(discovered); + } + } + } +} diff --git a/src/backend/sources/ingressNotifiers/WebhookNotifier.ts b/src/backend/sources/ingressNotifiers/WebhookNotifier.ts new file mode 100644 index 00000000..69b1b698 --- /dev/null +++ b/src/backend/sources/ingressNotifiers/WebhookNotifier.ts @@ -0,0 +1,39 @@ +import {IngressNotifier} from "./IngressNotifier"; +import {Request} from "express"; +import path from "path"; + +export class WebhookNotifier extends IngressNotifier { + + constructor() { + super('WebScrobbler'); + } + + seenSlugs: Record = {}; + notifyBySource(req: Request, isRaw: boolean): [boolean, (string | undefined)] { + + if(!isRaw) { + // let cleanPath = req.path; + // if(cleanPath.charAt(cleanPath.length - 1) === '/') { + // cleanPath = cleanPath.slice(0, -1); + // } + // const splitPath = cleanPath.split('/'); + // const slug = splitPath[cleanPath.length - 1]; + const parts = path.parse(req.path); + const slug = parts.name; + + if(this.seenSlugs[slug] === undefined) { + this.seenSlugs[slug] = true; + return [true, `Received valid data for API URL slug '${slug === 'webscrobbler' ? '(none)' : slug}' for the first time.`]; + } + } + + return [true, undefined]; + } + + notifyByRequest(req: Request, isRaw: boolean): string | undefined { + if(req.method !== 'POST') { + return `Expected POST request (webhook payload) but received ${req.method}`; + } + return; + } +} diff --git a/src/backend/utils/StringUtils.ts b/src/backend/utils/StringUtils.ts index 77dc6169..b5cb50d8 100644 --- a/src/backend/utils/StringUtils.ts +++ b/src/backend/utils/StringUtils.ts @@ -210,3 +210,7 @@ export const compareNormalizedStrings = (existing: string, candidate: string): S return stringSameness(normalExisting, orderedCandidateString, {transforms: [], strategies: [levenStrategy, diceStrategy]}); } + +export const getUrlSlug = (path: string) => { + +} diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts index b6f44a0d..9b2c9a72 100644 --- a/src/core/Atomic.ts +++ b/src/core/Atomic.ts @@ -4,7 +4,7 @@ import {ListenProgress} from "../backend/sources/PlayerState/ListenProgress"; export interface SourceStatusData { status: string; - type: "spotify" | "plex" | "tautulli" | "subsonic" | "jellyfin" | "lastfm" | "deezer" | "ytmusic" | "mpris" | "mopidy" | "listenbrainz" | "jriver" | "kodi"; + type: "spotify" | "plex" | "tautulli" | "subsonic" | "jellyfin" | "lastfm" | "deezer" | "ytmusic" | "mpris" | "mopidy" | "listenbrainz" | "jriver" | "kodi" | 'webscrobbler'; display: string; tracksDiscovered: number; name: string; @@ -101,6 +101,7 @@ export interface PlayMeta { newFromSource?: boolean url?: { web: string + origin?: string [key: string]: string } user?: string -- 2.51.2 From c8c5f5dcb799d2abe6fafdb8a98ac43a8f2be9c6 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 6 Oct 2023 09:44:29 -0400 Subject: [PATCH 03/16] docs(webscrobbler): Add disclaimer about hostname for firefox --- docsite/docs/configuration/configuration.md | 4 ++++ src/backend/sources/ScrobbleSources.ts | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docsite/docs/configuration/configuration.md b/docsite/docs/configuration/configuration.md index cc1a2289..882ba168 100644 --- a/docsite/docs/configuration/configuration.md +++ b/docsite/docs/configuration/configuration.md @@ -636,6 +636,10 @@ After installing the extension open the preferences/settings for it: Reload the extension after adding the webhook. +* **On Firefox** - Only FQNs (domain.tld), `localhost`, and `127.0.0.1` are supported for API URL due to [firefox requiring https](https://github.com/web-scrobbler/web-scrobbler/issues/4183#issuecomment-1749222006) +* **On Chromium-based Browsers** - Any domain will work for API URL +* All Other browsers are untested + #### Multiple Users If you would like use multiple WebScrobbler sources they can be matched using a **slug** at the end of the **API URL.** This requires using [a file-based config.](#file-based-configuration) diff --git a/src/backend/sources/ScrobbleSources.ts b/src/backend/sources/ScrobbleSources.ts index 2877474f..e6ca5797 100644 --- a/src/backend/sources/ScrobbleSources.ts +++ b/src/backend/sources/ScrobbleSources.ts @@ -309,7 +309,7 @@ export default class ScrobbleSources { blacklist: process.env.WEBSCROBBLER_BLACKLIST, whitelist: process.env.WEBSCROBBLER_WHITELIST } - if (!Object.values(mp).every(x => x === undefined) || wsShouldUse) { + if (!Object.values(ws).every(x => x === undefined) || wsShouldUse) { configs.push({ type: 'webscrobbler', name: 'unnamed', -- 2.51.2 From fbb67a8fe61cf54c867632ef28319e48942ae2d4 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Wed, 11 Oct 2023 12:38:43 -0400 Subject: [PATCH 04/16] feat: Implement dead letter endpoints * Implement backend api endpoints for dead letter CRUD * Include dead letter queue count in client status data * Implement dead letter client view * Partially working retry/remove actions for client (request is OK, RTK not working for update yet) --- src/backend/common/infrastructure/Atomic.ts | 6 +- .../scrobblers/AbstractScrobbleClient.ts | 44 +++++++------ src/backend/server/api.ts | 66 ++++++++++++++++++- src/client/App.tsx | 5 ++ .../statusCard/ClientStatusCard.tsx | 7 +- src/client/deadLetter/DeadPage.tsx | 61 +++++++++++++++++ src/client/deadLetter/deadLetterDucks.ts | 47 +++++++++++++ src/client/store.ts | 4 +- src/core/Atomic.ts | 22 ++++++- 9 files changed, 232 insertions(+), 30 deletions(-) create mode 100644 src/client/deadLetter/DeadPage.tsx create mode 100644 src/client/deadLetter/deadLetterDucks.ts diff --git a/src/backend/common/infrastructure/Atomic.ts b/src/backend/common/infrastructure/Atomic.ts index 53a2d87a..2330c38d 100644 --- a/src/backend/common/infrastructure/Atomic.ts +++ b/src/backend/common/infrastructure/Atomic.ts @@ -4,7 +4,7 @@ import {Logger} from '@foxxmd/winston'; import TupleMap from "../TupleMap"; import {Request, Response} from "express"; import {NextFunction, ParamsDictionary, Query} from "express-serve-static-core"; -import { LogLevel, logLevels, PlayMeta, PlayObject } from "../../../core/Atomic"; +import {LogLevel, logLevels, PlayMeta, PlayObject} from "../../../core/Atomic"; export type SourceType = 'spotify' | 'plex' | 'tautulli' | 'subsonic' | 'jellyfin' | 'lastfm' | 'deezer' | 'ytmusic' | 'mpris' | 'mopidy' | 'listenbrainz' | 'jriver' | 'kodi' | 'webscrobbler'; export const sourceTypes: SourceType[] = ['spotify', 'plex', 'tautulli', 'subsonic', 'jellyfin', 'lastfm', 'deezer', 'ytmusic', 'mpris', 'mopidy', 'listenbrainz', 'jriver', 'kodi', 'webscrobbler']; @@ -213,7 +213,3 @@ export const TIME_WEIGHT = 0.5; export const REFERENCE_WEIGHT = 0.5; export const DUP_SCORE_THRESHOLD = 1; -export interface SourceScrobble { - source: string - play: PlayObject -} diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts index 8a82d54f..b56f36fa 100644 --- a/src/backend/scrobblers/AbstractScrobbleClient.ts +++ b/src/backend/scrobblers/AbstractScrobbleClient.ts @@ -14,14 +14,14 @@ import { INITIALIZING, InitState, NOT_INITIALIZED, REFERENCE_WEIGHT, - ScrobbledPlayObject, SourceScrobble, TIME_WEIGHT, TITLE_WEIGHT, + ScrobbledPlayObject, TIME_WEIGHT, TITLE_WEIGHT, } from "../common/infrastructure/Atomic"; import winston, {Logger} from '@foxxmd/winston'; import { CommonClientConfig } from "../common/infrastructure/config/client/index"; import { ClientConfig } from "../common/infrastructure/config/client/clients"; import { Notifiers } from "../notifier/Notifiers"; import {FixedSizeList} from 'fixed-size-list'; -import { PlayObject, TrackStringOptions } from "../../core/Atomic"; +import {DeadLetterScrobble, PlayObject, QueuedScrobble, SourceScrobble, TrackStringOptions} from "../../core/Atomic"; import {buildTrackString, capitalize, truncateStringToLength} from "../../core/StringUtils"; import EventEmitter from "events"; import {compareScrobbleArtists, compareScrobbleTracks, normalizeStr} from "../utils/StringUtils"; @@ -62,8 +62,8 @@ export default abstract class AbstractScrobbleClient { scrobbleRetries: number = 0; scrobbling: boolean = false; userScrobblingStopSignal: undefined | any; - queuedScrobbles: (SourceScrobble & {id: string})[] = []; - deadLetterScrobbles: (SourceScrobble & {id: string, retries: number})[] = []; + queuedScrobbles: QueuedScrobble[] = []; + deadLetterScrobbles: DeadLetterScrobble[] = []; config: CommonClientConfig; logger: Logger; @@ -401,7 +401,8 @@ ${closestMatch.breakdowns.join('\n')}`); } } try { - return this.doScrobble(playObj); + throw new UpstreamError('a test', {showStopper: false}); + //return this.doScrobble(playObj); } finally { this.lastScrobbleAttempt = dayjs(); } @@ -588,29 +589,29 @@ ${closestMatch.breakdowns.join('\n')}`); return; } - const idsToRemove = []; + const removedIds = []; for (const deadScrobble of this.deadLetterScrobbles) { if (deadScrobble.retries < retries) { - const scrobbled = await this.processDeadLetterScrobble(deadScrobble.id); + const [scrobbled, dead] = await this.processDeadLetterScrobble(deadScrobble.id); if (scrobbled) { - idsToRemove.push(deadScrobble.id); + removedIds.push(deadScrobble.id); } } } - if (idsToRemove.length > 0) { - this.deadLetterScrobbles = this.deadLetterScrobbles.filter(x => !idsToRemove.includes(x.id)); - this.logger.info(`Removed ${idsToRemove.length} scrobbles from dead letter queue`, {leaf: 'Dead Letter'}); + if (removedIds.length > 0) { + this.logger.info(`Removed ${removedIds.length} scrobbles from dead letter queue`, {leaf: 'Dead Letter'}); } } - processDeadLetterScrobble = async (id: string) => { + processDeadLetterScrobble = async (id: string): Promise<[boolean, DeadLetterScrobble?]> => { + const deadScrobbleIndex = this.deadLetterScrobbles.findIndex(x => x.id === id); + const deadScrobble = this.deadLetterScrobbles[deadScrobbleIndex]; + if (!(await this.isReady())) { this.logger.warn('Cannot process dead letter scrobble because client is not ready.', {leaf: 'Dead Letter'}); - return; + return [false, deadScrobble]; } - const deadScrobbleIndex = this.deadLetterScrobbles.findIndex(x => x.id === id); - const deadScrobble = this.deadLetterScrobbles[deadScrobbleIndex]; - if (this.lastScrobbleCheck.unix() < this.getLatestQueuePlayDate().unix()) { + if (this.getLatestQueuePlayDate() !== undefined && this.lastScrobbleCheck.unix() < this.getLatestQueuePlayDate().unix()) { await this.refreshScrobbles(); } const [timeFrameValid, timeFrameValidLog] = this.timeFrameIsValid(deadScrobble.play); @@ -621,17 +622,22 @@ ${closestMatch.breakdowns.join('\n')}`); this.addScrobbledTrack(deadScrobble.play, scrobbledPlay); } catch (e) { deadScrobble.retries++; + deadScrobble.lastRetry = dayjs(); this.logger.error(`Could not scrobble ${buildTrackString(deadScrobble.play)} from Source '${deadScrobble.source}' due to error`, {leaf: 'Dead Letter'}); this.logger.error(e); this.deadLetterScrobbles[deadScrobbleIndex] = deadScrobble; - return false; + return [false, deadScrobble]; } finally { await sleep(1000); } } else if (!timeFrameValid) { this.logger.debug(`Will not scrobble ${buildTrackString(deadScrobble.play)} from Source '${deadScrobble.source}' because it ${timeFrameValidLog}`, {leaf: 'Dead Letter'}); } - return true; + if(deadScrobble !== undefined) { + this.removeDeadLetterScrobble(deadScrobble.id) + } + + return [true]; } removeDeadLetterScrobble = (id: string) => { @@ -658,7 +664,7 @@ ${closestMatch.breakdowns.join('\n')}`); this.queuedScrobbles.sort((a, b) => sortByOldestPlayDate(a.play, b.play)); } - protected addDeadLetterScrobble = (data: SourceScrobble & { id?: string, retries?: number }) => { + protected addDeadLetterScrobble = (data: QueuedScrobble) => { this.deadLetterScrobbles.push({id: nanoid(), retries: 0, ...data}); this.deadLetterScrobbles.sort((a, b) => sortByOldestPlayDate(a.play, b.play)); } diff --git a/src/backend/server/api.ts b/src/backend/server/api.ts index 1ed92bd1..c35654cb 100644 --- a/src/backend/server/api.ts +++ b/src/backend/server/api.ts @@ -3,7 +3,7 @@ import {getRoot} from "../ioc"; import {makeClientCheckMiddle, makeSourceCheckMiddle} from "./middleware"; import AbstractSource from "../sources/AbstractSource"; import { - ClientStatusData, + ClientStatusData, DeadLetterScrobble, LogInfo, LogInfoJson, LogLevel, @@ -197,7 +197,8 @@ export const setupApi = (app: ExpressWithAsync, logger: Logger, initialLogOutput hasAuth: requiresAuth, hasAuthInteraction: requiresAuthInteraction, authed, - initialized + initialized, + deadLetterScrobbles: x.deadLetterScrobbles.length }; if (!initialized) { base.status = 'Not Initialized'; @@ -225,6 +226,67 @@ export const setupApi = (app: ExpressWithAsync, logger: Logger, initialLogOutput return res.json(result); }); + app.getAsync('/api/dead', clientMiddleFunc(true), async (req, res, next) => { + const { + // @ts-expect-error TS(2339): Property 'scrobbleSource' does not exist on type '... Remove this comment to see the full error message + scrobbleClient: client, + } = req; + + let result: DeadLetterScrobble[] = []; + if (client !== undefined) { + result = (client as AbstractScrobbleClient).deadLetterScrobbles; + } + + return res.json(result); + }); + + app.putAsync('/api/dead/:id', clientMiddleFunc(true), async (req, res, next) => { + const { + // @ts-expect-error TS(2339): Property 'scrobbleSource' does not exist on type '... Remove this comment to see the full error message + scrobbleClient: client, + params: { + id + } = {} + } = req; + + const deadId = id as string; + + const deadScrobble = (client as AbstractScrobbleClient).deadLetterScrobbles.find(x => x.id === deadId); + + if(deadScrobble === undefined) { + return res.sendStatus(400); + } + + const [scrobbled, dead] = await (client as AbstractScrobbleClient).processDeadLetterScrobble(deadId); + + if(scrobbled) { + return res.sendStatus(200); + } + + return res.json(dead); + }); + + app.deleteAsync('/api/dead/:id', clientMiddleFunc(true), async (req, res, next) => { + const { + // @ts-expect-error TS(2339): Property 'scrobbleSource' does not exist on type '... Remove this comment to see the full error message + scrobbleClient: client, + params: { + id + } = {} + } = req; + + const deadId = id as string; + + const deadScrobble = (client as AbstractScrobbleClient).deadLetterScrobbles.find(x => x.id === deadId); + + if(deadScrobble === undefined) { + return res.sendStatus(400); + } + + (client as AbstractScrobbleClient).removeDeadLetterScrobble(deadId); + return res.sendStatus(200); + }); + app.getAsync('/api/scrobbled', clientMiddleFunc(false), async (req, res, next) => { const { // @ts-ignore diff --git a/src/client/App.tsx b/src/client/App.tsx index ff3546fe..b7c4c380 100644 --- a/src/client/App.tsx +++ b/src/client/App.tsx @@ -11,6 +11,7 @@ import {store} from './store'; import Dashboard from "./dashboard/dashboard"; import RecentPage from "./recent/RecentPage"; import ScrobbledPage from "./scrobbled/ScrobbledPage"; +import DeadPage from "./deadLetter/DeadPage"; function NoMatch() { let location = useLocation(); @@ -35,6 +36,10 @@ const router = createBrowserRouter([ path: "/scrobbled", element: , }, + { + path: "/dead", + element: , + }, { path: "*", element: diff --git a/src/client/components/statusCard/ClientStatusCard.tsx b/src/client/components/statusCard/ClientStatusCard.tsx index d2c45594..5e6ff689 100644 --- a/src/client/components/statusCard/ClientStatusCard.tsx +++ b/src/client/components/statusCard/ClientStatusCard.tsx @@ -30,7 +30,9 @@ const ClientStatusCard = (props: ClientStatusCardData) => { name, type, display, - status + status, + tracksDiscovered = 0, + deadLetterScrobbles = 0 } = {} } = props; let header: string | undefined = display; @@ -50,7 +52,8 @@ const ClientStatusCard = (props: ClientStatusCardData) => { // TODO links body = ( -
{scrobbled}: {data.tracksDiscovered}
+
{scrobbled}: {tracksDiscovered}
+
Failed Scrobbles: {deadLetterScrobbles}
{hasAuth ? (Re)authenticate or initialize : null} ); } diff --git a/src/client/deadLetter/DeadPage.tsx b/src/client/deadLetter/DeadPage.tsx new file mode 100644 index 00000000..73e481b9 --- /dev/null +++ b/src/client/deadLetter/DeadPage.tsx @@ -0,0 +1,61 @@ +import React, {useCallback} from 'react'; +import PlayDisplay from "../components/PlayDisplay"; +import {recentIncludes} from "../../core/Atomic"; +import {useSearchParams} from "react-router-dom"; +import { + useGetDeadQuery, + //useLazyProcessDeadSingleQuery, + //useLazyRemoveDeadSingleQuery, + useRemoveDeadSingleMutation, + useProcessDeadSingleMutation +} from "./deadLetterDucks"; +import dayjs from "dayjs"; +import {Simulate} from "react-dom/test-utils"; +import click = Simulate.click; + +const displayOpts = { + include: recentIncludes, + includeWeb: true +} + +const dead = () => { + let [searchParams, setSearchParams] = useSearchParams(); + const { + data = [], + error, + isLoading, + isSuccess + } = useGetDeadQuery({name: searchParams.get('name'), type: searchParams.get('type')}); + + // const [removeDeadFetch] = useLazyRemoveDeadSingleQuery(); + // const [retryDeadFetch] = useLazyProcessDeadSingleQuery(); + const [removeDeadFetch] = useRemoveDeadSingleMutation(); + const [retryDeadFetch] = useProcessDeadSingleMutation(); + + const retryDead = useCallback((id: string) => retryDeadFetch({name: searchParams.get('name'), type: searchParams.get('type'), id}), [retryDeadFetch, searchParams]); + const removeDead = useCallback((id: string) => removeDeadFetch({name: searchParams.get('name'), type: searchParams.get('type'), id}), [removeDeadFetch, searchParams]); + + return ( +
+
+
+

Failed Scrobbles +

+
+
+ {isSuccess && !isLoading && data.length === 0 ? 'No failed scrobbles!' : null} +
    {data.map(x => (
  • +
    +
    Source:{x.source.replace('Source -', '')}
    +
    Last Retried: {x.lastRetry === undefined ? 'Never' : dayjs.duration(dayjs().diff(dayjs(x.lastRetry))).humanize(true)}
    +
    Retries: {x.retries}
    +
    retryDead(x.id)} className="capitalize underline cursor-pointer">Retry
    +
    removeDead(x.id)} className="capitalize underline cursor-pointer">Remove
    +
    • ))}
+
+
+
+ ); +} + +export default dead; diff --git a/src/client/deadLetter/deadLetterDucks.ts b/src/client/deadLetter/deadLetterDucks.ts new file mode 100644 index 00000000..17be0f03 --- /dev/null +++ b/src/client/deadLetter/deadLetterDucks.ts @@ -0,0 +1,47 @@ +import {createApi, fetchBaseQuery} from "@reduxjs/toolkit/dist/query/react/index"; +import {DeadLetterScrobble, JsonPlayObject} from "../../core/Atomic"; +import {id} from "common-tags"; + +type DeadResponse = DeadLetterScrobble[]; +export const deadApi = createApi({ + reducerPath: 'deadApi', + baseQuery: fetchBaseQuery({ baseUrl: '/api/' }), + tagTypes: ['DeadLetters'], + endpoints: (builder) => ({ + getDead: builder.query({ + query: (params) => `dead?name=${params.name}&type=${params.type}`, + providesTags: ['DeadLetters'] + }), + processDeadSingle: builder.mutation | undefined, {name: string, type: string, id: string}>({ + query:(params) => ({ + url: `/dead/${params.id}`, + method: 'PUT', + params: { + name: params.name, + type: params.type + } + }) + }), + removeDeadSingle: builder.mutation | undefined, {name: string, type: string, id: string}>({ + query:(params) => ({ + url: `/dead/${params.id}`, + method: 'DELETE', + params: { + name: params.name, + type: params.type + }, + transformResponse: (response, meta, arg) => { + if(response === undefined) { + return undefined; + } else { + return response; + } + }, + invalidatesTags: ['DeadLetters'] + }), + }), + }), +}); + +//export const { useGetDeadQuery, useLazyProcessDeadSingleQuery, useLazyRemoveDeadSingleQuery } = deadApi; +export const { useGetDeadQuery, useProcessDeadSingleMutation, useRemoveDeadSingleMutation } = deadApi; diff --git a/src/client/store.ts b/src/client/store.ts index 32d88404..4b308774 100644 --- a/src/client/store.ts +++ b/src/client/store.ts @@ -7,6 +7,7 @@ import {logsReducer} from "./logs/logDucks"; import {logsApi} from "./logs/logsApi"; import {recentApi} from "./recent/recentDucks"; import {scrobbledApi} from "./scrobbled/scrobbledDucks"; +import {deadApi} from "./deadLetter/deadLetterDucks"; export const store = configureStore({ reducer: { @@ -14,6 +15,7 @@ export const store = configureStore({ [statusApi.reducerPath]: statusApi.reducer, [logsApi.reducerPath]: logsApi.reducer, [recentApi.reducerPath]: recentApi.reducer, + [deadApi.reducerPath]: deadApi.reducer, [scrobbledApi.reducerPath]: scrobbledApi.reducer, //parts: statusReducer clients: clientSlice.reducer, @@ -23,7 +25,7 @@ export const store = configureStore({ // Adding the api middleware enables caching, invalidation, polling, // and other useful features of `rtk-query`. middleware: (getDefaultMiddleware) => - getDefaultMiddleware().concat([statusApi.middleware, logsApi.middleware, recentApi.middleware, scrobbledApi.middleware]), + getDefaultMiddleware().concat([statusApi.middleware, logsApi.middleware, recentApi.middleware, scrobbledApi.middleware, deadApi.middleware]), }) // optional, but required for refetchOnFocus/refetchOnReconnect behaviors diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts index 9b2c9a72..bc37ce7c 100644 --- a/src/core/Atomic.ts +++ b/src/core/Atomic.ts @@ -20,6 +20,7 @@ export interface ClientStatusData { type: "maloja" | "lastfm" | "listenbrainz"; display: string; tracksDiscovered: number; + deadLetterScrobbles: number name: string; hasAuth: boolean; hasAuthInteraction: boolean; @@ -132,13 +133,17 @@ export interface PlayObject extends AmbPlayObject { } export interface JsonPlayObject extends AmbPlayObject { - playDate?: string + data: JsonPlayData } export interface ObjectPlayData extends PlayData { playDate?: Dayjs } +export interface JsonPlayData extends PlayData { + playDate?: Dayjs +} + export type LogLevel = "error" | "warn" | "info" | "verbose" | "debug"; export const logLevels = ['error', 'warn', 'info', 'verbose', 'debug']; @@ -180,3 +185,18 @@ export interface SourcePlayerObj { export interface SourcePlayerJson extends Omit { play: JsonPlayObject } + +export interface SourceScrobble { + source: string + play: PlayType +} + +export interface QueuedScrobble extends SourceScrobble { + id: string +} + +export interface DeadLetterScrobble extends QueuedScrobble { + id: string + retries: number + lastRetry?: RetryType +} -- 2.51.2 From 1c5530ee2afaa0a068ebba6df179b68bddea338e Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 13 Oct 2023 11:33:47 -0400 Subject: [PATCH 05/16] fix: deadletter endpoint responses Send empty response so RTK Query correctly parses response --- src/backend/server/api.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/backend/server/api.ts b/src/backend/server/api.ts index c35654cb..84b0defc 100644 --- a/src/backend/server/api.ts +++ b/src/backend/server/api.ts @@ -254,13 +254,13 @@ export const setupApi = (app: ExpressWithAsync, logger: Logger, initialLogOutput const deadScrobble = (client as AbstractScrobbleClient).deadLetterScrobbles.find(x => x.id === deadId); if(deadScrobble === undefined) { - return res.sendStatus(400); + return res.status(404).send(); } const [scrobbled, dead] = await (client as AbstractScrobbleClient).processDeadLetterScrobble(deadId); if(scrobbled) { - return res.sendStatus(200); + return res.status(200).send(); } return res.json(dead); @@ -280,11 +280,11 @@ export const setupApi = (app: ExpressWithAsync, logger: Logger, initialLogOutput const deadScrobble = (client as AbstractScrobbleClient).deadLetterScrobbles.find(x => x.id === deadId); if(deadScrobble === undefined) { - return res.sendStatus(400); + return res.status(404).send(); } (client as AbstractScrobbleClient).removeDeadLetterScrobble(deadId); - return res.sendStatus(200); + return res.status(200).send(); }); app.getAsync('/api/scrobbled', clientMiddleFunc(false), async (req, res, next) => { -- 2.51.2 From 0c90de78abe258d0fc5e47755523cc8c5ef284cc Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 13 Oct 2023 11:34:22 -0400 Subject: [PATCH 06/16] Implement full-fat deadletter reducer couldn't figure out a better way to handle this with just the api endpoint code --- .../scrobblers/AbstractScrobbleClient.ts | 3 +- src/client/deadLetter/DeadPage.tsx | 48 +++++++++---- src/client/deadLetter/deadLetterDucks.ts | 71 ++++++++++++++++--- src/client/store.ts | 3 +- 4 files changed, 101 insertions(+), 24 deletions(-) diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts index b56f36fa..3a8e40d3 100644 --- a/src/backend/scrobblers/AbstractScrobbleClient.ts +++ b/src/backend/scrobblers/AbstractScrobbleClient.ts @@ -401,8 +401,7 @@ ${closestMatch.breakdowns.join('\n')}`); } } try { - throw new UpstreamError('a test', {showStopper: false}); - //return this.doScrobble(playObj); + return this.doScrobble(playObj); } finally { this.lastScrobbleAttempt = dayjs(); } diff --git a/src/client/deadLetter/DeadPage.tsx b/src/client/deadLetter/DeadPage.tsx index 73e481b9..a6dfca6c 100644 --- a/src/client/deadLetter/DeadPage.tsx +++ b/src/client/deadLetter/DeadPage.tsx @@ -1,34 +1,40 @@ -import React, {useCallback} from 'react'; +import React, {useCallback, useEffect} from 'react'; import PlayDisplay from "../components/PlayDisplay"; import {recentIncludes} from "../../core/Atomic"; import {useSearchParams} from "react-router-dom"; import { useGetDeadQuery, - //useLazyProcessDeadSingleQuery, - //useLazyRemoveDeadSingleQuery, useRemoveDeadSingleMutation, - useProcessDeadSingleMutation + useProcessDeadSingleMutation, deadAdapter, clearDead } from "./deadLetterDucks"; import dayjs from "dayjs"; -import {Simulate} from "react-dom/test-utils"; -import click = Simulate.click; +import {RootState} from "../store"; +import {connect, ConnectedProps} from "react-redux"; const displayOpts = { include: recentIncludes, includeWeb: true } -const dead = () => { - let [searchParams, setSearchParams] = useSearchParams(); +const dead = (props: PropsFromRedux) => { const { data = [], - error, + clearDeadLetter + } = props; + + useEffect(() => { + return () => { + console.log('clearing dead letter'); + clearDeadLetter(); + } + }, []); + + let [searchParams, setSearchParams] = useSearchParams(); + const { isLoading, isSuccess } = useGetDeadQuery({name: searchParams.get('name'), type: searchParams.get('type')}); - // const [removeDeadFetch] = useLazyRemoveDeadSingleQuery(); - // const [retryDeadFetch] = useLazyProcessDeadSingleQuery(); const [removeDeadFetch] = useRemoveDeadSingleMutation(); const [retryDeadFetch] = useProcessDeadSingleMutation(); @@ -58,4 +64,22 @@ const dead = () => { ); } -export default dead; +const deadSelectors = deadAdapter.getSelectors(); + +const mapStateToProps = (state: RootState) => { + return { + data: deadSelectors.selectAll(state.deadLetter) + } +} + +const mapDispatchToProps = (dispatch) => { + return { + clearDeadLetter: () => dispatch(clearDead()) + } +} + +const connector = connect(mapStateToProps, mapDispatchToProps); + +type PropsFromRedux = ConnectedProps + +export default connector(dead); diff --git a/src/client/deadLetter/deadLetterDucks.ts b/src/client/deadLetter/deadLetterDucks.ts index 17be0f03..c30c72c5 100644 --- a/src/client/deadLetter/deadLetterDucks.ts +++ b/src/client/deadLetter/deadLetterDucks.ts @@ -1,19 +1,24 @@ import {createApi, fetchBaseQuery} from "@reduxjs/toolkit/dist/query/react/index"; import {DeadLetterScrobble, JsonPlayObject} from "../../core/Atomic"; import {id} from "common-tags"; +import {createAction, createEntityAdapter, createSlice} from "@reduxjs/toolkit"; type DeadResponse = DeadLetterScrobble[]; export const deadApi = createApi({ reducerPath: 'deadApi', - baseQuery: fetchBaseQuery({ baseUrl: '/api/' }), + baseQuery: fetchBaseQuery({baseUrl: '/api/'}), tagTypes: ['DeadLetters'], endpoints: (builder) => ({ - getDead: builder.query({ + getDead: builder.query({ query: (params) => `dead?name=${params.name}&type=${params.type}`, providesTags: ['DeadLetters'] }), - processDeadSingle: builder.mutation | undefined, {name: string, type: string, id: string}>({ - query:(params) => ({ + processDeadSingle: builder.mutation | undefined, { + name: string, + type: string, + id: string + }>({ + query: (params) => ({ url: `/dead/${params.id}`, method: 'PUT', params: { @@ -22,8 +27,12 @@ export const deadApi = createApi({ } }) }), - removeDeadSingle: builder.mutation | undefined, {name: string, type: string, id: string}>({ - query:(params) => ({ + removeDeadSingle: builder.mutation | undefined, { + name: string, + type: string, + id: string + }>({ + query: (params) => ({ url: `/dead/${params.id}`, method: 'DELETE', params: { @@ -31,7 +40,7 @@ export const deadApi = createApi({ type: params.type }, transformResponse: (response, meta, arg) => { - if(response === undefined) { + if (response === undefined) { return undefined; } else { return response; @@ -43,5 +52,49 @@ export const deadApi = createApi({ }), }); -//export const { useGetDeadQuery, useLazyProcessDeadSingleQuery, useLazyRemoveDeadSingleQuery } = deadApi; -export const { useGetDeadQuery, useProcessDeadSingleMutation, useRemoveDeadSingleMutation } = deadApi; +export const deadAdapter = createEntityAdapter>({ + selectId: (data) => data.id +}); + +export const deadSlice = createSlice({ + name: 'deadLetter', + initialState: deadAdapter.getInitialState(), + reducers: { + deadUpdated: deadAdapter.updateOne, + }, + extraReducers: (builder) => { + builder.addMatcher( + (action) => deadApi.endpoints.getDead.matchFulfilled(action), + (state, action) => { + deadAdapter.setAll(state, action.payload); + } + ) + .addMatcher( + (action) => deadApi.endpoints.removeDeadSingle.matchFulfilled(action), + (state, action) => { + deadAdapter.removeOne(state, action.meta.arg.originalArgs.id) + } + ) + .addMatcher( + (action) => deadApi.endpoints.processDeadSingle.matchFulfilled(action), + (state, action) => { + if (action.payload === undefined) { + deadAdapter.removeOne(state, action.meta.arg.originalArgs.id); + } else { + state.entities[action.meta.arg.originalArgs.id] = action.payload; + //deadAdapter.updateOne(state, action.meta.arg.originalArgs.id); + } + } + ) + .addMatcher( + (action) => clearDead.match(action), + (state, action) => { + state = deadAdapter.getInitialState(); + } + ) + } +}); + +export const clearDead = createAction('clearDead'); + +export const {useGetDeadQuery, useProcessDeadSingleMutation, useRemoveDeadSingleMutation} = deadApi; diff --git a/src/client/store.ts b/src/client/store.ts index 4b308774..2c5fd02d 100644 --- a/src/client/store.ts +++ b/src/client/store.ts @@ -7,7 +7,7 @@ import {logsReducer} from "./logs/logDucks"; import {logsApi} from "./logs/logsApi"; import {recentApi} from "./recent/recentDucks"; import {scrobbledApi} from "./scrobbled/scrobbledDucks"; -import {deadApi} from "./deadLetter/deadLetterDucks"; +import {deadApi, deadSlice} from "./deadLetter/deadLetterDucks"; export const store = configureStore({ reducer: { @@ -20,6 +20,7 @@ export const store = configureStore({ //parts: statusReducer clients: clientSlice.reducer, sources: sourceSlice.reducer, + deadLetter: deadSlice.reducer, logs: logsReducer }, // Adding the api middleware enables caching, invalidation, polling, -- 2.51.2 From 884134cab9d9573a5f79cf73dd4d996e806d1dc5 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 13 Oct 2023 11:38:38 -0400 Subject: [PATCH 07/16] Simplify dead letter data clear Just reset state on GET pending instead of using effect cleanup in react component --- src/client/deadLetter/DeadPage.tsx | 8 -------- src/client/deadLetter/deadLetterDucks.ts | 6 ++++++ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/client/deadLetter/DeadPage.tsx b/src/client/deadLetter/DeadPage.tsx index a6dfca6c..91ca933b 100644 --- a/src/client/deadLetter/DeadPage.tsx +++ b/src/client/deadLetter/DeadPage.tsx @@ -19,16 +19,8 @@ const displayOpts = { const dead = (props: PropsFromRedux) => { const { data = [], - clearDeadLetter } = props; - useEffect(() => { - return () => { - console.log('clearing dead letter'); - clearDeadLetter(); - } - }, []); - let [searchParams, setSearchParams] = useSearchParams(); const { isLoading, diff --git a/src/client/deadLetter/deadLetterDucks.ts b/src/client/deadLetter/deadLetterDucks.ts index c30c72c5..65486023 100644 --- a/src/client/deadLetter/deadLetterDucks.ts +++ b/src/client/deadLetter/deadLetterDucks.ts @@ -63,6 +63,12 @@ export const deadSlice = createSlice({ deadUpdated: deadAdapter.updateOne, }, extraReducers: (builder) => { + builder.addMatcher( + (action) => deadApi.endpoints.getDead.matchPending(action), + (state, action) => { + state = deadAdapter.getInitialState(); + } + ) builder.addMatcher( (action) => deadApi.endpoints.getDead.matchFulfilled(action), (state, action) => { -- 2.51.2 From 57f2ebf2b51f73bd7be3449a08498a770bb9dfd0 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 13 Oct 2023 12:12:46 -0400 Subject: [PATCH 08/16] feat: Add dead letter error to data --- package-lock.json | 9 ++++++ package.json | 1 + .../scrobblers/AbstractScrobbleClient.ts | 20 ++++++++----- src/client/components/loading/Loading.css | 8 +++++ src/client/components/loading/Loading.tsx | 30 +++++++++++++++++++ src/client/deadLetter/DeadPage.tsx | 13 ++++---- src/client/logs/LogsSection.css | 8 ----- src/client/logs/LogsSection.tsx | 19 ++---------- src/core/Atomic.ts | 1 + 9 files changed, 71 insertions(+), 38 deletions(-) create mode 100644 src/client/components/loading/Loading.css create mode 100644 src/client/components/loading/Loading.tsx diff --git a/package-lock.json b/package-lock.json index 38757dc7..811758b8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,6 +23,7 @@ "ajv": "^7.2.4", "better-sse": "^0.8.0", "body-parser": "^1.19.0", + "clsx": "^2.0.0", "common-tags": "^1.8.2", "compare-versions": "^4.1.2", "concat-stream": "^2.0.0", @@ -7424,6 +7425,14 @@ "node": ">=4" } }, + "node_modules/clsx": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.0.0.tgz", + "integrity": "sha512-rQ1+kcj+ttHG0MKVGBUXwayCCF1oh39BF5COIpRzuCEv8Mwjv0XucrI2ExNTOn9IlLifGClWQcU9BrZORvtw6Q==", + "engines": { + "node": ">=6" + } + }, "node_modules/co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", diff --git a/package.json b/package.json index 94a304e9..30f5071e 100644 --- a/package.json +++ b/package.json @@ -60,6 +60,7 @@ "ajv": "^7.2.4", "better-sse": "^0.8.0", "body-parser": "^1.19.0", + "clsx": "^2.0.0", "common-tags": "^1.8.2", "compare-versions": "^4.1.2", "concat-stream": "^2.0.0", diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts index 3a8e40d3..87d26a70 100644 --- a/src/backend/scrobblers/AbstractScrobbleClient.ts +++ b/src/backend/scrobblers/AbstractScrobbleClient.ts @@ -27,7 +27,7 @@ import EventEmitter from "events"; import {compareScrobbleArtists, compareScrobbleTracks, normalizeStr} from "../utils/StringUtils"; import {UpstreamError} from "../common/errors/UpstreamError"; import {nanoid} from "nanoid"; -import {ErrorWithCause} from "pony-cause"; +import {ErrorWithCause, messageWithCauses} from "pony-cause"; import {de} from "@faker-js/faker"; import {del} from "superagent"; @@ -537,8 +537,8 @@ ${closestMatch.breakdowns.join('\n')}`); this.addScrobbledTrack(currQueuedPlay.play, scrobbledPlay); } catch (e) { if (e instanceof UpstreamError && e.showStopper === false) { - this.addDeadLetterScrobble(currQueuedPlay); - this.logger.warn(`Could not scrobble ${buildTrackString(currQueuedPlay.play)} from Source '${currQueuedPlay.source}' but error was not show stopping. Adding scrobble to Dead Letter Queue and will retry on next heartbeat.`); + this.addDeadLetterScrobble(currQueuedPlay, e); + this.logger.warn(new ErrorWithCause(`Could not scrobble ${buildTrackString(currQueuedPlay.play)} from Source '${currQueuedPlay.source}' but error was not show stopping. Adding scrobble to Dead Letter Queue and will retry on next heartbeat.`, {cause: e})); } else { const processError = new ErrorWithCause('Error occurred while trying to scrobble', {cause: e}); //this.logger.error(processError); @@ -621,9 +621,9 @@ ${closestMatch.breakdowns.join('\n')}`); this.addScrobbledTrack(deadScrobble.play, scrobbledPlay); } catch (e) { deadScrobble.retries++; + deadScrobble.error = messageWithCauses(e); deadScrobble.lastRetry = dayjs(); - this.logger.error(`Could not scrobble ${buildTrackString(deadScrobble.play)} from Source '${deadScrobble.source}' due to error`, {leaf: 'Dead Letter'}); - this.logger.error(e); + this.logger.error(new ErrorWithCause(`Could not scrobble ${buildTrackString(deadScrobble.play)} from Source '${deadScrobble.source}' due to error`, {cause: e})); this.deadLetterScrobbles[deadScrobbleIndex] = deadScrobble; return [false, deadScrobble]; } finally { @@ -663,8 +663,14 @@ ${closestMatch.breakdowns.join('\n')}`); this.queuedScrobbles.sort((a, b) => sortByOldestPlayDate(a.play, b.play)); } - protected addDeadLetterScrobble = (data: QueuedScrobble) => { - this.deadLetterScrobbles.push({id: nanoid(), retries: 0, ...data}); + protected addDeadLetterScrobble = (data: QueuedScrobble, error: (Error | string) = 'Unspecified error') => { + let eString = ''; + if(typeof error === 'string') { + eString = error; + } else { + eString = messageWithCauses(error); + } + this.deadLetterScrobbles.push({id: nanoid(), retries: 0, error: eString, ...data}); this.deadLetterScrobbles.sort((a, b) => sortByOldestPlayDate(a.play, b.play)); } diff --git a/src/client/components/loading/Loading.css b/src/client/components/loading/Loading.css new file mode 100644 index 00000000..e14237f2 --- /dev/null +++ b/src/client/components/loading/Loading.css @@ -0,0 +1,8 @@ +.connected.loading { + display: inline; +} +.loading { + height: 35px; + fill: white; + display: none; +} diff --git a/src/client/components/loading/Loading.tsx b/src/client/components/loading/Loading.tsx new file mode 100644 index 00000000..225909e2 --- /dev/null +++ b/src/client/components/loading/Loading.tsx @@ -0,0 +1,30 @@ +import React from 'react'; +import clsx from 'clsx'; +import './Loading.css'; + +/*https://codepen.io/nikhil8krishnan/pen/rVoXJa*/ +export const Loading = (props: {show?: boolean}) => { + const {show = false} = props || {}; + const classes = ['loading']; + if(show) { + classes.push('connected'); + } + return + + + + ; +} + +export default Loading; diff --git a/src/client/deadLetter/DeadPage.tsx b/src/client/deadLetter/DeadPage.tsx index 91ca933b..930c7c85 100644 --- a/src/client/deadLetter/DeadPage.tsx +++ b/src/client/deadLetter/DeadPage.tsx @@ -1,4 +1,4 @@ -import React, {useCallback, useEffect} from 'react'; +import React, {useCallback, useEffect, useState} from 'react'; import PlayDisplay from "../components/PlayDisplay"; import {recentIncludes} from "../../core/Atomic"; import {useSearchParams} from "react-router-dom"; @@ -27,8 +27,8 @@ const dead = (props: PropsFromRedux) => { isSuccess } = useGetDeadQuery({name: searchParams.get('name'), type: searchParams.get('type')}); - const [removeDeadFetch] = useRemoveDeadSingleMutation(); - const [retryDeadFetch] = useProcessDeadSingleMutation(); + const [removeDeadFetch, removeResult] = useRemoveDeadSingleMutation(); + const [retryDeadFetch, processResult] = useProcessDeadSingleMutation(); const retryDead = useCallback((id: string) => retryDeadFetch({name: searchParams.get('name'), type: searchParams.get('type'), id}), [retryDeadFetch, searchParams]); const removeDead = useCallback((id: string) => removeDeadFetch({name: searchParams.get('name'), type: searchParams.get('type'), id}), [removeDeadFetch, searchParams]); @@ -45,10 +45,11 @@ const dead = (props: PropsFromRedux) => {
    {data.map(x => (
  • Source:{x.source.replace('Source -', '')}
    -
    Last Retried: {x.lastRetry === undefined ? 'Never' : dayjs.duration(dayjs().diff(dayjs(x.lastRetry))).humanize(true)}
    Retries: {x.retries}
    -
    retryDead(x.id)} className="capitalize underline cursor-pointer">Retry
    -
    removeDead(x.id)} className="capitalize underline cursor-pointer">Remove
    +
    Last Retried: {x.lastRetry === undefined ? 'Never' : dayjs.duration(dayjs().diff(dayjs(x.lastRetry))).humanize(true)}
    +
    Error: {x.error}
    +
    retryDead(x.id)} className="capitalize underline cursor-pointer max-w-fit">Retry
    +
    removeDead(x.id)} className="capitalize underline cursor-pointer max-w-fit">Remove
    • ))}
diff --git a/src/client/logs/LogsSection.css b/src/client/logs/LogsSection.css index f9add133..54dc1079 100644 --- a/src/client/logs/LogsSection.css +++ b/src/client/logs/LogsSection.css @@ -1,11 +1,3 @@ -.connected.loading { - display: inline; -} -.loading { - height: 35px; - fill: white; - display: none; -} .line { display: block; white-space: pre-wrap; diff --git a/src/client/logs/LogsSection.tsx b/src/client/logs/LogsSection.tsx index 87302bc2..449593ac 100644 --- a/src/client/logs/LogsSection.tsx +++ b/src/client/logs/LogsSection.tsx @@ -6,6 +6,7 @@ import LogLine from "./LogLine"; import {useGetLogsQuery, useLazySetLevelQuery, logsApi} from "./logsApi"; import {connect, ConnectedProps} from "react-redux"; import {RootState} from "../store"; +import Loading from "../components/loading/Loading"; let logBuffer: { message: string, id: string, level: string }[] = []; @@ -71,23 +72,7 @@ const LogsSection = (props: PropsFromRedux) => {

Log (Most Recent) - {/*https://codepen.io/nikhil8krishnan/pen/rVoXJa*/} - - - - - +

diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts index bc37ce7c..ebe7e0f0 100644 --- a/src/core/Atomic.ts +++ b/src/core/Atomic.ts @@ -199,4 +199,5 @@ export interface DeadLetterScrobble extends QueuedS id: string retries: number lastRetry?: RetryType + error: string } -- 2.51.2 From 62e9f7f620aab6d84f2e96f4712fbf2ee8e97d72 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 13 Oct 2023 14:28:24 -0400 Subject: [PATCH 09/16] feat: Log storage improvements and limit controls * Store logs by levels, fixes issue where rarer warn/info are pushed out of buffer by frequent debug/verbose * Use FixedSizeList instead of homegrown buffer * Implement API and ui controls for setting log limit --- src/backend/common/logging.ts | 15 ++++++++ src/backend/server/api.ts | 66 ++++++++++++++++++++++++--------- src/client/logs/LogsSection.tsx | 28 +++++++++++--- src/client/logs/logDucks.ts | 4 +- src/client/logs/logsApi.ts | 8 ++-- src/core/Atomic.ts | 1 + 6 files changed, 93 insertions(+), 29 deletions(-) diff --git a/src/backend/common/logging.ts b/src/backend/common/logging.ts index 169b1e43..b3e0a7ac 100644 --- a/src/backend/common/logging.ts +++ b/src/backend/common/logging.ts @@ -138,6 +138,7 @@ export const defaultFormat = (defaultLabel = 'App') => printf(({ durationMs, [SPLAT]: splatObj, stack, + id, ...rest }) => { const keys = Object.keys(rest); @@ -168,6 +169,9 @@ export const defaultFormat = (defaultLabel = 'App') => printf(({ return `${timestamp} ${level.padEnd(8)}: ${labelContent} ${msg}${stringifyValue !== '' ? ` ${stringifyValue}` : ''}${stackMsg}`; }); +// https://knowyourmeme.com/memes/cereal-guy +// this number will never overflow +let seqId: number = 0; export const labelledFormat = (labelName = 'App') => { const l = label({label: labelName, message: false}); return combine( @@ -176,6 +180,13 @@ export const labelledFormat = (labelName = 'App') => { format: () => dayjs().local().format(), } ), + { + transform: (info, opts) => { + info.id = seqId; + seqId++; + return info; + } + }, l, s, errorAwareFormat, @@ -215,6 +226,10 @@ export const isLogLineMinLevel = (log: string | LogInfo, minLevelText: LogLevel) return level <= minLevel; } +export const isLogLevelMinLevel = (levelStr: LogLevel, minLevelStr: LogLevel): boolean => { + return logLevels[levelStr] <= logLevels[minLevelStr]; +} + const isProbablyError = (val: any, explicitErrorName?: string) => { if(typeof val !== 'object' || val === null) { return false; diff --git a/src/backend/server/api.ts b/src/backend/server/api.ts index 84b0defc..2726df4a 100644 --- a/src/backend/server/api.ts +++ b/src/backend/server/api.ts @@ -11,7 +11,7 @@ import { SourceStatusData, } from "../../core/Atomic"; import {Logger} from "@foxxmd/winston"; -import {formatLogToHtml, getLogger, isLogLineMinLevel} from "../common/logging"; +import {formatLogToHtml, getLogger, isLogLevelMinLevel, isLogLineMinLevel} from "../common/logging"; import {MESSAGE} from "triple-beam"; import {Transform} from "stream"; import {createSession} from "better-sse"; @@ -28,13 +28,42 @@ import AbstractScrobbleClient from "../scrobblers/AbstractScrobbleClient"; import {sortByNewestPlayDate} from "../utils"; import bodyParser from "body-parser"; import {setupWebscrobblerRoutes} from "./webscrobblerRoutes"; +import {FixedSizeList} from 'fixed-size-list'; + +const maxBufferSize = 300; +const output: { + [key in LogLevel]: FixedSizeList +} = { + 'debug': new FixedSizeList(maxBufferSize), + 'verbose': new FixedSizeList(maxBufferSize), + 'info': new FixedSizeList(maxBufferSize), + 'warn': new FixedSizeList(maxBufferSize), + 'error': new FixedSizeList(maxBufferSize), +} + +const addToLogBuffer = (log: LogInfo) => { + output[log.level as LogLevel].add(log); +} -let output: LogInfo[] = [] +const getLogs = (minLevel: LogLevel, limit: number = maxBufferSize, sort: 'asc' | 'desc' = 'desc'): LogInfo[] => { + const allLogs: LogInfo[][] = []; + for(const level of Object.keys(output)) { + if(isLogLevelMinLevel(level as LogLevel, minLevel)) { + allLogs.push(output[level].data); + } + } + if(sort === 'desc') { + return allLogs.flat(1).sort((a, b) => b.id - a.id).slice(0, limit); + } + return allLogs.flat(1).sort((a, b) => a.id - b.id).slice(0, limit); +} -const availableLevels = ['error', 'warn', 'info', 'verbose', 'debug']; +const availableLevels: LogLevel[] = ['error', 'warn', 'info', 'verbose', 'debug']; export const setupApi = (app: ExpressWithAsync, logger: Logger, initialLogOutput: LogInfo[] = []) => { - output = initialLogOutput; + for(const log of initialLogOutput) { + addToLogBuffer(log); + } const root = getRoot(); //let logWebLevel: LogLevel = logger.level as LogLevel || (process.env.LOG_LEVEL || 'info') as LogLevel; @@ -60,8 +89,7 @@ export const setupApi = (app: ExpressWithAsync, logger: Logger, initialLogOutput const appLogger = getLogger({}, 'app'); appLogger.stream().on('log', (log: LogInfo) => { - output.unshift(log); - output = output.slice(0, 501); + addToLogBuffer(log); if(isLogLineMinLevel(log, logConfig.level)) { logObjectStream.write({message: log[MESSAGE], level: log.level}); } @@ -76,37 +104,39 @@ export const setupApi = (app: ExpressWithAsync, logger: Logger, initialLogOutput const clientRequiredMiddle = clientMiddleFunc(true); const sourceRequiredMiddle = sourceMiddleFunc(true); - const setLogWebLevel: ExpressHandler = async (req, res, next) => { + const setLogWebSettings: ExpressHandler = async (req, res, next) => { // @ts-ignore const sessionLevel: LogLevel | undefined = req.session.logLevel as LogLevel | undefined; if(sessionLevel !== undefined && logConfig.level !== sessionLevel) { logConfig.level = sessionLevel; } + // @ts-ignore + const sessionLimit: number | undefined = req.session.limit as Number | undefined; + if(sessionLimit !== undefined && logConfig.limit !== sessionLimit) { + logConfig.limit = sessionLimit; + } next(); } - app.get('/api/logs/stream', setLogWebLevel, async (req, res) => { + app.get('/api/logs/stream', setLogWebSettings, async (req, res) => { const session = await createSession(req, res); await session.stream(logObjectStream); }); - app.get('/api/logs', setLogWebLevel, async (req, res) => { - let slicedLog = output.filter(x => isLogLineMinLevel(x, logConfig.level)).slice(0, logConfig.limit + 1); - if (logConfig.sort === 'ascending') { - slicedLog.reverse(); - } + app.get('/api/logs', setLogWebSettings, async (req, res) => { + const slicedLog = getLogs(logConfig.level, logConfig.limit + 1, logConfig.sort === 'ascending' ? 'asc' : 'desc'); const jsonLogs: LogInfoJson[] = slicedLog.map(x => ({...x, formattedMessage: x[MESSAGE]})); return res.json({data: jsonLogs, settings: logConfig}); }); app.put('/api/logs', async (req, res) => { - logConfig.level = req.body.level as LogLevel; - let slicedLog = output.filter(x => isLogLineMinLevel(x, logConfig.level)).slice(0, logConfig.limit + 1); - if (logConfig.sort === 'ascending') { - slicedLog.reverse(); - } + logConfig.level = req.body.level as LogLevel | undefined ?? logConfig.level; + logConfig.limit = req.body.limit ?? logConfig.limit; + const slicedLog = getLogs(logConfig.level, logConfig.limit + 1, logConfig.sort === 'ascending' ? 'asc' : 'desc'); // @ts-ignore req.session.logLevel = logConfig.level; + // @ts-ignore + req.session.limit = logConfig.limit; const jsonLogs: LogInfoJson[] = slicedLog.map(x => ({...x, formattedMessage: x[MESSAGE]})); return res.json({data: jsonLogs, settings: logConfig}); }); diff --git a/src/client/logs/LogsSection.tsx b/src/client/logs/LogsSection.tsx index 449593ac..10972f4c 100644 --- a/src/client/logs/LogsSection.tsx +++ b/src/client/logs/LogsSection.tsx @@ -3,7 +3,7 @@ import './LogsSection.css'; import {FixedSizeList} from "fixed-size-list"; import {useEventSource, useEventSourceListener} from "@react-nano/use-event-source"; import LogLine from "./LogLine"; -import {useGetLogsQuery, useLazySetLevelQuery, logsApi} from "./logsApi"; +import {useGetLogsQuery, useLazySetLogSettingsQuery, logsApi} from "./logsApi"; import {connect, ConnectedProps} from "react-redux"; import {RootState} from "../store"; import Loading from "../components/loading/Loading"; @@ -33,11 +33,20 @@ const LogLevelButton = (props: LogLevelButtonProps) => { const className = active ? "mx-1" : "capitalize underline cursor-pointer mx-1"; return {name.toUpperCase()}; } +const LogLimitButton = (props: {val: number, active: boolean, onClick: Function}) => { + const {val, active, onClick} = props; + const click = useCallback(() => active ? null : onClick(val), [onClick, val, active]); + const className = active ? "mx-1" : "capitalize underline cursor-pointer mx-1"; + return {val}; +} const LogsSection = (props: PropsFromRedux) => { const { logs, settings, + settings: { + limit = 50 + } } = props; const [logList, setLogList] = useState(logBuffer); @@ -51,11 +60,14 @@ const LogsSection = (props: PropsFromRedux) => { setLogLevel(settings.level); }, [logs, settings, setLogList, setLogLevel]); - const [setLevel] = useLazySetLevelQuery(); + const [setSettings] = useLazySetLogSettingsQuery(); const fetchLevel = useCallback(async (val) => { - setLevel(val); - }, [setLogLevel]); + setSettings({level: val}); + }, [setSettings]); + const fetchLimit = useCallback(async (val) => { + setSettings({limit: val}); + }, [setSettings]); const [eventSource, eventSourceStatus] = useEventSource("api/logs/stream", false); useEventSourceListener(eventSource, ['messsage', 'stream'], evt => { @@ -78,7 +90,13 @@ const LogsSection = (props: PropsFromRedux) => {
Level : | | - + | + | + +
+
Limit : | + | +

diff --git a/src/client/logs/logDucks.ts b/src/client/logs/logDucks.ts index 30c78688..fa53f7c0 100644 --- a/src/client/logs/logDucks.ts +++ b/src/client/logs/logDucks.ts @@ -12,9 +12,9 @@ const initialState: LogsState = {data: [], settings: {level: 'debug', sort: 'asc const logsReducer = createReducer(initialState, (builder) => { builder .addMatcher( - (action) => logsApi.endpoints.getLogs.matchFulfilled(action) || logsApi.endpoints.setLevel.matchFulfilled(action), + (action) => logsApi.endpoints.getLogs.matchFulfilled(action) || logsApi.endpoints.setLogSettings.matchFulfilled(action), (state, action) => { - state.data = action.payload.data.slice(0, 50); + state.data = action.payload.data.slice(0, action.payload.settings.limit + 1); state.settings = action.payload.settings; } ) diff --git a/src/client/logs/logsApi.ts b/src/client/logs/logsApi.ts index 18b4ad54..7a33e375 100644 --- a/src/client/logs/logsApi.ts +++ b/src/client/logs/logsApi.ts @@ -8,14 +8,14 @@ export const logsApi = createApi({ getLogs: builder.query<{ data: LogInfoJson[], settings: LogOutputConfig }, undefined>({ query: () => `logs`, }), - setLevel: builder.query<{ data: LogInfoJson[], settings: LogOutputConfig }, string>({ - query: (level) => ({ + setLogSettings: builder.query<{ data: LogInfoJson[], settings: LogOutputConfig }, object>({ + query: (settings) => ({ url: '/logs', method: 'PUT', - body: {level} + body: settings }) }), }), }); -export const { useGetLogsQuery, useLazySetLevelQuery } = logsApi; +export const { useGetLogsQuery, useLazySetLogSettingsQuery } = logsApi; diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts index ebe7e0f0..f6025111 100644 --- a/src/core/Atomic.ts +++ b/src/core/Atomic.ts @@ -148,6 +148,7 @@ export type LogLevel = "error" | "warn" | "info" | "verbose" | "debug"; export const logLevels = ['error', 'warn', 'info', 'verbose', 'debug']; export interface LogInfo { + id: number message: string [MESSAGE]: string, level: string -- 2.51.2 From 2882f65b692428bdd8781258f4fc6fd797f340c4 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 19 Oct 2023 10:36:57 -0400 Subject: [PATCH 10/16] refactor: Reduce noisy logging at INFO level --- src/backend/common/vendor/LastfmApiClient.ts | 2 +- src/backend/scrobblers/AbstractScrobbleClient.ts | 4 +++- src/backend/tasks/heartbeatClients.ts | 2 +- src/backend/tasks/heartbeatSources.ts | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/backend/common/vendor/LastfmApiClient.ts b/src/backend/common/vendor/LastfmApiClient.ts index f67305a4..c50801f4 100644 --- a/src/backend/common/vendor/LastfmApiClient.ts +++ b/src/backend/common/vendor/LastfmApiClient.ts @@ -144,7 +144,7 @@ export default class LastfmApiClient extends AbstractApiClient { testAuth = async () => { if (this.client.sessionKey === undefined) { - this.logger.info('No session key found. User interaction for authentication required.'); + this.logger.warn('No session key found. User interaction for authentication required.'); this.logger.info(`Redirect URL that will be used on auth callback: '${this.redirectUri}'`); return false; } diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts index 87d26a70..02f27522 100644 --- a/src/backend/scrobblers/AbstractScrobbleClient.ts +++ b/src/backend/scrobblers/AbstractScrobbleClient.ts @@ -583,10 +583,12 @@ ${closestMatch.breakdowns.join('\n')}`); const retries = attemptWithRetries ?? deadLetterRetries; const processable = this.deadLetterScrobbles.filter(x => x.retries < retries); - this.logger.info(`${processable.length} of ${this.deadLetterScrobbles.length} dead scrobbles have less than ${retries} retries, ${processable.length === 0 ? 'will skip processing.': 'processing now...'}`, {leaf: 'Dead Letter'}); + const queueStatus = `${processable.length} of ${this.deadLetterScrobbles.length} dead scrobbles have less than ${retries} retries, ${processable.length === 0 ? 'will skip processing.': 'processing now...'}`; if (processable.length === 0) { + this.logger.verbose(queueStatus, {leaf: 'Dead Letter'}); return; } + this.logger.info(queueStatus, {leaf: 'Dead Letter'}); const removedIds = []; for (const deadScrobble of this.deadLetterScrobbles) { diff --git a/src/backend/tasks/heartbeatClients.ts b/src/backend/tasks/heartbeatClients.ts index 90619d0c..9083e9f6 100644 --- a/src/backend/tasks/heartbeatClients.ts +++ b/src/backend/tasks/heartbeatClients.ts @@ -24,7 +24,7 @@ export const createHeartbeatClientsTask = (clients: ScrobbleClients, parentLogge return 0; } }).then(({results, errors}) => { - logger.info(`Checked Dead letter queue for ${clients.clients.length} clients.`); + logger.verbose(`Checked Dead letter queue for ${clients.clients.length} clients.`); const restarted = results.reduce((acc, curr) => acc += curr, 0); if (restarted > 0) { logger.info(`Attempted to restart ${restarted} clients that were not processing scrobbles.`); diff --git a/src/backend/tasks/heartbeatSources.ts b/src/backend/tasks/heartbeatSources.ts index 07eb4d1c..c468f1b5 100644 --- a/src/backend/tasks/heartbeatSources.ts +++ b/src/backend/tasks/heartbeatSources.ts @@ -21,7 +21,7 @@ export const createHeartbeatSourcesTask = (sources: ScrobbleSources, parentLogge } return 0; }).then(({results, errors}) => { - logger.info(`Checked ${sources.sources.length} sources for restart signals.`); + logger.verbose(`Checked ${sources.sources.length} sources for restart signals.`); const restarted = results.reduce((acc, curr) => acc += curr, 0); if (restarted > 0) { logger.info(`Attempted to restart ${restarted} sources that were not polling.`); -- 2.51.2 From 0143ae62aa60565d984413c927fc34f53cb24bca Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 19 Oct 2023 11:03:04 -0400 Subject: [PATCH 11/16] feat: Implement retry/remove all dead letter scrobbles * Implement API endpoints * Implement front-end logic and actions on dead letter page --- .../scrobblers/AbstractScrobbleClient.ts | 7 ++++- src/backend/server/api.ts | 29 ++++++++++++++++--- src/client/deadLetter/DeadPage.tsx | 16 ++++++++-- src/client/deadLetter/deadLetterDucks.ts | 27 +++++++++++++++-- 4 files changed, 68 insertions(+), 11 deletions(-) diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts index 02f27522..e6c48318 100644 --- a/src/backend/scrobblers/AbstractScrobbleClient.ts +++ b/src/backend/scrobblers/AbstractScrobbleClient.ts @@ -646,10 +646,15 @@ ${closestMatch.breakdowns.join('\n')}`); if (index === -1) { this.logger.warn(`No scrobble found with ID ${id}`, {leaf: 'Dead Letter'}); } - this.logger.debug(`Removed scrobble ${buildTrackString(this.deadLetterScrobbles[index].play)} from queue`, {leaf: 'Dead Letter'}); + this.logger.info(`Removed scrobble ${buildTrackString(this.deadLetterScrobbles[index].play)} from queue`, {leaf: 'Dead Letter'}); this.deadLetterScrobbles.splice(index, 1); } + removeDeadLetterScrobbles = () => { + this.deadLetterScrobbles = []; + this.logger.info('Removed all scrobbles from queue', {leaf: 'Dead Letter'}); + } + protected getLatestQueuePlayDate = () => { if (this.queuedScrobbles.length === 0) { return undefined; diff --git a/src/backend/server/api.ts b/src/backend/server/api.ts index 2726df4a..99ac791e 100644 --- a/src/backend/server/api.ts +++ b/src/backend/server/api.ts @@ -262,10 +262,20 @@ export const setupApi = (app: ExpressWithAsync, logger: Logger, initialLogOutput scrobbleClient: client, } = req; - let result: DeadLetterScrobble[] = []; - if (client !== undefined) { - result = (client as AbstractScrobbleClient).deadLetterScrobbles; - } + let result: DeadLetterScrobble[] = (client as AbstractScrobbleClient).deadLetterScrobbles; + + return res.json(result); + }); + + app.putAsync('/api/dead', clientMiddleFunc(true), async (req, res, next) => { + const { + // @ts-expect-error TS(2339): Property 'scrobbleSource' does not exist on type '... Remove this comment to see the full error message + scrobbleClient: client, + } = req; + + await (client as AbstractScrobbleClient).processDeadLetterQueue(1000); + + let result: DeadLetterScrobble[] = (client as AbstractScrobbleClient).deadLetterScrobbles; return res.json(result); }); @@ -296,6 +306,17 @@ export const setupApi = (app: ExpressWithAsync, logger: Logger, initialLogOutput return res.json(dead); }); + app.deleteAsync('/api/dead', clientMiddleFunc(true), async (req, res, next) => { + const { + // @ts-expect-error TS(2339): Property 'scrobbleSource' does not exist on type '... Remove this comment to see the full error message + scrobbleClient: client, + } = req; + + (client as AbstractScrobbleClient).removeDeadLetterScrobbles(); + + return res.json([]); + }); + app.deleteAsync('/api/dead/:id', clientMiddleFunc(true), async (req, res, next) => { const { // @ts-expect-error TS(2339): Property 'scrobbleSource' does not exist on type '... Remove this comment to see the full error message diff --git a/src/client/deadLetter/DeadPage.tsx b/src/client/deadLetter/DeadPage.tsx index 930c7c85..9e06753e 100644 --- a/src/client/deadLetter/DeadPage.tsx +++ b/src/client/deadLetter/DeadPage.tsx @@ -5,11 +5,16 @@ import {useSearchParams} from "react-router-dom"; import { useGetDeadQuery, useRemoveDeadSingleMutation, - useProcessDeadSingleMutation, deadAdapter, clearDead + useProcessDeadSingleMutation, + useLazyProcessDeadQuery, + useLazyRemoveDeadQuery, + deadAdapter, + clearDead, } from "./deadLetterDucks"; import dayjs from "dayjs"; import {RootState} from "../store"; import {connect, ConnectedProps} from "react-redux"; +import {id} from "common-tags"; const displayOpts = { include: recentIncludes, @@ -29,15 +34,20 @@ const dead = (props: PropsFromRedux) => { const [removeDeadFetch, removeResult] = useRemoveDeadSingleMutation(); const [retryDeadFetch, processResult] = useProcessDeadSingleMutation(); + const [removeAllDeadFetch] = useLazyRemoveDeadQuery(); + const [retryAllDeadFetch] = useLazyProcessDeadQuery(); const retryDead = useCallback((id: string) => retryDeadFetch({name: searchParams.get('name'), type: searchParams.get('type'), id}), [retryDeadFetch, searchParams]); const removeDead = useCallback((id: string) => removeDeadFetch({name: searchParams.get('name'), type: searchParams.get('type'), id}), [removeDeadFetch, searchParams]); + const removeAllDead = useCallback(() => removeAllDeadFetch({name: searchParams.get('name'), type: searchParams.get('type')}), [removeAllDeadFetch, searchParams]); + const retryAllDead = useCallback(() => retryAllDeadFetch({name: searchParams.get('name'), type: searchParams.get('type')}), [retryAllDeadFetch, searchParams]); return (
-

Failed Scrobbles +

Failed Scrobbles - retryAllDead()} className="capitalize underline cursor-pointer max-w-fit">Retry All + | removeAllDead()} className="capitalize underline cursor-pointer max-w-fit">Remove All

@@ -46,7 +56,7 @@ const dead = (props: PropsFromRedux) => {
Source:{x.source.replace('Source -', '')}
Retries: {x.retries}
-
Last Retried: {x.lastRetry === undefined ? 'Never' : dayjs.duration(dayjs().diff(dayjs(x.lastRetry))).humanize(true)}
+
Last Retried: {x.lastRetry === undefined ? 'Never' : dayjs.duration(dayjs(x.lastRetry).diff(dayjs())).humanize(true)}
Error: {x.error}
retryDead(x.id)} className="capitalize underline cursor-pointer max-w-fit">Retry
removeDead(x.id)} className="capitalize underline cursor-pointer max-w-fit">Remove
diff --git a/src/client/deadLetter/deadLetterDucks.ts b/src/client/deadLetter/deadLetterDucks.ts index 65486023..275a8a59 100644 --- a/src/client/deadLetter/deadLetterDucks.ts +++ b/src/client/deadLetter/deadLetterDucks.ts @@ -1,6 +1,5 @@ import {createApi, fetchBaseQuery} from "@reduxjs/toolkit/dist/query/react/index"; import {DeadLetterScrobble, JsonPlayObject} from "../../core/Atomic"; -import {id} from "common-tags"; import {createAction, createEntityAdapter, createSlice} from "@reduxjs/toolkit"; type DeadResponse = DeadLetterScrobble[]; @@ -13,6 +12,17 @@ export const deadApi = createApi({ query: (params) => `dead?name=${params.name}&type=${params.type}`, providesTags: ['DeadLetters'] }), + processDead: builder.query({ + query: (params) => ({ + url: `dead`, + method: 'PUT', + params: { + name: params.name, + type: params.type + } + }), + providesTags: ['DeadLetters'] + }), processDeadSingle: builder.mutation | undefined, { name: string, type: string, @@ -27,6 +37,17 @@ export const deadApi = createApi({ } }) }), + removeDead: builder.query({ + query: (params) => ({ + url: `dead`, + method: 'DELETE', + params: { + name: params.name, + type: params.type + } + }), + providesTags: ['DeadLetters'] + }), removeDeadSingle: builder.mutation | undefined, { name: string, type: string, @@ -70,7 +91,7 @@ export const deadSlice = createSlice({ } ) builder.addMatcher( - (action) => deadApi.endpoints.getDead.matchFulfilled(action), + (action) => deadApi.endpoints.getDead.matchFulfilled(action) || deadApi.endpoints.processDead.matchFulfilled(action) || deadApi.endpoints.removeDead.matchFulfilled(action), (state, action) => { deadAdapter.setAll(state, action.payload); } @@ -103,4 +124,4 @@ export const deadSlice = createSlice({ export const clearDead = createAction('clearDead'); -export const {useGetDeadQuery, useProcessDeadSingleMutation, useRemoveDeadSingleMutation} = deadApi; +export const {useGetDeadQuery, useProcessDeadSingleMutation, useRemoveDeadSingleMutation, useLazyProcessDeadQuery, useLazyRemoveDeadQuery} = deadApi; -- 2.51.2 From 49e9fd4b3b2cd6d170a55c37dc0620d77bbf19ba Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 19 Oct 2023 11:41:06 -0400 Subject: [PATCH 12/16] feat: Implement live updates for dead letter and scrobble counts --- .../scrobblers/AbstractScrobbleClient.ts | 4 +- src/backend/server/api.ts | 5 +++ src/client/App.tsx | 37 ++++++++++++++++++- src/client/deadLetter/deadLetterDucks.ts | 7 ++++ src/client/status/StatusSection.tsx | 17 +-------- src/client/status/ducks.ts | 28 +++++++++++--- 6 files changed, 74 insertions(+), 24 deletions(-) diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts index e6c48318..14104068 100644 --- a/src/backend/scrobblers/AbstractScrobbleClient.ts +++ b/src/backend/scrobblers/AbstractScrobbleClient.ts @@ -677,8 +677,10 @@ ${closestMatch.breakdowns.join('\n')}`); } else { eString = messageWithCauses(error); } - this.deadLetterScrobbles.push({id: nanoid(), retries: 0, error: eString, ...data}); + const deadData = {id: nanoid(), retries: 0, error: eString, ...data}; + this.deadLetterScrobbles.push(deadData); this.deadLetterScrobbles.sort((a, b) => sortByOldestPlayDate(a.play, b.play)); + this.emitEvent('deadLetter', {dead: deadData}); } diff --git a/src/backend/server/api.ts b/src/backend/server/api.ts index 99ac791e..b58905f4 100644 --- a/src/backend/server/api.ts +++ b/src/backend/server/api.ts @@ -148,6 +148,11 @@ export const setupApi = (app: ExpressWithAsync, logger: Logger, initialLogOutput session.push({event: eventName, ...payload}, payload.from); } }); + scrobbleClients.emitter.on('*', (payload: any, eventName: string) => { + if(payload.from !== undefined) { + session.push({event: eventName, ...payload}, payload.from); + } + }); }); setupTautulliRoutes(app, logger, scrobbleSources); diff --git a/src/client/App.tsx b/src/client/App.tsx index b7c4c380..b4b09582 100644 --- a/src/client/App.tsx +++ b/src/client/App.tsx @@ -5,13 +5,15 @@ import { createBrowserRouter, RouterProvider, useLocation, } from "react-router-dom"; -import { Provider } from 'react-redux' +import {connect, ConnectedProps, Provider} from 'react-redux' import './App.css'; import {store} from './store'; import Dashboard from "./dashboard/dashboard"; import RecentPage from "./recent/RecentPage"; import ScrobbledPage from "./scrobbled/ScrobbledPage"; import DeadPage from "./deadLetter/DeadPage"; +import {clientUpdate, sourceUpdate} from "./status/ducks"; +import {useEventSource, useEventSourceListener} from "@react-nano/use-event-source"; function NoMatch() { let location = useLocation(); @@ -46,6 +48,38 @@ const router = createBrowserRouter([ } ]); +const mapDispatchToProps = (dispatch) => { + return { + updateSource: (payload) => dispatch(sourceUpdate(payload)), + updateClient: (payload) => dispatch(clientUpdate(payload)) + } +} + +const connector = connect(null, mapDispatchToProps); + +type PropsFromRedux = ConnectedProps; + +const Global = (props: PropsFromRedux) => { + const { + updateSource, + updateClient + } = props; + + const [sourceEventSource, eventSourceStatus] = useEventSource("api/events", false); + useEventSourceListener(sourceEventSource, ['source', 'client'], evt => { + const data = JSON.parse(evt.data); + if(data.from === 'source') { + updateSource(data); + } else if(data.from === 'client') { + updateClient(data); + } + }, [updateSource, updateClient]); + + return ; +} + +const ConnectedGlobal = connector(Global); + function App() { return ( @@ -63,6 +97,7 @@ function App() {
+
diff --git a/src/client/deadLetter/deadLetterDucks.ts b/src/client/deadLetter/deadLetterDucks.ts index 275a8a59..af30c3b1 100644 --- a/src/client/deadLetter/deadLetterDucks.ts +++ b/src/client/deadLetter/deadLetterDucks.ts @@ -1,6 +1,7 @@ import {createApi, fetchBaseQuery} from "@reduxjs/toolkit/dist/query/react/index"; import {DeadLetterScrobble, JsonPlayObject} from "../../core/Atomic"; import {createAction, createEntityAdapter, createSlice} from "@reduxjs/toolkit"; +import {ApiEventPayload, clientUpdate} from "../status/ducks"; type DeadResponse = DeadLetterScrobble[]; export const deadApi = createApi({ @@ -119,6 +120,12 @@ export const deadSlice = createSlice({ state = deadAdapter.getInitialState(); } ) + /*.addMatcher( + (action) => clientUpdate.match(action) && action.payload.event === 'deadLetter', + (state, action) => { + state.entities[(action.payload as ApiEventPayload).data.dead.id] = (action.payload as ApiEventPayload).data.dead; + } + )*/ } }); diff --git a/src/client/status/StatusSection.tsx b/src/client/status/StatusSection.tsx index 7f6d4bb5..647c590b 100644 --- a/src/client/status/StatusSection.tsx +++ b/src/client/status/StatusSection.tsx @@ -4,26 +4,13 @@ import StatusCardSkeleton from "../components/statusCard/StatusCardSkeleton"; import SourceStatusCard from "../components/statusCard/SourceStatusCard"; import ClientStatusCard from "../components/statusCard/ClientStatusCard"; import {useGetStatusQuery} from "./statusApi"; -import {clientAdapter, clientUpdate, sourceAdapter, sourceUpdate} from "./ducks"; +import {clientAdapter, sourceAdapter} from "./ducks"; import {RootState} from "../store"; -import {useEventSource, useEventSourceListener} from "@react-nano/use-event-source"; const StatusSection = (props: PropsFromRedux) => { const { - updateSource, - updateClient } = props; const {data, error, isLoading} = useGetStatusQuery(undefined); - const [sourceEventSource, eventSourceStatus] = useEventSource("api/events", false); - useEventSourceListener(sourceEventSource, ['source', 'client'], evt => { - const data = JSON.parse(evt.data); - if(data.from === 'source') { - updateSource(data); - } else if(data.from === 'client') { - updateClient(data); - } - }, [updateSource]); - return (
@@ -48,8 +35,6 @@ const mapStateToProps = (state: RootState) => { const mapDispatchToProps = (dispatch) => { return { - updateSource: (payload) => dispatch(sourceUpdate(payload)), - updateClient: (payload) => dispatch(clientUpdate(payload)) } } diff --git a/src/client/status/ducks.ts b/src/client/status/ducks.ts index 52eee732..64cd93ad 100644 --- a/src/client/status/ducks.ts +++ b/src/client/status/ducks.ts @@ -46,18 +46,24 @@ const sourceSlice = createSlice({ .addMatcher( (action) => sourceUpdate.match(action) && action.payload.event === 'discovered', (state, action) => { - state.entities[action.payload.id].tracksDiscovered = state.entities[action.payload.id].tracksDiscovered + 1; + if(state.entities[action.payload.id] !== undefined) { + state.entities[action.payload.id].tracksDiscovered = state.entities[action.payload.id].tracksDiscovered + 1; + } } ).addMatcher( (action) => sourceUpdate.match(action) && action.payload.event === 'playerUpdate', (state, action) => { - const playerState = action.payload.data as SourcePlayerJson; - state.entities[action.payload.id].players[playerState.platformId] = playerState; + if(state.entities[action.payload.id] !== undefined) { + const playerState = action.payload.data as SourcePlayerJson; + state.entities[action.payload.id].players[playerState.platformId] = playerState; + } }).addMatcher( (action) => sourceUpdate.match(action) && action.payload.event === 'playerDelete', (state, action) => { - const playerState = action.payload.data as {platformId: string}; - delete state.entities[action.payload.id].players[playerState.platformId]; + if(state.entities[action.payload.id] !== undefined) { + const playerState = action.payload.data as {platformId: string}; + delete state.entities[action.payload.id].players[playerState.platformId]; + } } ) } @@ -79,7 +85,17 @@ const clientSlice = createSlice({ .addMatcher( (action) => clientUpdate.match(action) && action.payload.event === 'scrobble', (state, action) => { - state.entities[action.payload.id].tracksDiscovered = state.entities[action.payload.id].tracksDiscovered + 1; + if(state.entities[action.payload.id] !== undefined) { + state.entities[action.payload.id].tracksDiscovered = state.entities[action.payload.id].tracksDiscovered + 1; + } + } + ) + .addMatcher( + (action) => clientUpdate.match(action) && action.payload.event === 'deadLetter', + (state, action) => { + if(state.entities[action.payload.id] !== undefined) { + state.entities[action.payload.id].deadLetterScrobbles = state.entities[action.payload.id].deadLetterScrobbles + 1; + } } ) } -- 2.51.2 From 3aab6a207b8b2e4fdababe93efe4ccaaea42633f Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 19 Oct 2023 11:53:48 -0400 Subject: [PATCH 13/16] feat: Implement live client/source status updates --- src/backend/scrobblers/AbstractScrobbleClient.ts | 3 +++ src/backend/sources/AbstractSource.ts | 3 +++ src/client/status/ducks.ts | 16 ++++++++++++++++ 3 files changed, 22 insertions(+) diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts index 14104068..717de209 100644 --- a/src/backend/scrobblers/AbstractScrobbleClient.ts +++ b/src/backend/scrobblers/AbstractScrobbleClient.ts @@ -510,6 +510,7 @@ ${closestMatch.breakdowns.join('\n')}`); protected doStopScrobbling = (reason: string = 'system') => { this.scrobbling = false; this.userScrobblingStopSignal = undefined; + this.emitEvent('statusChange', {status: 'Idle'}); this.logger.info(`Stopped scrobble processing due to: ${reason}`); } @@ -520,6 +521,7 @@ ${closestMatch.breakdowns.join('\n')}`); return true; } this.logger.info('Scrobble processing started'); + this.emitEvent('statusChange', {status: 'Running'}); try { this.scrobbling = true; @@ -563,6 +565,7 @@ ${closestMatch.breakdowns.join('\n')}`); } catch (e) { this.logger.error('Scrobble processing interrupted'); this.logger.error(e); + this.emitEvent('statusChange', {status: 'Idle'}); this.scrobbling = false; throw e; } diff --git a/src/backend/sources/AbstractSource.ts b/src/backend/sources/AbstractSource.ts index f79f678b..e5bd34ff 100644 --- a/src/backend/sources/AbstractSource.ts +++ b/src/backend/sources/AbstractSource.ts @@ -330,6 +330,7 @@ export default abstract class AbstractSource { protected doStopPolling = (reason: string = 'system') => { this.polling = false; this.userPollingStopSignal = undefined; + this.emitEvent('statusChange', {status: 'Idle'}); this.logger.info(`Stopped polling due to: ${reason}`); } @@ -340,6 +341,7 @@ export default abstract class AbstractSource { return true; } this.logger.info('Polling started'); + this.emitEvent('statusChange', {status: 'Running'}); this.notify({title: `${this.identifier} - Polling Started`, message: 'Polling Started', priority: 'info'}); this.lastActivityAt = dayjs(); let checkCount = 0; @@ -416,6 +418,7 @@ export default abstract class AbstractSource { } catch (e) { this.logger.error('Error occurred while polling'); this.logger.error(e); + this.emitEvent('statusChange', {status: 'Idle'}); this.polling = false; throw e; } diff --git a/src/client/status/ducks.ts b/src/client/status/ducks.ts index 64cd93ad..7cdb14a6 100644 --- a/src/client/status/ducks.ts +++ b/src/client/status/ducks.ts @@ -66,6 +66,14 @@ const sourceSlice = createSlice({ } } ) + .addMatcher( + (action) => sourceUpdate.match(action) && action.payload.event === 'statusChange', + (state, action) => { + if(state.entities[action.payload.id] !== undefined) { + state.entities[action.payload.id].status = action.payload.data.status; + } + } + ) } }); const clientSlice = createSlice({ @@ -98,6 +106,14 @@ const clientSlice = createSlice({ } } ) + .addMatcher( + (action) => clientUpdate.match(action) && action.payload.event === 'statusChange', + (state, action) => { + if(state.entities[action.payload.id] !== undefined) { + state.entities[action.payload.id].status = action.payload.data.status; + } + } + ) } }); -- 2.51.2 From fe9f49779c8b1d7ded51e4749c3f71ea5ff9d655 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 19 Oct 2023 13:40:43 -0400 Subject: [PATCH 14/16] feat: Implement queued scrobbles display * Show number of queued scrobbles for clients in ui * Implement live updates to queue count --- .../scrobblers/AbstractScrobbleClient.ts | 5 ++++- src/backend/server/api.ts | 5 +++-- .../statusCard/ClientStatusCard.tsx | 6 +++-- src/client/status/ducks.ts | 22 ++++++++++++++++--- src/core/Atomic.ts | 3 ++- 5 files changed, 32 insertions(+), 9 deletions(-) diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts index 717de209..0ddd5eac 100644 --- a/src/backend/scrobblers/AbstractScrobbleClient.ts +++ b/src/backend/scrobblers/AbstractScrobbleClient.ts @@ -553,6 +553,7 @@ ${closestMatch.breakdowns.join('\n')}`); // processing play may have changed index while we were scrobbling const pIndex = this.queuedScrobbles.findIndex(x => x.id === currQueuedPlay.id); if (pIndex !== -1) { + this.emitEvent('scrobbleDequeued', {queuedScrobble: currQueuedPlay}) this.queuedScrobbles.splice(pIndex, 1); } } @@ -668,7 +669,9 @@ ${closestMatch.breakdowns.join('\n')}`); queueScrobble = (data: PlayObject | PlayObject[], source: string) => { const plays = Array.isArray(data) ? data : [data]; for(const p of plays) { - this.queuedScrobbles.push({id: nanoid(), source, play: p}); + const queuedPlay = {id: nanoid(), source, play: p} + this.emitEvent('scrobbleQueued', {queuedPlay: queuedPlay}); + this.queuedScrobbles.push(queuedPlay); } this.queuedScrobbles.sort((a, b) => sortByOldestPlayDate(a.play, b.play)); } diff --git a/src/backend/server/api.ts b/src/backend/server/api.ts index b58905f4..5aa9deba 100644 --- a/src/backend/server/api.ts +++ b/src/backend/server/api.ts @@ -227,13 +227,14 @@ export const setupApi = (app: ExpressWithAsync, logger: Logger, initialLogOutput status: '', type, display: capitalize(type), - tracksDiscovered: tracksScrobbled, + scrobbled: tracksScrobbled, name, hasAuth: requiresAuth, hasAuthInteraction: requiresAuthInteraction, authed, initialized, - deadLetterScrobbles: x.deadLetterScrobbles.length + deadLetterScrobbles: x.deadLetterScrobbles.length, + queued: x.queuedScrobbles.length }; if (!initialized) { base.status = 'Not Initialized'; diff --git a/src/client/components/statusCard/ClientStatusCard.tsx b/src/client/components/statusCard/ClientStatusCard.tsx index 5e6ff689..6e0bacd3 100644 --- a/src/client/components/statusCard/ClientStatusCard.tsx +++ b/src/client/components/statusCard/ClientStatusCard.tsx @@ -31,7 +31,8 @@ const ClientStatusCard = (props: ClientStatusCardData) => { type, display, status, - tracksDiscovered = 0, + scrobbled: scrobbledCount = 0, + queued = 0, deadLetterScrobbles = 0 } = {} } = props; @@ -52,7 +53,8 @@ const ClientStatusCard = (props: ClientStatusCardData) => { // TODO links body = ( -
{scrobbled}: {tracksDiscovered}
+
{scrobbled}: {scrobbledCount}
+
Queued Scrobbles: {queued}
Failed Scrobbles: {deadLetterScrobbles}
{hasAuth ? (Re)authenticate or initialize : null} ); diff --git a/src/client/status/ducks.ts b/src/client/status/ducks.ts index 7cdb14a6..d96604c7 100644 --- a/src/client/status/ducks.ts +++ b/src/client/status/ducks.ts @@ -47,7 +47,7 @@ const sourceSlice = createSlice({ (action) => sourceUpdate.match(action) && action.payload.event === 'discovered', (state, action) => { if(state.entities[action.payload.id] !== undefined) { - state.entities[action.payload.id].tracksDiscovered = state.entities[action.payload.id].tracksDiscovered + 1; + state.entities[action.payload.id].tracksDiscovered++; } } ).addMatcher( @@ -94,7 +94,7 @@ const clientSlice = createSlice({ (action) => clientUpdate.match(action) && action.payload.event === 'scrobble', (state, action) => { if(state.entities[action.payload.id] !== undefined) { - state.entities[action.payload.id].tracksDiscovered = state.entities[action.payload.id].tracksDiscovered + 1; + state.entities[action.payload.id].scrobbled++; } } ) @@ -102,7 +102,7 @@ const clientSlice = createSlice({ (action) => clientUpdate.match(action) && action.payload.event === 'deadLetter', (state, action) => { if(state.entities[action.payload.id] !== undefined) { - state.entities[action.payload.id].deadLetterScrobbles = state.entities[action.payload.id].deadLetterScrobbles + 1; + state.entities[action.payload.id].deadLetterScrobbles++; } } ) @@ -114,6 +114,22 @@ const clientSlice = createSlice({ } } ) + .addMatcher( + (action) => clientUpdate.match(action) && action.payload.event === 'scrobbleQueued', + (state, action) => { + if(state.entities[action.payload.id] !== undefined) { + state.entities[action.payload.id].queued++; + } + } + ) + .addMatcher( + (action) => clientUpdate.match(action) && action.payload.event === 'scrobbleDequeued', + (state, action) => { + if(state.entities[action.payload.id] !== undefined) { + state.entities[action.payload.id].queued--; + } + } + ) } }); diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts index f6025111..29e93092 100644 --- a/src/core/Atomic.ts +++ b/src/core/Atomic.ts @@ -19,8 +19,9 @@ export interface ClientStatusData { status: string; type: "maloja" | "lastfm" | "listenbrainz"; display: string; - tracksDiscovered: number; + scrobbled: number; deadLetterScrobbles: number + queued: number name: string; hasAuth: boolean; hasAuthInteraction: boolean; -- 2.51.2 From 0f6638dfdf5fa5c24b739641ee607b062d7f3708 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Tue, 24 Oct 2023 08:56:14 -0400 Subject: [PATCH 15/16] docs: Add queued scrobble feature highlight --- README.md | 1 + docsite/src/pages/index.mdx | 1 + 2 files changed, 2 insertions(+) diff --git a/README.md b/README.md index e17fb83a..8da712c9 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ A javascript app to scrobble music you listened to, to [Maloja](https://github.c * Monitor status of Sources and Clients using [webhooks (Gotify or Ntfy)](/docsite/docs/configuration/configuration.md#webhook-configurations) or [healthcheck endpoint](/docsite/docs/configuration/configuration.md#health-endpoint) * Supports configuring for single or multiple users (scrobbling for your friends and family!) * Web server interface for stats, basic control, and detailed logs +* Graceful network and client failure handling (queued scrobbles that auto-retry) * Smart handling of credentials (persistent, authorization through app) * Easy configuration through ENVs or JSON * Install using [Docker images for x86/ARM](/docsite/docs/installation/installation.md#docker), [flatpak](/docsite/docs/installation/installation.md#flatpak), or [locally with NodeJS](/docsite/docs/installation/installation.md#nodejs) diff --git a/docsite/src/pages/index.mdx b/docsite/src/pages/index.mdx index ba2e3c19..3612d24a 100644 --- a/docsite/src/pages/index.mdx +++ b/docsite/src/pages/index.mdx @@ -33,6 +33,7 @@ A javascript app to scrobble music you listened to, to [Maloja](https://github.c * Monitor status of Sources and Clients using [webhooks (Gotify or Ntfy)](docs/configuration#webhook-configurations) or [healthcheck endpoint](docs/configuration#health-endpoint) * Supports configuring for single or multiple users (scrobbling for your friends and family!) * Web server interface for stats, basic control, and detailed logs +* Graceful network and client failure handling (queued scrobbles that auto-retry) * Smart handling of credentials (persistent, authorization through app) * Easy configuration through ENVs or JSON * Install using [Docker images for x86/ARM](docs/installation#docker), [flatpak](docs/installation#flatpak), or [locally with NodeJS](docs/installation#nodejs) -- 2.51.2 From 9616860e236f0e325bc2d0d58809cb10220e1a4a Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Tue, 24 Oct 2023 09:20:04 -0400 Subject: [PATCH 16/16] chore: Bump version for release --- flatpak/io.github.foxxmd.multiscrobbler.metainfo.xml | 1 + package.json | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/flatpak/io.github.foxxmd.multiscrobbler.metainfo.xml b/flatpak/io.github.foxxmd.multiscrobbler.metainfo.xml index e6807d43..569355bd 100644 --- a/flatpak/io.github.foxxmd.multiscrobbler.metainfo.xml +++ b/flatpak/io.github.foxxmd.multiscrobbler.metainfo.xml @@ -41,6 +41,7 @@ + diff --git a/package.json b/package.json index 30f5071e..b7f86ebe 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "multi-scrobbler", - "version": "0.6.0", + "version": "0.6.1", "description": "scrobble plays from multiple sources to multiple clients", "scripts": { "schema": "npm run -s schema-aio & npm run -s schema-source & npm run -s schema-client & npm run -s schema-aiosource & npm run -s schema-aioclient",