diff --git a/package-lock.json b/package-lock.json
index 2fc86ec2..393ee019 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -48,6 +48,7 @@
"react-redux": "^8.1.2",
"react-router-dom": "^6.15.0",
"safe-stable-stringify": "^1.1.1",
+ "sass": "^1.66.1",
"spotify-web-api-node": "^5.0.2",
"superagent": "^8.0.9",
"tailwindcss": "^3.3.3",
@@ -11896,6 +11897,11 @@
"url": "https://opencollective.com/immer"
}
},
+ "node_modules/immutable": {
+ "version": "4.3.4",
+ "resolved": "https://registry.npmjs.org/immutable/-/immutable-4.3.4.tgz",
+ "integrity": "sha512-fsXeu4J4i6WNWSikpI88v/PcVflZz+6kMhUfIwc5SY+poQRPnaf5V7qds6SUyUN3cVxEzuCab7QIoLOQ+DQ1wA=="
+ },
"node_modules/import-fresh": {
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz",
@@ -20793,6 +20799,22 @@
"integrity": "sha512-ZRwKbh/eQ6w9vmTjkuG0Ioi3HBwPFce0O+v//ve+aOq1oeCy7jMV2qzzAlpsNuqpqCBjjriM1lbtZbF/Q8jVyA==",
"dev": true
},
+ "node_modules/sass": {
+ "version": "1.66.1",
+ "resolved": "https://registry.npmjs.org/sass/-/sass-1.66.1.tgz",
+ "integrity": "sha512-50c+zTsZOJVgFfTgwwEzkjA3/QACgdNsKueWPyAR0mRINIvLAStVQBbPg14iuqEQ74NPDbXzJARJ/O4SI1zftA==",
+ "dependencies": {
+ "chokidar": ">=3.0.0 <4.0.0",
+ "immutable": "^4.0.0",
+ "source-map-js": ">=0.6.2 <2.0.0"
+ },
+ "bin": {
+ "sass": "sass.js"
+ },
+ "engines": {
+ "node": ">=14.0.0"
+ }
+ },
"node_modules/sass-loader": {
"version": "12.6.0",
"resolved": "https://registry.npmjs.org/sass-loader/-/sass-loader-12.6.0.tgz",
diff --git a/package.json b/package.json
index d2724f1b..1eca3922 100644
--- a/package.json
+++ b/package.json
@@ -85,6 +85,7 @@
"react-redux": "^8.1.2",
"react-router-dom": "^6.15.0",
"safe-stable-stringify": "^1.1.1",
+ "sass": "^1.66.1",
"spotify-web-api-node": "^5.0.2",
"superagent": "^8.0.9",
"tailwindcss": "^3.3.3",
diff --git a/src/client/components/player/Player.tsx b/src/client/components/player/Player.tsx
new file mode 100644
index 00000000..a29fa4df
--- /dev/null
+++ b/src/client/components/player/Player.tsx
@@ -0,0 +1,340 @@
+import React, {useState, useCallback} from 'react';
+import './player.scss';
+import PlayerTimestamp from "./PlayerTimestamp";
+import PlayerControls from "./PlayerControls";
+
+export interface PlayerProps {
+ tracks: Track[]
+
+}
+
+export interface Track {
+ name: string
+ artist: string
+ album: string
+ year: number
+ duration: number
+ artwork: string
+}
+
+export const sampleTracks: Track[] = [
+ {
+ name: "Somebody Hates Me",
+ artist: "Reel Big Fish",
+ album: "Why Do They Rock So Hard?",
+ year: 1998,
+ duration: 210,
+ artwork: "https://is5-ssl.mzstatic.com/image/thumb/Music/59/db/b5/mzi.nloomwwj.jpg/268x0w.jpg"
+ },
+ {
+ name: "Beer",
+ artist: "Reel Big Fish",
+ album: "We're Not Happy Til You're Not Happy",
+ year: 2005,
+ duration: 233,
+ artwork: "https://upload.wikimedia.org/wikipedia/en/9/9d/Reel_Big_Fish_-_We%27re_Not_Happy_%27til_You%27re_Not_Happy_cover.jpg"
+ },
+ {
+ name: "We Will Fall Together",
+ artist: "Streetlight Manifesto",
+ album: "Somewhere In The Between",
+ year: 2007,
+ duration: 289,
+ artwork: "https://upload.wikimedia.org/wikipedia/en/9/95/Streetlight_Manifesto_-_Somewhere_in_the_Between.jpg"
+ },
+ {
+ name: "Welcome To The Jungle",
+ artist: "Guns N' Roses",
+ album: "Appetite For Destruction",
+ year: 1987,
+ duration: 277,
+ artwork: "https://upload.wikimedia.org/wikipedia/en/6/60/GunsnRosesAppetiteforDestructionalbumcover.jpg"
+ },
+ {
+ name: "Estranged",
+ artist: "Guns N' Roses",
+ album: "Use Your Illusion II",
+ year: 1991,
+ duration: 563,
+ artwork: "https://www.lifeofvinyl.com/images/super/0720642442012.jpg"
+ },
+ {
+ name: "Cheer Up!",
+ artist: "Reel Big Fish",
+ album: "Cheer Up!",
+ year: 2002,
+ duration: 163,
+ artwork: "https://upload.wikimedia.org/wikipedia/en/f/f9/Reel_Big_Fish_-_Cheer_Up%21_cover.jpg"
+ }
+];
+
+const Player = (props: PlayerProps) => {
+ const [playStatus, setPlayStatus] = useState(false);
+ const [currentTrack, setCurrentTrack] = useState(0);
+ const [nextTrack, setNextTrack] = useState(1);
+ const [prevTrack, setPrevTrack] = useState(props.tracks.length - 1);
+ const [currentTime, setCurrentTime] = useState(0);
+ const [sliding, setSliding] = useState(0);
+ const [viewMode, setViewMode] = useState('player');
+
+ const toggleViewMode = useCallback(() => {
+ let newViewMode = "";
+ switch(viewMode) {
+ case "player":
+ newViewMode = "playlist";
+ break;
+ case "playlist":
+ newViewMode = "player"
+ break;
+ }
+ setViewMode(newViewMode);
+ }, [viewMode, setViewMode]);
+
+ const togglePlay = useCallback(() => {},[playStatus, setPlayStatus]);
+ const changeTrack = useCallback((track) => {}, [props.tracks, nextTrack, prevTrack, setCurrentTrack, setSliding, setCurrentTrack, setNextTrack, setPrevTrack]);
+ const toggleNextTrack = useCallback(() => {}, [nextTrack, changeTrack]);
+ const togglePrevTrack = useCallback(() => {}, [prevTrack, changeTrack]);
+ const setTime = useCallback((time) => {}, [props.tracks, currentTrack, setNextTrack, setCurrentTime]);
+
+ const aPrevTrack = props.tracks[prevTrack];
+ const aCurrentTrack = props.tracks[currentTrack];
+ const aNextTrack = props.tracks[nextTrack];
+
+ let slidingStatus = "";
+ switch (sliding) {
+ case -1:
+ slidingStatus = "is-sliding-prev";
+ break;
+ case 1:
+ slidingStatus = "is-sliding-next";
+ break;
+ }
+
+ let playlistIcon = "fa fa-fw ";
+ if (viewMode == "playlist") {
+ playlistIcon += "fa-times";
+ } else {
+ playlistIcon += "fa-bars";
+ }
+
+ return (
+
+
+
+
+
+
+
+
+
+
+ {/*
+
+
+
+ */}
+
+ {/* */}
+
+
+ {aCurrentTrack.name}
+ {aCurrentTrack.artist}
+
+
+ {/* {}} />*/}
+
+
+ );
+}
+
+export default Player;
+
+/*
+export class PlayerC extends React.Component {
+ static get defaultProps() {
+ return {
+
+ }
+ }
+
+ constructor(props, defaultProps) {
+ super(props, defaultProps);
+
+ this.state = {
+ playStatus: 0,
+ currentTrack: 0,
+ nextTrack: 1,
+ prevTrack: this.props.tracks.length-1,
+ currentTime: 0,
+ sliding: 0,
+ likedTracks: [],
+ viewMode: "player"
+ }
+
+ this.togglePlay = this.togglePlay.bind(this);
+ this.toggleViewMode = this.toggleViewMode.bind(this);
+ this.nextTrack = this.nextTrack.bind(this);
+ this.prevTrack = this.prevTrack.bind(this);
+ this.likeTrack = this.likeTrack.bind(this);
+ }
+
+ toggleViewMode() {
+ let viewMode = "";
+ switch(this.state.viewMode) {
+ case "player":
+ viewMode = "playlist";
+ break;
+ case "playlist":
+ viewMode = "player"
+ break;
+ }
+
+ this.setState({
+ viewMode: viewMode
+ });
+ }
+
+ togglePlay() {
+ let status = this.state.playStatus;
+ if (status == 0) {
+ // Play
+ status = 1;
+ this.timer = setInterval(() => {
+ this.setTime(this.state.currentTime + 1);
+ }, 1000);
+ } else {
+ // Pause
+ status = 0;
+ clearInterval(this.timer);
+ }
+ this.setState({ playStatus: status });
+ }
+
+ changeTrack(track, dir) {
+ if (this.state.sliding == 0) {
+ let this_track = track;
+ let next_track = track + 1;
+ let prev_track = track - 1;
+
+ if (next_track >= this.props.tracks.length) next_track = 0;
+ if (prev_track < 0) prev_track = this.props.tracks.length - 1;
+
+ this.setState({
+ sliding: dir
+ });
+
+ setTimeout(() => {
+ this.setState({
+ sliding: 0,
+ currentTrack: this_track,
+ nextTrack: next_track,
+ prevTrack: prev_track,
+ currentTime: 0
+ });
+ }, 500);
+ }
+ }
+
+ nextTrack() {
+ this.changeTrack(this.state.nextTrack, 1);
+ }
+
+ prevTrack() {
+ if (this.state.currentTime < 2) {
+ this.changeTrack(this.state.prevTrack, -1);
+ } else {
+ this.setState({
+ currentTime: 0
+ });
+ }
+ }
+
+ likeTrack() {
+ let likedTracks = this.state.likedTracks;
+ let found = false;
+ for (let i = 0; i < this.state.likedTracks.length; i++) {
+ if (this.state.likedTracks[i] == this.state.currentTrack) {
+ found = true;
+ likedTracks.splice(i, 1);
+ break;
+ }
+ }
+
+ if (!found) {
+ likedTracks.push(this.state.currentTrack);
+ }
+
+ this.setState({
+ likedTracks: likedTracks
+ });
+ }
+
+ setTime(time) {
+ time = Math.floor(time);
+ if (time > this.props.tracks[this.state.currentTrack].duration) {
+ this.nextTrack();
+ } else {
+ this.setState({ currentTime: time });
+ }
+ }
+
+ render() {
+ const prevTrack = this.props.tracks[this.state.prevTrack];
+ const currentTrack = this.props.tracks[this.state.currentTrack];
+ const nextTrack = this.props.tracks[this.state.nextTrack];
+
+ let sliding = "";
+ switch (this.state.sliding) {
+ case -1:
+ sliding = "is-sliding-prev";
+ break;
+ case 1:
+ sliding = "is-sliding-next";
+ break;
+ }
+
+ let playlistIcon = "fa fa-fw ";
+ if (this.state.viewMode == "playlist") {
+ playlistIcon += "fa-times";
+ } else {
+ playlistIcon += "fa-bars";
+ }
+
+ let isLiked = false;
+ for (let i = 0; i < this.state.likedTracks.length; i++) {
+ if (this.state.likedTracks[i] == this.state.currentTrack) {
+ isLiked = true;
+ }
+ }
+
+ return (
+
+
+
+
+
+
+
+ {currentTrack.name}
+ {currentTrack.artist}
+
+
+
+
+
+ );
+ }
+}
+*/
diff --git a/src/client/components/player/PlayerControls.tsx b/src/client/components/player/PlayerControls.tsx
new file mode 100644
index 00000000..caff1a33
--- /dev/null
+++ b/src/client/components/player/PlayerControls.tsx
@@ -0,0 +1,111 @@
+import React from "react";
+import './controls.scss';
+
+export interface ControlsProps {
+ isPlaying: boolean
+ isLiked: boolean
+ prevTrack: () => any
+ togglePlay: () => any
+ nextTrack: () => any
+ likeTrack: () => any
+}
+
+const Controls = (props: ControlsProps) => {
+ let playClass = "fa fa-fw ";
+ if (!props.isPlaying) {
+ playClass += "fa-play";
+ } else {
+ playClass += "fa-pause";
+ }
+
+ let likeClass = "button like ";
+ let heartClass = "fa fa-fw ";
+ if (!props.isLiked) {
+ heartClass += "fa-heart-o";
+ } else {
+ likeClass += "is-liked";
+ heartClass += "fa-heart";
+ }
+
+ return(
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ );
+}
+
+/*export class ControlsC extends React.Component {
+ render() {
+ let playClass = "fa fa-fw ";
+ if (!this.props.isPlaying) {
+ playClass += "fa-play";
+ } else {
+ playClass += "fa-pause";
+ }
+
+ let likeClass = "button like ";
+ let heartClass = "fa fa-fw ";
+ if (!this.props.isLiked) {
+ heartClass += "fa-heart-o";
+ } else {
+ likeClass += "is-liked";
+ heartClass += "fa-heart";
+ }
+
+ return(
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ );
+ }
+}*/
+
+export default Controls;
diff --git a/src/client/components/player/PlayerTimestamp.tsx b/src/client/components/player/PlayerTimestamp.tsx
new file mode 100644
index 00000000..11ec7c7f
--- /dev/null
+++ b/src/client/components/player/PlayerTimestamp.tsx
@@ -0,0 +1,62 @@
+import React from "react";
+import './timestamp.scss';
+
+export interface TimestampProps {
+ current: number
+ duration: number
+}
+
+const convertTime = (rawTime: number) => {
+ let mins = Math.floor(rawTime / 60);
+ let seconds = rawTime - (mins * 60);
+ let secStr: string = seconds.toString();
+ if (seconds < 10) {
+ secStr = "0" + seconds;
+ }
+ return mins + ":" + secStr;
+}
+
+const Timestamp = (props: TimestampProps) => {
+ return(
+
+
+ {convertTime(props.current)}
+
+
+
+ {convertTime(props.duration - props.current)}
+
+
+ );
+}
+/*export class TimestampC extends React.Component {
+ convertTime(time) {
+ let mins = Math.floor(time / 60);
+ let seconds = time - (mins * 60);
+ if (seconds < 10) {
+ seconds = "0" + seconds;
+ }
+ time = mins + ":" + seconds;
+ return time;
+ }
+
+ render() {
+ return(
+
+
+ {this.convertTime(this.props.current)}
+
+
+
+ {this.convertTime(this.props.duration - this.props.current)}
+
+
+ );
+ }
+}*/
+
+export default Timestamp;
diff --git a/src/client/components/player/Playlist.tsx b/src/client/components/player/Playlist.tsx
new file mode 100644
index 00000000..3a929cee
--- /dev/null
+++ b/src/client/components/player/Playlist.tsx
@@ -0,0 +1,33 @@
+import React from 'react';
+/*export class Playlist extends React.Component {
+ render() {
+ const tracks = this.props.tracks;
+
+ let isHidden = "";
+ if (!this.props.isVisible) {
+ isHidden = "is-hidden";
+ }
+
+ return(
+
+ {tracks.map((track, i) => {
+ return(
+ this.props.changeTrack(i)}>
+
+
+
+
+
+
+
+
{track.name}
+
{track.artist}
+
+
+
+ )
+ })}
+
+ );
+ }
+}*/
diff --git a/src/client/components/player/README.md b/src/client/components/player/README.md
new file mode 100644
index 00000000..138bd96d
--- /dev/null
+++ b/src/client/components/player/README.md
@@ -0,0 +1 @@
+Player designed based on [React Music Player UI](https://codepen.io/keithpickering/pen/wjYJKM) by [Keith Pickering](https://keithpickering.github.io/)
diff --git a/src/client/components/player/controls.scss b/src/client/components/player/controls.scss
new file mode 100644
index 00000000..57759951
--- /dev/null
+++ b/src/client/components/player/controls.scss
@@ -0,0 +1,40 @@
+@use './playerBase.scss';
+$spacing: 1.5rem;
+$primary: #00ACC1;
+
+
+.controls {
+ display: flex;
+ justify-content: space-around;
+ align-items: center;
+ height: $spacing*2.5;
+ text-align: center;
+ background: $primary;
+
+ .button {
+ height: 100%;
+ width: $spacing*2.5;
+ border: 0;
+ border-radius: 0;
+ transition: background 250ms;
+ box-shadow: none !important;
+ background: transparent;
+ color: white;
+
+ &:hover {
+ background: lighten($primary, 4%);
+ }
+
+ &.play {
+ font-size: 1.6rem;
+ }
+
+ &.like.is-liked {
+ background: #F06292;
+ }
+
+ .icon {
+ //transition: transform 150ms;
+ }
+ }
+}
diff --git a/src/client/components/player/player.scss b/src/client/components/player/player.scss
new file mode 100644
index 00000000..be8d39a7
--- /dev/null
+++ b/src/client/components/player/player.scss
@@ -0,0 +1,113 @@
+@use './playerBase.scss';
+$spacing: 1.5rem;
+$primary: #00ACC1;
+
+
+.player {
+ color: black;
+ overflow: hidden;
+ max-width: 90%;
+ max-height: 90vh;
+ //min-height: 487px;
+ border-radius: 5px;
+ background: white;
+ box-shadow: 0 10px 20px rgba(0,0,0,0.19), 0 6px 6px rgba(0,0,0,0.23);
+
+ &__bg {
+ position: fixed;
+ top: -40px;
+ right: -40px;
+ bottom: -40px;
+ left: -40px;
+ background-size: cover;
+ background-position: center;
+ filter: blur(20px);
+ z-index: -2;
+
+ &:after {
+ content: "";
+ position: absolute;
+ top: -40px;
+ right: -40px;
+ bottom: -40px;
+ left: -40px;
+ background: rgba(#263238, 0.8);
+ transition: background 1s;
+
+ [class*="is-sliding"] & {
+ background: rgba(#263238, 1);
+ }
+ }
+
+ &.prev,
+ &.next {
+ opacity: 0;
+ z-index: -1;
+ }
+ }
+
+ &__head {
+ padding: $spacing/2 $spacing;
+ }
+
+ &__art {
+ position: relative;
+ padding-bottom: 100%;
+ width: 100%;
+ border-bottom: 1px solid #ECEFF1;
+ background: darken(#263238, 10%);
+
+ .is-sliding-next & {
+ transform: translate(-100%, 0);
+ transition: transform 500ms ease-in-out;
+ }
+
+ .is-sliding-prev & {
+ transform: translate(100%, 0);
+ transition: transform 500ms ease-in-out;
+ }
+
+ > img {
+ position: absolute;
+ top: 0;
+ left: 0;
+ display: block;
+ width: 100%;
+ margin: 0 auto;
+
+ &.prev {
+ left: -100%;
+ }
+
+ &.next {
+ left: 100%;
+ }
+ }
+ }
+
+ &__body {
+ text-align: center;
+ padding: $spacing;
+ transition: all 250ms;
+
+ [class*="is-sliding"] & {
+ opacity: 0;
+ transform: scale(0.95);
+ }
+
+ .title {
+ position: relative;
+ width: 100%;
+ font-size: 1.2rem;
+ white-space: nowrap;
+ }
+
+ .subtitle {
+ font-size: 0.75rem;
+ color: #90A4AE;
+ padding-top: $spacing/4;
+ margin-bottom: $spacing;
+ text-transform: uppercase;
+ }
+ }
+}
diff --git a/src/client/components/player/playerBase.scss b/src/client/components/player/playerBase.scss
new file mode 100644
index 00000000..a0919afa
--- /dev/null
+++ b/src/client/components/player/playerBase.scss
@@ -0,0 +1,10 @@
+$spacing: 1.5rem;
+$primary: #00ACC1;
+
+.button {
+ &:active {
+ > .icon {
+ transform: scale(0.85);
+ }
+ }
+}
diff --git a/src/client/components/player/playlist.scss b/src/client/components/player/playlist.scss
new file mode 100644
index 00000000..8360194b
--- /dev/null
+++ b/src/client/components/player/playlist.scss
@@ -0,0 +1,62 @@
+$spacing: 1.5rem;
+$primary: #00ACC1;
+
+
+.playlist {
+ position: absolute;
+ top: 0;
+ bottom: 0;
+ left: 0;
+ right: 0;
+ padding: $spacing;
+ padding-right: $spacing*2;
+ overflow: auto;
+ background: #263238;
+
+ &__item.media {
+ align-items: center;
+ border-top: 2px solid #37474F;
+
+ &:first-child {
+ border-top: 0;
+ }
+
+ .title,
+ .subtitle {
+ white-space: nowrap;
+ max-width: 100%;
+ overflow: hidden;
+ text-overflow: ellipsis;
+ }
+
+ .title {
+ font-size: 0.85rem;
+ color: #B0BEC5;
+ }
+
+ .subtitle {
+ margin-top: -$spacing/2;
+ font-size: 0.75rem;
+ text-transform: uppercase;
+ color: #607D8B
+ }
+ }
+}
+
+.toggle-playlist {
+ position: absolute;
+ top: $spacing/2;
+ right: $spacing/2;
+ border: 0;
+ outline: 0;
+ background: transparent;
+ text-shadow: rgba(black, 0.75) 0 0 5px;
+ box-shadow: none !important;
+ color: white !important;
+ z-index: 1;
+ transition: all 500ms;
+
+ [class*="is-sliding"] & {
+ opacity: 0;
+ }
+}
diff --git a/src/client/components/player/timestamp.scss b/src/client/components/player/timestamp.scss
new file mode 100644
index 00000000..fe055ebd
--- /dev/null
+++ b/src/client/components/player/timestamp.scss
@@ -0,0 +1,46 @@
+@use './playerBase.scss';
+$spacing: 1.5rem;
+$primary: #00ACC1;
+
+
+.timestamp {
+ position: relative;
+ width: 100%;
+ height: $spacing;
+
+ &__progress,
+ &__current,
+ &__total {
+ position: absolute;
+ top: 0;
+ font-size: 0.75rem;
+ }
+
+ &__progress {
+ position: relative;
+ top: $spacing/4;
+ left: 50%;
+ height: $spacing/4;
+ width: calc(100% - #{$spacing*3});
+ transform: translate(-50%, 0);
+ border-radius: 5px;
+ overflow: hidden;
+ background: #ECEFF1;
+
+ > div {
+ position: absolute;
+ top: 0;
+ left: 0;
+ bottom: 0;
+ background: $primary;
+ }
+ }
+
+ &__current {
+ left: 0;
+ }
+
+ &__total {
+ right: 0;
+ }
+}
--
2.54.0
From f7195fb40c0446218d41794d73ec40e4c1de8120 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Fri, 8 Sep 2023 14:47:53 -0400
Subject: [PATCH 02/82] feat: Rework source api to include player states
* Create and implement api interface for player state data
* Serialize memory source players to object indexed by platform id
* Refactor event emitters to separate event data better from event metadata
* Update source slice when player update event is broadcast
---
.../scrobblers/AbstractScrobbleClient.ts | 2 +-
src/backend/server/api.ts | 4 +++-
src/backend/sources/AbstractSource.ts | 6 +++---
src/backend/sources/MemorySource.ts | 14 ++++++++++++-
.../PlayerState/AbstractPlayerState.ts | 20 +++++++++++++++++-
src/client/status/ducks.ts | 10 +++++++--
src/core/Atomic.ts | 21 +++++++++++++++++++
7 files changed, 68 insertions(+), 9 deletions(-)
diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts
index f9e3bb33..890ddda6 100644
--- a/src/backend/scrobblers/AbstractScrobbleClient.ts
+++ b/src/backend/scrobblers/AbstractScrobbleClient.ts
@@ -376,7 +376,7 @@ export default abstract class AbstractScrobbleClient {
public emitEvent = (eventName: string, payload: object) => {
this.emitter.emit(eventName, {
- ...payload,
+ data: payload,
type: this.type,
name: this.name,
from: 'client'
diff --git a/src/backend/server/api.ts b/src/backend/server/api.ts
index 4fb3519d..8807502f 100644
--- a/src/backend/server/api.ts
+++ b/src/backend/server/api.ts
@@ -22,6 +22,7 @@ import {setupJellyfinRoutes} from "./jellyfinRoutes";
import {setupDeezerRoutes} from "./deezerRoutes";
import {setupAuthRoutes} from "./auth";
import { ExpressHandler } from "../common/infrastructure/Atomic";
+import MemorySource from "../sources/MemorySource.js";
let output: LogInfo[] = []
@@ -106,7 +107,7 @@ export const setupApi = (app: ExpressWithAsync, logger: Logger, initialLogOutput
const session = await createSession(req, res);
scrobbleSources.emitter.on('*', (payload: any, eventName: string) => {
if(payload.from !== undefined) {
- session.push({...payload.data, event: eventName}, payload.from);
+ session.push({event: eventName, ...payload}, payload.from);
}
});
});
@@ -143,6 +144,7 @@ export const setupApi = (app: ExpressWithAsync, logger: Logger, initialLogOutput
hasAuth: requiresAuth,
hasAuthInteraction: requiresAuthInteraction,
authed,
+ players: x instanceof MemorySource ? x.playersToObject() : {}
};
if (!initialized) {
base.status = 'Not Initialized';
diff --git a/src/backend/sources/AbstractSource.ts b/src/backend/sources/AbstractSource.ts
index 48dbdfeb..8d81f431 100644
--- a/src/backend/sources/AbstractSource.ts
+++ b/src/backend/sources/AbstractSource.ts
@@ -383,12 +383,12 @@ export default abstract class AbstractSource {
}
}
- public emitEvent = (eventName: string, payload: object) => {
+ public emitEvent = (eventName: string, payload: object = {}) => {
this.emitter.emit(eventName, {
- ...payload,
type: this.type,
name: this.name,
- from: 'source'
+ from: 'source',
+ data: payload,
});
}
}
diff --git a/src/backend/sources/MemorySource.ts b/src/backend/sources/MemorySource.ts
index 72579087..0e64713c 100644
--- a/src/backend/sources/MemorySource.ts
+++ b/src/backend/sources/MemorySource.ts
@@ -27,7 +27,7 @@ import TupleMap from "../common/TupleMap";
import { AbstractPlayerState, PlayerStateOptions } from "./PlayerState/AbstractPlayerState";
import { GenericPlayerState } from "./PlayerState/GenericPlayerState";
import {Logger} from "@foxxmd/winston";
-import { PlayObject } from "../../core/Atomic";
+import {PlayObject, SourcePlayerObj} from "../../core/Atomic";
import { buildTrackString } from "../../core/StringUtils";
export default class MemorySource extends AbstractSource {
@@ -58,6 +58,17 @@ export default class MemorySource extends AbstractSource {
players: Map = new Map();
+ playersToObject = (): Record => {
+ if(this.players.size === 0) {
+ return {};
+ }
+ const record: Record = {};
+ for(const [k,v] of this.players.entries()) {
+ record[k] = v.getApiState();
+ }
+ return record;
+ }
+
getFlatCandidateRecentlyPlayed = (): PlayObject[] => {
// TODO sort?
return Array.from(this.candidateRecentlyPlayed.values()).flat();
@@ -154,6 +165,7 @@ export default class MemorySource extends AbstractSource {
}
if(this.config.options?.logPlayerState === true) {
player.logSummary();
+ this.emitEvent('playerUpdate', player.getApiState());
}
}
for (const deadId of deadPlatformIds) {
diff --git a/src/backend/sources/PlayerState/AbstractPlayerState.ts b/src/backend/sources/PlayerState/AbstractPlayerState.ts
index bea0f9ee..4de82f84 100644
--- a/src/backend/sources/PlayerState/AbstractPlayerState.ts
+++ b/src/backend/sources/PlayerState/AbstractPlayerState.ts
@@ -9,7 +9,7 @@ import dayjs, {Dayjs} from "dayjs";
import { formatNumber, genGroupIdStr, playObjDataMatch, progressBar } from "../../utils";
import {Logger} from "@foxxmd/winston";
import { ListenProgress } from "./ListenProgress";
-import { ListenRange, PlayData, PlayObject } from "../../../core/Atomic";
+import {ListenRange, PlayData, PlayObject, SourcePlayerObj} from "../../../core/Atomic";
import { buildTrackString } from "../../../core/StringUtils";
export interface PlayerStateIntervals {
@@ -274,4 +274,22 @@ export abstract class AbstractPlayerState {
logSummary() {
this.logger.debug(this.textSummary());
}
+
+ getApiState(): SourcePlayerObj {
+ return {
+ platformId: this.platformIdStr,
+ play: this.getPlayedObject(),
+ playLastUpdatedAt: this.playLastUpdatedAt.toISOString(),
+ playFirstSeenAt: this.playFirstSeenAt.toISOString(),
+ playerLastUpdatedAt: this.stateLastUpdatedAt.toISOString(),
+ position: this.currentListenRange !== undefined && this.currentListenRange[1].position !== undefined ? this.currentListenRange[1].position : undefined,
+ listenedDuration: this.getListenDuration(),
+ status: {
+ reported: this.reportedStatus,
+ calculated: this.calculatedStatus,
+ stale: this.isUpdateStale(),
+ orphaned: this.isOrphaned()
+ }
+ }
+ }
}
diff --git a/src/client/status/ducks.ts b/src/client/status/ducks.ts
index 3374da7f..abc932d3 100644
--- a/src/client/status/ducks.ts
+++ b/src/client/status/ducks.ts
@@ -7,7 +7,7 @@ import {
} from '@reduxjs/toolkit'
import { Api } from '@reduxjs/toolkit/query/react';
import {statusApi} from "./statusApi";
-import {ClientStatusData, SourceStatusData} from "../../core/Atomic";
+import {ClientStatusData, SourcePlayerJson, SourceStatusData} from "../../core/Atomic";
export interface ApiEventPayload {
type: string,
@@ -48,7 +48,13 @@ const sourceSlice = createSlice({
(state, action) => {
state.entities[action.payload.id].tracksDiscovered = state.entities[action.payload.id].tracksDiscovered + 1;
}
- )
+ ).addMatcher(
+ (action) => sourceUpdate.match(action) && action.payload.event === 'playerUpdate',
+ (state, action) => {
+ const playerState = action.payload.data as SourcePlayerJson;
+ state.entities[action.payload.id].players[playerState.platformId] = playerState;
+ }
+ )
}
});
const clientSlice = createSlice({
diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts
index a7768840..71cfb61f 100644
--- a/src/core/Atomic.ts
+++ b/src/core/Atomic.ts
@@ -11,6 +11,7 @@ export interface SourceStatusData {
hasAuth: boolean;
hasAuthInteraction: boolean;
authed: boolean;
+ players: Record
}
export interface ClientStatusData {
@@ -149,3 +150,23 @@ export interface LogOutputConfig {
export interface LogInfoJson extends LogInfo {
formattedMessage: string
}
+
+export interface SourcePlayerObj {
+ platformId: string,
+ play: PlayObject,
+ playFirstSeenAt: string,
+ playLastUpdatedAt: string,
+ playerLastUpdatedAt: string
+ position?: number
+ listenedDuration: number
+ status: {
+ reported: string
+ calculated: string
+ stale: boolean
+ orphaned: boolean
+ }
+}
+
+export interface SourcePlayerJson extends Omit {
+ play: JsonPlayObject
+}
--
2.54.0
From 9cc820c76e027288121203ed961ff9d9591b4b80 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Fri, 8 Sep 2023 16:33:23 -0400
Subject: [PATCH 03/82] feat: More player polish and additional info pane
---
package-lock.json | 52 ++-
package.json | 3 +
src/backend/server/api.ts | 2 +-
src/client/components/player/Player.tsx | 332 ++----------------
src/client/components/player/PlayerInfo.tsx | 40 +++
.../components/player/PlayerTimestamp.tsx | 4 +-
src/client/components/player/Playlist.tsx | 33 --
src/client/components/player/controls.scss | 2 +-
src/client/components/player/player.scss | 31 +-
.../player/{playlist.scss => trackInfo.scss} | 24 +-
.../statusCard/SourceStatusCard.tsx | 12 +-
.../statusCard/StatusCardSkeleton.tsx | 2 +-
12 files changed, 185 insertions(+), 352 deletions(-)
create mode 100644 src/client/components/player/PlayerInfo.tsx
delete mode 100644 src/client/components/player/Playlist.tsx
rename src/client/components/player/{playlist.scss => trackInfo.scss} (74%)
diff --git a/package-lock.json b/package-lock.json
index 393ee019..0b9c7fca 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -10,6 +10,9 @@
"license": "MIT",
"dependencies": {
"@awaitjs/express": "^0.6.3",
+ "@fortawesome/fontawesome-svg-core": "^6.4.2",
+ "@fortawesome/free-solid-svg-icons": "^6.4.2",
+ "@fortawesome/react-fontawesome": "^0.2.0",
"@foxxmd/string-sameness": "^0.1.3",
"@foxxmd/winston": "3.3.31",
"@foxxmd/youtube-music-ts-api": "1.5.1",
@@ -2684,6 +2687,51 @@
"node": "^12.22.0 || ^14.17.0 || >=16.0.0"
}
},
+ "node_modules/@fortawesome/fontawesome-common-types": {
+ "version": "6.4.2",
+ "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-common-types/-/fontawesome-common-types-6.4.2.tgz",
+ "integrity": "sha512-1DgP7f+XQIJbLFCTX1V2QnxVmpLdKdzzo2k8EmvDOePfchaIGQ9eCHj2up3/jNEbZuBqel5OxiaOJf37TWauRA==",
+ "hasInstallScript": true,
+ "engines": {
+ "node": ">=6"
+ }
+ },
+ "node_modules/@fortawesome/fontawesome-svg-core": {
+ "version": "6.4.2",
+ "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-svg-core/-/fontawesome-svg-core-6.4.2.tgz",
+ "integrity": "sha512-gjYDSKv3TrM2sLTOKBc5rH9ckje8Wrwgx1CxAPbN5N3Fm4prfi7NsJVWd1jklp7i5uSCVwhZS5qlhMXqLrpAIg==",
+ "hasInstallScript": true,
+ "dependencies": {
+ "@fortawesome/fontawesome-common-types": "6.4.2"
+ },
+ "engines": {
+ "node": ">=6"
+ }
+ },
+ "node_modules/@fortawesome/free-solid-svg-icons": {
+ "version": "6.4.2",
+ "resolved": "https://registry.npmjs.org/@fortawesome/free-solid-svg-icons/-/free-solid-svg-icons-6.4.2.tgz",
+ "integrity": "sha512-sYwXurXUEQS32fZz9hVCUUv/xu49PEJEyUOsA51l6PU/qVgfbTb2glsTEaJngVVT8VqBATRIdh7XVgV1JF1LkA==",
+ "hasInstallScript": true,
+ "dependencies": {
+ "@fortawesome/fontawesome-common-types": "6.4.2"
+ },
+ "engines": {
+ "node": ">=6"
+ }
+ },
+ "node_modules/@fortawesome/react-fontawesome": {
+ "version": "0.2.0",
+ "resolved": "https://registry.npmjs.org/@fortawesome/react-fontawesome/-/react-fontawesome-0.2.0.tgz",
+ "integrity": "sha512-uHg75Rb/XORTtVt7OS9WoK8uM276Ufi7gCzshVWkUJbHhh3svsUUeqXerrM96Wm7fRiDzfKRwSoahhMIkGAYHw==",
+ "dependencies": {
+ "prop-types": "^15.8.1"
+ },
+ "peerDependencies": {
+ "@fortawesome/fontawesome-svg-core": "~1 || ~6",
+ "react": ">=16.3"
+ }
+ },
"node_modules/@foxxmd/string-sameness": {
"version": "0.1.3",
"resolved": "https://registry.npmjs.org/@foxxmd/string-sameness/-/string-sameness-0.1.3.tgz",
@@ -19392,7 +19440,6 @@
"version": "15.8.1",
"resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz",
"integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==",
- "dev": true,
"dependencies": {
"loose-envify": "^1.4.0",
"object-assign": "^4.1.1",
@@ -19402,8 +19449,7 @@
"node_modules/prop-types/node_modules/react-is": {
"version": "16.13.1",
"resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz",
- "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==",
- "dev": true
+ "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ=="
},
"node_modules/proxy-addr": {
"version": "2.0.7",
diff --git a/package.json b/package.json
index 1eca3922..bf14394c 100644
--- a/package.json
+++ b/package.json
@@ -47,6 +47,9 @@
"homepage": ".",
"dependencies": {
"@awaitjs/express": "^0.6.3",
+ "@fortawesome/fontawesome-svg-core": "^6.4.2",
+ "@fortawesome/free-solid-svg-icons": "^6.4.2",
+ "@fortawesome/react-fontawesome": "^0.2.0",
"@foxxmd/string-sameness": "^0.1.3",
"@foxxmd/winston": "3.3.31",
"@foxxmd/youtube-music-ts-api": "1.5.1",
diff --git a/src/backend/server/api.ts b/src/backend/server/api.ts
index 8807502f..f05b37df 100644
--- a/src/backend/server/api.ts
+++ b/src/backend/server/api.ts
@@ -144,7 +144,7 @@ export const setupApi = (app: ExpressWithAsync, logger: Logger, initialLogOutput
hasAuth: requiresAuth,
hasAuthInteraction: requiresAuthInteraction,
authed,
- players: x instanceof MemorySource ? x.playersToObject() : {}
+ players: 'players' in x ? (x as MemorySource).playersToObject() : {}
};
if (!initialized) {
base.status = 'Not Initialized';
diff --git a/src/client/components/player/Player.tsx b/src/client/components/player/Player.tsx
index a29fa4df..8e2a8c6b 100644
--- a/src/client/components/player/Player.tsx
+++ b/src/client/components/player/Player.tsx
@@ -1,11 +1,13 @@
-import React, {useState, useCallback} from 'react';
+import React, {useState, useCallback, Fragment} from 'react';
import './player.scss';
import PlayerTimestamp from "./PlayerTimestamp";
-import PlayerControls from "./PlayerControls";
+import {SourcePlayerJson} from "../../../core/Atomic";
+import PlayerInfo from "./PlayerInfo";
+import {FontAwesomeIcon} from "@fortawesome/react-fontawesome";
+import {faBars, faTimes} from '@fortawesome/free-solid-svg-icons'
export interface PlayerProps {
- tracks: Track[]
-
+ data: SourcePlayerJson
}
export interface Track {
@@ -17,64 +19,22 @@ export interface Track {
artwork: string
}
-export const sampleTracks: Track[] = [
- {
- name: "Somebody Hates Me",
- artist: "Reel Big Fish",
- album: "Why Do They Rock So Hard?",
- year: 1998,
- duration: 210,
- artwork: "https://is5-ssl.mzstatic.com/image/thumb/Music/59/db/b5/mzi.nloomwwj.jpg/268x0w.jpg"
- },
- {
- name: "Beer",
- artist: "Reel Big Fish",
- album: "We're Not Happy Til You're Not Happy",
- year: 2005,
- duration: 233,
- artwork: "https://upload.wikimedia.org/wikipedia/en/9/9d/Reel_Big_Fish_-_We%27re_Not_Happy_%27til_You%27re_Not_Happy_cover.jpg"
- },
- {
- name: "We Will Fall Together",
- artist: "Streetlight Manifesto",
- album: "Somewhere In The Between",
- year: 2007,
- duration: 289,
- artwork: "https://upload.wikimedia.org/wikipedia/en/9/95/Streetlight_Manifesto_-_Somewhere_in_the_Between.jpg"
- },
- {
- name: "Welcome To The Jungle",
- artist: "Guns N' Roses",
- album: "Appetite For Destruction",
- year: 1987,
- duration: 277,
- artwork: "https://upload.wikimedia.org/wikipedia/en/6/60/GunsnRosesAppetiteforDestructionalbumcover.jpg"
- },
- {
- name: "Estranged",
- artist: "Guns N' Roses",
- album: "Use Your Illusion II",
- year: 1991,
- duration: 563,
- artwork: "https://www.lifeofvinyl.com/images/super/0720642442012.jpg"
- },
- {
- name: "Cheer Up!",
- artist: "Reel Big Fish",
- album: "Cheer Up!",
- year: 2002,
- duration: 163,
- artwork: "https://upload.wikimedia.org/wikipedia/en/f/f9/Reel_Big_Fish_-_Cheer_Up%21_cover.jpg"
- }
-];
-
const Player = (props: PlayerProps) => {
- const [playStatus, setPlayStatus] = useState(false);
- const [currentTrack, setCurrentTrack] = useState(0);
- const [nextTrack, setNextTrack] = useState(1);
- const [prevTrack, setPrevTrack] = useState(props.tracks.length - 1);
- const [currentTime, setCurrentTime] = useState(0);
- const [sliding, setSliding] = useState(0);
+ const {
+ data,
+ } = props;
+
+ const {
+ play: {
+ data: {
+ track = '???',
+ artists = ['???'],
+ duration = 0
+ } = {}
+ } = {},
+ play
+ } = data;
+
const [viewMode, setViewMode] = useState('player');
const toggleViewMode = useCallback(() => {
@@ -90,251 +50,23 @@ const Player = (props: PlayerProps) => {
setViewMode(newViewMode);
}, [viewMode, setViewMode]);
- const togglePlay = useCallback(() => {},[playStatus, setPlayStatus]);
- const changeTrack = useCallback((track) => {}, [props.tracks, nextTrack, prevTrack, setCurrentTrack, setSliding, setCurrentTrack, setNextTrack, setPrevTrack]);
- const toggleNextTrack = useCallback(() => {}, [nextTrack, changeTrack]);
- const togglePrevTrack = useCallback(() => {}, [prevTrack, changeTrack]);
- const setTime = useCallback((time) => {}, [props.tracks, currentTrack, setNextTrack, setCurrentTime]);
-
- const aPrevTrack = props.tracks[prevTrack];
- const aCurrentTrack = props.tracks[currentTrack];
- const aNextTrack = props.tracks[nextTrack];
-
- let slidingStatus = "";
- switch (sliding) {
- case -1:
- slidingStatus = "is-sliding-prev";
- break;
- case 1:
- slidingStatus = "is-sliding-next";
- break;
- }
-
- let playlistIcon = "fa fa-fw ";
- if (viewMode == "playlist") {
- playlistIcon += "fa-times";
- } else {
- playlistIcon += "fa-bars";
- }
-
return (
-
-
-
-
-
-
-
-
-
-
- {/*
-
-
-
- */}
-
- {/* */}
-
+
+
+
+
+
+
- {aCurrentTrack.name}
- {aCurrentTrack.artist}
-
+ {track}
+ {artists.join(' / ')}
+
- {/*
{}} />*/}
+
+
);
}
export default Player;
-
-/*
-export class PlayerC extends React.Component {
- static get defaultProps() {
- return {
-
- }
- }
-
- constructor(props, defaultProps) {
- super(props, defaultProps);
-
- this.state = {
- playStatus: 0,
- currentTrack: 0,
- nextTrack: 1,
- prevTrack: this.props.tracks.length-1,
- currentTime: 0,
- sliding: 0,
- likedTracks: [],
- viewMode: "player"
- }
-
- this.togglePlay = this.togglePlay.bind(this);
- this.toggleViewMode = this.toggleViewMode.bind(this);
- this.nextTrack = this.nextTrack.bind(this);
- this.prevTrack = this.prevTrack.bind(this);
- this.likeTrack = this.likeTrack.bind(this);
- }
-
- toggleViewMode() {
- let viewMode = "";
- switch(this.state.viewMode) {
- case "player":
- viewMode = "playlist";
- break;
- case "playlist":
- viewMode = "player"
- break;
- }
-
- this.setState({
- viewMode: viewMode
- });
- }
-
- togglePlay() {
- let status = this.state.playStatus;
- if (status == 0) {
- // Play
- status = 1;
- this.timer = setInterval(() => {
- this.setTime(this.state.currentTime + 1);
- }, 1000);
- } else {
- // Pause
- status = 0;
- clearInterval(this.timer);
- }
- this.setState({ playStatus: status });
- }
-
- changeTrack(track, dir) {
- if (this.state.sliding == 0) {
- let this_track = track;
- let next_track = track + 1;
- let prev_track = track - 1;
-
- if (next_track >= this.props.tracks.length) next_track = 0;
- if (prev_track < 0) prev_track = this.props.tracks.length - 1;
-
- this.setState({
- sliding: dir
- });
-
- setTimeout(() => {
- this.setState({
- sliding: 0,
- currentTrack: this_track,
- nextTrack: next_track,
- prevTrack: prev_track,
- currentTime: 0
- });
- }, 500);
- }
- }
-
- nextTrack() {
- this.changeTrack(this.state.nextTrack, 1);
- }
-
- prevTrack() {
- if (this.state.currentTime < 2) {
- this.changeTrack(this.state.prevTrack, -1);
- } else {
- this.setState({
- currentTime: 0
- });
- }
- }
-
- likeTrack() {
- let likedTracks = this.state.likedTracks;
- let found = false;
- for (let i = 0; i < this.state.likedTracks.length; i++) {
- if (this.state.likedTracks[i] == this.state.currentTrack) {
- found = true;
- likedTracks.splice(i, 1);
- break;
- }
- }
-
- if (!found) {
- likedTracks.push(this.state.currentTrack);
- }
-
- this.setState({
- likedTracks: likedTracks
- });
- }
-
- setTime(time) {
- time = Math.floor(time);
- if (time > this.props.tracks[this.state.currentTrack].duration) {
- this.nextTrack();
- } else {
- this.setState({ currentTime: time });
- }
- }
-
- render() {
- const prevTrack = this.props.tracks[this.state.prevTrack];
- const currentTrack = this.props.tracks[this.state.currentTrack];
- const nextTrack = this.props.tracks[this.state.nextTrack];
-
- let sliding = "";
- switch (this.state.sliding) {
- case -1:
- sliding = "is-sliding-prev";
- break;
- case 1:
- sliding = "is-sliding-next";
- break;
- }
-
- let playlistIcon = "fa fa-fw ";
- if (this.state.viewMode == "playlist") {
- playlistIcon += "fa-times";
- } else {
- playlistIcon += "fa-bars";
- }
-
- let isLiked = false;
- for (let i = 0; i < this.state.likedTracks.length; i++) {
- if (this.state.likedTracks[i] == this.state.currentTrack) {
- isLiked = true;
- }
- }
-
- return (
-
-
-
-
-
-
-
- {currentTrack.name}
- {currentTrack.artist}
-
-
-
-
-
- );
- }
-}
-*/
diff --git a/src/client/components/player/PlayerInfo.tsx b/src/client/components/player/PlayerInfo.tsx
new file mode 100644
index 00000000..bff6d6d0
--- /dev/null
+++ b/src/client/components/player/PlayerInfo.tsx
@@ -0,0 +1,40 @@
+import React, {Fragment} from 'react';
+import './trackInfo.scss';
+import {SourcePlayerJson} from "../../../core/Atomic";
+
+export interface PlayerInfoProps {
+ data: SourcePlayerJson
+ isVisible: boolean
+}
+
+const PlayerInfo = (props: PlayerInfoProps) => {
+ const {
+ isVisible = true,
+ data,
+ data: {
+ play,
+ } = {}
+ } = props;
+
+ let isHidden = "";
+ if (!isVisible) {
+ isHidden = "hidden";
+ }
+
+ if (play === undefined || play === null) {
+ return ;
+ }
+
+ return (
+
+
+
Player ID: {data.platformId}
+
Player Updated: {data.playerLastUpdatedAt}
+
Track Seen: {data.playFirstSeenAt}
+
Track Updated: {data.playLastUpdatedAt}
+
+
+ );
+}
+export default PlayerInfo;
diff --git a/src/client/components/player/PlayerTimestamp.tsx b/src/client/components/player/PlayerTimestamp.tsx
index 11ec7c7f..c353f6aa 100644
--- a/src/client/components/player/PlayerTimestamp.tsx
+++ b/src/client/components/player/PlayerTimestamp.tsx
@@ -20,13 +20,13 @@ const Timestamp = (props: TimestampProps) => {
return(
- {convertTime(props.current)}
+ {convertTime(Math.floor(props.current))}
- {convertTime(props.duration - props.current)}
+ {convertTime(Math.floor(props.duration) - Math.floor(props.current))}
);
diff --git a/src/client/components/player/Playlist.tsx b/src/client/components/player/Playlist.tsx
deleted file mode 100644
index 3a929cee..00000000
--- a/src/client/components/player/Playlist.tsx
+++ /dev/null
@@ -1,33 +0,0 @@
-import React from 'react';
-/*export class Playlist extends React.Component {
- render() {
- const tracks = this.props.tracks;
-
- let isHidden = "";
- if (!this.props.isVisible) {
- isHidden = "is-hidden";
- }
-
- return(
-
- {tracks.map((track, i) => {
- return(
- this.props.changeTrack(i)}>
-
-
-
-
-
-
-
-
{track.name}
-
{track.artist}
-
-
-
- )
- })}
-
- );
- }
-}*/
diff --git a/src/client/components/player/controls.scss b/src/client/components/player/controls.scss
index 57759951..71da75a1 100644
--- a/src/client/components/player/controls.scss
+++ b/src/client/components/player/controls.scss
@@ -1,5 +1,5 @@
@use './playerBase.scss';
-$spacing: 1.5rem;
+$spacing: 1rem;
$primary: #00ACC1;
diff --git a/src/client/components/player/player.scss b/src/client/components/player/player.scss
index be8d39a7..0ddf1a73 100644
--- a/src/client/components/player/player.scss
+++ b/src/client/components/player/player.scss
@@ -1,17 +1,39 @@
@use './playerBase.scss';
-$spacing: 1.5rem;
+$spacing: 0.5rem;
$primary: #00ACC1;
.player {
color: black;
overflow: hidden;
- max-width: 90%;
+ //max-width: 90%;
max-height: 90vh;
//min-height: 487px;
- border-radius: 5px;
+ border-radius: 2px;
background: white;
- box-shadow: 0 10px 20px rgba(0,0,0,0.19), 0 6px 6px rgba(0,0,0,0.23);
+ //box-shadow: 0 10px 20px rgba(0,0,0,0.19), 0 6px 6px rgba(0,0,0,0.23);
+
+ &__wrapper {
+ position: relative;
+ }
+
+ .toggle-playlist {
+ position: absolute;
+ top: $spacing/2;
+ right: $spacing/0.5;
+ border: 0;
+ outline: 0;
+ background: transparent;
+ text-shadow: rgba(black, 0.75) 0 0 5px;
+ box-shadow: none !important;
+ color: white !important;
+ z-index: 1;
+ transition: all 500ms;
+
+ [class*="is-sliding"] & {
+ opacity: 0;
+ }
+ }
&__bg {
position: fixed;
@@ -88,6 +110,7 @@ $primary: #00ACC1;
&__body {
text-align: center;
padding: $spacing;
+ //position: relative;
transition: all 250ms;
[class*="is-sliding"] & {
diff --git a/src/client/components/player/playlist.scss b/src/client/components/player/trackInfo.scss
similarity index 74%
rename from src/client/components/player/playlist.scss
rename to src/client/components/player/trackInfo.scss
index 8360194b..de2b4d6a 100644
--- a/src/client/components/player/playlist.scss
+++ b/src/client/components/player/trackInfo.scss
@@ -3,15 +3,31 @@ $primary: #00ACC1;
.playlist {
+ color: white;
+ font-size:0.80rem;
position: absolute;
top: 0;
bottom: 0;
left: 0;
right: 0;
- padding: $spacing;
- padding-right: $spacing*2;
- overflow: auto;
- background: #263238;
+ padding: $spacing/1.7;
+ padding-right: $spacing/1.5;
+ //overflow: auto;
+ //background: #282c34;
+
+ .playlist_body {
+ display: flex;
+ flex-wrap: wrap;
+
+ .full {
+ width: 100%;
+ flex: 100%;
+ }
+ .half {
+ width: 100%;
+ flex: 50%;
+ }
+ }
&__item.media {
align-items: center;
diff --git a/src/client/components/statusCard/SourceStatusCard.tsx b/src/client/components/statusCard/SourceStatusCard.tsx
index 84cce58d..844d765c 100644
--- a/src/client/components/statusCard/SourceStatusCard.tsx
+++ b/src/client/components/statusCard/SourceStatusCard.tsx
@@ -5,6 +5,7 @@ import {Link} from "react-router-dom";
import {sourceAdapter} from "../../status/ducks";
import {RootState} from "../../store";
import {connect, ConnectedProps} from "react-redux";
+import Player from "../player/Player";
export interface SourceStatusCardData extends StatusCardSkeletonData, PropsFromRedux {
loading?: boolean
@@ -25,6 +26,7 @@ const SourceStatusCard = (props: SourceStatusCardData) => {
},[data]);
if(data !== undefined)
{
+ const className = data.type !== 'spotify' ? 'md:pt-6' : '';
const {
display,
name,
@@ -34,18 +36,22 @@ const SourceStatusCard = (props: SourceStatusCardData) => {
status,
tracksDiscovered,
hasAuthInteraction,
- type
+ type,
+ players = {}
} = data;
header = `(Source) ${display} - ${name}`
+ const platformIds = Object.keys(players);
+
// TODO links
- body = (
+ body = (
+ {platformIds.map(x =>
)}
Status: {status}
Tracks Discovered (since app started): {tracksDiscovered}
{canPoll && (!hasAuth || authed) ? See recently played tracks returned by API
: null}
{canPoll && hasAuthInteraction ? (Re)authenticate and (re)start polling : null}
{canPoll && (!hasAuth || authed) ? Restart Polling
: null}
- );
+ );
}
return (
diff --git a/src/client/components/statusCard/StatusCardSkeleton.tsx b/src/client/components/statusCard/StatusCardSkeleton.tsx
index fc4840ef..3464ca5b 100644
--- a/src/client/components/statusCard/StatusCardSkeleton.tsx
+++ b/src/client/components/statusCard/StatusCardSkeleton.tsx
@@ -19,7 +19,7 @@ const StatusCardSkeleton = (props: PropsWithChildren) =>
{loading || header === undefined ? : header}
-
--
2.54.0
From b9f04deb283665235f40cf62f41fd31d7770e573 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Fri, 8 Sep 2023 16:35:20 -0400
Subject: [PATCH 04/82] Always emit player state
---
src/backend/sources/MemorySource.ts | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/backend/sources/MemorySource.ts b/src/backend/sources/MemorySource.ts
index 0e64713c..e3513a44 100644
--- a/src/backend/sources/MemorySource.ts
+++ b/src/backend/sources/MemorySource.ts
@@ -165,8 +165,8 @@ export default class MemorySource extends AbstractSource {
}
if(this.config.options?.logPlayerState === true) {
player.logSummary();
- this.emitEvent('playerUpdate', player.getApiState());
}
+ this.emitEvent('playerUpdate', player.getApiState());
}
for (const deadId of deadPlatformIds) {
this.players.delete(deadId);
--
2.54.0
From 3762f95bae8b6f05d57ca354a2dfba5adc2c0a64 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Mon, 11 Sep 2023 12:34:09 -0400
Subject: [PATCH 05/82] fix(player state): Fix stale/dead logic checks
* Stale should not reset to stale when status is orphaned
* Dead check should be in seconds, not minutes
---
src/backend/sources/PlayerState/AbstractPlayerState.ts | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/backend/sources/PlayerState/AbstractPlayerState.ts b/src/backend/sources/PlayerState/AbstractPlayerState.ts
index 4de82f84..7d454e1f 100644
--- a/src/backend/sources/PlayerState/AbstractPlayerState.ts
+++ b/src/backend/sources/PlayerState/AbstractPlayerState.ts
@@ -62,7 +62,7 @@ export abstract class AbstractPlayerState {
checkStale() {
const isStale = this.isUpdateStale();
- if (isStale && this.calculatedStatus !== CALCULATED_PLAYER_STATUSES.stale) {
+ if (isStale && ![CALCULATED_PLAYER_STATUSES.stale, CALCULATED_PLAYER_STATUSES.orphaned].includes(this.calculatedStatus)) {
this.calculatedStatus = CALCULATED_PLAYER_STATUSES.stale;
this.logger.debug(`Stale after no Play updates for ${Math.abs(dayjs().diff(this.playLastUpdatedAt, 'seconds'))} seconds`);
// end current listening sessions
@@ -76,7 +76,7 @@ export abstract class AbstractPlayerState {
}
isDead() {
- return dayjs().diff(this.stateLastUpdatedAt, 'minutes') >= this.stateIntervalOptions.orphanedInterval * 2;
+ return dayjs().diff(this.stateLastUpdatedAt, 'seconds') >= this.stateIntervalOptions.orphanedInterval * 2;
}
checkOrphaned() {
--
2.54.0
From a7faaaac862a9701f9aaaaf1ed711a7624c413d5 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Mon, 11 Sep 2023 12:35:41 -0400
Subject: [PATCH 06/82] refactor(player state): Refactor handling of
non-updated players
* Check for stale/orphaned/dead in independent loop so they are not dependent on getting source updates to update
* Only emit events if player state has updated
---
package-lock.json | 8 +++
package.json | 2 +
src/backend/sources/MemorySource.ts | 89 +++++++++++++++++++++--------
3 files changed, 76 insertions(+), 23 deletions(-)
diff --git a/package-lock.json b/package-lock.json
index 0b9c7fca..8a4a14b6 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -42,6 +42,7 @@
"mopidy": "^1.3.0",
"normalize-url": "^6.1.0",
"ntfy": "1.0.5",
+ "object-hash": "^3.0.0",
"p-event": "^4.2.0",
"passport": "^0.6.0",
"passport-deezer": "^0.2.0",
@@ -83,6 +84,7 @@
"@types/jscodeshift": "^0.11.6",
"@types/mocha": "^9.1.0",
"@types/node": "^18.0.0",
+ "@types/object-hash": "^3.0.0",
"@types/passport": "^1.0.12",
"@types/react": "^18.2.18",
"@types/react-dom": "^18.2.7",
@@ -4867,6 +4869,12 @@
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.17.12.tgz",
"integrity": "sha512-d6xjC9fJ/nSnfDeU0AMDsaJyb1iHsqCSOdi84w4u+SlN/UgQdY5tRhpMzaFYsI4mnpvgTivEaQd0yOUhAtOnEQ=="
},
+ "node_modules/@types/object-hash": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/@types/object-hash/-/object-hash-3.0.0.tgz",
+ "integrity": "sha512-lHH/AKLbBWaKUYuWzzB1M/9XzN94AhG1B2wuoXPTLL6B4KLOfEfGOoTq2u5FVbqgnhCPpgPULr1qH1eYs68ZWA==",
+ "dev": true
+ },
"node_modules/@types/parse-json": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz",
diff --git a/package.json b/package.json
index bf14394c..c49a15c7 100644
--- a/package.json
+++ b/package.json
@@ -79,6 +79,7 @@
"mopidy": "^1.3.0",
"normalize-url": "^6.1.0",
"ntfy": "1.0.5",
+ "object-hash": "^3.0.0",
"p-event": "^4.2.0",
"passport": "^0.6.0",
"passport-deezer": "^0.2.0",
@@ -120,6 +121,7 @@
"@types/jscodeshift": "^0.11.6",
"@types/mocha": "^9.1.0",
"@types/node": "^18.0.0",
+ "@types/object-hash": "^3.0.0",
"@types/passport": "^1.0.12",
"@types/react": "^18.2.18",
"@types/react-dom": "^18.2.7",
diff --git a/src/backend/sources/MemorySource.ts b/src/backend/sources/MemorySource.ts
index e3513a44..5f6abbb1 100644
--- a/src/backend/sources/MemorySource.ts
+++ b/src/backend/sources/MemorySource.ts
@@ -4,7 +4,6 @@ import {
sortByOldestPlayDate,
toProgressAwarePlayObject,
getProgress,
- genGroupIdStrFromPlay,
playPassesScrobbleThreshold,
timePassesScrobbleThreshold,
thresholdResultSummary,
@@ -16,12 +15,12 @@ import dayjs from "dayjs";
import {
asPlayerStateData,
DeviceId,
- GroupedPlays,
+ GroupedPlays, InternalConfig,
PlayerStateData,
PlayPlatformId,
PlayUserId,
ProgressAwarePlayObject,
- ScrobbleThresholdResult,
+ ScrobbleThresholdResult, SourceType,
} from "../common/infrastructure/Atomic";
import TupleMap from "../common/TupleMap";
import { AbstractPlayerState, PlayerStateOptions } from "./PlayerState/AbstractPlayerState";
@@ -29,6 +28,10 @@ import { GenericPlayerState } from "./PlayerState/GenericPlayerState";
import {Logger} from "@foxxmd/winston";
import {PlayObject, SourcePlayerObj} from "../../core/Atomic";
import { buildTrackString } from "../../core/StringUtils";
+import {SimpleIntervalJob, Task, ToadScheduler} from "toad-scheduler";
+import {SourceConfig} from "../common/infrastructure/config/source/sources";
+import {EventEmitter} from "events";
+import objectHash from 'object-hash';
export default class MemorySource extends AbstractSource {
/*
@@ -57,6 +60,43 @@ export default class MemorySource extends AbstractSource {
candidateRecentlyPlayed: GroupedPlays = new TupleMap
players: Map = new Map();
+ playerState: Map = new Map();
+
+ scheduler: ToadScheduler = new ToadScheduler();
+
+ constructor(type: SourceType, name: string, config: SourceConfig, internal: InternalConfig, emitter: EventEmitter) {
+ super(type, name, config, internal, emitter);
+ this.scheduler.addSimpleIntervalJob(new SimpleIntervalJob({seconds: 15}, new Task('Player Cleanup', () => {
+ this.cleanupPlayers();
+ })));
+ }
+
+ cleanupPlayers = () => {
+ const deadPlatformIds: string[] = [];
+ for (const [key, player] of this.players.entries()) {
+ // no communication from the source was received for this player
+ const isStale = player.checkStale();
+ if (isStale && player.checkOrphaned() && player.isDead()) {
+ player.logger.debug(`Removed after being orphaned for ${dayjs.duration(player.stateIntervalOptions.orphanedInterval, 'seconds').asMinutes()} minutes`);
+ deadPlatformIds.push(player.platformIdStr);
+ this.emitEvent('playerDelete', {platformId: player.platformIdStr});
+ } else if (isStale) {
+ const state = player.getApiState();
+ // @ts-ignore
+ const stateHash = objectHash.sha1(state);
+ if(stateHash !== this.playerState.get(key)) {
+ this.playerState.set(key, stateHash);
+ this.emitEvent('playerUpdate', state);
+ }
+ if(this.config.options?.logPlayerState === true) {
+ player.logSummary();
+ }
+ }
+ }
+ for (const deadId of deadPlatformIds) {
+ this.deletePlayer(deadId);
+ }
+ }
playersToObject = (): Record => {
if(this.players.size === 0) {
@@ -78,6 +118,20 @@ export default class MemorySource extends AbstractSource {
return new GenericPlayerState(logger, id, opts);
}
+ setNewPlayer = (idStr: string, logger: Logger, id: PlayPlatformId, opts: PlayerStateOptions = {}) => {
+ this.players.set(idStr, this.getNewPlayer(this.logger, id, {
+ staleInterval: (this.config.data.interval ?? 30) * 3,
+ orphanedInterval: (this.config.data.maxInterval ?? 60) * 5,
+ ...opts
+ }));
+ this.playerState.set(idStr, '');
+ }
+
+ deletePlayer = (id: string) => {
+ this.players.delete(id);
+ this.playerState.delete(id);
+ }
+
processRecentPlaysNew = (datas: (PlayObject | PlayerStateData)[]) => {
const {
@@ -89,20 +143,15 @@ export default class MemorySource extends AbstractSource {
const newStatefulPlays: PlayObject[] = [];
// create any new players from incoming data
- //const incomingPlatformIds: PlayPlatformId[] = [];
for (const data of datas) {
const id = getPlatformIdFromData(data);
const idStr = genGroupIdStr(id);
if (!this.players.has(idStr)) {
- //incomingPlatformIds.push(id);
- this.players.set(idStr, this.getNewPlayer(this.logger, id, {
- staleInterval: (this.config.data.interval ?? 30) * 3,
- orphanedInterval: (this.config.data.maxInterval ?? 60) * 5
- }));
+ this.setNewPlayer(idStr, this.logger, id);
}
}
- const deadPlatformIds: string[] = [];
+ //const deadPlatformIds: string[] = [];
for (const [key, player] of this.players.entries()) {
@@ -155,21 +204,15 @@ export default class MemorySource extends AbstractSource {
}
}
}
- } else {
- // no communication from the source was received for this player
- player.checkStale();
- if (player.checkOrphaned() && player.isDead()) {
- player.logger.debug(`Removed after being orphaned for ${dayjs.duration(player.stateIntervalOptions.orphanedInterval, 'seconds').asMinutes()} minutes`);
- deadPlatformIds.push(player.platformIdStr);
+
+ if(this.config.options?.logPlayerState === true) {
+ player.logSummary();
}
+ const apiState = player.getApiState();
+ // @ts-ignore
+ this.playerState.set(key, objectHash.sha1(apiState))
+ this.emitEvent('playerUpdate', apiState);
}
- if(this.config.options?.logPlayerState === true) {
- player.logSummary();
- }
- this.emitEvent('playerUpdate', player.getApiState());
- }
- for (const deadId of deadPlatformIds) {
- this.players.delete(deadId);
}
return newStatefulPlays;
--
2.54.0
From d1060e724e084f807807aaa9ed72a70565f29625 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Mon, 11 Sep 2023 12:36:10 -0400
Subject: [PATCH 07/82] feat(ui): Remove player when dead
---
src/client/status/ducks.ts | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/src/client/status/ducks.ts b/src/client/status/ducks.ts
index abc932d3..52eee732 100644
--- a/src/client/status/ducks.ts
+++ b/src/client/status/ducks.ts
@@ -53,6 +53,11 @@ const sourceSlice = createSlice({
(state, action) => {
const playerState = action.payload.data as SourcePlayerJson;
state.entities[action.payload.id].players[playerState.platformId] = playerState;
+ }).addMatcher(
+ (action) => sourceUpdate.match(action) && action.payload.event === 'playerDelete',
+ (state, action) => {
+ const playerState = action.payload.data as {platformId: string};
+ delete state.entities[action.payload.id].players[playerState.platformId];
}
)
}
--
2.54.0
From d95fef09a3b27e961db5ddd4d84bf6ba11583700 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Mon, 11 Sep 2023 12:38:10 -0400
Subject: [PATCH 08/82] refactor(player state): Remove old play logic
Recent plays are dead. Long live recent plays.
---
src/backend/sources/JRiverSource.ts | 2 +-
src/backend/sources/JellyfinSource.ts | 2 +-
src/backend/sources/KodiSource.ts | 2 +-
src/backend/sources/MPRISSource.ts | 2 +-
src/backend/sources/MemorySource.ts | 157 +-------------------------
src/backend/sources/MopidySource.ts | 2 +-
src/backend/sources/SpotifySource.ts | 2 +-
src/backend/sources/SubsonicSource.ts | 2 +-
8 files changed, 8 insertions(+), 163 deletions(-)
diff --git a/src/backend/sources/JRiverSource.ts b/src/backend/sources/JRiverSource.ts
index 72b0261e..fcbd625c 100644
--- a/src/backend/sources/JRiverSource.ts
+++ b/src/backend/sources/JRiverSource.ts
@@ -141,7 +141,7 @@ export class JRiverSource extends MemorySource {
}
}
- return this.processRecentPlaysNew(play);
+ return this.processRecentPlays(play);
}
}
diff --git a/src/backend/sources/JellyfinSource.ts b/src/backend/sources/JellyfinSource.ts
index aab60915..3fd41b6a 100644
--- a/src/backend/sources/JellyfinSource.ts
+++ b/src/backend/sources/JellyfinSource.ts
@@ -350,7 +350,7 @@ export default class JellyfinSource extends MemorySource {
scrobbleOpts.checkAll = true;
} else {
- newPlays = this.processRecentPlaysNew([playObj]);
+ newPlays = this.processRecentPlays([playObj]);
}
if(newPlays.length > 0) {
diff --git a/src/backend/sources/KodiSource.ts b/src/backend/sources/KodiSource.ts
index 91ec5e56..9cb5d46a 100644
--- a/src/backend/sources/KodiSource.ts
+++ b/src/backend/sources/KodiSource.ts
@@ -70,7 +70,7 @@ export class KodiSource extends MemorySource {
let play = await this.client.getRecentlyPlayed(options);
- return this.processRecentPlaysNew(play);
+ return this.processRecentPlays(play);
}
}
diff --git a/src/backend/sources/MPRISSource.ts b/src/backend/sources/MPRISSource.ts
index c09e7309..44006458 100644
--- a/src/backend/sources/MPRISSource.ts
+++ b/src/backend/sources/MPRISSource.ts
@@ -218,7 +218,7 @@ export class MPRISSource extends MemorySource {
if(options.display === true) {
return deduped;
}
- return this.processRecentPlaysNew(deduped);
+ return this.processRecentPlays(deduped);
}
}
diff --git a/src/backend/sources/MemorySource.ts b/src/backend/sources/MemorySource.ts
index 5f6abbb1..32a33995 100644
--- a/src/backend/sources/MemorySource.ts
+++ b/src/backend/sources/MemorySource.ts
@@ -42,23 +42,6 @@ export default class MemorySource extends AbstractSource {
* * where these timestamps don't have enough granularity (IE second accuracy)
* such as subsonic and jellyfin */
- /**
- * Tracks we are tracked that we are confident qualified as being played based on:
- *
- * - MS saw the track for the first time while it was running/polling
- * - Continued to see this same track through consecutive polling/ingress events for AT LEAST 30 seconds
- * - If the play info contains playback position data we also check if that has progressed at least 30 seconds
- *
- * These are tracks that are actually used by the source to scrobble to clients
- * */
- //statefulRecentlyPlayed: GroupedPlays = new Map();
- /**
- * Tracks that are actively being tracked (discovered from source recently) to see if they will "qualify" as being played.
- *
- * Once a track qualifies it is added to statefuls.
- * */
- candidateRecentlyPlayed: GroupedPlays = new TupleMap
-
players: Map = new Map();
playerState: Map = new Map();
@@ -109,11 +92,6 @@ export default class MemorySource extends AbstractSource {
return record;
}
- getFlatCandidateRecentlyPlayed = (): PlayObject[] => {
- // TODO sort?
- return Array.from(this.candidateRecentlyPlayed.values()).flat();
- }
-
getNewPlayer = (logger: Logger, id: PlayPlatformId, opts: PlayerStateOptions) => {
return new GenericPlayerState(logger, id, opts);
}
@@ -132,7 +110,7 @@ export default class MemorySource extends AbstractSource {
this.playerState.delete(id);
}
- processRecentPlaysNew = (datas: (PlayObject | PlayerStateData)[]) => {
+ processRecentPlays = (datas: (PlayObject | PlayerStateData)[]) => {
const {
data: {
@@ -218,139 +196,6 @@ export default class MemorySource extends AbstractSource {
return newStatefulPlays;
}
- processRecentPlays = (plays: PlayObject[], useExistingPlayDate = false) => {
-
- const {
- data: {
- scrobbleThresholds = {}
- } = {}
- } = this.config;
-
- let newStatefulPlays: PlayObject[] = [];
- // if we can't trust existing play dates (like for subsonic where there is no timestamp) then
- // we format new plays with locked play date IE the first time we have "seen" this track
-
- // -- otherwise, for sources like Spotify that accurately report when track started to play, we can use existing dates
- const flatLockedPlays = useExistingPlayDate ? plays : plays.map((p: any) => {
- const {data: {playDate, ...restData}, ...rest} = p;
- return {data: {...restData, playDate: dayjs()}, ...rest};
- });
-
- // group by device-user
- const groupedLockedPlays = flatLockedPlays.reduce((acc: GroupedPlays, curr: ProgressAwarePlayObject) => {
- const id = genGroupId(curr);
- acc.set(id, (acc.get(id) ?? []).concat(curr));
- return acc;
- }, new Map());
-
- for (const [groupId, lockedPlays] of groupedLockedPlays.entries()) {
- const groupIdStr = `${groupId[0]}-${groupId[1]}`;
- let cRecentlyPlayed = this.candidateRecentlyPlayed.get(groupId) ?? [];
- // if no candidates exist new plays are new candidates
- if (cRecentlyPlayed.length === 0) {
- this.logger.debug(`[Platform ${groupIdStr}] No prior candidate recent plays!`)
- // update activity date here so that polling interval decreases *before* we get a new valid play
- // so that we don't miss a play due to long polling interval
- this.lastActivityAt = dayjs();
- const progressAware: ProgressAwarePlayObject[] = [];
- for (const p of lockedPlays) {
- progressAware.push(toProgressAwarePlayObject(p));
- this.logger.debug(`[Platform ${groupIdStr}] Adding new locked play: ${buildTrackString(p, {include: ['trackId', 'artist', 'track']})}`);
- }
- this.candidateRecentlyPlayed.set(groupId, progressAware)
- } else {
- // otherwise determine new tracks (not found in prior candidates)
- const newTracks = lockedPlays.filter((x: any) => cRecentlyPlayed.every(y => !playObjDataMatch(y, x)));
- const newProgressAwareTracks: ProgressAwarePlayObject[] = [];
- if (newTracks.length > 0) {
- // update activity date here so that polling interval decreases *before* we get a new valid play
- // so that we don't miss a play due to long polling interval
- this.lastActivityAt = dayjs();
- this.logger.debug(`[Platform ${groupIdStr}] New plays found that do not match existing candidates.`)
- for (const p of newTracks) {
- this.logger.debug(`[Platform ${groupIdStr}] Adding new locked play: ${buildTrackString(p, {include: ['trackId', 'artist', 'track']})}`);
- newProgressAwareTracks.push(toProgressAwarePlayObject(p));
- }
- }
- // filter prior candidates based on new recently played
- cRecentlyPlayed = cRecentlyPlayed.filter(x => {
- const candidateMatchedLocked = lockedPlays.some((y: any) => playObjDataMatch(x, y));
- if (!candidateMatchedLocked) {
- this.logger.debug(`[Platform ${groupIdStr}] Existing candidate not found in locked plays will be removed: ${buildTrackString(x, {include: ['trackId', 'artist', 'track']})}`);
- }
- return candidateMatchedLocked;
- });
- // and then combine still playing with new tracks
- cRecentlyPlayed = cRecentlyPlayed.concat(newProgressAwareTracks);
- cRecentlyPlayed.sort(sortByOldestPlayDate);
-
- this.candidateRecentlyPlayed.set(groupId, cRecentlyPlayed);
-
- //const sRecentlyPlayed = this.statefulRecentlyPlayed.get(groupId) ?? [];
-
- // now we check if all candidates pass tests for having been tracked long enough:
- // * Has been tracked for at least [duration] seconds or [percentage] of track duration
- // * If it has playback position data then it must also have progressed at least [duration] seconds or [percentage] of track duration progress since our initial tracking data
- for (const candidate of cRecentlyPlayed) {
- let thresholdResults: ScrobbleThresholdResult;
- thresholdResults = playPassesScrobbleThreshold(candidate, scrobbleThresholds);
- const {passes: firstSeenValid} = thresholdResults;
- let progressValid = firstSeenValid;
- if (firstSeenValid) {
- // check if we can get progress as well
- const matchingLockedPlay = lockedPlays.find(x => playObjDataMatch(x, candidate));
- // this should always be found but checking just in case
- if (matchingLockedPlay !== undefined) {
- const progress = getProgress(candidate, matchingLockedPlay);
- if (progress !== undefined) {
- thresholdResults = timePassesScrobbleThreshold(scrobbleThresholds, progress, candidate.data.duration);
- const {passes: progressPasses} = thresholdResults;
- progressValid = progressPasses;
- }
- }
- }
-
- if (firstSeenValid && progressValid) {
- // a prior candidate has been playing for more than 30 seconds and passed progress test, time to check statefuls
-
- const matchingRecent = this.existingDiscovered(candidate); //sRecentlyPlayed.find(x => playObjDataMatch(x, candidate));
- let stPrefix = `[Platform ${groupId}] (Stateful Play) ${buildTrackString(candidate, {include: ['trackId', 'artist', 'track']})}`;
- if (matchingRecent === undefined) {
- this.logger.debug(`${stPrefix} added after ${thresholdResultSummary(thresholdResults)} and not matching any prior plays`);
- newStatefulPlays.push(candidate);
- //sRecentlyPlayed.push(candidate);
- } else {
- const {data: {playDate, duration}} = candidate;
- const {data: {playDate: rplayDate}} = matchingRecent;
- if (!playDate.isSame(rplayDate)) {
- if (duration !== undefined) {
- if (playDate.isAfter(rplayDate.add(duration, 's'))) {
- this.logger.debug(`${stPrefix} added after ${thresholdResultSummary(thresholdResults)} and having a different timestamp than a prior play`);
- newStatefulPlays.push(candidate);
- //sRecentlyPlayed.push(candidate);
- }
- } else {
- const discoveredPlays = this.getRecentlyDiscoveredPlaysByPlatform(genGroupId(candidate));
- if (discoveredPlays.length === 0 || !playObjDataMatch(discoveredPlays[0], candidate)) {
- // if most recent stateful play is not this track we'll add it
- this.logger.debug(`${stPrefix} added after ${thresholdResultSummary(thresholdResults)}. Matched other recent play but could not determine time frame due to missing duration. Allowed due to not being last played track.`);
- newStatefulPlays.push(candidate);
- }
- //sRecentlyPlayed.push(candidate);
- }
- }
- }
- }
- }
- //sRecentlyPlayed.sort(sortByPlayDate);
- //this.statefulRecentlyPlayed.set(groupId, sRecentlyPlayed);
- }
- }
- return newStatefulPlays;
- //return this.getFlatStatefulRecentlyPlayed();
- //return newStatefulPlays;
- }
-
recentlyPlayedTrackIsValid = (playObj: any) => {
return playObj.data.playDate.isBefore(dayjs().subtract(30, 's'));
}
diff --git a/src/backend/sources/MopidySource.ts b/src/backend/sources/MopidySource.ts
index 1b007a3b..1d1bce27 100644
--- a/src/backend/sources/MopidySource.ts
+++ b/src/backend/sources/MopidySource.ts
@@ -209,7 +209,7 @@ export class MopidySource extends MemorySource {
play
}
- return this.processRecentPlaysNew([playerState]);
+ return this.processRecentPlays([playerState]);
}
}
diff --git a/src/backend/sources/SpotifySource.ts b/src/backend/sources/SpotifySource.ts
index 30598b34..15e09497 100644
--- a/src/backend/sources/SpotifySource.ts
+++ b/src/backend/sources/SpotifySource.ts
@@ -283,7 +283,7 @@ export default class SpotifySource extends MemorySource {
plays.push(currPlay);
}
}
- return this.processRecentPlaysNew(plays);
+ return this.processRecentPlays(plays);
}
getPlayHistory = async (options: RecentlyPlayedOptions = {}) => {
diff --git a/src/backend/sources/SubsonicSource.ts b/src/backend/sources/SubsonicSource.ts
index 76dd5f22..027aba9b 100644
--- a/src/backend/sources/SubsonicSource.ts
+++ b/src/backend/sources/SubsonicSource.ts
@@ -194,6 +194,6 @@ export class SubsonicSource extends MemorySource {
} = resp;
// sometimes subsonic sources will return the same track as being played twice on the same player, need to remove this so we don't duplicate plays
const deduped = removeDuplicates(entry.map(SubsonicSource.formatPlayObj));
- return this.processRecentPlaysNew(deduped);
+ return this.processRecentPlays(deduped);
}
}
--
2.54.0
From f1cd66d3e8c8fb0f600e8ba94b3dc9ceeeed70f4 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Mon, 11 Sep 2023 13:20:58 -0400
Subject: [PATCH 09/82] feat(player state): Preserve position when player is
not in stopped status
Makes for a nicer UI experience to see where the player was paused
---
.../sources/PlayerState/AbstractPlayerState.ts | 18 +++++++++++++++++-
1 file changed, 17 insertions(+), 1 deletion(-)
diff --git a/src/backend/sources/PlayerState/AbstractPlayerState.ts b/src/backend/sources/PlayerState/AbstractPlayerState.ts
index 7d454e1f..8c270772 100644
--- a/src/backend/sources/PlayerState/AbstractPlayerState.ts
+++ b/src/backend/sources/PlayerState/AbstractPlayerState.ts
@@ -275,6 +275,22 @@ export abstract class AbstractPlayerState {
this.logger.debug(this.textSummary());
}
+ getPosition() {
+ if(this.calculatedStatus === 'stopped') {
+ return undefined;
+ }
+ let lastRange: [ListenProgress, ListenProgress] | undefined;
+ if(this.currentListenRange !== undefined) {
+ lastRange = this.currentListenRange;
+ } else if(this.listenRanges.length > 0) {
+ lastRange = this.listenRanges[this.listenRanges.length - 1];
+ }
+ if(lastRange === undefined || lastRange[1] === undefined || lastRange[1].position === undefined) {
+ return undefined;
+ }
+ return lastRange[1].position;
+ }
+
getApiState(): SourcePlayerObj {
return {
platformId: this.platformIdStr,
@@ -282,7 +298,7 @@ export abstract class AbstractPlayerState {
playLastUpdatedAt: this.playLastUpdatedAt.toISOString(),
playFirstSeenAt: this.playFirstSeenAt.toISOString(),
playerLastUpdatedAt: this.stateLastUpdatedAt.toISOString(),
- position: this.currentListenRange !== undefined && this.currentListenRange[1].position !== undefined ? this.currentListenRange[1].position : undefined,
+ position: this.getPosition(),
listenedDuration: this.getListenDuration(),
status: {
reported: this.reportedStatus,
--
2.54.0
From dcff9ee73d6118131ae413a15a2548be08cafcf0 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Mon, 11 Sep 2023 13:21:24 -0400
Subject: [PATCH 10/82] feat(player state): Implement player status and
listened duration in UI
---
.../common/vendor/AbstractApiClient.ts | 3 ++-
.../scrobblers/AbstractScrobbleClient.ts | 3 +--
src/backend/scrobblers/LastfmScrobbler.ts | 3 +--
.../scrobblers/ListenbrainzScrobbler.ts | 3 +--
src/backend/scrobblers/MalojaScrobbler.ts | 3 +--
src/backend/server/api.ts | 4 +--
src/backend/sources/AbstractSource.ts | 3 +--
src/backend/utils.ts | 11 ++++----
src/client/components/player/Player.tsx | 26 +++++++++++++++++--
src/client/components/player/PlayerInfo.tsx | 7 ++++-
src/client/components/player/player.scss | 4 +++
src/client/components/player/timestamp.scss | 2 +-
src/core/StringUtils.ts | 4 ++-
13 files changed, 52 insertions(+), 24 deletions(-)
diff --git a/src/backend/common/vendor/AbstractApiClient.ts b/src/backend/common/vendor/AbstractApiClient.ts
index bcbe86cb..8fd170e7 100644
--- a/src/backend/common/vendor/AbstractApiClient.ts
+++ b/src/backend/common/vendor/AbstractApiClient.ts
@@ -1,8 +1,9 @@
-import { capitalize, mergeArr } from "../../utils";
+import { mergeArr } from "../../utils";
import {Logger} from '@foxxmd/winston';
import { FormatPlayObjectOptions } from "../infrastructure/Atomic";
import winston from '@foxxmd/winston';
import { PlayObject } from "../../../core/Atomic";
+import {capitalize} from "../../../core/StringUtils";
export default abstract class AbstractApiClient {
name: string;
diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts
index 890ddda6..63d3e228 100644
--- a/src/backend/scrobblers/AbstractScrobbleClient.ts
+++ b/src/backend/scrobblers/AbstractScrobbleClient.ts
@@ -1,6 +1,5 @@
import dayjs, {Dayjs} from "dayjs";
import {
- capitalize,
isPlayTemporallyClose,
mergeArr,
playObjDataMatch,
@@ -21,7 +20,7 @@ import { ClientConfig } from "../common/infrastructure/config/client/clients";
import { Notifiers } from "../notifier/Notifiers";
import {FixedSizeList} from 'fixed-size-list';
import { PlayObject, TrackStringOptions } from "../../core/Atomic";
-import { buildTrackString, truncateStringToLength } from "../../core/StringUtils";
+import {buildTrackString, capitalize, truncateStringToLength} from "../../core/StringUtils";
import EventEmitter from "events";
export default abstract class AbstractScrobbleClient {
diff --git a/src/backend/scrobblers/LastfmScrobbler.ts b/src/backend/scrobblers/LastfmScrobbler.ts
index 883c01ad..c623bc2e 100644
--- a/src/backend/scrobblers/LastfmScrobbler.ts
+++ b/src/backend/scrobblers/LastfmScrobbler.ts
@@ -2,7 +2,6 @@ import AbstractScrobbleClient from "./AbstractScrobbleClient";
import dayjs from 'dayjs';
import {
- capitalize,
playObjDataMatch,
removeUndefinedKeys,
setIntersection,
@@ -16,7 +15,7 @@ import {TrackScrobbleResponse, UserGetRecentTracksResponse} from "lastfm-node-cl
import { Notifiers } from "../notifier/Notifiers";
import {Logger} from '@foxxmd/winston';
import { PlayObject, TrackStringOptions } from "../../core/Atomic";
-import { buildTrackString } from "../../core/StringUtils";
+import {buildTrackString, capitalize} from "../../core/StringUtils";
import EventEmitter from "events";
export default class LastfmScrobbler extends AbstractScrobbleClient {
diff --git a/src/backend/scrobblers/ListenbrainzScrobbler.ts b/src/backend/scrobblers/ListenbrainzScrobbler.ts
index c34636b2..46ccf2bd 100644
--- a/src/backend/scrobblers/ListenbrainzScrobbler.ts
+++ b/src/backend/scrobblers/ListenbrainzScrobbler.ts
@@ -2,7 +2,6 @@ import dayjs from 'dayjs';
import AbstractScrobbleClient from "./AbstractScrobbleClient";
import {
- capitalize,
playObjDataMatch,
removeUndefinedKeys,
setIntersection,
@@ -16,7 +15,7 @@ import {Logger} from '@foxxmd/winston';
import { ListenBrainzClientConfig } from "../common/infrastructure/config/client/listenbrainz";
import { ListenbrainzApiClient } from "../common/vendor/ListenbrainzApiClient";
import { PlayObject, TrackStringOptions } from "../../core/Atomic";
-import { buildTrackString } from "../../core/StringUtils";
+import {buildTrackString, capitalize} from "../../core/StringUtils";
import EventEmitter from "events";
export default class ListenbrainzScrobbler extends AbstractScrobbleClient {
diff --git a/src/backend/scrobblers/MalojaScrobbler.ts b/src/backend/scrobblers/MalojaScrobbler.ts
index a381f025..6bfba90d 100644
--- a/src/backend/scrobblers/MalojaScrobbler.ts
+++ b/src/backend/scrobblers/MalojaScrobbler.ts
@@ -8,7 +8,6 @@ import {
sleep,
sortByOldestPlayDate,
parseRetryAfterSecsFromObj,
- capitalize,
isPlayTemporallyClose,
} from "../utils";
import { FormatPlayObjectOptions, INITIALIZING } from "../common/infrastructure/Atomic";
@@ -22,7 +21,7 @@ import {
MalojaScrobbleV3RequestData, MalojaScrobbleV3ResponseData, MalojaV2ScrobbleData, MalojaV3ScrobbleData
} from "../common/vendor/maloja/interfaces";
import { PlayObject, TrackStringOptions } from "../../core/Atomic";
-import { buildTrackString } from "../../core/StringUtils";
+import {buildTrackString, capitalize} from "../../core/StringUtils";
import EventEmitter from "events";
const feat = ["ft.", "ft", "feat.", "feat", "featuring", "Ft.", "Ft", "Feat.", "Feat", "Featuring"];
diff --git a/src/backend/server/api.ts b/src/backend/server/api.ts
index f05b37df..fd6988f5 100644
--- a/src/backend/server/api.ts
+++ b/src/backend/server/api.ts
@@ -1,6 +1,5 @@
import {ExpressWithAsync} from "@awaitjs/express";
import {getRoot} from "../ioc";
-import { capitalize } from "../utils";
import {makeClientCheckMiddle, makeSourceCheckMiddle} from "./middleware";
import AbstractSource from "../sources/AbstractSource";
import {
@@ -22,7 +21,8 @@ import {setupJellyfinRoutes} from "./jellyfinRoutes";
import {setupDeezerRoutes} from "./deezerRoutes";
import {setupAuthRoutes} from "./auth";
import { ExpressHandler } from "../common/infrastructure/Atomic";
-import MemorySource from "../sources/MemorySource.js";
+import MemorySource from "../sources/MemorySource";
+import {capitalize} from "../../core/StringUtils";
let output: LogInfo[] = []
diff --git a/src/backend/sources/AbstractSource.ts b/src/backend/sources/AbstractSource.ts
index 8d81f431..e932c96d 100644
--- a/src/backend/sources/AbstractSource.ts
+++ b/src/backend/sources/AbstractSource.ts
@@ -1,6 +1,5 @@
import dayjs, {Dayjs} from "dayjs";
import {
- capitalize,
isPlayTemporallyClose,
genGroupId,
genGroupIdStrFromPlay,
@@ -30,7 +29,7 @@ import {EventEmitter} from "events";
import {FixedSizeList} from "fixed-size-list";
import TupleMap from "../common/TupleMap";
import { PlayObject } from "../../core/Atomic";
-import { buildTrackString } from "../../core/StringUtils";
+import {buildTrackString, capitalize} from "../../core/StringUtils";
export interface RecentlyPlayedOptions {
limit?: number
diff --git a/src/backend/utils.ts b/src/backend/utils.ts
index 326bc3f5..38145bca 100644
--- a/src/backend/utils.ts
+++ b/src/backend/utils.ts
@@ -7,7 +7,8 @@ import {TimeoutError, WebapiError} from "spotify-web-api-node/src/response-error
import Ajv, {Schema} from 'ajv';
import {
asPlayerStateData,
- DEFAULT_SCROBBLE_DURATION_THRESHOLD, DELIMITERS,
+ DEFAULT_SCROBBLE_DURATION_THRESHOLD,
+ DELIMITERS,
lowGranularitySources,
NO_DEVICE,
NO_USER,
@@ -23,11 +24,12 @@ import {Request} from "express";
import pathUtil from "path";
import {ErrorWithCause} from "pony-cause";
import backoffStrategies from '@kenyip/backoff-strategies';
-import { ScrobbleThresholds } from "./common/infrastructure/config/source/index";
+import {ScrobbleThresholds} from "./common/infrastructure/config/source/index";
import {replaceResultTransformer, stripIndentTransformer, TemplateTag, trimResultTransformer} from 'common-tags';
import {Duration} from "dayjs/plugin/duration.js";
-import { ListenRange, PlayObject } from "../core/Atomic";
+import {ListenRange, PlayObject} from "../core/Atomic";
import address from "address";
+
dayjs.extend(utc);
export async function readJson(this: any, path: any, {throwOnNotFound = true} = {}) {
@@ -177,9 +179,6 @@ const sentenceLengthWeight = (length: number) => {
return (Math.log(length) / 0.20) - 5;
}
-export const capitalize = (str: any) => {
- return str.charAt(0).toUpperCase() + str.slice(1)
-}
/**
* Check if two play objects are the same by comparing non time-related data using most-to-least specific/confidence
*
diff --git a/src/client/components/player/Player.tsx b/src/client/components/player/Player.tsx
index 8e2a8c6b..61081eb9 100644
--- a/src/client/components/player/Player.tsx
+++ b/src/client/components/player/Player.tsx
@@ -6,6 +6,8 @@ import PlayerInfo from "./PlayerInfo";
import {FontAwesomeIcon} from "@fortawesome/react-fontawesome";
import {faBars, faTimes} from '@fortawesome/free-solid-svg-icons'
+import {capitalize} from "../../../core/StringUtils";
+
export interface PlayerProps {
data: SourcePlayerJson
}
@@ -30,11 +32,27 @@ const Player = (props: PlayerProps) => {
track = '???',
artists = ['???'],
duration = 0
- } = {}
+ } = {},
} = {},
- play
+ play,
+ listenedDuration,
+ status: {
+ calculated = '???',
+ reported,
+ stale,
+ orphaned
+ }
} = data;
+ let durPer = null;
+ if(duration !== undefined && duration !== null) {
+ if(listenedDuration === 0) {
+ durPer = ' (0%)';
+ } else {
+ durPer = ` (${((listenedDuration/duration) * 100).toFixed(0)}%)`;
+ }
+ }
+
const [viewMode, setViewMode] = useState('player');
const toggleViewMode = useCallback(() => {
@@ -61,6 +79,10 @@ const Player = (props: PlayerProps) => {
{track}
{artists.join(' / ')}
+
+
Status: {capitalize(calculated)}
+
Listened: {listenedDuration.toFixed(0)}s{durPer}
+
diff --git a/src/client/components/player/PlayerInfo.tsx b/src/client/components/player/PlayerInfo.tsx
index bff6d6d0..d338e405 100644
--- a/src/client/components/player/PlayerInfo.tsx
+++ b/src/client/components/player/PlayerInfo.tsx
@@ -13,7 +13,11 @@ const PlayerInfo = (props: PlayerInfoProps) => {
data,
data: {
play,
- } = {}
+ status: {
+ calculated,
+ reported
+ }
+ } = {},
} = props;
let isHidden = "";
@@ -33,6 +37,7 @@ const PlayerInfo = (props: PlayerInfoProps) => {
Player Updated: {data.playerLastUpdatedAt}
Track Seen: {data.playFirstSeenAt}
Track Updated: {data.playLastUpdatedAt}
+ Status: Calculated -> {calculated.toUpperCase()} | Reported -> {reported.toUpperCase()}
);
diff --git a/src/client/components/player/player.scss b/src/client/components/player/player.scss
index 0ddf1a73..ffd632d1 100644
--- a/src/client/components/player/player.scss
+++ b/src/client/components/player/player.scss
@@ -118,6 +118,10 @@ $primary: #00ACC1;
transform: scale(0.95);
}
+ .stats {
+ font-size: 0.75rem;
+ }
+
.title {
position: relative;
width: 100%;
diff --git a/src/client/components/player/timestamp.scss b/src/client/components/player/timestamp.scss
index fe055ebd..2d649824 100644
--- a/src/client/components/player/timestamp.scss
+++ b/src/client/components/player/timestamp.scss
@@ -1,6 +1,6 @@
@use './playerBase.scss';
$spacing: 1.5rem;
-$primary: #00ACC1;
+$primary: #556a77;
.timestamp {
diff --git a/src/core/StringUtils.ts b/src/core/StringUtils.ts
index f59413e0..b4192658 100644
--- a/src/core/StringUtils.ts
+++ b/src/core/StringUtils.ts
@@ -1,5 +1,4 @@
import dayjs, {Dayjs} from "dayjs";
-import React, {createElement, ReactElement} from "react";
import utc from "dayjs/plugin/utc.js";
import isBetween from "dayjs/plugin/isBetween.js";
import relativeTime from "dayjs/plugin/relativeTime.js";
@@ -96,3 +95,6 @@ export const slice = (str: string, index: number, count: number, add?: string):
return str.slice(0, index) + (add || "") + str.slice(index + count);
}
+export const capitalize = (str: any) => {
+ return str.charAt(0).toUpperCase() + str.slice(1)
+}
--
2.54.0
From 0fb192dd4105a82be01f94f21ef456c1094c81cd Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Tue, 12 Sep 2023 10:16:16 -0400
Subject: [PATCH 11/82] refactor(ui): Prevent nodemon from restarting on
tailwind config change
---
nodemon.json | 3 ++-
tailwind.config.js | 4 +++-
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/nodemon.json b/nodemon.json
index c7d09f69..620fcbd3 100644
--- a/nodemon.json
+++ b/nodemon.json
@@ -1,3 +1,4 @@
{
- "ext": "js"
+ "ext": "js",
+ "ignore": ["tailwind.config.js"]
}
diff --git a/tailwind.config.js b/tailwind.config.js
index c0958ec4..186aad44 100644
--- a/tailwind.config.js
+++ b/tailwind.config.js
@@ -4,8 +4,10 @@ module.exports = {
"./src/**/*.{js,jsx,ts,tsx}",
],
theme: {
+ container: {
+ padding: '1rem',
+ },
extend: {},
},
plugins: [],
}
-
--
2.54.0
From 81894d691b934157301740afb727f0b5d2a36896 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Tue, 12 Sep 2023 11:07:39 -0400
Subject: [PATCH 12/82] feat(ui): Compact ui a bit
* Reduce padding on card headers and bodies
* Improve log experience
* Reduce log font size and convert to mono
* Use proper log message parsing to remove dangerous html set
---
.../components/statusCard/StatusCardSkeleton.tsx | 4 ++--
src/client/logs/LogLine.tsx | 8 +++++++-
src/client/logs/LogsSection.css | 6 ++++++
src/client/logs/LogsSection.tsx | 8 ++++----
src/client/status/StatusSection.tsx | 2 +-
src/client/utils/index.tsx | 13 +++++++++++++
6 files changed, 33 insertions(+), 8 deletions(-)
diff --git a/src/client/components/statusCard/StatusCardSkeleton.tsx b/src/client/components/statusCard/StatusCardSkeleton.tsx
index 3464ca5b..6250d654 100644
--- a/src/client/components/statusCard/StatusCardSkeleton.tsx
+++ b/src/client/components/statusCard/StatusCardSkeleton.tsx
@@ -16,10 +16,10 @@ const StatusCardSkeleton = (props: PropsWithChildren) =>
return (
-
+
{loading || header === undefined ? : header}
-
diff --git a/src/client/logs/LogLine.tsx b/src/client/logs/LogLine.tsx
index a40804a9..e2dc232f 100644
--- a/src/client/logs/LogLine.tsx
+++ b/src/client/logs/LogLine.tsx
@@ -1,4 +1,5 @@
import React, {PropsWithChildren} from 'react';
+import {parseLogLine} from "../utils/index";
const breakSymbol = '
';
@@ -26,7 +27,12 @@ const getClass = (level: string) => {
}
}
const LogLine = (props: PropsWithChildren<{level: string, message: string}>) => {
- return
+ const lineParts = parseLogLine(props.message);
+ const level = props.level ?? lineParts.level;
+ const levelClass = getClass(level);
+ return (
+
{lineParts.timestamp} {level.padEnd(7, ' ')} : {lineParts.message}
+ )
};
export default LogLine;
diff --git a/src/client/logs/LogsSection.css b/src/client/logs/LogsSection.css
index e14237f2..f9add133 100644
--- a/src/client/logs/LogsSection.css
+++ b/src/client/logs/LogsSection.css
@@ -6,3 +6,9 @@
fill: white;
display: none;
}
+.line {
+ display: block;
+ white-space: pre-wrap;
+ font-size: 0.675rem;
+ line-height: 0.9rem;
+}
diff --git a/src/client/logs/LogsSection.tsx b/src/client/logs/LogsSection.tsx
index f8feb3a8..87302bc2 100644
--- a/src/client/logs/LogsSection.tsx
+++ b/src/client/logs/LogsSection.tsx
@@ -60,7 +60,7 @@ const LogsSection = (props: PropsFromRedux) => {
useEventSourceListener(eventSource, ['messsage', 'stream'], evt => {
const data = JSON.parse(evt.data);
// @ts-ignore
- list.add({message: data.message, id: evt.lastEventId});
+ list.add({message: data.message, id: evt.lastEventId, level: data.level});
setLogList(Array.from(list.data));
//console.log(evt);
}, [setLogList]);
@@ -69,7 +69,7 @@ const LogsSection = (props: PropsFromRedux) => {
return (
-
+
Log (Most Recent)
{/*https://codepen.io/nikhil8krishnan/pen/rVoXJa*/}
{
-
+
Level : |
|
-
+
{
logList.map(x =>
)
}
diff --git a/src/client/status/StatusSection.tsx b/src/client/status/StatusSection.tsx
index 54d19813..26c58884 100644
--- a/src/client/status/StatusSection.tsx
+++ b/src/client/status/StatusSection.tsx
@@ -26,7 +26,7 @@ const StatusSection = (props: PropsFromRedux) => {
return (
-
+
{isLoading ? : undefined}
{props.sourceIds.map(x => )}
diff --git a/src/client/utils/index.tsx b/src/client/utils/index.tsx
index ce4e1e17..54a0b7b6 100644
--- a/src/client/utils/index.tsx
+++ b/src/client/utils/index.tsx
@@ -20,3 +20,16 @@ export const buildTrackStringReactOptions: TrackStringOptions
= {
}
}
}
+
+const LOG_LINE_REGEX = new RegExp(/(?\S+)\s+(?\w+)\s*:\s*(?.*)/);
+export const parseLogLine = (line: string) => {
+ const match = line.match(LOG_LINE_REGEX);
+ if (match === null) {
+ return undefined;
+ }
+ return {
+ timestamp: match.groups.timestamp,
+ level: match.groups.level,
+ message: match.groups.message
+ }
+}
--
2.54.0
From a524e18820e404412591e1282361d20f4df7f3b9 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Tue, 12 Sep 2023 12:03:49 -0400
Subject: [PATCH 13/82] feat(ui): More ui compaction
* Better use of header
* Show name on separate line
* Only show client/source if ambiguous
* Move status into right side of header
* Reduce padding/margin on card grid
---
src/client/components/player/Player.tsx | 2 --
.../components/skeleton/SkeletonTitle.tsx | 10 ++++++---
.../statusCard/ClientStatusCard.tsx | 14 +++++++++----
.../statusCard/SourceStatusCard.tsx | 20 +++++++++++-------
.../statusCard/StatusCardSkeleton.tsx | 21 ++++++++++++++++---
.../components/statusCard/statusCard.scss | 7 +++++++
src/client/status/StatusSection.tsx | 4 ++--
7 files changed, 57 insertions(+), 21 deletions(-)
create mode 100644 src/client/components/statusCard/statusCard.scss
diff --git a/src/client/components/player/Player.tsx b/src/client/components/player/Player.tsx
index 61081eb9..4c4f223d 100644
--- a/src/client/components/player/Player.tsx
+++ b/src/client/components/player/Player.tsx
@@ -69,7 +69,6 @@ const Player = (props: PlayerProps) => {
}, [viewMode, setViewMode]);
return (
-
@@ -87,7 +86,6 @@ const Player = (props: PlayerProps) => {
-
);
}
diff --git a/src/client/components/skeleton/SkeletonTitle.tsx b/src/client/components/skeleton/SkeletonTitle.tsx
index 797d255a..1344013f 100644
--- a/src/client/components/skeleton/SkeletonTitle.tsx
+++ b/src/client/components/skeleton/SkeletonTitle.tsx
@@ -1,8 +1,12 @@
import React from 'react';
-
-const SkeletonTitle = () => {
+export interface SkeletonTitleProps {
+ show?: boolean
+}
+const SkeletonTitle = (props?: SkeletonTitleProps) => {
+ const {show = true} = props;
+ const cn = `max-w-sm animate-pulse${show ? '' : ' hidden'}`;
return (
-
+
);
diff --git a/src/client/components/statusCard/ClientStatusCard.tsx b/src/client/components/statusCard/ClientStatusCard.tsx
index f9815f61..d7c4f920 100644
--- a/src/client/components/statusCard/ClientStatusCard.tsx
+++ b/src/client/components/statusCard/ClientStatusCard.tsx
@@ -14,8 +14,14 @@ const ClientStatusCard = (props: ClientStatusCardData) => {
const {
loading = false,
data,
+ data: {
+ name,
+ type,
+ display,
+ status
+ } = {}
} = props;
- let header: string | undefined = undefined;
+ let header: string | undefined = display;
let body =
;
if(data !== undefined) {
const {
@@ -23,17 +29,17 @@ const ClientStatusCard = (props: ClientStatusCardData) => {
name,
type
} = data;
- header = `(Client) ${data.display} - ${data.name}`
+ if(type === 'lastfm' || type === 'listenbrainz')
+ header = `${display} (Client)`;
// TODO links
body = (
- Status: {data.status}
Tracks Scrobbled (since app started): {data.tracksDiscovered}
{hasAuth ? (Re)authenticate or initialize : null}
);
}
return (
-
+
{body}
);
diff --git a/src/client/components/statusCard/SourceStatusCard.tsx b/src/client/components/statusCard/SourceStatusCard.tsx
index 844d765c..ea0b9596 100644
--- a/src/client/components/statusCard/SourceStatusCard.tsx
+++ b/src/client/components/statusCard/SourceStatusCard.tsx
@@ -6,6 +6,7 @@ import {sourceAdapter} from "../../status/ducks";
import {RootState} from "../../store";
import {connect, ConnectedProps} from "react-redux";
import Player from "../player/Player";
+import './statusCard.scss';
export interface SourceStatusCardData extends StatusCardSkeletonData, PropsFromRedux {
loading?: boolean
@@ -14,9 +15,14 @@ export interface SourceStatusCardData extends StatusCardSkeletonData, PropsFromR
const SourceStatusCard = (props: SourceStatusCardData) => {
const {
loading = false,
- data
+ data,
+ data: {
+ display,
+ name,
+ status,
+ } = {}
} = props;
- let header: string | undefined = undefined;
+ let header: string | undefined = display;
let body = ;
const poll = useCallback(async () => {
const params = new URLSearchParams({type: data.type, name: data.name});
@@ -26,7 +32,6 @@ const SourceStatusCard = (props: SourceStatusCardData) => {
},[data]);
if(data !== undefined)
{
- const className = data.type !== 'spotify' ? 'md:pt-6' : '';
const {
display,
name,
@@ -39,14 +44,15 @@ const SourceStatusCard = (props: SourceStatusCardData) => {
type,
players = {}
} = data;
- header = `(Source) ${display} - ${name}`
+ if(type === 'listenbrainz' || type === 'lastfm') {
+ header = `${display} (Source)`;
+ }
const platformIds = Object.keys(players);
// TODO links
- body = (
+ body = (
{platformIds.map(x =>
)}
- Status: {status}
Tracks Discovered (since app started): {tracksDiscovered}
{canPoll && (!hasAuth || authed) ? See recently played tracks returned by API
: null}
{canPoll && hasAuthInteraction ? (Re)authenticate and (re)start polling : null}
@@ -54,7 +60,7 @@ const SourceStatusCard = (props: SourceStatusCardData) => {
);
}
return (
-
+
{body}
);
diff --git a/src/client/components/statusCard/StatusCardSkeleton.tsx b/src/client/components/statusCard/StatusCardSkeleton.tsx
index 6250d654..b0df1b46 100644
--- a/src/client/components/statusCard/StatusCardSkeleton.tsx
+++ b/src/client/components/statusCard/StatusCardSkeleton.tsx
@@ -5,19 +5,34 @@ import SkeletonParagraph from "../skeleton/SkeletonParagraph";
export interface StatusCardSkeletonData {
loading?: boolean
header?: string
+ title?: string
+ subtitle?: string
+ status?: string
}
const StatusCardSkeleton = (props: PropsWithChildren) => {
const {
loading,
header,
+ status = null,
+ title,
+ subtitle,
children
} = props || {};
+ const showLoading = loading || (header === undefined && title === undefined);
return (
-
-
-
{loading || header === undefined ? : header}
+
+
+
+
+
+ {header ?
{header}
: null}
+ {title ?
{title}
: null}
+ {subtitle ?
{subtitle}
: null}
+
+
{status}
+
{loading ?
: children}
diff --git a/src/client/components/statusCard/statusCard.scss b/src/client/components/statusCard/statusCard.scss
new file mode 100644
index 00000000..9e82fe21
--- /dev/null
+++ b/src/client/components/statusCard/statusCard.scss
@@ -0,0 +1,7 @@
+.statusCardBody {
+ article {
+ &:last-of-type {
+ margin-bottom: 1rem;
+ }
+ }
+}
diff --git a/src/client/status/StatusSection.tsx b/src/client/status/StatusSection.tsx
index 26c58884..7f6d4bb5 100644
--- a/src/client/status/StatusSection.tsx
+++ b/src/client/status/StatusSection.tsx
@@ -26,11 +26,11 @@ const StatusSection = (props: PropsFromRedux) => {
return (
-
+
{isLoading ? : undefined}
{props.sourceIds.map(x => )}
-
+
{!isLoading ? props.clientIds.map(x => ) : null}
--
2.54.0
From c7b2dcbbbe2c400ea253f00fc40ac1a1890b7fd6 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Tue, 12 Sep 2023 12:11:39 -0400
Subject: [PATCH 14/82] fix(ui): Use normal match parameters
groups are undefined probably due to minification?
---
src/client/utils/index.tsx | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/client/utils/index.tsx b/src/client/utils/index.tsx
index 54a0b7b6..ba8104d5 100644
--- a/src/client/utils/index.tsx
+++ b/src/client/utils/index.tsx
@@ -28,8 +28,8 @@ export const parseLogLine = (line: string) => {
return undefined;
}
return {
- timestamp: match.groups.timestamp,
- level: match.groups.level,
- message: match.groups.message
+ timestamp: match[1],
+ level: match[2],
+ message: match[3]
}
}
--
2.54.0
From 40a7d9e39873d613ba791f89953ae12a226016d1 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Tue, 12 Sep 2023 14:57:41 -0400
Subject: [PATCH 15/82] feat(ui): Make player dates human friendly
---
src/client/components/player/PlayerInfo.tsx | 7 ++--
src/client/utils/index.tsx | 41 ++++++++++++++++++++-
src/index.tsx | 5 +++
3 files changed, 49 insertions(+), 4 deletions(-)
diff --git a/src/client/components/player/PlayerInfo.tsx b/src/client/components/player/PlayerInfo.tsx
index d338e405..00c7adc7 100644
--- a/src/client/components/player/PlayerInfo.tsx
+++ b/src/client/components/player/PlayerInfo.tsx
@@ -1,6 +1,7 @@
import React, {Fragment} from 'react';
import './trackInfo.scss';
import {SourcePlayerJson} from "../../../core/Atomic";
+import {isoToHuman} from "../../utils/index";
export interface PlayerInfoProps {
data: SourcePlayerJson
@@ -34,9 +35,9 @@ const PlayerInfo = (props: PlayerInfoProps) => {
Player ID: {data.platformId}
-
Player Updated: {data.playerLastUpdatedAt}
-
Track Seen: {data.playFirstSeenAt}
-
Track Updated: {data.playLastUpdatedAt}
+
Player Updated: {isoToHuman(data.playerLastUpdatedAt, {includeRelative: true})}
+
Track Seen: {isoToHuman(data.playFirstSeenAt, {includeRelative: true})}
+
Track Updated: {isoToHuman(data.playLastUpdatedAt, {includeRelative: true})}
Status: Calculated -> {calculated.toUpperCase()} | Reported -> {reported.toUpperCase()}
diff --git a/src/client/utils/index.tsx b/src/client/utils/index.tsx
index ba8104d5..1d2c1167 100644
--- a/src/client/utils/index.tsx
+++ b/src/client/utils/index.tsx
@@ -1,13 +1,14 @@
import {TrackStringOptions} from "../../core/Atomic";
import React, {ReactElement, Fragment} from "react";
import {defaultBuildTrackStringTransformers} from "../../core/StringUtils";
+import dayjs, {Dayjs} from 'dayjs';
export const buildTrackStringReactOptions: TrackStringOptions = {
transformers: {
...defaultBuildTrackStringTransformers,
reducer: arr => {
const allFrags = arr.map((x, index) => {
- if(typeof x === 'string') {
+ if (typeof x === 'string') {
return {x} ;
} else {
return x;
@@ -33,3 +34,41 @@ export const parseLogLine = (line: string) => {
message: match[3]
}
}
+
+export interface DateFormatOptions {
+ includeRelative?: boolean
+ includeDate?: boolean
+}
+
+export const DAYJS_TIMEZ_FORMAT = 'HH:mm:ssZ';
+export const DAYJS_TIME_FORMAT = 'HH:mm:ss';
+export const DAYJS_DATE_FORMAT = 'YYYY-MM-DD';
+
+export const isoToHuman = (iso: string, opts?: DateFormatOptions) => {
+ const {
+ includeRelative = false,
+ includeDate
+ } = opts;
+
+ let parts = [];
+
+ const date = dayjs(iso);
+ let useDate = includeDate;
+ let now: Dayjs;
+ if(useDate === undefined) {
+ now = dayjs();
+ if(now.isSame(date, 'day')) {
+ useDate = false;
+ }
+ }
+ if(useDate) {
+ parts.push(date.format(DAYJS_DATE_FORMAT));
+ }
+ parts.push(date.format(DAYJS_TIME_FORMAT));
+
+ if(includeRelative) {
+ parts.push(`(${date.fromNow()})`);
+ }
+
+ return parts.join(' ');
+}
diff --git a/src/index.tsx b/src/index.tsx
index ed91c7dd..0b2d43d6 100644
--- a/src/index.tsx
+++ b/src/index.tsx
@@ -3,6 +3,11 @@ import ReactDOM from 'react-dom/client';
import './client/index.css';
import App from './client/App';
import reportWebVitals from './client/reportWebVitals';
+import dayjs from 'dayjs';
+import utc from 'dayjs/plugin/utc.js';
+import relativeTime from 'dayjs/plugin/relativeTime.js';
+dayjs.extend(utc)
+dayjs.extend(relativeTime);
const root = ReactDOM.createRoot(
document.getElementById('root') as HTMLElement
--
2.54.0
From 0197e70a1a75cb9d2a786966f3dc31cb072eb8aa Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Tue, 12 Sep 2023 14:58:20 -0400
Subject: [PATCH 16/82] feat(ui): Remove unnecessary wording for track stats
---
src/client/components/statusCard/ClientStatusCard.tsx | 2 +-
src/client/components/statusCard/SourceStatusCard.tsx | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/client/components/statusCard/ClientStatusCard.tsx b/src/client/components/statusCard/ClientStatusCard.tsx
index d7c4f920..01e1140d 100644
--- a/src/client/components/statusCard/ClientStatusCard.tsx
+++ b/src/client/components/statusCard/ClientStatusCard.tsx
@@ -34,7 +34,7 @@ const ClientStatusCard = (props: ClientStatusCardData) => {
// TODO links
body = (
- Tracks Scrobbled (since app started): {data.tracksDiscovered}
+ Tracks Scrobbled: {data.tracksDiscovered}
{hasAuth ? (Re)authenticate or initialize : null}
);
}
diff --git a/src/client/components/statusCard/SourceStatusCard.tsx b/src/client/components/statusCard/SourceStatusCard.tsx
index ea0b9596..523e1761 100644
--- a/src/client/components/statusCard/SourceStatusCard.tsx
+++ b/src/client/components/statusCard/SourceStatusCard.tsx
@@ -53,7 +53,7 @@ const SourceStatusCard = (props: SourceStatusCardData) => {
// TODO links
body = (
{platformIds.map(x =>
)}
- Tracks Discovered (since app started): {tracksDiscovered}
+ Tracks Discovered: {tracksDiscovered}
{canPoll && (!hasAuth || authed) ? See recently played tracks returned by API
: null}
{canPoll && hasAuthInteraction ? (Re)authenticate and (re)start polling : null}
{canPoll && (!hasAuth || authed) ? Restart Polling
: null}
--
2.54.0
From 3cbbc0837fcd50e2ae24d91997d4c7dfd406213b Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Tue, 12 Sep 2023 15:46:49 -0400
Subject: [PATCH 17/82] feat(ui): Expand displayed plays
* Implement API for fetching client recently scrobbled
* Reduce text noise by combining track stats and link for recent
---
.../scrobblers/AbstractScrobbleClient.ts | 4 ++
src/backend/server/api.ts | 45 ++++++++++++++-----
src/backend/server/middleware.ts | 32 ++++++-------
src/client/App.tsx | 5 +++
.../statusCard/ClientStatusCard.tsx | 9 +++-
.../statusCard/SourceStatusCard.tsx | 5 ++-
src/client/recent/RecentPage.tsx | 15 ++++---
src/client/scrobbled/ScrobbledPage.tsx | 37 +++++++++++++++
src/client/scrobbled/scrobbledDucks.ts | 18 ++++++++
src/client/store.ts | 4 +-
src/core/Atomic.ts | 3 ++
11 files changed, 140 insertions(+), 37 deletions(-)
create mode 100644 src/client/scrobbled/ScrobbledPage.tsx
create mode 100644 src/client/scrobbled/scrobbledDucks.ts
diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts
index 63d3e228..58aa92f3 100644
--- a/src/backend/scrobblers/AbstractScrobbleClient.ts
+++ b/src/backend/scrobblers/AbstractScrobbleClient.ts
@@ -180,6 +180,10 @@ export default abstract class AbstractScrobbleClient {
this.scrobbledPlayObjs = new FixedSizeList(this.MAX_STORED_SCROBBLES, this.scrobbledPlayObjs.data.filter(x => this.timeFrameIsValid(x.play)[0])) ;
}
+ getScrobbledPlays = () => {
+ return this.scrobbledPlayObjs.data.map(x => x.scrobble);
+ }
+
cleanSourceSearchTitle = (playObj: PlayObject) => {
const {
data: {
diff --git a/src/backend/server/api.ts b/src/backend/server/api.ts
index fd6988f5..9fafc4ce 100644
--- a/src/backend/server/api.ts
+++ b/src/backend/server/api.ts
@@ -7,7 +7,7 @@ import {
LogInfo,
LogInfoJson,
LogLevel,
- LogOutputConfig,
+ LogOutputConfig, PlayObject,
SourceStatusData,
} from "../../core/Atomic";
import {Logger} from "@foxxmd/winston";
@@ -20,9 +20,11 @@ import {setupPlexRoutes} from "./plexRoutes";
import {setupJellyfinRoutes} from "./jellyfinRoutes";
import {setupDeezerRoutes} from "./deezerRoutes";
import {setupAuthRoutes} from "./auth";
-import { ExpressHandler } from "../common/infrastructure/Atomic";
+import {ExpressHandler, ScrobbledPlayObject} from "../common/infrastructure/Atomic";
import MemorySource from "../sources/MemorySource";
import {capitalize} from "../../core/StringUtils";
+import {source} from "common-tags";
+import AbstractScrobbleClient from "../scrobblers/AbstractScrobbleClient.js";
let output: LogInfo[] = []
@@ -65,8 +67,11 @@ export const setupApi = (app: ExpressWithAsync, logger: Logger, initialLogOutput
const scrobbleSources = root.get('sources');
const scrobbleClients = root.get('clients');
- const clientCheckMiddle = makeClientCheckMiddle(scrobbleClients);
- const sourceCheckMiddle = makeSourceCheckMiddle(scrobbleSources);
+ const clientMiddleFunc = makeClientCheckMiddle(scrobbleClients);
+ const sourceMiddleFunc = makeSourceCheckMiddle(scrobbleSources);
+
+ const clientRequiredMiddle = clientMiddleFunc(true);
+ const sourceRequiredMiddle = sourceMiddleFunc(true);
const setLogWebLevel: ExpressHandler = async (req, res, next) => {
// @ts-ignore
@@ -116,7 +121,7 @@ export const setupApi = (app: ExpressWithAsync, logger: Logger, initialLogOutput
setupPlexRoutes(app, logger, scrobbleSources);
setupJellyfinRoutes(app, logger, scrobbleSources);
setupDeezerRoutes(app, logger, scrobbleSources);
- setupAuthRoutes(app, logger, sourceCheckMiddle, clientCheckMiddle, scrobbleSources, scrobbleClients);
+ setupAuthRoutes(app, logger, sourceRequiredMiddle, clientRequiredMiddle, scrobbleSources, scrobbleClients);
app.getAsync('/api/status', async (req, res, next) => {
@@ -175,6 +180,9 @@ export const setupApi = (app: ExpressWithAsync, logger: Logger, initialLogOutput
tracksDiscovered: tracksScrobbled,
name,
hasAuth: requiresAuth,
+ hasAuthInteraction: requiresAuthInteraction,
+ authed,
+ initialized
};
if (!initialized) {
base.status = 'Not Initialized';
@@ -188,22 +196,35 @@ export const setupApi = (app: ExpressWithAsync, logger: Logger, initialLogOutput
return res.json({sources: sourceData, clients: clientData});
});
- app.getAsync('/api/recent', sourceCheckMiddle, async (req, res, next) => {
+ app.getAsync('/api/recent', sourceMiddleFunc(false), async (req, res, next) => {
const {
// @ts-expect-error TS(2339): Property 'scrobbleSource' does not exist on type '... Remove this comment to see the full error message
scrobbleSource: source,
} = req;
- if (!source.canPoll) {
- return res.status(400).send(`Specified source cannot retrieve recent plays (${source.type})`);
+
+ let result: PlayObject[] = [];
+ if (source !== undefined) {
+ result = (source as AbstractSource).getFlatRecentlyDiscoveredPlays();
+ }
+
+ return res.json(result);
+ });
+
+ app.getAsync('/api/scrobbled', clientMiddleFunc(false), async (req, res, next) => {
+ const {
+ // @ts-ignore
+ scrobbleClient: client,
+ } = req;
+
+ let result: PlayObject[] = [];
+ if (client !== undefined) {
+ result = (client as AbstractScrobbleClient).getScrobbledPlays();
}
- const result = (source as AbstractSource).getFlatRecentlyDiscoveredPlays();
- //const artistTruncFunc = truncateStringToLength(Math.min(40, longestString(result.map((x: any) => x.data.artists.join(' / ')).flat())));
- //const trackLength = longestString(result.map((x: any) => x.data.track))
return res.json(result);
});
- app.use('/api/poll', sourceCheckMiddle);
+ app.use('/api/poll', sourceRequiredMiddle);
app.getAsync('/api/poll', async function (req, res) {
// @ts-expect-error TS(2339): Property 'scrobbleSource' does not exist on type '... Remove this comment to see the full error message
const source = req.scrobbleSource as AbstractSource;
diff --git a/src/backend/server/middleware.ts b/src/backend/server/middleware.ts
index 3eedaa62..cd39d08d 100644
--- a/src/backend/server/middleware.ts
+++ b/src/backend/server/middleware.ts
@@ -1,6 +1,6 @@
import { ExpressHandler } from "../common/infrastructure/Atomic";
-export const makeSourceCheckMiddle = (sources: any): ExpressHandler => (req: any, res: any, next: any) => {
+export const makeSourceCheckMiddle = (sources: any) => (required: boolean ): ExpressHandler => (req: any, res: any, next: any) => {
const {
query: {
// @ts-expect-error TS(2525): Initializer provides no value for this binding ele... Remove this comment to see the full error message
@@ -10,22 +10,23 @@ export const makeSourceCheckMiddle = (sources: any): ExpressHandler => (req: any
} = {}
} = req;
- if (name === undefined) {
+ if (required && name === undefined) {
return res.status(404).send('Source name must be defined');
- }
+ } else if(name !== undefined) {
+ const source = sources.getByNameAndType(name, type);
- const source = sources.getByNameAndType(name, type);
+ if (source === undefined) {
+ return res.status(404).send(`No source with the name [${name}] and type [${type}`);
+ }
- if (source === undefined) {
- return res.status(404).send(`No source with the name [${name}] and type [${type}`);
+ req.sourceName = name;
+ req.scrobbleSource = source;
}
- req.sourceName = name;
- req.scrobbleSource = source;
next();
}
-export const makeClientCheckMiddle = (clients: any): ExpressHandler => (req: any, res: any, next: any) => {
+export const makeClientCheckMiddle = (clients: any) => (required: boolean): ExpressHandler => (req: any, res: any, next: any) => {
const {
query: {
// @ts-expect-error TS(2525): Initializer provides no value for this binding ele... Remove this comment to see the full error message
@@ -33,16 +34,17 @@ export const makeClientCheckMiddle = (clients: any): ExpressHandler => (req: any
} = {}
} = req;
- if (name === undefined) {
+ if (required && name === undefined) {
return res.status(404).send('Client name must be defined');
- }
+ } else if (name !== undefined) {
+ const client = clients.getByName(name);
- const client = clients.getByName(name);
+ if (client === undefined) {
+ return res.status(404).send(`No client with the name: ${name}`);
+ }
- if (client === undefined) {
- return res.status(404).send(`No client with the name: ${name}`);
+ req.scrobbleClient = client;
}
- req.scrobbleClient = client;
next();
}
diff --git a/src/client/App.tsx b/src/client/App.tsx
index c7cdde0c..ff3546fe 100644
--- a/src/client/App.tsx
+++ b/src/client/App.tsx
@@ -10,6 +10,7 @@ import './App.css';
import {store} from './store';
import Dashboard from "./dashboard/dashboard";
import RecentPage from "./recent/RecentPage";
+import ScrobbledPage from "./scrobbled/ScrobbledPage";
function NoMatch() {
let location = useLocation();
@@ -30,6 +31,10 @@ const router = createBrowserRouter([
path: "/recent",
element: ,
},
+ {
+ path: "/scrobbled",
+ element: ,
+ },
{
path: "*",
element:
diff --git a/src/client/components/statusCard/ClientStatusCard.tsx b/src/client/components/statusCard/ClientStatusCard.tsx
index 01e1140d..f838b2ed 100644
--- a/src/client/components/statusCard/ClientStatusCard.tsx
+++ b/src/client/components/statusCard/ClientStatusCard.tsx
@@ -4,6 +4,7 @@ import SkeletonParagraph from "../skeleton/SkeletonParagraph";
import {clientAdapter} from "../../status/ducks";
import {RootState} from "../../store";
import {connect, ConnectedProps} from "react-redux";
+import {Link} from "react-router-dom";
export interface ClientStatusCardData extends StatusCardSkeletonData, PropsFromRedux {
loading?: boolean
@@ -27,14 +28,18 @@ const ClientStatusCard = (props: ClientStatusCardData) => {
const {
hasAuth,
name,
- type
+ type,
+ authed,
+ initialized
} = data;
if(type === 'lastfm' || type === 'listenbrainz')
header = `${display} (Client)`;
+ const scrobbled = initialized && (!hasAuth || (hasAuth && authed)) ? Tracks Scrobbled : Tracks Scrobbled
+
// TODO links
body = (
- Tracks Scrobbled: {data.tracksDiscovered}
+ {scrobbled}: {data.tracksDiscovered}
{hasAuth ? (Re)authenticate or initialize : null}
);
}
diff --git a/src/client/components/statusCard/SourceStatusCard.tsx b/src/client/components/statusCard/SourceStatusCard.tsx
index 523e1761..087d5d60 100644
--- a/src/client/components/statusCard/SourceStatusCard.tsx
+++ b/src/client/components/statusCard/SourceStatusCard.tsx
@@ -50,11 +50,12 @@ const SourceStatusCard = (props: SourceStatusCardData) => {
const platformIds = Object.keys(players);
+ const discovered = (!hasAuth || authed) ? Tracks Discovered : Tracks Discovered ;
+
// TODO links
body = (
{platformIds.map(x =>
)}
- Tracks Discovered: {tracksDiscovered}
- {canPoll && (!hasAuth || authed) ? See recently played tracks returned by API
: null}
+ {discovered}: {tracksDiscovered}
{canPoll && hasAuthInteraction ? (Re)authenticate and (re)start polling : null}
{canPoll && (!hasAuth || authed) ? Restart Polling
: null}
);
diff --git a/src/client/recent/RecentPage.tsx b/src/client/recent/RecentPage.tsx
index f9b38605..82dd4d1f 100644
--- a/src/client/recent/RecentPage.tsx
+++ b/src/client/recent/RecentPage.tsx
@@ -11,16 +11,21 @@ const displayOpts = {
const recent = () => {
let [searchParams, setSearchParams] = useSearchParams();
- const { data = [], error, isLoading, isSuccess} = useGetRecentQuery({name: searchParams.get('name'), type: searchParams.get('type')});
+ const {
+ data = [],
+ error,
+ isLoading,
+ isSuccess
+ } = useGetRecentQuery({name: searchParams.get('name'), type: searchParams.get('type')});
return (
-
-
-
+
+
+
Recently Played
-
+
{isSuccess && !isLoading && data.length === 0 ? 'No recently played tracks!' : null}
diff --git a/src/client/scrobbled/ScrobbledPage.tsx b/src/client/scrobbled/ScrobbledPage.tsx
new file mode 100644
index 00000000..56b301ed
--- /dev/null
+++ b/src/client/scrobbled/ScrobbledPage.tsx
@@ -0,0 +1,37 @@
+import React from 'react';
+import PlayDisplay from "../components/PlayDisplay";
+import {recentIncludes} from "../../core/Atomic";
+import {useSearchParams} from "react-router-dom";
+import {useGetRecentQuery} from "./scrobbledDucks";
+
+const displayOpts = {
+ include: recentIncludes,
+ includeWeb: true
+}
+
+const scrobbled = () => {
+ let [searchParams, setSearchParams] = useSearchParams();
+ const {
+ data = [],
+ error,
+ isLoading,
+ isSuccess
+ } = useGetRecentQuery({name: searchParams.get('name'), type: searchParams.get('type')});
+
+ return (
+
+
+
+
Recently Scrobbled
+
+
+
+ {isSuccess && !isLoading && data.length === 0 ? 'No recently scrobbled tracks!' : null}
+
+
+
+
+ );
+}
+
+export default scrobbled;
diff --git a/src/client/scrobbled/scrobbledDucks.ts b/src/client/scrobbled/scrobbledDucks.ts
new file mode 100644
index 00000000..22213ba9
--- /dev/null
+++ b/src/client/scrobbled/scrobbledDucks.ts
@@ -0,0 +1,18 @@
+import {createApi, fetchBaseQuery} from "@reduxjs/toolkit/dist/query/react/index";
+import {JsonPlayObject} from "../../core/Atomic";
+
+type ScrobbledResponse = (JsonPlayObject & { index: number })[];
+export const scrobbledApi = createApi({
+ reducerPath: 'scrobbledApi',
+ baseQuery: fetchBaseQuery({ baseUrl: '/api/' }),
+ endpoints: (builder) => ({
+ getRecent: builder.query
({
+ query: (params) => `scrobbled?name=${params.name}&type=${params.type}`,
+ transformResponse: (response: ScrobbledResponse, meta, arg) => {
+ return response.map((x, index) => ({...x, index: index + 1}))
+ }
+ }),
+ }),
+});
+
+export const { useGetRecentQuery } = scrobbledApi;
diff --git a/src/client/store.ts b/src/client/store.ts
index 0c7a3c93..32d88404 100644
--- a/src/client/store.ts
+++ b/src/client/store.ts
@@ -6,6 +6,7 @@ import {clientSlice, sourceSlice} from "./status/ducks";
import {logsReducer} from "./logs/logDucks";
import {logsApi} from "./logs/logsApi";
import {recentApi} from "./recent/recentDucks";
+import {scrobbledApi} from "./scrobbled/scrobbledDucks";
export const store = configureStore({
reducer: {
@@ -13,6 +14,7 @@ export const store = configureStore({
[statusApi.reducerPath]: statusApi.reducer,
[logsApi.reducerPath]: logsApi.reducer,
[recentApi.reducerPath]: recentApi.reducer,
+ [scrobbledApi.reducerPath]: scrobbledApi.reducer,
//parts: statusReducer
clients: clientSlice.reducer,
sources: sourceSlice.reducer,
@@ -21,7 +23,7 @@ export const store = configureStore({
// Adding the api middleware enables caching, invalidation, polling,
// and other useful features of `rtk-query`.
middleware: (getDefaultMiddleware) =>
- getDefaultMiddleware().concat([statusApi.middleware, logsApi.middleware, recentApi.middleware]),
+ getDefaultMiddleware().concat([statusApi.middleware, logsApi.middleware, recentApi.middleware, scrobbledApi.middleware]),
})
// optional, but required for refetchOnFocus/refetchOnReconnect behaviors
diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts
index 71cfb61f..5163b89a 100644
--- a/src/core/Atomic.ts
+++ b/src/core/Atomic.ts
@@ -21,6 +21,9 @@ export interface ClientStatusData {
tracksDiscovered: number;
name: string;
hasAuth: boolean;
+ hasAuthInteraction: boolean;
+ authed: boolean;
+ initialized: boolean;
}
export type PlayObjectIncludeTypes = 'time' | 'artist' | 'track' | 'timeFromNow' | 'trackId';
--
2.54.0
From f7dae32f503b253e4890a317b711f526ca6547a1 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Tue, 12 Sep 2023 16:01:32 -0400
Subject: [PATCH 18/82] feat(maloja): Generate web links for scrobbled tracks
---
src/backend/scrobblers/MalojaScrobbler.ts | 11 +++++++++--
src/backend/server/api.ts | 3 ++-
2 files changed, 11 insertions(+), 3 deletions(-)
diff --git a/src/backend/scrobblers/MalojaScrobbler.ts b/src/backend/scrobblers/MalojaScrobbler.ts
index 6bfba90d..2c6e4dd4 100644
--- a/src/backend/scrobblers/MalojaScrobbler.ts
+++ b/src/backend/scrobblers/MalojaScrobbler.ts
@@ -23,6 +23,7 @@ import {
import { PlayObject, TrackStringOptions } from "../../core/Atomic";
import {buildTrackString, capitalize} from "../../core/StringUtils";
import EventEmitter from "events";
+import normalizeUrl from "normalize-url";
const feat = ["ft.", "ft", "feat.", "feat", "featuring", "Ft.", "Ft", "Feat.", "Feat", "Featuring"];
@@ -31,6 +32,7 @@ export default class MalojaScrobbler extends AbstractScrobbleClient {
requiresAuth = true;
serverIsHealthy = false;
serverVersion: any;
+ webUrl: string;
declare config: MalojaClientConfig
@@ -43,6 +45,7 @@ export default class MalojaScrobbler extends AbstractScrobbleClient {
if (url === undefined) {
throw new Error("Missing 'url' for Maloja config");
}
+ this.webUrl = normalizeUrl(url);
}
static formatPlayObj(obj: MalojaScrobbleData, options: FormatPlayObjectOptions = {}): PlayObject {
@@ -52,7 +55,7 @@ export default class MalojaScrobbler extends AbstractScrobbleClient {
duration,
time;
- const {serverVersion} = options;
+ const {serverVersion, url} = options;
if(serverVersion === undefined || compareVersions(serverVersion, '3.0.0') >= 0) {
// scrobble data structure changed for v3
@@ -106,6 +109,7 @@ export default class MalojaScrobbler extends AbstractScrobbleClient {
const aStrings = aString.split(',');
return [...acc, ...aStrings];
}, []);
+ const urlParams = new URLSearchParams([['artist', artists[0]], ['title', title]]);
return {
data: {
artists: [...new Set(artistStrings)] as string[],
@@ -116,11 +120,14 @@ export default class MalojaScrobbler extends AbstractScrobbleClient {
},
meta: {
source: 'Maloja',
+ url: {
+ web: `${url}/track?${urlParams.toString()}`
+ }
}
}
}
- formatPlayObj = (obj: any, options: FormatPlayObjectOptions = {}) => MalojaScrobbler.formatPlayObj(obj, {serverVersion: this.serverVersion});
+ formatPlayObj = (obj: any, options: FormatPlayObjectOptions = {}) => MalojaScrobbler.formatPlayObj(obj, {serverVersion: this.serverVersion, url: this.webUrl});
callApi = async (req: any, retries = 0) => {
const {
diff --git a/src/backend/server/api.ts b/src/backend/server/api.ts
index 9fafc4ce..d7c80b16 100644
--- a/src/backend/server/api.ts
+++ b/src/backend/server/api.ts
@@ -25,6 +25,7 @@ import MemorySource from "../sources/MemorySource";
import {capitalize} from "../../core/StringUtils";
import {source} from "common-tags";
import AbstractScrobbleClient from "../scrobblers/AbstractScrobbleClient.js";
+import {sortByNewestPlayDate} from "../utils.js";
let output: LogInfo[] = []
@@ -218,7 +219,7 @@ export const setupApi = (app: ExpressWithAsync, logger: Logger, initialLogOutput
let result: PlayObject[] = [];
if (client !== undefined) {
- result = (client as AbstractScrobbleClient).getScrobbledPlays();
+ result = [...(client as AbstractScrobbleClient).getScrobbledPlays()].sort(sortByNewestPlayDate);
}
return res.json(result);
--
2.54.0
From 590610022d32c6ba92bc9dd8a0ad06304bde9e2e Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Wed, 13 Sep 2023 10:49:54 -0400
Subject: [PATCH 19/82] feat!: Set default scrobble thresholds to standard
values #73
https://github.com/krateng/maloja/blob/master/API.md#scrobbling-guideline
https://www.last.fm/api/scrobbling
The defaults are 50% of the track or 4 minutes.
---
src/backend/common/infrastructure/Atomic.ts | 11 +++++++-
.../infrastructure/config/source/index.ts | 25 ++++++++++++++-----
src/backend/common/schema/aio-source.json | 12 ++++++---
src/backend/common/schema/aio.json | 12 ++++++---
src/backend/common/schema/source.json | 12 ++++++---
src/backend/utils.ts | 19 +++++++-------
6 files changed, 63 insertions(+), 28 deletions(-)
diff --git a/src/backend/common/infrastructure/Atomic.ts b/src/backend/common/infrastructure/Atomic.ts
index 4862905f..f00b2b44 100644
--- a/src/backend/common/infrastructure/Atomic.ts
+++ b/src/backend/common/infrastructure/Atomic.ts
@@ -147,7 +147,16 @@ export const asLogOptions = (obj: LogConfig = {}): obj is LogOptions => {
// https://stackoverflow.com/questions/40510611/typescript-interface-require-one-of-two-properties-to-exist#comment116238286_49725198
export type RequireAtLeastOne = Omit & { [ P in R ] : Required> & Partial> }[R];
-export const DEFAULT_SCROBBLE_DURATION_THRESHOLD: number = 30;
+/**
+ * https://www.last.fm/api/scrobbling (When is a scrobble a scrobble?)
+ * https://github.com/krateng/maloja/blob/master/API.md#scrobbling-guideline
+ * */
+export const DEFAULT_SCROBBLE_DURATION_THRESHOLD: number = 240;
+/**
+ * https://www.last.fm/api/scrobbling (When is a scrobble a scrobble?)
+ * https://github.com/krateng/maloja/blob/master/API.md#scrobbling-guideline
+ * */
+export const DEFAULT_SCROBBLE_PERCENT_THRESHOLD: number = 50;
export interface ScrobbleThresholdResult {
passes: boolean
diff --git a/src/backend/common/infrastructure/config/source/index.ts b/src/backend/common/infrastructure/config/source/index.ts
index 9723a80f..d99eec66 100644
--- a/src/backend/common/infrastructure/config/source/index.ts
+++ b/src/backend/common/infrastructure/config/source/index.ts
@@ -12,18 +12,31 @@ export interface SourceRetryOptions extends RequestRetryOptions {
export interface ScrobbleThresholds {
/**
- * The number of seconds before a track should be considered scrobbled.
+ * The number of seconds a track has been listened to before it should be considered scrobbled.
*
- * @default 30
- * @examples [30]
+ * Set to null to disable.
+ *
+ * @see https://www.last.fm/api/scrobbling (When is a scrobble a scrobble?)
+ * @seehttps://github.com/krateng/maloja/blob/master/API.md#scrobbling-guideline
+ *
+ * @default 240
+ * @examples [240]
* */
- duration?: number
+ duration?: number | null
/**
- * The percentage of a track that should have been seen played before it should be scrobbled. Only used if the Source provides information about how long the track is.
+ * The percentage (as an integer) of a track that should have been seen played before it should be scrobbled. Only used if the Source provides information about how long the track is.
+ *
+ * Set to null to disable.
*
* NOTE: This should be used with care when the Source is a "polling" type (has an 'interval' property). If the track is short and the interval is too high MS may ignore the track if percentage is high because it had not "seen" the track for long enough from first discovery, even if you have been playing the track for longer.
+ *
+ * @see https://www.last.fm/api/scrobbling (When is a scrobble a scrobble?)
+ * @see https://github.com/krateng/maloja/blob/master/API.md#scrobbling-guideline
+ *
+ * @default 50
+ * @examples [50]
* */
- percent?: number
+ percent?: number | null
}
export interface CommonSourceOptions {
diff --git a/src/backend/common/schema/aio-source.json b/src/backend/common/schema/aio-source.json
index 823d10d0..fe5c6450 100644
--- a/src/backend/common/schema/aio-source.json
+++ b/src/backend/common/schema/aio-source.json
@@ -1270,16 +1270,20 @@
"ScrobbleThresholds": {
"properties": {
"duration": {
- "default": 30,
- "description": "The number of seconds before a track should be considered scrobbled.",
+ "default": 240,
+ "description": "The number of seconds a track has been listened to before it should be considered scrobbled.\n\nSet to null to disable.",
"examples": [
- 30
+ 240
],
"title": "duration",
"type": "number"
},
"percent": {
- "description": "The percentage of a track that should have been seen played before it should be scrobbled. Only used if the Source provides information about how long the track is.\n\nNOTE: This should be used with care when the Source is a \"polling\" type (has an 'interval' property). If the track is short and the interval is too high MS may ignore the track if percentage is high because it had not \"seen\" the track for long enough from first discovery, even if you have been playing the track for longer.",
+ "default": 50,
+ "description": "The percentage (as an integer) of a track that should have been seen played before it should be scrobbled. Only used if the Source provides information about how long the track is.\n\nSet to null to disable.\n\nNOTE: This should be used with care when the Source is a \"polling\" type (has an 'interval' property). If the track is short and the interval is too high MS may ignore the track if percentage is high because it had not \"seen\" the track for long enough from first discovery, even if you have been playing the track for longer.",
+ "examples": [
+ 50
+ ],
"title": "percent",
"type": "number"
}
diff --git a/src/backend/common/schema/aio.json b/src/backend/common/schema/aio.json
index adfe84be..75c16acb 100644
--- a/src/backend/common/schema/aio.json
+++ b/src/backend/common/schema/aio.json
@@ -1931,16 +1931,20 @@
"ScrobbleThresholds": {
"properties": {
"duration": {
- "default": 30,
- "description": "The number of seconds before a track should be considered scrobbled.",
+ "default": 240,
+ "description": "The number of seconds a track has been listened to before it should be considered scrobbled.\n\nSet to null to disable.",
"examples": [
- 30
+ 240
],
"title": "duration",
"type": "number"
},
"percent": {
- "description": "The percentage of a track that should have been seen played before it should be scrobbled. Only used if the Source provides information about how long the track is.\n\nNOTE: This should be used with care when the Source is a \"polling\" type (has an 'interval' property). If the track is short and the interval is too high MS may ignore the track if percentage is high because it had not \"seen\" the track for long enough from first discovery, even if you have been playing the track for longer.",
+ "default": 50,
+ "description": "The percentage (as an integer) of a track that should have been seen played before it should be scrobbled. Only used if the Source provides information about how long the track is.\n\nSet to null to disable.\n\nNOTE: This should be used with care when the Source is a \"polling\" type (has an 'interval' property). If the track is short and the interval is too high MS may ignore the track if percentage is high because it had not \"seen\" the track for long enough from first discovery, even if you have been playing the track for longer.",
+ "examples": [
+ 50
+ ],
"title": "percent",
"type": "number"
}
diff --git a/src/backend/common/schema/source.json b/src/backend/common/schema/source.json
index 87128bdc..6ddd54ab 100644
--- a/src/backend/common/schema/source.json
+++ b/src/backend/common/schema/source.json
@@ -1239,16 +1239,20 @@
"ScrobbleThresholds": {
"properties": {
"duration": {
- "default": 30,
- "description": "The number of seconds before a track should be considered scrobbled.",
+ "default": 240,
+ "description": "The number of seconds a track has been listened to before it should be considered scrobbled.\n\nSet to null to disable.",
"examples": [
- 30
+ 240
],
"title": "duration",
"type": "number"
},
"percent": {
- "description": "The percentage of a track that should have been seen played before it should be scrobbled. Only used if the Source provides information about how long the track is.\n\nNOTE: This should be used with care when the Source is a \"polling\" type (has an 'interval' property). If the track is short and the interval is too high MS may ignore the track if percentage is high because it had not \"seen\" the track for long enough from first discovery, even if you have been playing the track for longer.",
+ "default": 50,
+ "description": "The percentage (as an integer) of a track that should have been seen played before it should be scrobbled. Only used if the Source provides information about how long the track is.\n\nSet to null to disable.\n\nNOTE: This should be used with care when the Source is a \"polling\" type (has an 'interval' property). If the track is short and the interval is too high MS may ignore the track if percentage is high because it had not \"seen\" the track for long enough from first discovery, even if you have been playing the track for longer.",
+ "examples": [
+ 50
+ ],
"title": "percent",
"type": "number"
}
diff --git a/src/backend/utils.ts b/src/backend/utils.ts
index 38145bca..3db38a7f 100644
--- a/src/backend/utils.ts
+++ b/src/backend/utils.ts
@@ -7,7 +7,7 @@ import {TimeoutError, WebapiError} from "spotify-web-api-node/src/response-error
import Ajv, {Schema} from 'ajv';
import {
asPlayerStateData,
- DEFAULT_SCROBBLE_DURATION_THRESHOLD,
+ DEFAULT_SCROBBLE_DURATION_THRESHOLD, DEFAULT_SCROBBLE_PERCENT_THRESHOLD,
DELIMITERS,
lowGranularitySources,
NO_DEVICE,
@@ -622,15 +622,16 @@ export const playPassesScrobbleThreshold = (play: PlayObject, thresholds: Scrobb
export const timePassesScrobbleThreshold = (thresholds: ScrobbleThresholds, secondsTracked: number, playDuration?: number): ScrobbleThresholdResult => {
let durationPasses = undefined,
- durationThreshold = (thresholds.duration ?? DEFAULT_SCROBBLE_DURATION_THRESHOLD),
+ durationThreshold: number | null = thresholds.duration ?? DEFAULT_SCROBBLE_DURATION_THRESHOLD,
percentPasses = undefined,
- percent: number | undefined = undefined;
+ percentThreshold: number | null = thresholds.percent ?? DEFAULT_SCROBBLE_PERCENT_THRESHOLD,
+ percent: number | undefined;
- if (thresholds.percent !== undefined && playDuration !== undefined) {
- percent = (secondsTracked / playDuration) * 100;
- percentPasses = percent >= thresholds.percent;
+ if (percentThreshold !== null && playDuration !== undefined && playDuration !== 0) {
+ percent = Math.round(((secondsTracked / playDuration) * 100));
+ percentPasses = percent >= percentThreshold;
}
- if (thresholds.duration !== undefined || percentPasses === undefined) {
+ if (durationThreshold !== null || percentPasses === undefined) {
durationPasses = secondsTracked >= durationThreshold;
}
@@ -644,7 +645,7 @@ export const timePassesScrobbleThreshold = (thresholds: ScrobbleThresholds, seco
percent: {
passes: percentPasses,
value: percent,
- threshold: thresholds.percent
+ threshold: percentThreshold
}
}
}
@@ -655,7 +656,7 @@ export const thresholdResultSummary = (result: ScrobbleThresholdResult) => {
parts.push(`tracked time of ${result.duration.value}s (wanted ${result.duration.threshold}s)`);
}
if(result.percent.passes !== undefined) {
- parts.push(`tracked percent of ${(result.percent.value).toFixed(2)}% (wanted ${result.percent.threshold})`)
+ parts.push(`tracked percent of ${(result.percent.value).toFixed(2)}% (wanted ${result.percent.threshold}%)`)
}
return `${result.passes ? 'met' : 'did not meet'} thresholds with ${parts.join(' and')}`;
--
2.54.0
From c40acdd7c7c2c395611d4bf3962abfc0c46d60c1 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Wed, 13 Sep 2023 11:36:22 -0400
Subject: [PATCH 20/82] feat!: Decrease default polling interval
* Due to increased scrobbled thresholds in 590610022d32c6ba92bc9dd8a0ad06304bde9e2e we need to pol more frequently to make sure more of the track is accurately monitored
* Decreased default from 30 to 10 seconds
* Decreased max interval from 60 to 30 seconds
---
src/backend/common/infrastructure/Atomic.ts | 3 ++
.../common/infrastructure/config/common.ts | 8 ++---
.../infrastructure/config/source/spotify.ts | 8 +++--
src/backend/common/schema/aio-source.json | 34 +++++++++----------
src/backend/common/schema/aio.json | 34 +++++++++----------
src/backend/common/schema/source.json | 34 +++++++++----------
src/backend/sources/AbstractSource.ts | 3 +-
src/backend/sources/JRiverSource.ts | 4 +--
src/backend/sources/KodiSource.ts | 4 +--
src/backend/sources/MopidySource.ts | 4 +--
src/backend/sources/SpotifySource.ts | 7 ++--
src/backend/sources/SubsonicSource.ts | 4 +--
12 files changed, 72 insertions(+), 75 deletions(-)
diff --git a/src/backend/common/infrastructure/Atomic.ts b/src/backend/common/infrastructure/Atomic.ts
index f00b2b44..00033577 100644
--- a/src/backend/common/infrastructure/Atomic.ts
+++ b/src/backend/common/infrastructure/Atomic.ts
@@ -158,6 +158,9 @@ export const DEFAULT_SCROBBLE_DURATION_THRESHOLD: number = 240;
* */
export const DEFAULT_SCROBBLE_PERCENT_THRESHOLD: number = 50;
+export const DEFAULT_POLLING_INTERVAL: number = 10;
+export const DEFAULT_POLLING_MAX_INTERVAL = 30;
+
export interface ScrobbleThresholdResult {
passes: boolean
duration: {
diff --git a/src/backend/common/infrastructure/config/common.ts b/src/backend/common/infrastructure/config/common.ts
index 33ffa481..5ba98ca7 100644
--- a/src/backend/common/infrastructure/config/common.ts
+++ b/src/backend/common/infrastructure/config/common.ts
@@ -30,16 +30,16 @@ export interface PollingOptions {
/**
* How long to wait before polling the source API for new tracks (in seconds)
*
- * @default 30
- * @examples [30]
+ * @default 10
+ * @examples [10]
* */
interval?: number
/**
* When there has been no new activity from the Source API multi-scrobbler will gradually increase the wait time between polling up to this value (in seconds)
*
- * @default 60
- * @examples [60]
+ * @default 30
+ * @examples [30]
* */
maxInterval?: number
}
diff --git a/src/backend/common/infrastructure/config/source/spotify.ts b/src/backend/common/infrastructure/config/source/spotify.ts
index 9c2948c4..3bdc7e5c 100644
--- a/src/backend/common/infrastructure/config/source/spotify.ts
+++ b/src/backend/common/infrastructure/config/source/spotify.ts
@@ -29,10 +29,12 @@ export interface SpotifySourceData extends CommonSourceData, PollingOptions {
* Reading:
* * https://developer.spotify.com/documentation/web-api/guides/rate-limits/
* * https://medium.com/mendix/limiting-your-amount-of-calls-in-mendix-most-of-the-time-rest-835dde55b10e
- * * The rate limit is ~180 req/min
+ * * Rate limit may ~180 req/min
+ * * https://community.spotify.com/t5/Spotify-for-Developers/Web-API-ratelimit/m-p/5503150/highlight/true#M7930
+ * * Informally indicated as 20 req/sec? Probably for burstiness
*
- * @default 30
- * @examples [30]
+ * @default 10
+ * @examples [10]
* */
interval?: number
}
diff --git a/src/backend/common/schema/aio-source.json b/src/backend/common/schema/aio-source.json
index fe5c6450..0fc5aec4 100644
--- a/src/backend/common/schema/aio-source.json
+++ b/src/backend/common/schema/aio-source.json
@@ -166,19 +166,19 @@
"JRiverData": {
"properties": {
"interval": {
- "default": 30,
+ "default": 10,
"description": "How long to wait before polling the source API for new tracks (in seconds)",
"examples": [
- 30
+ 10
],
"title": "interval",
"type": "number"
},
"maxInterval": {
- "default": 60,
+ "default": 30,
"description": "When there has been no new activity from the Source API multi-scrobbler will gradually increase the wait time between polling up to this value (in seconds)",
"examples": [
- 60
+ 30
],
"title": "maxInterval",
"type": "number"
@@ -444,19 +444,19 @@
"KodiData": {
"properties": {
"interval": {
- "default": 30,
+ "default": 10,
"description": "How long to wait before polling the source API for new tracks (in seconds)",
"examples": [
- 30
+ 10
],
"title": "interval",
"type": "number"
},
"maxInterval": {
- "default": 60,
+ "default": 30,
"description": "When there has been no new activity from the Source API multi-scrobbler will gradually increase the wait time between polling up to this value (in seconds)",
"examples": [
- 60
+ 30
],
"title": "maxInterval",
"type": "number"
@@ -1431,19 +1431,19 @@
"type": "string"
},
"interval": {
- "default": 30,
- "description": "How long to wait before polling the source API for new tracks (in seconds)\n\nIt is unlikely you should need to change this unless you scrobble many very short tracks often\n\nReading:\n* https://developer.spotify.com/documentation/web-api/guides/rate-limits/\n* https://medium.com/mendix/limiting-your-amount-of-calls-in-mendix-most-of-the-time-rest-835dde55b10e\n * The rate limit is ~180 req/min",
+ "default": 10,
+ "description": "How long to wait before polling the source API for new tracks (in seconds)\n\nIt is unlikely you should need to change this unless you scrobble many very short tracks often\n\nReading:\n* https://developer.spotify.com/documentation/web-api/guides/rate-limits/\n* https://medium.com/mendix/limiting-your-amount-of-calls-in-mendix-most-of-the-time-rest-835dde55b10e\n * Rate limit may ~180 req/min\n* https://community.spotify.com/t5/Spotify-for-Developers/Web-API-ratelimit/m-p/5503150/highlight/true#M7930\n * Informally indicated as 20 req/sec? Probably for burstiness",
"examples": [
- 30
+ 10
],
"title": "interval",
"type": "number"
},
"maxInterval": {
- "default": 60,
+ "default": 30,
"description": "When there has been no new activity from the Source API multi-scrobbler will gradually increase the wait time between polling up to this value (in seconds)",
"examples": [
- 60
+ 30
],
"title": "maxInterval",
"type": "number"
@@ -1692,19 +1692,19 @@
"type": "string"
},
"interval": {
- "default": 30,
+ "default": 10,
"description": "How long to wait before polling the source API for new tracks (in seconds)",
"examples": [
- 30
+ 10
],
"title": "interval",
"type": "number"
},
"maxInterval": {
- "default": 60,
+ "default": 30,
"description": "When there has been no new activity from the Source API multi-scrobbler will gradually increase the wait time between polling up to this value (in seconds)",
"examples": [
- 60
+ 30
],
"title": "maxInterval",
"type": "number"
diff --git a/src/backend/common/schema/aio.json b/src/backend/common/schema/aio.json
index 75c16acb..7cbe5d05 100644
--- a/src/backend/common/schema/aio.json
+++ b/src/backend/common/schema/aio.json
@@ -288,19 +288,19 @@
"JRiverData": {
"properties": {
"interval": {
- "default": 30,
+ "default": 10,
"description": "How long to wait before polling the source API for new tracks (in seconds)",
"examples": [
- 30
+ 10
],
"title": "interval",
"type": "number"
},
"maxInterval": {
- "default": 60,
+ "default": 30,
"description": "When there has been no new activity from the Source API multi-scrobbler will gradually increase the wait time between polling up to this value (in seconds)",
"examples": [
- 60
+ 30
],
"title": "maxInterval",
"type": "number"
@@ -566,19 +566,19 @@
"KodiData": {
"properties": {
"interval": {
- "default": 30,
+ "default": 10,
"description": "How long to wait before polling the source API for new tracks (in seconds)",
"examples": [
- 30
+ 10
],
"title": "interval",
"type": "number"
},
"maxInterval": {
- "default": 60,
+ "default": 30,
"description": "When there has been no new activity from the Source API multi-scrobbler will gradually increase the wait time between polling up to this value (in seconds)",
"examples": [
- 60
+ 30
],
"title": "maxInterval",
"type": "number"
@@ -2101,19 +2101,19 @@
"type": "string"
},
"interval": {
- "default": 30,
- "description": "How long to wait before polling the source API for new tracks (in seconds)\n\nIt is unlikely you should need to change this unless you scrobble many very short tracks often\n\nReading:\n* https://developer.spotify.com/documentation/web-api/guides/rate-limits/\n* https://medium.com/mendix/limiting-your-amount-of-calls-in-mendix-most-of-the-time-rest-835dde55b10e\n * The rate limit is ~180 req/min",
+ "default": 10,
+ "description": "How long to wait before polling the source API for new tracks (in seconds)\n\nIt is unlikely you should need to change this unless you scrobble many very short tracks often\n\nReading:\n* https://developer.spotify.com/documentation/web-api/guides/rate-limits/\n* https://medium.com/mendix/limiting-your-amount-of-calls-in-mendix-most-of-the-time-rest-835dde55b10e\n * Rate limit may ~180 req/min\n* https://community.spotify.com/t5/Spotify-for-Developers/Web-API-ratelimit/m-p/5503150/highlight/true#M7930\n * Informally indicated as 20 req/sec? Probably for burstiness",
"examples": [
- 30
+ 10
],
"title": "interval",
"type": "number"
},
"maxInterval": {
- "default": 60,
+ "default": 30,
"description": "When there has been no new activity from the Source API multi-scrobbler will gradually increase the wait time between polling up to this value (in seconds)",
"examples": [
- 60
+ 30
],
"title": "maxInterval",
"type": "number"
@@ -2373,19 +2373,19 @@
"type": "string"
},
"interval": {
- "default": 30,
+ "default": 10,
"description": "How long to wait before polling the source API for new tracks (in seconds)",
"examples": [
- 30
+ 10
],
"title": "interval",
"type": "number"
},
"maxInterval": {
- "default": 60,
+ "default": 30,
"description": "When there has been no new activity from the Source API multi-scrobbler will gradually increase the wait time between polling up to this value (in seconds)",
"examples": [
- 60
+ 30
],
"title": "maxInterval",
"type": "number"
diff --git a/src/backend/common/schema/source.json b/src/backend/common/schema/source.json
index 6ddd54ab..ba054e3d 100644
--- a/src/backend/common/schema/source.json
+++ b/src/backend/common/schema/source.json
@@ -199,19 +199,19 @@
"JRiverData": {
"properties": {
"interval": {
- "default": 30,
+ "default": 10,
"description": "How long to wait before polling the source API for new tracks (in seconds)",
"examples": [
- 30
+ 10
],
"title": "interval",
"type": "number"
},
"maxInterval": {
- "default": 60,
+ "default": 30,
"description": "When there has been no new activity from the Source API multi-scrobbler will gradually increase the wait time between polling up to this value (in seconds)",
"examples": [
- 60
+ 30
],
"title": "maxInterval",
"type": "number"
@@ -461,19 +461,19 @@
"KodiData": {
"properties": {
"interval": {
- "default": 30,
+ "default": 10,
"description": "How long to wait before polling the source API for new tracks (in seconds)",
"examples": [
- 30
+ 10
],
"title": "interval",
"type": "number"
},
"maxInterval": {
- "default": 60,
+ "default": 30,
"description": "When there has been no new activity from the Source API multi-scrobbler will gradually increase the wait time between polling up to this value (in seconds)",
"examples": [
- 60
+ 30
],
"title": "maxInterval",
"type": "number"
@@ -1315,19 +1315,19 @@
"type": "string"
},
"interval": {
- "default": 30,
- "description": "How long to wait before polling the source API for new tracks (in seconds)\n\nIt is unlikely you should need to change this unless you scrobble many very short tracks often\n\nReading:\n* https://developer.spotify.com/documentation/web-api/guides/rate-limits/\n* https://medium.com/mendix/limiting-your-amount-of-calls-in-mendix-most-of-the-time-rest-835dde55b10e\n * The rate limit is ~180 req/min",
+ "default": 10,
+ "description": "How long to wait before polling the source API for new tracks (in seconds)\n\nIt is unlikely you should need to change this unless you scrobble many very short tracks often\n\nReading:\n* https://developer.spotify.com/documentation/web-api/guides/rate-limits/\n* https://medium.com/mendix/limiting-your-amount-of-calls-in-mendix-most-of-the-time-rest-835dde55b10e\n * Rate limit may ~180 req/min\n* https://community.spotify.com/t5/Spotify-for-Developers/Web-API-ratelimit/m-p/5503150/highlight/true#M7930\n * Informally indicated as 20 req/sec? Probably for burstiness",
"examples": [
- 30
+ 10
],
"title": "interval",
"type": "number"
},
"maxInterval": {
- "default": 60,
+ "default": 30,
"description": "When there has been no new activity from the Source API multi-scrobbler will gradually increase the wait time between polling up to this value (in seconds)",
"examples": [
- 60
+ 30
],
"title": "maxInterval",
"type": "number"
@@ -1560,19 +1560,19 @@
"type": "string"
},
"interval": {
- "default": 30,
+ "default": 10,
"description": "How long to wait before polling the source API for new tracks (in seconds)",
"examples": [
- 30
+ 10
],
"title": "interval",
"type": "number"
},
"maxInterval": {
- "default": 60,
+ "default": 30,
"description": "When there has been no new activity from the Source API multi-scrobbler will gradually increase the wait time between polling up to this value (in seconds)",
"examples": [
- 60
+ 30
],
"title": "maxInterval",
"type": "number"
diff --git a/src/backend/sources/AbstractSource.ts b/src/backend/sources/AbstractSource.ts
index e932c96d..719bcef4 100644
--- a/src/backend/sources/AbstractSource.ts
+++ b/src/backend/sources/AbstractSource.ts
@@ -11,6 +11,7 @@ import {
sortByOldestPlayDate,
} from "../utils";
import {
+ DEFAULT_POLLING_INTERVAL, DEFAULT_POLLING_MAX_INTERVAL,
DeviceId,
GroupedFixedPlays,
GroupedPlays,
@@ -307,7 +308,7 @@ export default abstract class AbstractSource {
let checkCount = 0;
let checksOverThreshold = 0;
- const {interval = 30, checkActiveFor = 300, maxInterval = 60} = this.config.data;
+ const {interval = DEFAULT_POLLING_INTERVAL, checkActiveFor = 300, maxInterval = DEFAULT_POLLING_MAX_INTERVAL} = this.config.data;
const maxBackoff = maxInterval - interval;
let sleepTime = interval;
diff --git a/src/backend/sources/JRiverSource.ts b/src/backend/sources/JRiverSource.ts
index fcbd625c..788b802e 100644
--- a/src/backend/sources/JRiverSource.ts
+++ b/src/backend/sources/JRiverSource.ts
@@ -22,11 +22,9 @@ export class JRiverSource extends MemorySource {
data,
} = config;
const {
- interval = 10,
- maxInterval = 30,
...rest
} = data || {};
- super('jriver', name, {...config, data: {interval, maxInterval, ...rest}}, internal, emitter);
+ super('jriver', name, {...config, data: {...rest}}, internal, emitter);
const {
data: {
diff --git a/src/backend/sources/KodiSource.ts b/src/backend/sources/KodiSource.ts
index 9cb5d46a..007a0ea4 100644
--- a/src/backend/sources/KodiSource.ts
+++ b/src/backend/sources/KodiSource.ts
@@ -17,11 +17,9 @@ export class KodiSource extends MemorySource {
data,
} = config;
const {
- interval = 10,
- maxInterval = 30,
...rest
} = data || {};
- super('kodi', name, {...config, data: {interval, maxInterval, ...rest}}, internal, emitter);
+ super('kodi', name, {...config, data: {...rest}}, internal, emitter);
this.requiresAuth = true;
this.canPoll = true;
diff --git a/src/backend/sources/MopidySource.ts b/src/backend/sources/MopidySource.ts
index 1d1bce27..e5dffec5 100644
--- a/src/backend/sources/MopidySource.ts
+++ b/src/backend/sources/MopidySource.ts
@@ -39,11 +39,9 @@ export class MopidySource extends MemorySource {
albumBlacklist = ['Soundcloud'],
uriWhitelist = [],
uriBlacklist = [],
- interval = 10,
- maxInterval = 30,
...rest
} = data;
- super('mopidy', name, {...config, data: {interval, maxInterval, ...rest}}, internal, emitter);
+ super('mopidy', name, {...config, data: {...rest}}, internal, emitter);
this.albumBlacklist = albumBlacklist.map(x => x.toLocaleLowerCase());
this.uriWhitelist = uriWhitelist.map(x => x.toLocaleLowerCase());
diff --git a/src/backend/sources/SpotifySource.ts b/src/backend/sources/SpotifySource.ts
index 15e09497..61e46756 100644
--- a/src/backend/sources/SpotifySource.ts
+++ b/src/backend/sources/SpotifySource.ts
@@ -11,6 +11,7 @@ import SpotifyWebApi from "spotify-web-api-node";
import AbstractSource, { RecentlyPlayedOptions } from "./AbstractSource";
import { SpotifySourceConfig } from "../common/infrastructure/config/source/spotify";
import {
+ DEFAULT_POLLING_INTERVAL,
FormatPlayObjectOptions,
InternalConfig,
NO_USER,
@@ -51,16 +52,14 @@ export default class SpotifySource extends MemorySource {
super('spotify', name, config, internal, emitter);
const {
data: {
- interval = 15,
+ interval = DEFAULT_POLLING_INTERVAL,
} = {}
} = config;
if (interval < 5) {
- this.logger.warn('Interval should be 5 seconds or above...😬 preferably 15');
+ this.logger.warn('Interval should probably be 5 seconds or above! Spotify may return 429 response (too many requests)');
}
- this.config.data.interval = interval;
-
this.workingCredsPath = `${this.configDir}/currentCreds-${name}.json`;
this.canPoll = true;
}
diff --git a/src/backend/sources/SubsonicSource.ts b/src/backend/sources/SubsonicSource.ts
index 027aba9b..c0fd900a 100644
--- a/src/backend/sources/SubsonicSource.ts
+++ b/src/backend/sources/SubsonicSource.ts
@@ -24,12 +24,10 @@ export class SubsonicSource extends MemorySource {
// default to quick interval so we can get a decently accurate nowPlaying
const {
data: {
- interval = 10,
- maxInterval = 30,
...restData
} = {}
} = config;
- const subsonicConfig = {...config, data: {...restData, internal, maxInterval}};
+ const subsonicConfig = {...config, data: {...restData}};
super('subsonic', name, subsonicConfig, internal,emitter);
const {data: {user, password, url} = {}} = this.config;
--
2.54.0
From d0fd225d99ad270a5a12e8ccf654aaf5af25fa50 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Wed, 13 Sep 2023 11:40:29 -0400
Subject: [PATCH 21/82] chore: Use constant for default retry multiplier
---
src/backend/common/infrastructure/Atomic.ts | 2 ++
src/backend/common/vendor/JRiverApiClient.ts | 3 ++-
src/backend/common/vendor/LastfmApiClient.ts | 4 ++--
src/backend/common/vendor/ListenbrainzApiClient.ts | 4 ++--
src/backend/scrobblers/MalojaScrobbler.ts | 4 ++--
src/backend/sources/AbstractSource.ts | 4 ++--
src/backend/sources/DeezerSource.ts | 4 ++--
src/backend/sources/SubsonicSource.ts | 4 ++--
8 files changed, 16 insertions(+), 13 deletions(-)
diff --git a/src/backend/common/infrastructure/Atomic.ts b/src/backend/common/infrastructure/Atomic.ts
index 00033577..e551104e 100644
--- a/src/backend/common/infrastructure/Atomic.ts
+++ b/src/backend/common/infrastructure/Atomic.ts
@@ -161,6 +161,8 @@ export const DEFAULT_SCROBBLE_PERCENT_THRESHOLD: number = 50;
export const DEFAULT_POLLING_INTERVAL: number = 10;
export const DEFAULT_POLLING_MAX_INTERVAL = 30;
+export const DEFAULT_RETRY_MULTIPLIER: number = 1.5;
+
export interface ScrobbleThresholdResult {
passes: boolean
duration: {
diff --git a/src/backend/common/vendor/JRiverApiClient.ts b/src/backend/common/vendor/JRiverApiClient.ts
index cfb22610..d050be50 100644
--- a/src/backend/common/vendor/JRiverApiClient.ts
+++ b/src/backend/common/vendor/JRiverApiClient.ts
@@ -3,6 +3,7 @@ import {JRiverData} from "../infrastructure/config/source/jriver";
import request, {Request, Response} from 'superagent';
import xml2js from 'xml2js';
import {ErrorWithCause} from "pony-cause";
+import {DEFAULT_RETRY_MULTIPLIER} from "../infrastructure/Atomic.js";
const parser = new xml2js.Parser({'async': true});
@@ -107,7 +108,7 @@ export class JRiverApiClient extends AbstractApiClient {
callApi = async (req: Request, retries = 0): Promise => {
const {
maxRequestRetries = 2,
- retryMultiplier = 1.5
+ retryMultiplier = DEFAULT_RETRY_MULTIPLIER
} = this.config;
if (this.token !== undefined) {
diff --git a/src/backend/common/vendor/LastfmApiClient.ts b/src/backend/common/vendor/LastfmApiClient.ts
index ad00b40c..193c9400 100644
--- a/src/backend/common/vendor/LastfmApiClient.ts
+++ b/src/backend/common/vendor/LastfmApiClient.ts
@@ -2,7 +2,7 @@ import LastFm, {AuthGetSessionResponse, TrackObject, UserGetInfoResponse} from "
import AbstractApiClient from "./AbstractApiClient";
import dayjs from "dayjs";
import { readJson, sleep, writeFile } from "../../utils";
-import { FormatPlayObjectOptions } from "../infrastructure/Atomic";
+import {DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions} from "../infrastructure/Atomic";
import { LastfmData } from "../infrastructure/config/client/lastfm";
import { PlayObject } from "../../../core/Atomic";
@@ -81,7 +81,7 @@ export default class LastfmApiClient extends AbstractApiClient {
callApi = async (func: any, retries = 0): Promise => {
const {
maxRequestRetries = 2,
- retryMultiplier = 1.5
+ retryMultiplier = DEFAULT_RETRY_MULTIPLIER
} = this.config;
try {
diff --git a/src/backend/common/vendor/ListenbrainzApiClient.ts b/src/backend/common/vendor/ListenbrainzApiClient.ts
index fee7faba..b593f42e 100644
--- a/src/backend/common/vendor/ListenbrainzApiClient.ts
+++ b/src/backend/common/vendor/ListenbrainzApiClient.ts
@@ -1,7 +1,7 @@
import AbstractApiClient from "./AbstractApiClient";
import request, {Request} from 'superagent';
import { ListenBrainzClientData } from "../infrastructure/config/client/listenbrainz";
-import { DELIMITERS, FormatPlayObjectOptions } from "../infrastructure/Atomic";
+import {DEFAULT_RETRY_MULTIPLIER, DELIMITERS, FormatPlayObjectOptions} from "../infrastructure/Atomic";
import dayjs from "dayjs";
import { stringSameness } from '@foxxmd/string-sameness';
import {
@@ -130,7 +130,7 @@ export class ListenbrainzApiClient extends AbstractApiClient {
callApi = async (req: Request, retries = 0): Promise => {
const {
maxRequestRetries = 2,
- retryMultiplier = 1.5
+ retryMultiplier = DEFAULT_RETRY_MULTIPLIER
} = this.config;
try {
diff --git a/src/backend/scrobblers/MalojaScrobbler.ts b/src/backend/scrobblers/MalojaScrobbler.ts
index 2c6e4dd4..3f375658 100644
--- a/src/backend/scrobblers/MalojaScrobbler.ts
+++ b/src/backend/scrobblers/MalojaScrobbler.ts
@@ -10,7 +10,7 @@ import {
parseRetryAfterSecsFromObj,
isPlayTemporallyClose,
} from "../utils";
-import { FormatPlayObjectOptions, INITIALIZING } from "../common/infrastructure/Atomic";
+import {DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions, INITIALIZING} from "../common/infrastructure/Atomic";
import { MalojaClientConfig } from "../common/infrastructure/config/client/maloja";
import { Notifiers } from "../notifier/Notifiers";
import {Logger} from '@foxxmd/winston';
@@ -132,7 +132,7 @@ export default class MalojaScrobbler extends AbstractScrobbleClient {
callApi = async (req: any, retries = 0) => {
const {
maxRequestRetries = 1,
- retryMultiplier = 1.5
+ retryMultiplier = DEFAULT_RETRY_MULTIPLIER
} = this.config.data;
try {
diff --git a/src/backend/sources/AbstractSource.ts b/src/backend/sources/AbstractSource.ts
index 719bcef4..53278547 100644
--- a/src/backend/sources/AbstractSource.ts
+++ b/src/backend/sources/AbstractSource.ts
@@ -11,7 +11,7 @@ import {
sortByOldestPlayDate,
} from "../utils";
import {
- DEFAULT_POLLING_INTERVAL, DEFAULT_POLLING_MAX_INTERVAL,
+ DEFAULT_POLLING_INTERVAL, DEFAULT_POLLING_MAX_INTERVAL, DEFAULT_RETRY_MULTIPLIER,
DeviceId,
GroupedFixedPlays,
GroupedPlays,
@@ -237,7 +237,7 @@ export default abstract class AbstractSource {
const {
data: {
maxPollRetries = 5,
- retryMultiplier = 1,
+ retryMultiplier = DEFAULT_RETRY_MULTIPLIER,
} = {},
} = this.config;
diff --git a/src/backend/sources/DeezerSource.ts b/src/backend/sources/DeezerSource.ts
index 55c74422..1271ee2a 100644
--- a/src/backend/sources/DeezerSource.ts
+++ b/src/backend/sources/DeezerSource.ts
@@ -10,7 +10,7 @@ import {Strategy as DeezerStrategy} from 'passport-deezer';
import AbstractSource, { RecentlyPlayedOptions } from "./AbstractSource";
import dayjs from "dayjs";
import { DeezerSourceConfig } from "../common/infrastructure/config/source/deezer";
-import { FormatPlayObjectOptions, InternalConfig } from "../common/infrastructure/Atomic";
+import {DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions, InternalConfig} from "../common/infrastructure/Atomic";
import EventEmitter from "events";
import { PlayObject } from "../../core/Atomic";
@@ -116,7 +116,7 @@ export default class DeezerSource extends AbstractSource {
callApi = async (req: any, retries = 0) => {
const {
maxRequestRetries = 1,
- retryMultiplier = 1.5
+ retryMultiplier = DEFAULT_RETRY_MULTIPLIER
} = this.config.data;
req.query({
diff --git a/src/backend/sources/SubsonicSource.ts b/src/backend/sources/SubsonicSource.ts
index c0fd900a..ccc81dea 100644
--- a/src/backend/sources/SubsonicSource.ts
+++ b/src/backend/sources/SubsonicSource.ts
@@ -5,7 +5,7 @@ import isSameOrAfter from "dayjs/plugin/isSameOrAfter.js";
import { parseRetryAfterSecsFromObj, removeDuplicates, sleep } from "../utils";
import MemorySource from "./MemorySource";
import { SubSonicSourceConfig } from "../common/infrastructure/config/source/subsonic";
-import { FormatPlayObjectOptions, InternalConfig } from "../common/infrastructure/Atomic";
+import {DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions, InternalConfig} from "../common/infrastructure/Atomic";
import { RecentlyPlayedOptions } from "./AbstractSource";
import EventEmitter from "events";
import { PlayObject } from "../../core/Atomic";
@@ -82,7 +82,7 @@ export class SubsonicSource extends MemorySource {
user,
password,
maxRequestRetries = 1,
- retryMultiplier = 1.5
+ retryMultiplier = DEFAULT_RETRY_MULTIPLIER
} = this.config.data;
--
2.54.0
From aa2c565afd46d853fb11f2d9b100c66e35e93d1f Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Wed, 13 Sep 2023 11:59:27 -0400
Subject: [PATCH 22/82] fix(ui): Log line parsing missing line terminator
capture
---
src/client/utils/index.tsx | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/client/utils/index.tsx b/src/client/utils/index.tsx
index 1d2c1167..0f5c4a14 100644
--- a/src/client/utils/index.tsx
+++ b/src/client/utils/index.tsx
@@ -22,7 +22,7 @@ export const buildTrackStringReactOptions: TrackStringOptions = {
}
}
-const LOG_LINE_REGEX = new RegExp(/(?\S+)\s+(?\w+)\s*:\s*(?.*)/);
+const LOG_LINE_REGEX = new RegExp(/(?\S+)\s+(?\w+)\s*:\s*(?(?:.|\n)*)/, 'm');
export const parseLogLine = (line: string) => {
const match = line.match(LOG_LINE_REGEX);
if (match === null) {
--
2.54.0
From 6e8ea96f32543c3c1e92cb6fcfcb589dbce6d014 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Wed, 13 Sep 2023 12:19:58 -0400
Subject: [PATCH 23/82] fix: Fix js endings
---
src/backend/common/vendor/JRiverApiClient.ts | 2 +-
src/backend/ioc.ts | 1 -
src/backend/server/api.ts | 4 ++--
src/client/components/PlayDisplay.tsx | 2 +-
4 files changed, 4 insertions(+), 5 deletions(-)
diff --git a/src/backend/common/vendor/JRiverApiClient.ts b/src/backend/common/vendor/JRiverApiClient.ts
index d050be50..ebdda124 100644
--- a/src/backend/common/vendor/JRiverApiClient.ts
+++ b/src/backend/common/vendor/JRiverApiClient.ts
@@ -3,7 +3,7 @@ import {JRiverData} from "../infrastructure/config/source/jriver";
import request, {Request, Response} from 'superagent';
import xml2js from 'xml2js';
import {ErrorWithCause} from "pony-cause";
-import {DEFAULT_RETRY_MULTIPLIER} from "../infrastructure/Atomic.js";
+import {DEFAULT_RETRY_MULTIPLIER} from "../infrastructure/Atomic";
const parser = new xml2js.Parser({'async': true});
diff --git a/src/backend/ioc.ts b/src/backend/ioc.ts
index d33c9ccb..e9573885 100644
--- a/src/backend/ioc.ts
+++ b/src/backend/ioc.ts
@@ -8,7 +8,6 @@ import {EventEmitter} from "events";
import {logPath} from "./common/logging";
import {Container} from '@foxxmd/winston';
import { WildcardEmitter } from "./common/WildcardEmitter";
-//import ScrobbleClients from "./clients/ScrobbleClients.js";
const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`);
//const port = process.env.PORT ?? 9078;
diff --git a/src/backend/server/api.ts b/src/backend/server/api.ts
index d7c80b16..2bf673d3 100644
--- a/src/backend/server/api.ts
+++ b/src/backend/server/api.ts
@@ -24,8 +24,8 @@ import {ExpressHandler, ScrobbledPlayObject} from "../common/infrastructure/Atom
import MemorySource from "../sources/MemorySource";
import {capitalize} from "../../core/StringUtils";
import {source} from "common-tags";
-import AbstractScrobbleClient from "../scrobblers/AbstractScrobbleClient.js";
-import {sortByNewestPlayDate} from "../utils.js";
+import AbstractScrobbleClient from "../scrobblers/AbstractScrobbleClient";
+import {sortByNewestPlayDate} from "../utils";
let output: LogInfo[] = []
diff --git a/src/client/components/PlayDisplay.tsx b/src/client/components/PlayDisplay.tsx
index d2da5d1a..a97ef42c 100644
--- a/src/client/components/PlayDisplay.tsx
+++ b/src/client/components/PlayDisplay.tsx
@@ -1,6 +1,6 @@
import React, {Fragment, PropsWithChildren, ReactElement} from 'react';
-import {JsonPlayObject, TrackStringOptions} from "../../core/Atomic.js";
+import {JsonPlayObject, TrackStringOptions} from "../../core/Atomic";
import {buildTrackString} from "../../core/StringUtils";
import {buildTrackStringReactOptions} from "../utils/index";
--
2.54.0
From bbb4555c0ebb3cf2038a8b102b11200ed08b800a Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Wed, 13 Sep 2023 13:21:44 -0400
Subject: [PATCH 24/82] chore(docker)!: Bump lsio base image version to 3.18
* Updates alpine 3.17 -> 3.18
* Main reason is to potentially fix DNS issues related to musl #88
* https://github.com/nodejs/node/issues/15780#issuecomment-334496907 and https://bell-sw.com/blog/how-to-deal-with-alpine-dns-issues/
* Resolved in musl 1.2.4
* https://gitlab.alpinelinux.org/alpine/tsc/-/issues/43
* https://www.openwall.com/lists/musl/2023/05/02/1
* Bumps npm 9.1.2 -> 9.6.6
New/changed ENVs used by LSIO may require removing "stored" ENVs if using portainer
---
Dockerfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Dockerfile b/Dockerfile
index 071a8b3c..300693f4 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM lsiobase/alpine:3.17 as base
+FROM lsiobase/alpine:3.18 as base
ENV TZ=Etc/GMT
--
2.54.0
From d438c8b256bd2d876d417d1ba5ed3c93ed023ccb Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Wed, 13 Sep 2023 14:29:38 -0400
Subject: [PATCH 25/82] fix(docker): Normalize ownership of node_modules files
files in node_module can have wierd ownership based on how a package owner published the package. If this occurs a rootless container build system could potentially have issues with subui/gid mapping for non-standard (high number) ids.
During image build recursively chown all node_modules files to normalize ownership and avoid this problem.
Fixes #96
---
Dockerfile | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 300693f4..498d9e3f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -24,7 +24,8 @@ ENV CONFIG_DIR=$data_dir
COPY docker/root/ /
-RUN npm install -g patch-package
+RUN npm install -g patch-package \
+ && chown -R root:root /usr/local/lib/node_modules/patch-package
WORKDIR /app
@@ -39,7 +40,8 @@ COPY --chown=abc:abc package*.json tsconfig.json ./
COPY --chown=abc:abc patches ./patches
-RUN npm install
+RUN npm install \
+ && chown -R root:root node_modules
#RUN yarn install
COPY --chown=abc:abc . /app
@@ -72,7 +74,7 @@ ENV IS_DOCKER=true
RUN npm install --omit=dev \
&& npm cache clean --force \
- && chown abc:abc node_modules \
+ && chown -R abc:abc node_modules \
&& rm -rf node_modules/ts-node \
&& rm -rf node_modules/typescript \
&& rm -rf node_modules/@types
--
2.54.0
From ba161b7b9277377c62083b35cadbe24ec5aff371 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Thu, 14 Sep 2023 09:07:36 -0400
Subject: [PATCH 26/82] chore: Remove unused dashboard backend endpoint
---
src/backend/server/api.ts | 79 ---------------------------------------
1 file changed, 79 deletions(-)
diff --git a/src/backend/server/api.ts b/src/backend/server/api.ts
index 2bf673d3..f985d6a4 100644
--- a/src/backend/server/api.ts
+++ b/src/backend/server/api.ts
@@ -267,83 +267,4 @@ export const setupApi = (app: ExpressWithAsync, logger: Logger, initialLogOutput
logger.debug(`Server received ${req.method} request from ${remote}${proxyRemote !== undefined ? ` (${proxyRemote})` : ''}${ua !== undefined ? ` (UA: ${ua})` : ''} to unknown route: ${req.url}`);
return res.sendStatus(404);
});
-
- app.getAsync('/dashboard', async function (req, res) {
- let slicedLog = output.filter(x => isLogLineMinLevel(x, logConfig.level)).slice(0, logConfig.limit + 1).map(x => formatLogToHtml(x[MESSAGE]));
- if (logConfig.sort === 'ascending') {
- slicedLog.reverse();
- }
- // TODO links for re-trying auth and variables for signalling it (and API recently played)
- const sourceData = scrobbleSources.sources.map((x) => {
- const {
- type,
- tracksDiscovered = 0,
- name,
- canPoll = false,
- polling = false,
- initialized = false,
- requiresAuth = false,
- requiresAuthInteraction = false,
- authed = false,
- } = x;
- const base = {
- status: '',
- type,
- display: capitalize(type),
- tracksDiscovered,
- name,
- canPoll,
- hasAuth: requiresAuth,
- hasAuthInteraction: requiresAuthInteraction,
- authed,
- };
- if(!initialized) {
- base.status = 'Not Initialized';
- } else if(requiresAuth && !authed) {
- base.status = requiresAuthInteraction ? 'Auth Interaction Required' : 'Authentication Failed Or Not Attempted'
- } else if(canPoll) {
- base.status = polling ? 'Running' : 'Idle';
- } else {
- base.status = tracksDiscovered > 0 ? 'Received Data' : 'Awaiting Data'
- }
- return base;
- });
- const clientData = scrobbleClients.clients.map((x) => {
- const {
- type,
- tracksScrobbled = 0,
- name,
- initialized = false,
- requiresAuth = false,
- requiresAuthInteraction = false,
- authed = false,
- } = x;
- const base = {
- status: '',
- type,
- display: capitalize(type),
- tracksDiscovered: tracksScrobbled,
- name,
- hasAuth: requiresAuth,
- };
- if(!initialized) {
- base.status = 'Not Initialized';
- } else if(requiresAuth && !authed) {
- base.status = requiresAuthInteraction ? 'Auth Interaction Required' : 'Authentication Failed Or Not Attempted'
- } else {
- base.status = tracksScrobbled > 0 ? 'Received Data' : 'Awaiting Data';
- }
- return base;
- })
- res.render('status', {
- sources: sourceData,
- clients: clientData,
- logs: {
- output: slicedLog,
- limit: [10, 20, 50, 100].map(x => `${x} `).join(' | '),
- sort: ['ascending', 'descending'].map(x => `${x} `).join(' | '),
- level: availableLevels.map(x => `${x} `).join(' | ')
- }
- });
- })
}
--
2.54.0
From 2d2319fa27634d91d9154c6850f07f8e6bba56eb Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Thu, 14 Sep 2023 09:08:01 -0400
Subject: [PATCH 27/82] fix(ui): Fix wrong route for client auth #97
---
src/client/components/statusCard/ClientStatusCard.tsx | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/client/components/statusCard/ClientStatusCard.tsx b/src/client/components/statusCard/ClientStatusCard.tsx
index f838b2ed..e4719dac 100644
--- a/src/client/components/statusCard/ClientStatusCard.tsx
+++ b/src/client/components/statusCard/ClientStatusCard.tsx
@@ -40,7 +40,7 @@ const ClientStatusCard = (props: ClientStatusCardData) => {
// TODO links
body = (
{scrobbled}: {data.tracksDiscovered}
- {hasAuth ? (Re)authenticate or initialize : null}
+ {hasAuth ? (Re)authenticate or initialize : null}
);
}
return (
--
2.54.0
From fa90a3b821c026e691912edf7d571623c375ec53 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Thu, 14 Sep 2023 10:05:24 -0400
Subject: [PATCH 28/82] fix: Load variables from .env for backend
Was being used for frontend (cra) but not applied to backend
---
package-lock.json | 6 +++---
package.json | 5 +++--
src/backend/index.ts | 1 +
3 files changed, 7 insertions(+), 5 deletions(-)
diff --git a/package-lock.json b/package-lock.json
index 8a4a14b6..e8ae97ef 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "multi-scrobbler",
- "version": "0.5.1",
+ "version": "0.5.2",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "multi-scrobbler",
- "version": "0.5.1",
+ "version": "0.5.2",
"license": "MIT",
"dependencies": {
"@awaitjs/express": "^0.6.3",
@@ -29,6 +29,7 @@
"concat-stream": "^2.0.0",
"dayjs": "^1.10.4",
"dbus-next": "0.10.2",
+ "dotenv": "^10.0.0",
"es6-error": "^4.1.1",
"express": "^4.17.1",
"express-session": "^1.17.2",
@@ -8855,7 +8856,6 @@
"version": "10.0.0",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-10.0.0.tgz",
"integrity": "sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q==",
- "dev": true,
"engines": {
"node": ">=10"
}
diff --git a/package.json b/package.json
index 69e00b67..4163df2c 100644
--- a/package.json
+++ b/package.json
@@ -19,8 +19,8 @@
"dev:server:webpack": "webpack --config webpack.config.server.js",
"dev:server:nodemon": "rm -f build/server.js && wait-on -l build/server.js && nodemon build/server.js",
"build": "npm run -s build:client && npm run -s build:server",
- "build:client": "react-scripts build",
- "build:server": "webpack --config webpack.config.server.js",
+ "build:client": "NODE_ENV=production react-scripts build",
+ "build:server": "NODE_ENV=production webpack --config webpack.config.server.js",
"start": "node build/server.js",
"fileEndings": "jscodeshift --transformFrom js --transformTo none --importTypes relative --extensions=ts --parser tsx --transform codeshift/transform.ts src/backend"
},
@@ -66,6 +66,7 @@
"concat-stream": "^2.0.0",
"dayjs": "^1.10.4",
"dbus-next": "0.10.2",
+ "dotenv": "^10.0.0",
"es6-error": "^4.1.1",
"express": "^4.17.1",
"express-session": "^1.17.2",
diff --git a/src/backend/index.ts b/src/backend/index.ts
index b6dcb672..5b01ea50 100644
--- a/src/backend/index.ts
+++ b/src/backend/index.ts
@@ -1,3 +1,4 @@
+import 'dotenv/config';
import {Logger} from '@foxxmd/winston';
import dayjs from 'dayjs';
import utc from 'dayjs/plugin/utc.js';
--
2.54.0
From 8e36c27ec98e9453ad28a6df80cf160daac398d5 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Thu, 14 Sep 2023 10:06:42 -0400
Subject: [PATCH 29/82] fix(backend): Fix how port is determined and used for
localUrl
---
src/backend/ioc.ts | 34 ++++++++++++++++++----------------
src/backend/server/index.ts | 7 ++++---
2 files changed, 22 insertions(+), 19 deletions(-)
diff --git a/src/backend/ioc.ts b/src/backend/ioc.ts
index e9573885..6fb9e76c 100644
--- a/src/backend/ioc.ts
+++ b/src/backend/ioc.ts
@@ -1,15 +1,12 @@
import {createContainer} from "iti";
import path from "path";
-import {projectDir} from "./common/index";
+import {configDir, projectDir} from "./common/index";
import ScrobbleClients from "./scrobblers/ScrobbleClients";
import ScrobbleSources from "./sources/ScrobbleSources";
import {Notifiers} from "./notifier/Notifiers";
import {EventEmitter} from "events";
import {logPath} from "./common/logging";
-import {Container} from '@foxxmd/winston';
import { WildcardEmitter } from "./common/WildcardEmitter";
-
-const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`);
//const port = process.env.PORT ?? 9078;
/*let logPath = path.resolve(projectDir, `./logs`);
@@ -20,28 +17,33 @@ if(typeof process.env.CONFIG_DIR === 'string') {
let root: ReturnType;
const createRoot = (port: number | string | undefined) => {
+ const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`);
+ const envPort = process.env.PORT;
return createContainer().add({
configDir: configDir,
logDir: logPath,
- localUrl: `http://localhost:${port}`,
+ //localUrl: `http://localhost:${port}`,
isProd: process.env.NODE_ENV !== undefined && (process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'prod'),
configPort: port,
apiPort: process.env.API_PORT ?? 9079,
- mainPort: process.env.PORT ?? 3000,
+ mainPort: envPort ?? 3000,
clientEmitter: () => new WildcardEmitter(),
sourceEmitter: () => new WildcardEmitter(),
notifierEmitter: () => new EventEmitter(),
- }).add((items) => ({
- clients: () => new ScrobbleClients(items.clientEmitter, items.sourceEmitter, items.configDir),
- sources: () => new ScrobbleSources(items.sourceEmitter, items.localUrl, items.configDir),
- notifiers: () => new Notifiers(items.notifierEmitter, items.clientEmitter, items.sourceEmitter),
- port: () => {
- if(items.configPort !== undefined) {
- return items.configPort;
- }
- return items.isProd ? items.mainPort : items.apiPort;
+ }).add((items) => {
+ let usedPort = items.configPort ?? envPort;
+ if (usedPort === undefined) {
+ usedPort = items.isProd ? items.mainPort : items.apiPort;
+ }
+ const localUrl = `http://localhost:${items.mainPort}`;
+ return {
+ port: usedPort,
+ clients: () => new ScrobbleClients(items.clientEmitter, items.sourceEmitter, localUrl, items.configDir),
+ sources: () => new ScrobbleSources(items.sourceEmitter, localUrl, items.configDir),
+ notifiers: () => new Notifiers(items.notifierEmitter, items.clientEmitter, items.sourceEmitter),
+ localUrl,
}
- }));
+ });
}
export const getRoot = (port?: number | string) => {
diff --git a/src/backend/server/index.ts b/src/backend/server/index.ts
index 7b2dc799..c63906ba 100644
--- a/src/backend/server/index.ts
+++ b/src/backend/server/index.ts
@@ -52,7 +52,8 @@ export const initServer = async (parentLogger: Logger, initialOutput: LogInfo[]
res.sendFile(path.join(buildDir, "index.html"));
});
- app.listen(port);
+ const backendPort = isProd ? port : apiPort;
+ app.listen(backendPort);
const addy = getAddress();
const addresses: string[] = [];
@@ -71,10 +72,10 @@ export const initServer = async (parentLogger: Logger, initialOutput: LogInfo[]
switch (k) {
case 'host':
case 'v4':
- addresses.push(`---> ${k === 'host' ? 'Local'.padEnd(14, ' ') : 'Network'.padEnd(14, ' ')} http://${v}:${port}`);
+ addresses.push(`---> ${k === 'host' ? 'Local'.padEnd(14, ' ') : 'Network'.padEnd(14, ' ')} http://${v}:${backendPort}`);
break;
case 'v6':
- addresses.push(`---> Network (IPv6) http://[${v}]:${port}`);
+ addresses.push(`---> Network (IPv6) http://[${v}]:${backendPort}`);
}
}
}
--
2.54.0
From 3f2cb8ac0c256e0249c3be7e622ea58af49ff6f2 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Thu, 14 Sep 2023 10:07:12 -0400
Subject: [PATCH 30/82] fix: Pass localUrl to scrobble clients for use by
lastfm
---
src/backend/scrobblers/ScrobbleClients.ts | 12 +++++++-----
1 file changed, 7 insertions(+), 5 deletions(-)
diff --git a/src/backend/scrobblers/ScrobbleClients.ts b/src/backend/scrobblers/ScrobbleClients.ts
index 800e3b7a..cf23a71c 100644
--- a/src/backend/scrobblers/ScrobbleClients.ts
+++ b/src/backend/scrobblers/ScrobbleClients.ts
@@ -19,7 +19,7 @@ import { LastfmClientConfig } from "../common/infrastructure/config/client/lastf
import { Notifiers } from "../notifier/Notifiers";
import AbstractScrobbleClient from "./AbstractScrobbleClient";
import {EventEmitter} from "events";
-import winston from '@foxxmd/winston';
+import winston, {Logger} from '@foxxmd/winston';
import ListenbrainzScrobbler from "./ListenbrainzScrobbler";
import { ListenBrainzClientConfig } from "../common/infrastructure/config/client/listenbrainz";
import {ErrorWithCause} from "pony-cause";
@@ -35,17 +35,19 @@ export default class ScrobbleClients {
/** @type AbstractScrobbleClient[] */
clients: (MalojaScrobbler | LastfmScrobbler)[] = [];
- logger;
- configDir;
+ logger: Logger;
+ configDir: string;
+ localUrl: string;
emitter: WildcardEmitter;
sourceEmitter: WildcardEmitter;
- constructor(emitter: WildcardEmitter, sourceEmitter: WildcardEmitter, configDir: any) {
+ constructor(emitter: WildcardEmitter, sourceEmitter: WildcardEmitter, localUrl: string, configDir: string) {
this.emitter = emitter;
this.sourceEmitter = sourceEmitter;
this.configDir = configDir;
+ this.localUrl = localUrl;
this.logger = winston.loggers.get('app').child({labels: ['Scrobblers']}, mergeArr);
this.sourceEmitter.on('discoveredToScrobble', async (payload: { data: (PlayObject | PlayObject[]), options: { forceRefresh?: boolean, checkTime?: Dayjs, scrobbleTo?: string[], scrobbleFrom?: string } }) => {
@@ -161,7 +163,7 @@ export default class ScrobbleClients {
mode: 'single',
configureAs: 'client',
// @ts-ignore
- data: lfm
+ data: {...lfm, redirectUri: lfm.redirectUri ?? `${this.localUrl}/lastfm/callback`}
})
}
break;
--
2.54.0
From e9eb7d357fc78e1c9db4709fcae39c1229054462 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Thu, 14 Sep 2023 10:07:44 -0400
Subject: [PATCH 31/82] fix*(scrobblers): Test auth when client completes auth
callback
---
src/backend/server/auth.ts | 1 +
1 file changed, 1 insertion(+)
diff --git a/src/backend/server/auth.ts b/src/backend/server/auth.ts
index a1b0d960..8d593230 100644
--- a/src/backend/server/auth.ts
+++ b/src/backend/server/auth.ts
@@ -76,6 +76,7 @@ export const setupAuthRoutes = (app: ExpressWithAsync, logger: Logger, sourceMid
try {
await entity.api.authenticate(token);
await entity.initialize();
+ await entity.testAuth();
return res.send('OK');
} catch (e) {
return res.send(e.message);
--
2.54.0
From b8ebd8ca7e2950c307a1e459f1825df5bdd206c3 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Thu, 14 Sep 2023 10:34:07 -0400
Subject: [PATCH 32/82] chore: Ensure production mode for webpack without
relying on env
---
package.json | 2 +-
webpack.config.server.js | 70 +++++++++++++++++++++++-----------------
2 files changed, 42 insertions(+), 30 deletions(-)
diff --git a/package.json b/package.json
index 4163df2c..43b14a9c 100644
--- a/package.json
+++ b/package.json
@@ -20,7 +20,7 @@
"dev:server:nodemon": "rm -f build/server.js && wait-on -l build/server.js && nodemon build/server.js",
"build": "npm run -s build:client && npm run -s build:server",
"build:client": "NODE_ENV=production react-scripts build",
- "build:server": "NODE_ENV=production webpack --config webpack.config.server.js",
+ "build:server": "webpack --env production --config webpack.config.server.js",
"start": "node build/server.js",
"fileEndings": "jscodeshift --transformFrom js --transformTo none --importTypes relative --extensions=ts --parser tsx --transform codeshift/transform.ts src/backend"
},
diff --git a/webpack.config.server.js b/webpack.config.server.js
index 61ec133a..9607a2ec 100644
--- a/webpack.config.server.js
+++ b/webpack.config.server.js
@@ -6,34 +6,46 @@ const nodeExternals = require("webpack-node-externals");
const entry = { server: "./src/backend/index.ts" };
-module.exports = {
- mode: process.env.NODE_ENV !== undefined ? process.env.NODE_ENV : "development",
- target: "node",
- devtool: "source-map",
- entry: entry,
- output: {
- path: path.resolve(__dirname, "build"),
- filename: "[name].js",
- },
- resolve: {
- extensions: [".ts", ".tsx", ".js"],
- },
- // don't compile node_modules
- externals: [nodeExternals()],
- module: {
- rules: [
- {
- test: /\.ts$/,
- use: [
- {
- loader: "ts-loader",
- options: {
- // use the tsconfig in the server directory
- configFile: "src/backend/tsconfig.json",
+module.exports = (env) => {
+ //console.log(`webpack env: ${env.production}`);
+ let mode = 'development';
+ if(env.production) {
+ process.env.NODE_ENV = 'production';
+ mode = 'production';
+ }
+ if(process.env.NODE_ENV !== undefined) {
+ mode = process.env.NODE_ENV;
+ }
+ //console.log(`NODE_ENV: ${process.env.NODE_ENV}`);
+ return {
+ mode: mode,
+ target: "node",
+ devtool: "source-map",
+ entry: entry,
+ output: {
+ path: path.resolve(__dirname, "build"),
+ filename: "[name].js",
+ },
+ resolve: {
+ extensions: [".ts", ".tsx", ".js"],
+ },
+ // don't compile node_modules
+ externals: [nodeExternals()],
+ module: {
+ rules: [
+ {
+ test: /\.ts$/,
+ use: [
+ {
+ loader: "ts-loader",
+ options: {
+ // use the tsconfig in the server directory
+ configFile: "src/backend/tsconfig.json",
+ },
},
- },
- ],
- },
- ],
- },
+ ],
+ },
+ ],
+ },
+ }
};
--
2.54.0
From 6dc40e50ad47f29ccf7872401fc3d92e797b6867 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Thu, 14 Sep 2023 12:38:20 -0400
Subject: [PATCH 33/82] feat: Implement Base URL
* Allows users to define a base URL to be used for default redirect URI generation
* Convenience mostly for docker users so they do not have to explicitly define redirect uri for each source/client
* Use in UI start up logging hints
---
docker-compose.yml | 4 +++
docsite/docs/configuration/configuration.md | 15 +++++++++
docsite/docs/installation/installation.md | 18 ++++++++++-
docsite/static/configExample.json | 1 +
.../common/infrastructure/config/aioConfig.ts | 14 +++++++++
src/backend/common/schema/aio.json | 11 +++++++
src/backend/common/vendor/LastfmApiClient.ts | 8 ++---
src/backend/index.ts | 3 +-
src/backend/ioc.ts | 31 ++++++++++++-------
src/backend/server/index.ts | 9 +++++-
src/backend/sources/DeezerSource.ts | 7 ++++-
src/backend/sources/LastfmSource.ts | 2 +-
src/backend/sources/SpotifySource.ts | 9 ++++++
13 files changed, 111 insertions(+), 21 deletions(-)
diff --git a/docker-compose.yml b/docker-compose.yml
index 1936fc05..4e54a122 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -13,6 +13,10 @@ services:
#- MALOJA_API_KEY=
#- PUID=1000 # required if running docker on linux host, see main README Docker setup instructions
#- PGID=1000 # required if running docker on linux host, see main README Docker setup instructions
+
+ # set if using a source/client with redirect URI that you have not explicitly set and MS is NOT running on the same machine that you will view the dashboard from
+ # EX: You will view MS dashboard at 'http://192.168.0.101:9078' -> set BASE_URL=http://192.168.0.101:9078
+ #- BASE_URL=http://MyHostIP:9078
volumes:
- /path_on_host/multi-scrobbler-config:/config
#networks:
diff --git a/docsite/docs/configuration/configuration.md b/docsite/docs/configuration/configuration.md
index cce28c9c..45a69ae3 100644
--- a/docsite/docs/configuration/configuration.md
+++ b/docsite/docs/configuration/configuration.md
@@ -168,6 +168,21 @@ Example directory structure:
See the [/config](https://github.com/FoxxMD/multi-scrobbler/blob/master/config) directory of this project for examples of each type of config file or reference specific files below.
+# Application Options
+
+These options affect multi-scrobbler's behavior and are not specific to any source/client.
+
+#### Base URL
+
+Defines the URL that is used to generate default redirect URLs for authentication on [spotify](#spotify), [lastfm](#lastfm), and [deezer](#deezer) -- as well as some logging hints.
+
+* Default => `http://localhost`
+* Set with [ENV](#env-based-configuration) `BASE_URL` or `baseUrl` [all-in-one configuration](#all-in-one-file-configuration)
+
+EX: Lastfm Redirect Url is `BASE_URL:PORT/lastfm/callback` -- Set `BASE_URL=http://192.168.0.101` => Redirect URL is `http://192.168.0.101:9078/lastfm/callback` (when no other redirectUri is specified for [lastfm configuration](#lastfm))
+
+Useful when running with [docker](../installation/installation.md#docker) so that you do not need to specify redirect URLs for each configuration.
+
# Source Configurations
## [Spotify](https://www.spotify.com)
diff --git a/docsite/docs/installation/installation.md b/docsite/docs/installation/installation.md
index ee65be0d..765f25d0 100644
--- a/docsite/docs/installation/installation.md
+++ b/docsite/docs/installation/installation.md
@@ -84,7 +84,7 @@ Recommended configuration steps for docker or docker-compose usage:
#### Storage
-You **must** bind a host directory into the container for storing configurations and credentials. Otherwise these will be lost when the container is updated.
+You **must** bind a host directory into the container for storing configurations and credentials. Otherwise, these will be lost when the container is updated.
* [Using `-v` method for docker](https://docs.docker.com/storage/bind-mounts/#start-a-container-with-a-bind-mount): `-v /path/on/host/config:/config`
* [Using docker-compose](https://docs.docker.com/compose/compose-file/compose-file-v3/#short-syntax-3): `- /path/on/host/config:/config`
@@ -98,6 +98,20 @@ The default container port is `9078`. To map container to host port:
* With [docker](https://docs.docker.com/engine/reference/commandline/run/#publish): `-p 9078:9078` (first port is the port on the host to use)
* With [docker-compose](https://docs.docker.com/compose/compose-file/compose-file-v3/#short-syntax-1): `- "9078:9078"`
+##### Base URL
+
+Optionally, when
+
+* using a [Source or Client](../configuration/configuration.md) that has a "Redirect URI" that you have not explicitly defined
+* and
+ * using a bridge network or
+ * installing MS on a different machine than the one used to view the dashboard
+
+set the [Base URL](../configuration/configuration.md#base-url) as the IP of the host machine. (This is the IP you would use to view the dashboard in a browser)
+
+* With docker: `-e BASE_URL="http://hostMachineIP"` (first port is the port on the host to use)
+* With docker-compose: [see comments in docker-compose.yml](../../../docker-compose.yml)
+
#### Other
* (Optionally) set the [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) for the container using the environmental variable `TZ` ([docker](https://docs.docker.com/engine/reference/commandline/run/#env)) ([docker-compose](https://docs.docker.com/compose/compose-file/compose-file-v3/#environment))
@@ -120,6 +134,8 @@ To get the UID and GID for the current user run these commands from a terminal:
## Docker Usage Examples
+If installing on a different machine make sure all redirect URIs are defined or that you have set a [Base URL](#base-url).
+
### Using [env-based](../configuration/configuration.md#env-based-configuration) configuration
```bash
diff --git a/docsite/static/configExample.json b/docsite/static/configExample.json
index bd49c58f..b4b50f09 100644
--- a/docsite/static/configExample.json
+++ b/docsite/static/configExample.json
@@ -8,6 +8,7 @@
"maxRequestRetries": 1,
"retryMultiplier": 1.5
},
+ "baseUrl": "http://localhost",
"sources": [
{
"type": "spotify",
diff --git a/src/backend/common/infrastructure/config/aioConfig.ts b/src/backend/common/infrastructure/config/aioConfig.ts
index 7e468b55..9ff3c07b 100644
--- a/src/backend/common/infrastructure/config/aioConfig.ts
+++ b/src/backend/common/infrastructure/config/aioConfig.ts
@@ -28,6 +28,20 @@ export interface AIOConfig {
* */
port?: number
+ /**
+ * Set the Base URL the application should assume the UI is served from.
+ *
+ * This will affect how default redirect URLs are generated (spotify, lastfm, deezer) and some logging messages.
+ *
+ * It will NOT set the actual interface/IP that the application is listening on.
+ *
+ * This can also be set using the BASE_URL environmental variable.
+ *
+ * @default "http://localhost"
+ * @examples ["http://localhost", "http://192.168.0.101", "https://ms.myDomain.tld"]
+ * */
+ baseUrl?: string
+
logging?: LogOptions
/**
diff --git a/src/backend/common/schema/aio.json b/src/backend/common/schema/aio.json
index 7cbe5d05..94ec3e58 100644
--- a/src/backend/common/schema/aio.json
+++ b/src/backend/common/schema/aio.json
@@ -2479,6 +2479,17 @@
}
},
"properties": {
+ "baseUrl": {
+ "default": "http://localhost",
+ "description": "Set the Base URL the application should assume the UI is served from.\n\nThis will affect how default redirect URLs are generated (spotify, lastfm, deezer) and some logging messages.\n\nIt will NOT set the actual interface/IP that the application is listening on.\n\nThis can also be set using the BASE_URL environmental variable.",
+ "examples": [
+ "http://localhost",
+ "http://192.168.0.101",
+ "https://ms.myDomain.tld"
+ ],
+ "title": "baseUrl",
+ "type": "string"
+ },
"clientDefaults": {
"$ref": "#/definitions/RequestRetryOptions",
"title": "clientDefaults"
diff --git a/src/backend/common/vendor/LastfmApiClient.ts b/src/backend/common/vendor/LastfmApiClient.ts
index 193c9400..f67305a4 100644
--- a/src/backend/common/vendor/LastfmApiClient.ts
+++ b/src/backend/common/vendor/LastfmApiClient.ts
@@ -25,10 +25,10 @@ export default class LastfmApiClient extends AbstractApiClient {
user?: string;
declare config: LastfmData;
- constructor(name: any, config: Partial & {configDir: string}, options = {}) {
+ constructor(name: any, config: Partial & {configDir: string, localUrl: string}, options = {}) {
super('lastfm', name, config, options);
const {redirectUri, apiKey, secret, session, configDir} = config;
- this.redirectUri = `${redirectUri}?state=${name}`;
+ this.redirectUri = `${redirectUri ?? `${config.localUrl}/lastfm/callback`}?state=${name}`;
if (apiKey === undefined) {
this.logger.warn("'apiKey' not found in config!");
}
@@ -109,8 +109,7 @@ export default class LastfmApiClient extends AbstractApiClient {
}
getAuthUrl = () => {
- const redir = `${this.config.redirectUri}?state=${this.name}`;
- return `http://www.last.fm/api/auth/?api_key=${this.config.apiKey}&cb=${encodeURIComponent(redir)}`
+ return `http://www.last.fm/api/auth/?api_key=${this.config.apiKey}&cb=${encodeURIComponent(this.redirectUri)}`
}
authenticate = async (token: any) => {
@@ -146,6 +145,7 @@ export default class LastfmApiClient extends AbstractApiClient {
testAuth = async () => {
if (this.client.sessionKey === undefined) {
this.logger.info('No session key found. User interaction for authentication required.');
+ this.logger.info(`Redirect URL that will be used on auth callback: '${this.redirectUri}'`);
return false;
}
try {
diff --git a/src/backend/index.ts b/src/backend/index.ts
index 5b01ea50..7164eb45 100644
--- a/src/backend/index.ts
+++ b/src/backend/index.ts
@@ -58,7 +58,6 @@ const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`)
const {
webhooks = [],
- port,
logging = {},
debugMode,
} = (config || {}) as AIOConfig;
@@ -72,7 +71,7 @@ const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`)
process.env.DEBUG_MODE = b.toString();
}
- const root = getRoot(port);
+ const root = getRoot(config);
logger = getLogger(logging, 'app');
diff --git a/src/backend/ioc.ts b/src/backend/ioc.ts
index 6fb9e76c..7c7acdaa 100644
--- a/src/backend/ioc.ts
+++ b/src/backend/ioc.ts
@@ -7,22 +7,25 @@ import {Notifiers} from "./notifier/Notifiers";
import {EventEmitter} from "events";
import {logPath} from "./common/logging";
import { WildcardEmitter } from "./common/WildcardEmitter";
-//const port = process.env.PORT ?? 9078;
-
-/*let logPath = path.resolve(projectDir, `./logs`);
-if(typeof process.env.CONFIG_DIR === 'string') {
- logPath = path.resolve(process.env.CONFIG_DIR, './logs');
-}*/
+import normalizeUrl from 'normalize-url';
let root: ReturnType;
-const createRoot = (port: number | string | undefined) => {
+export interface RootOptions {
+ baseUrl?: string,
+ port?: string | number
+}
+
+const createRoot = (options?: RootOptions) => {
+ const {
+ port,
+ baseUrl = process.env.BASE_URL,
+ } = options || {};
const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`);
const envPort = process.env.PORT;
return createContainer().add({
configDir: configDir,
logDir: logPath,
- //localUrl: `http://localhost:${port}`,
isProd: process.env.NODE_ENV !== undefined && (process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'prod'),
configPort: port,
apiPort: process.env.API_PORT ?? 9079,
@@ -35,20 +38,26 @@ const createRoot = (port: number | string | undefined) => {
if (usedPort === undefined) {
usedPort = items.isProd ? items.mainPort : items.apiPort;
}
- const localUrl = `http://localhost:${items.mainPort}`;
+ const base = normalizeUrl(baseUrl ?? 'http://localhost', {removeSingleSlash: true});
+ const u = new URL(base);
+ let localUrl = u.toString();
+ if(u.port === '' && u.pathname === '/') {
+ localUrl = `${u.origin}:${items.mainPort}`;
+ }
return {
port: usedPort,
clients: () => new ScrobbleClients(items.clientEmitter, items.sourceEmitter, localUrl, items.configDir),
sources: () => new ScrobbleSources(items.sourceEmitter, localUrl, items.configDir),
notifiers: () => new Notifiers(items.notifierEmitter, items.clientEmitter, items.sourceEmitter),
localUrl,
+ hasDefinedBaseUrl: baseUrl !== undefined
}
});
}
-export const getRoot = (port?: number | string) => {
+export const getRoot = (options?: RootOptions) => {
if(root === undefined) {
- root = createRoot(port);
+ root = createRoot(options);
}
return root;
}
diff --git a/src/backend/server/index.ts b/src/backend/server/index.ts
index c63906ba..f13e3fdd 100644
--- a/src/backend/server/index.ts
+++ b/src/backend/server/index.ts
@@ -11,6 +11,7 @@ import { setupApi } from "./api";
import { getAddress, mergeArr, parseBool } from "../utils";
import {stripIndents} from "common-tags";
import {ErrorWithCause} from "pony-cause";
+import {or} from "ajv/dist/compile/codegen/index.js";
const buildDir = path.join(process.cwd() + "/build");
@@ -42,6 +43,8 @@ export const initServer = async (parentLogger: Logger, initialOutput: LogInfo[]
const apiPort = root.get('apiPort');
const mainPort = root.get('mainPort');
const port = root.get('port');
+ const local = root.get('localUrl');
+ const localDefined = root.get('hasDefinedBaseUrl');
setupApi(app, logger, initialOutput);
@@ -63,7 +66,7 @@ export const initServer = async (parentLogger: Logger, initialOutput: LogInfo[]
--- HINT ---
MS is likely being run in a container with BRIDGE networking which means the above addresses are not accessible from outside this container.
To ensure the container is accessible make sure you have mapped the *container* port ${port} to a *host* port. https://foxxmd.github.io/multi-scrobbler/docs/installation#networking
- The container will then be accessible at http://HOST_MACHINE_IP:HOST_PORT
+ The container will then be accessible at http://HOST_MACHINE_IP:HOST_PORT${localDefined ? ` (or ${local} since you defined this!)` : ''}
--- HINT ---
`;
}
@@ -84,6 +87,10 @@ export const initServer = async (parentLogger: Logger, initialOutput: LogInfo[]
${addresses.join('\n')}${dockerHint !== '' ? `\n${dockerHint}` : ''}`
logger.info(start);
+
+ if(localDefined) {
+ logger.info(`User-defined base URL for UI and redirect URLs (spotify, deezer, lastfm): ${local}`)
+ }
} catch (e) {
logger.error(new ErrorWithCause('Server crashed with uncaught exception', {cause: e}));
}
diff --git a/src/backend/sources/DeezerSource.ts b/src/backend/sources/DeezerSource.ts
index 1271ee2a..df62931d 100644
--- a/src/backend/sources/DeezerSource.ts
+++ b/src/backend/sources/DeezerSource.ts
@@ -22,6 +22,7 @@ export default class DeezerSource extends AbstractSource {
requiresAuthInteraction = true;
baseUrl = 'https://api.deezer.com';
+ redirectUri: string;
declare config: DeezerSourceConfig;
@@ -30,6 +31,7 @@ export default class DeezerSource extends AbstractSource {
const {
data: {
interval = 60,
+ redirectUri,
} = {},
} = config;
@@ -38,6 +40,7 @@ export default class DeezerSource extends AbstractSource {
}
this.config.data.interval = interval;
+ this.redirectUri = redirectUri || `${this.localUrl}/deezer/callback`;
this.workingCredsPath = `${this.configDir}/currentCreds-${name}.json`;
this.canPoll = true;
@@ -92,6 +95,8 @@ export default class DeezerSource extends AbstractSource {
} else if(this.config.data.clientSecret === undefined) {
throw new Error('clientSecret must be defined when accessToken is not present');
}
+ this.logger.info(`No access token is present. User interaction for authentication is required.`);
+ this.logger.info(`Redirect URL that will be used on auth callback: '${this.redirectUri}'`);
}
this.initialized = true;
return this.initialized;
@@ -185,7 +190,7 @@ export default class DeezerSource extends AbstractSource {
return new DeezerStrategy({
clientID: this.config.data.clientId,
clientSecret: this.config.data.clientSecret,
- callbackURL: this.config.data.redirectUri || `${this.localUrl}/deezer/callback`,
+ callbackURL: this.redirectUri,
scope: ['listening_history','offline_access'],
}, (accessToken: any, refreshToken: any, profile: any, done: any) => {
// return done(null, {
diff --git a/src/backend/sources/LastfmSource.ts b/src/backend/sources/LastfmSource.ts
index 225dd325..43034cf1 100644
--- a/src/backend/sources/LastfmSource.ts
+++ b/src/backend/sources/LastfmSource.ts
@@ -18,7 +18,7 @@ export default class LastfmSource extends AbstractSource {
constructor(name: any, config: LastfmClientConfig, internal: InternalConfig, emitter: EventEmitter) {
super('lastfm', name, config, internal, emitter);
this.canPoll = true;
- this.api = new LastfmApiClient(name, {...config.data, configDir: internal.configDir});
+ this.api = new LastfmApiClient(name, {...config.data, configDir: internal.configDir, localUrl: internal.localUrl});
}
static formatPlayObj(obj: any, options: FormatPlayObjectOptions = {}): PlayObject {
diff --git a/src/backend/sources/SpotifySource.ts b/src/backend/sources/SpotifySource.ts
index 61e46756..5fbfeb35 100644
--- a/src/backend/sources/SpotifySource.ts
+++ b/src/backend/sources/SpotifySource.ts
@@ -214,6 +214,11 @@ export default class SpotifySource extends MemorySource {
throw new Error('Failed to initialize a Spotify source');
}
+ if(accessToken === undefined || refreshToken === undefined) {
+ this.logger.info(`No access or refresh token is present. User interaction for authentication is required.`);
+ this.logger.info(`Redirect URL that will be used on auth callback: '${rdUri}'`);
+ }
+
this.spotifyApi = new SpotifyWebApi(apiConfig);
}
@@ -227,6 +232,10 @@ export default class SpotifySource extends MemorySource {
testAuth = async () => {
try {
+ if(undefined === this.spotifyApi.getAccessToken()) {
+ this.authed = false;
+ return;
+ }
await this.callApi>(((api: any) => api.getMe()));
this.authed = true;
} catch (e) {
--
2.54.0
From d5dfc33fce1ce4a7b9280fd8705cf5d564298fee Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Thu, 14 Sep 2023 12:51:16 -0400
Subject: [PATCH 34/82] fix(deezer): Fix deezer passport generation
---
src/backend/server/deezerRoutes.ts | 9 +++++----
src/backend/sources/DeezerSource.ts | 2 ++
src/backend/sources/ScrobbleSources.ts | 4 ++--
3 files changed, 9 insertions(+), 6 deletions(-)
diff --git a/src/backend/server/deezerRoutes.ts b/src/backend/server/deezerRoutes.ts
index beb78cf2..c60c7b9d 100644
--- a/src/backend/server/deezerRoutes.ts
+++ b/src/backend/server/deezerRoutes.ts
@@ -11,10 +11,10 @@ import passport from "passport";
export const setupDeezerRoutes = (app: ExpressWithAsync, logger: Logger, scrobbleSources: ScrobbleSources) => {
// initialize deezer strategies
- const deezerSources = scrobbleSources.getByType('deezer') as DeezerSource[];
- for(const d of deezerSources) {
- passport.use(`deezer-${d.name}`, d.generatePassportStrategy());
- }
+ // const deezerSources = scrobbleSources.getByType('deezer') as DeezerSource[];
+ // for(const d of deezerSources) {
+ // passport.use(`deezer-${d.name}`, d.generatePassportStrategy());
+ // }
// something about the deezer passport strategy makes express continue with the response even though it should wait for accesstoken callback and userprofile fetching
// so to get around this add an additional middleware that loops/sleeps until we should have fetched everything ¯\_(ツ)_/¯
@@ -34,6 +34,7 @@ export const setupDeezerRoutes = (app: ExpressWithAsync, logger: Logger, scrobbl
return res.send('Error with deezer credentials storage');
} else if(entity.config.data.accessToken !== undefined) {
// start polling
+ await entity.testAuth();
entity.poll()
return res.redirect('/');
} else {
diff --git a/src/backend/sources/DeezerSource.ts b/src/backend/sources/DeezerSource.ts
index df62931d..c02dc479 100644
--- a/src/backend/sources/DeezerSource.ts
+++ b/src/backend/sources/DeezerSource.ts
@@ -1,4 +1,5 @@
import request from 'superagent';
+import passport from "passport";
import {
parseRetryAfterSecsFromObj,
readJson,
@@ -99,6 +100,7 @@ export default class DeezerSource extends AbstractSource {
this.logger.info(`Redirect URL that will be used on auth callback: '${this.redirectUri}'`);
}
this.initialized = true;
+ passport.use(`deezer-${this.name}`, this.generatePassportStrategy());
return this.initialized;
}
diff --git a/src/backend/sources/ScrobbleSources.ts b/src/backend/sources/ScrobbleSources.ts
index 0d85927a..0f630f8b 100644
--- a/src/backend/sources/ScrobbleSources.ts
+++ b/src/backend/sources/ScrobbleSources.ts
@@ -230,8 +230,8 @@ export default class ScrobbleSources {
break;
case 'deezer':
const d = {
- clientId: process.env.DEEZER_APP_ID,
- clientSecret: process.env.DEEZER_SECRET_KEY,
+ clientId: process.env.DEEZER_CLIENT_ID,
+ clientSecret: process.env.DEEZER_CLIENT_SECRET,
redirectUri: process.env.DEEZER_REDIRECT_URI,
accessToken: process.env.DEEZER_ACCESS_TOKEN,
};
--
2.54.0
From 3fb5d6aae907b5e86207d74fc6069546fb4fc1fa Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Mon, 18 Sep 2023 14:40:22 -0400
Subject: [PATCH 35/82] fix(ui): Remove duration percent display when duration
is zero
---
src/client/components/player/Player.tsx | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/client/components/player/Player.tsx b/src/client/components/player/Player.tsx
index 4c4f223d..59e2476e 100644
--- a/src/client/components/player/Player.tsx
+++ b/src/client/components/player/Player.tsx
@@ -45,7 +45,7 @@ const Player = (props: PlayerProps) => {
} = data;
let durPer = null;
- if(duration !== undefined && duration !== null) {
+ if(duration !== undefined && duration !== null && duration !== 0) {
if(listenedDuration === 0) {
durPer = ' (0%)';
} else {
--
2.54.0
From 7dece447dfde155e1850f691ae5c9a677a411029 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Mon, 18 Sep 2023 14:56:23 -0400
Subject: [PATCH 36/82] feat(player): Add sourceOfTruth variable to control
logging and future use
Will not be used by listenbrainz source for actual scrobbling so make logs less noisy by suppressing what would normally be logging on newely added tracks from player
---
src/backend/sources/MemorySource.ts | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
diff --git a/src/backend/sources/MemorySource.ts b/src/backend/sources/MemorySource.ts
index 32a33995..e203448c 100644
--- a/src/backend/sources/MemorySource.ts
+++ b/src/backend/sources/MemorySource.ts
@@ -34,6 +34,9 @@ import {EventEmitter} from "events";
import objectHash from 'object-hash';
export default class MemorySource extends AbstractSource {
+
+ playerSourceOfTruth: boolean = true;
+
/*
* MemorySource uses its own state to maintain a list of recently played tracks and determine if a track is valid.
* This is necessary for any source that
@@ -159,7 +162,9 @@ export default class MemorySource extends AbstractSource {
const matchingRecent = this.existingDiscovered(candidate); //sRecentlyPlayed.find(x => playObjDataMatch(x, candidate));
let stPrefix = `${buildTrackString(candidate, {include: ['trackId', 'artist', 'track']})}`;
if (matchingRecent === undefined) {
- player.logger.debug(`${stPrefix} added after ${thresholdResultSummary(thresholdResults)} and not matching any prior plays`);
+ if(this.playerSourceOfTruth) {
+ player.logger.debug(`${stPrefix} added after ${thresholdResultSummary(thresholdResults)} and not matching any prior plays`);
+ }
newStatefulPlays.push(candidate);
} else {
const {data: {playDate, duration}} = candidate;
@@ -167,14 +172,18 @@ export default class MemorySource extends AbstractSource {
if (!playDate.isSame(rplayDate)) {
if (duration !== undefined) {
if (playDate.isAfter(rplayDate.add(duration, 's'))) {
- player.logger.debug(`${stPrefix} added after ${thresholdResultSummary(thresholdResults)} and having a different timestamp than a prior play`);
+ if(this.playerSourceOfTruth) {
+ player.logger.debug(`${stPrefix} added after ${thresholdResultSummary(thresholdResults)} and having a different timestamp than a prior play`);
+ }
newStatefulPlays.push(candidate);
}
} else {
const discoveredPlays = this.getRecentlyDiscoveredPlaysByPlatform(genGroupId(candidate));
if (discoveredPlays.length === 0 || !playObjDataMatch(discoveredPlays[0], candidate)) {
// if most recent stateful play is not this track we'll add it
- player.logger.debug(`${stPrefix} added after ${thresholdResultSummary(thresholdResults)}. Matched other recent play but could not determine time frame due to missing duration. Allowed due to not being last played track.`);
+ if(this.playerSourceOfTruth) {
+ player.logger.debug(`${stPrefix} added after ${thresholdResultSummary(thresholdResults)}. Matched other recent play but could not determine time frame due to missing duration. Allowed due to not being last played track.`);
+ }
newStatefulPlays.push(candidate);
}
}
--
2.54.0
From 735264cb2b01b26c74a62f088f67ea054fd7c533 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Mon, 18 Sep 2023 14:58:18 -0400
Subject: [PATCH 37/82] feat(listenbrainz): Add player support #74
* Use listenbrainz 'playing-now' endpoint for MS player implementation
* Do not use MS player as source of truth -- continue to use LZ listen history
---
.../config/source/listenbrainz.ts | 3 +-
src/backend/common/schema/aio-source.json | 18 ++++++++++++
src/backend/common/schema/aio.json | 18 ++++++++++++
src/backend/common/schema/source.json | 18 ++++++++++++
.../common/vendor/ListenbrainzApiClient.ts | 29 ++++++++++++++++++-
src/backend/sources/ListenbrainzSource.ts | 15 ++++++++--
src/backend/sources/SubsonicSource.ts | 1 -
7 files changed, 97 insertions(+), 5 deletions(-)
diff --git a/src/backend/common/infrastructure/config/source/listenbrainz.ts b/src/backend/common/infrastructure/config/source/listenbrainz.ts
index b8bdd3a7..1e7ffff5 100644
--- a/src/backend/common/infrastructure/config/source/listenbrainz.ts
+++ b/src/backend/common/infrastructure/config/source/listenbrainz.ts
@@ -1,7 +1,8 @@
import { CommonSourceConfig, CommonSourceData } from "./index";
import { ListenBrainzData } from "../client/listenbrainz";
+import {PollingOptions} from "../common.js";
-export interface ListenBrainzSourceData extends ListenBrainzData, CommonSourceData {
+export interface ListenBrainzSourceData extends ListenBrainzData, CommonSourceData, PollingOptions {
}
export interface ListenBrainzSourceConfig extends CommonSourceConfig {
diff --git a/src/backend/common/schema/aio-source.json b/src/backend/common/schema/aio-source.json
index 0fc5aec4..3bc52375 100644
--- a/src/backend/common/schema/aio-source.json
+++ b/src/backend/common/schema/aio-source.json
@@ -759,6 +759,24 @@
},
"ListenBrainzSourceData": {
"properties": {
+ "interval": {
+ "default": 10,
+ "description": "How long to wait before polling the source API for new tracks (in seconds)",
+ "examples": [
+ 10
+ ],
+ "title": "interval",
+ "type": "number"
+ },
+ "maxInterval": {
+ "default": 30,
+ "description": "When there has been no new activity from the Source API multi-scrobbler will gradually increase the wait time between polling up to this value (in seconds)",
+ "examples": [
+ 30
+ ],
+ "title": "maxInterval",
+ "type": "number"
+ },
"maxPollRetries": {
"default": 5,
"description": "default # of automatic polling restarts on error",
diff --git a/src/backend/common/schema/aio.json b/src/backend/common/schema/aio.json
index 94ec3e58..29ccb45a 100644
--- a/src/backend/common/schema/aio.json
+++ b/src/backend/common/schema/aio.json
@@ -1119,6 +1119,24 @@
},
"ListenBrainzSourceData": {
"properties": {
+ "interval": {
+ "default": 10,
+ "description": "How long to wait before polling the source API for new tracks (in seconds)",
+ "examples": [
+ 10
+ ],
+ "title": "interval",
+ "type": "number"
+ },
+ "maxInterval": {
+ "default": 30,
+ "description": "When there has been no new activity from the Source API multi-scrobbler will gradually increase the wait time between polling up to this value (in seconds)",
+ "examples": [
+ 30
+ ],
+ "title": "maxInterval",
+ "type": "number"
+ },
"maxPollRetries": {
"default": 5,
"description": "default # of automatic polling restarts on error",
diff --git a/src/backend/common/schema/source.json b/src/backend/common/schema/source.json
index ba054e3d..25b45d72 100644
--- a/src/backend/common/schema/source.json
+++ b/src/backend/common/schema/source.json
@@ -752,6 +752,24 @@
},
"ListenBrainzSourceData": {
"properties": {
+ "interval": {
+ "default": 10,
+ "description": "How long to wait before polling the source API for new tracks (in seconds)",
+ "examples": [
+ 10
+ ],
+ "title": "interval",
+ "type": "number"
+ },
+ "maxInterval": {
+ "default": 30,
+ "description": "When there has been no new activity from the Source API multi-scrobbler will gradually increase the wait time between polling up to this value (in seconds)",
+ "examples": [
+ 30
+ ],
+ "title": "maxInterval",
+ "type": "number"
+ },
"maxPollRetries": {
"default": 5,
"description": "default # of automatic polling restarts on error",
diff --git a/src/backend/common/vendor/ListenbrainzApiClient.ts b/src/backend/common/vendor/ListenbrainzApiClient.ts
index b593f42e..173621ca 100644
--- a/src/backend/common/vendor/ListenbrainzApiClient.ts
+++ b/src/backend/common/vendor/ListenbrainzApiClient.ts
@@ -5,6 +5,7 @@ import {DEFAULT_RETRY_MULTIPLIER, DELIMITERS, FormatPlayObjectOptions} from "../
import dayjs from "dayjs";
import { stringSameness } from '@foxxmd/string-sameness';
import {
+ combinePartsToString,
containsDelimiters,
findDelimiters,
normalizeStr,
@@ -185,9 +186,30 @@ export class ListenbrainzApiClient extends AbstractApiClient {
}
}
+ getPlayingNow = async (user?: string): Promise => {
+ try {
+
+ const resp = await this.callApi(request
+ .get(`${this.url}1/user/${user ?? this.config.username}/playing-now`)
+ // this endpoint can take forever, sometimes, and we want to make sure we timeout in a reasonable amount of time for polling sources to continue trying to scrobble
+ .timeout({
+ response: 15000, // wait 15 seconds before timeout if server doesn't response at all
+ deadline: 30000 // wait 30 seconds overall for request to complete
+ }));
+ const {body: {payload}} = resp as any;
+ // const data = payload as ListensResponse;
+ // if(data.listens.length > 0) {}
+ // return data.listens[0];
+ return payload as ListensResponse;
+ } catch (e) {
+ throw e;
+ }
+ }
+
getRecentlyPlayed = async (maxTracks: number, user?: string): Promise => {
try {
const resp = await this.getUserListens(maxTracks, user);
+ const now = await this.getPlayingNow(user);
return resp.listens.map(x => ListenbrainzApiClient.listenResponseToPlay(x));
} catch (e) {
this.logger.error(`Error encountered while getting User listens | Error => ${e.message}`);
@@ -464,6 +486,10 @@ export class ListenbrainzApiClient extends AbstractApiClient {
recording_mbid: aRecordingMbid,
duration: aDuration,
duration_ms: aDurationMs,
+ music_service_name,
+ music_service,
+ submission_client,
+ submission_client_version
} = {},
mbid_mapping: {
recording_mbid: mRecordingMbid
@@ -509,7 +535,8 @@ export class ListenbrainzApiClient extends AbstractApiClient {
meta: {
source: 'listenbrainz',
trackId,
- playId
+ playId,
+ deviceId: combinePartsToString([music_service_name ?? music_service, submission_client, submission_client_version])
}
}
}
diff --git a/src/backend/sources/ListenbrainzSource.ts b/src/backend/sources/ListenbrainzSource.ts
index fd928936..e2c6e586 100644
--- a/src/backend/sources/ListenbrainzSource.ts
+++ b/src/backend/sources/ListenbrainzSource.ts
@@ -3,8 +3,9 @@ import { FormatPlayObjectOptions, INITIALIZING, InternalConfig } from "../common
import EventEmitter from "events";
import { ListenBrainzSourceConfig } from "../common/infrastructure/config/source/listenbrainz";
import { ListenbrainzApiClient } from "../common/vendor/ListenbrainzApiClient";
+import MemorySource from "./MemorySource.js";
-export default class ListenbrainzSource extends AbstractSource {
+export default class ListenbrainzSource extends MemorySource {
api: ListenbrainzApiClient;
requiresAuth = true;
@@ -13,9 +14,17 @@ export default class ListenbrainzSource extends AbstractSource {
declare config: ListenBrainzSourceConfig;
constructor(name: any, config: ListenBrainzSourceConfig, internal: InternalConfig, emitter: EventEmitter) {
- super('listenbrainz', name, config, internal, emitter);
+ const {
+ data: {
+ interval = 15,
+ maxInterval = 60,
+ ...restData
+ } = {}
+ } = config;
+ super('listenbrainz', name, {...config, data: {interval, maxInterval, ...restData}}, internal, emitter);
this.canPoll = true;
this.api = new ListenbrainzApiClient(name, config.data);
+ this.playerSourceOfTruth = false;
}
static formatPlayObj = (obj: any, options: FormatPlayObjectOptions = {}) => ListenbrainzApiClient.formatPlayObj(obj, options);
@@ -52,6 +61,8 @@ export default class ListenbrainzSource extends AbstractSource {
getRecentlyPlayed = async(options: RecentlyPlayedOptions = {}) => {
const {limit = 20} = options;
+ const now = await this.api.getPlayingNow();
+ this.processRecentPlays(now.listens.map(x => ListenbrainzSource.formatPlayObj(x)));
return await this.api.getRecentlyPlayed(limit);
}
}
diff --git a/src/backend/sources/SubsonicSource.ts b/src/backend/sources/SubsonicSource.ts
index ccc81dea..1eb16c1e 100644
--- a/src/backend/sources/SubsonicSource.ts
+++ b/src/backend/sources/SubsonicSource.ts
@@ -21,7 +21,6 @@ export class SubsonicSource extends MemorySource {
declare config: SubSonicSourceConfig;
constructor(name: any, config: SubSonicSourceConfig, internal: InternalConfig, emitter: EventEmitter) {
- // default to quick interval so we can get a decently accurate nowPlaying
const {
data: {
...restData
--
2.54.0
From a48610274731e128d66c953730b32e9b1af9b5d7 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Mon, 18 Sep 2023 15:19:12 -0400
Subject: [PATCH 38/82] feat(lastfm): Add player support
* Use lastfm 'nowPlaying' property in user listens for MS player implementation
* Do not use MS player as source of truth -- continue to use lfm listen history
---
.../infrastructure/config/source/lastfm.ts | 3 +-
.../config/source/listenbrainz.ts | 2 +-
src/backend/common/schema/aio-source.json | 18 +++++++
src/backend/common/schema/aio.json | 18 +++++++
src/backend/common/schema/source.json | 18 +++++++
src/backend/server/index.ts | 1 -
src/backend/sources/LastfmSource.ts | 47 ++++++++++++-------
src/backend/sources/ListenbrainzSource.ts | 2 +-
src/backend/sources/ScrobbleSources.ts | 2 +-
src/core/Atomic.ts | 2 +
10 files changed, 92 insertions(+), 21 deletions(-)
diff --git a/src/backend/common/infrastructure/config/source/lastfm.ts b/src/backend/common/infrastructure/config/source/lastfm.ts
index 1c09a8dd..98fad3fc 100644
--- a/src/backend/common/infrastructure/config/source/lastfm.ts
+++ b/src/backend/common/infrastructure/config/source/lastfm.ts
@@ -1,7 +1,8 @@
import { LastfmData } from "../client/lastfm";
import { CommonSourceConfig, CommonSourceData } from "./index";
+import {PollingOptions} from "../common";
-export interface LastFmSourceData extends CommonSourceData, LastfmData{}
+export interface LastFmSourceData extends CommonSourceData, PollingOptions, LastfmData{}
export interface LastfmSourceConfig extends CommonSourceConfig {
/**
diff --git a/src/backend/common/infrastructure/config/source/listenbrainz.ts b/src/backend/common/infrastructure/config/source/listenbrainz.ts
index 1e7ffff5..5f793ff0 100644
--- a/src/backend/common/infrastructure/config/source/listenbrainz.ts
+++ b/src/backend/common/infrastructure/config/source/listenbrainz.ts
@@ -1,6 +1,6 @@
import { CommonSourceConfig, CommonSourceData } from "./index";
import { ListenBrainzData } from "../client/listenbrainz";
-import {PollingOptions} from "../common.js";
+import {PollingOptions} from "../common";
export interface ListenBrainzSourceData extends ListenBrainzData, CommonSourceData, PollingOptions {
}
diff --git a/src/backend/common/schema/aio-source.json b/src/backend/common/schema/aio-source.json
index 3bc52375..42163f25 100644
--- a/src/backend/common/schema/aio-source.json
+++ b/src/backend/common/schema/aio-source.json
@@ -635,6 +635,24 @@
"title": "apiKey",
"type": "string"
},
+ "interval": {
+ "default": 10,
+ "description": "How long to wait before polling the source API for new tracks (in seconds)",
+ "examples": [
+ 10
+ ],
+ "title": "interval",
+ "type": "number"
+ },
+ "maxInterval": {
+ "default": 30,
+ "description": "When there has been no new activity from the Source API multi-scrobbler will gradually increase the wait time between polling up to this value (in seconds)",
+ "examples": [
+ 30
+ ],
+ "title": "maxInterval",
+ "type": "number"
+ },
"maxPollRetries": {
"default": 5,
"description": "default # of automatic polling restarts on error",
diff --git a/src/backend/common/schema/aio.json b/src/backend/common/schema/aio.json
index 29ccb45a..6570e563 100644
--- a/src/backend/common/schema/aio.json
+++ b/src/backend/common/schema/aio.json
@@ -757,6 +757,24 @@
"title": "apiKey",
"type": "string"
},
+ "interval": {
+ "default": 10,
+ "description": "How long to wait before polling the source API for new tracks (in seconds)",
+ "examples": [
+ 10
+ ],
+ "title": "interval",
+ "type": "number"
+ },
+ "maxInterval": {
+ "default": 30,
+ "description": "When there has been no new activity from the Source API multi-scrobbler will gradually increase the wait time between polling up to this value (in seconds)",
+ "examples": [
+ 30
+ ],
+ "title": "maxInterval",
+ "type": "number"
+ },
"maxPollRetries": {
"default": 5,
"description": "default # of automatic polling restarts on error",
diff --git a/src/backend/common/schema/source.json b/src/backend/common/schema/source.json
index 25b45d72..0f8d9647 100644
--- a/src/backend/common/schema/source.json
+++ b/src/backend/common/schema/source.json
@@ -588,6 +588,24 @@
"title": "apiKey",
"type": "string"
},
+ "interval": {
+ "default": 10,
+ "description": "How long to wait before polling the source API for new tracks (in seconds)",
+ "examples": [
+ 10
+ ],
+ "title": "interval",
+ "type": "number"
+ },
+ "maxInterval": {
+ "default": 30,
+ "description": "When there has been no new activity from the Source API multi-scrobbler will gradually increase the wait time between polling up to this value (in seconds)",
+ "examples": [
+ 30
+ ],
+ "title": "maxInterval",
+ "type": "number"
+ },
"maxPollRetries": {
"default": 5,
"description": "default # of automatic polling restarts on error",
diff --git a/src/backend/server/index.ts b/src/backend/server/index.ts
index f13e3fdd..f3177438 100644
--- a/src/backend/server/index.ts
+++ b/src/backend/server/index.ts
@@ -11,7 +11,6 @@ import { setupApi } from "./api";
import { getAddress, mergeArr, parseBool } from "../utils";
import {stripIndents} from "common-tags";
import {ErrorWithCause} from "pony-cause";
-import {or} from "ajv/dist/compile/codegen/index.js";
const buildDir = path.join(process.cwd() + "/build");
diff --git a/src/backend/sources/LastfmSource.ts b/src/backend/sources/LastfmSource.ts
index 43034cf1..868c05f6 100644
--- a/src/backend/sources/LastfmSource.ts
+++ b/src/backend/sources/LastfmSource.ts
@@ -1,24 +1,34 @@
-import AbstractSource, { RecentlyPlayedOptions } from "./AbstractSource";
+import { RecentlyPlayedOptions } from "./AbstractSource";
import LastfmApiClient from "../common/vendor/LastfmApiClient";
import { sortByOldestPlayDate } from "../utils";
-import { LastfmClientConfig } from "../common/infrastructure/config/client/lastfm";
import { FormatPlayObjectOptions, InternalConfig } from "../common/infrastructure/Atomic";
-import {UserGetRecentTracksResponse} from "lastfm-node-client";
+import {TrackObject, UserGetRecentTracksResponse} from "lastfm-node-client";
import EventEmitter from "events";
import { PlayObject } from "../../core/Atomic";
+import MemorySource from "./MemorySource";
+import {LastfmSourceConfig} from "../common/infrastructure/config/source/lastfm";
+import dayjs from "dayjs";
-export default class LastfmSource extends AbstractSource {
+export default class LastfmSource extends MemorySource {
api: LastfmApiClient;
requiresAuth = true;
requiresAuthInteraction = true;
- declare config: LastfmClientConfig;
+ declare config: LastfmSourceConfig;
- constructor(name: any, config: LastfmClientConfig, internal: InternalConfig, emitter: EventEmitter) {
- super('lastfm', name, config, internal, emitter);
+ constructor(name: any, config: LastfmSourceConfig, internal: InternalConfig, emitter: EventEmitter) {
+ const {
+ data: {
+ interval = 15,
+ maxInterval = 60,
+ ...restData
+ } = {}
+ } = config;
+ super('lastfm', name, {...config, data: {interval, maxInterval, ...restData}}, internal, emitter);
this.canPoll = true;
this.api = new LastfmApiClient(name, {...config.data, configDir: internal.configDir, localUrl: internal.localUrl});
+ this.playerSourceOfTruth = false;
}
static formatPlayObj(obj: any, options: FormatPlayObjectOptions = {}): PlayObject {
@@ -42,7 +52,7 @@ export default class LastfmSource extends AbstractSource {
}
- getRecentlyPlayed = async(options: RecentlyPlayedOptions = {}) => {
+ getRecentlyPlayed = async(options: RecentlyPlayedOptions = {}): Promise => {
const {limit = 20} = options;
const resp = await this.api.callApi((client: any) => client.userGetRecentTracks({user: this.api.user, limit, extended: true}));
const {
@@ -51,7 +61,7 @@ export default class LastfmSource extends AbstractSource {
}
} = resp;
- return list.reduce((acc: any, x: any) => {
+ const plays = list.reduce((acc: PlayObject[], x: TrackObject) => {
try {
const formatted = LastfmApiClient.formatPlayObj(x);
const {
@@ -64,13 +74,11 @@ export default class LastfmSource extends AbstractSource {
nowPlaying,
}
} = formatted;
- if(nowPlaying === true) {
- // if the track is "now playing" it doesn't get a timestamp so we can't determine when it started playing
- // and don't want to accidentally count the same track at different timestamps by artificially assigning it 'now' as a timestamp
- // so we'll just ignore it in the context of recent tracks since really we only want "tracks that have already finished being played" anyway
- this.logger.debug("Ignoring 'now playing' track returned from Last.fm client", {track, mbid});
- return acc;
- } else if(playDate === undefined) {
+ if(playDate === undefined) {
+ if(nowPlaying === true) {
+ formatted.data.playDate = dayjs();
+ return acc.concat(formatted);
+ }
this.logger.warn(`Last.fm recently scrobbled track did not contain a timestamp, omitting from time frame check`, {track, mbid});
return acc;
}
@@ -82,5 +90,12 @@ export default class LastfmSource extends AbstractSource {
return acc;
}
}, []).sort(sortByOldestPlayDate);
+ // if the track is "now playing" it doesn't get a timestamp so we can't determine when it started playing
+ // and don't want to accidentally count the same track at different timestamps by artificially assigning it 'now' as a timestamp
+ // so we'll just ignore it in the context of recent tracks since really we only want "tracks that have already finished being played" anyway
+ const history = plays.filter(x => x.meta.nowPlaying !== true);
+ const now = plays.filter(x => x.meta.nowPlaying === true);
+ this.processRecentPlays(now);
+ return history;
}
}
diff --git a/src/backend/sources/ListenbrainzSource.ts b/src/backend/sources/ListenbrainzSource.ts
index e2c6e586..994b9e72 100644
--- a/src/backend/sources/ListenbrainzSource.ts
+++ b/src/backend/sources/ListenbrainzSource.ts
@@ -3,7 +3,7 @@ import { FormatPlayObjectOptions, INITIALIZING, InternalConfig } from "../common
import EventEmitter from "events";
import { ListenBrainzSourceConfig } from "../common/infrastructure/config/source/listenbrainz";
import { ListenbrainzApiClient } from "../common/vendor/ListenbrainzApiClient";
-import MemorySource from "./MemorySource.js";
+import MemorySource from "./MemorySource";
export default class ListenbrainzSource extends MemorySource {
diff --git a/src/backend/sources/ScrobbleSources.ts b/src/backend/sources/ScrobbleSources.ts
index 0f630f8b..2fca2a25 100644
--- a/src/backend/sources/ScrobbleSources.ts
+++ b/src/backend/sources/ScrobbleSources.ts
@@ -448,7 +448,7 @@ export default class ScrobbleSources {
newSource = await new JellyfinSource(name, compositeConfig as JellySourceConfig, internal, this.emitter);
break;
case 'lastfm':
- newSource = await new LastfmSource(name, compositeConfig as LastfmClientConfig, internal, this.emitter);
+ newSource = await new LastfmSource(name, compositeConfig as LastfmSourceConfig, internal, this.emitter);
break;
case 'deezer':
newSource = await new DeezerSource(name, compositeConfig as DeezerSourceConfig, internal, this.emitter);
diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts
index 5163b89a..1ebf8b8d 100644
--- a/src/core/Atomic.ts
+++ b/src/core/Atomic.ts
@@ -112,6 +112,8 @@ export interface PlayMeta {
* */
deviceId?: string
+ nowPlaying?: boolean
+
[key: string]: any
}
--
2.54.0
From 860d2d953f130c49a71b1ab5940c261c8fb69f95 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Tue, 19 Sep 2023 09:22:22 -0400
Subject: [PATCH 39/82] fix(youtube): Use updated yt music library #98
---
package-lock.json | 20 ++++++++++----------
package.json | 4 ++--
src/backend/sources/YTMusicSource.ts | 7 +++----
3 files changed, 15 insertions(+), 16 deletions(-)
diff --git a/package-lock.json b/package-lock.json
index e8ae97ef..214a4ce3 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -15,7 +15,6 @@
"@fortawesome/react-fontawesome": "^0.2.0",
"@foxxmd/string-sameness": "^0.1.3",
"@foxxmd/winston": "3.3.31",
- "@foxxmd/youtube-music-ts-api": "1.5.1",
"@kenyip/backoff-strategies": "^1.0.4",
"@react-nano/use-event-source": "^0.13.0",
"@reduxjs/toolkit": "^1.9.5",
@@ -65,7 +64,8 @@
"winston-duplex": "0.1.3",
"winston-null": "^2.0.0",
"winston-transport": "^4.4.0",
- "xml2js": "^0.4.23"
+ "xml2js": "^0.4.23",
+ "youtube-music-ts-api": "^1.7.0"
},
"devDependencies": {
"@babel/plugin-proposal-private-property-in-object": "^7.21.11",
@@ -2768,14 +2768,6 @@
"node": ">= 6.4.0"
}
},
- "node_modules/@foxxmd/youtube-music-ts-api": {
- "version": "1.5.1",
- "resolved": "https://registry.npmjs.org/@foxxmd/youtube-music-ts-api/-/youtube-music-ts-api-1.5.1.tgz",
- "integrity": "sha512-0VvYM42I6sDaHtUNAinxdhFW+GqmTRUJLy08jshdgZZhGwZTmU0p/RYTubMZXstVVlA+VGSU3ceseA4K+SytmQ==",
- "dependencies": {
- "sha1": "^1.1.1"
- }
- },
"node_modules/@hapi/hoek": {
"version": "9.3.0",
"resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-9.3.0.tgz",
@@ -24906,6 +24898,14 @@
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
+ },
+ "node_modules/youtube-music-ts-api": {
+ "version": "1.7.0",
+ "resolved": "https://registry.npmjs.org/youtube-music-ts-api/-/youtube-music-ts-api-1.7.0.tgz",
+ "integrity": "sha512-IaH14moHLbXTs3e8cM/Zpld+Jf4pQeYslCGmm6karfQSJ5OQ6h1dZ01qblpzKgqZrcbRF3tg7ASAYNDpB0RaFw==",
+ "dependencies": {
+ "sha1": "^1.1.1"
+ }
}
}
}
diff --git a/package.json b/package.json
index 43b14a9c..8d8ebe32 100644
--- a/package.json
+++ b/package.json
@@ -52,7 +52,6 @@
"@fortawesome/react-fontawesome": "^0.2.0",
"@foxxmd/string-sameness": "^0.1.3",
"@foxxmd/winston": "3.3.31",
- "@foxxmd/youtube-music-ts-api": "1.5.1",
"@kenyip/backoff-strategies": "^1.0.4",
"@react-nano/use-event-source": "^0.13.0",
"@reduxjs/toolkit": "^1.9.5",
@@ -102,7 +101,8 @@
"winston-duplex": "0.1.3",
"winston-null": "^2.0.0",
"winston-transport": "^4.4.0",
- "xml2js": "^0.4.23"
+ "xml2js": "^0.4.23",
+ "youtube-music-ts-api": "^1.7.0"
},
"devDependencies": {
"@babel/plugin-proposal-private-property-in-object": "^7.21.11",
diff --git a/src/backend/sources/YTMusicSource.ts b/src/backend/sources/YTMusicSource.ts
index 49e4cccf..3907c47b 100644
--- a/src/backend/sources/YTMusicSource.ts
+++ b/src/backend/sources/YTMusicSource.ts
@@ -1,14 +1,13 @@
-// @ts-ignore
-import YouTubeMusic from "@foxxmd/youtube-music-ts-api";
+import YouTubeMusic from "youtube-music-ts-api";
import AbstractSource, { RecentlyPlayedOptions } from "./AbstractSource";
import { FormatPlayObjectOptions, InternalConfig } from "../common/infrastructure/Atomic";
// @ts-ignore
-import {IYouTubeMusicAuthenticated} from "@foxxmd/youtube-music-ts-api/interfaces-primary";
+import {IYouTubeMusicAuthenticated} from "youtube-music-ts-api/interfaces-primary";
import dayjs from "dayjs";
import { parseDurationFromTimestamp, playObjDataMatch } from "../utils";
// @ts-ignore
-import {IPlaylistDetail, ITrackDetail} from "@foxxmd/youtube-music-ts-api/interfaces-supplementary";
+import {IPlaylistDetail, ITrackDetail} from "youtube-music-ts-api/interfaces-supplementary";
import { YTMusicSourceConfig } from "../common/infrastructure/config/source/ytmusic";
import EventEmitter from "events";
import { PlayObject } from "../../core/Atomic";
--
2.54.0
From ac2d453aa24b7d9e77609820437f3cec3cf79c76 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Wed, 20 Sep 2023 11:31:15 -0400
Subject: [PATCH 40/82] feat(config): Implement property to control whether a
source/client is used
* QoL improvement -- Instead of needing to rename a config file so it is not read by MS a user can now use the 'enable' property in all source/client configs to enable or disable loading the config
* Added property to all config examples
---
config/config.json.example | 2 +
config/deezer.json.example | 1 +
config/jellyfin.json.example | 1 +
config/jriver.json.example | 1 +
config/kodi.json.example | 1 +
config/lastfm.json.example | 1 +
config/listenbrainz.json.example | 1 +
config/maloja.json.example | 1 +
config/mopidy.json.example | 1 +
config/mpris.json.example | 1 +
config/plex.json.example | 1 +
config/spotify.json.example | 1 +
config/subsonic.json.example | 1 +
config/tautulli.json.example | 1 +
config/ytmusic.json.example | 1 +
.../common/infrastructure/config/common.ts | 7 +
src/backend/common/schema/aio-client.json | 27 ++++
src/backend/common/schema/aio-source.json | 117 ++++++++++++++
src/backend/common/schema/aio.json | 144 ++++++++++++++++++
src/backend/common/schema/client.json | 27 ++++
src/backend/common/schema/source.json | 117 ++++++++++++++
src/backend/scrobblers/ScrobbleClients.ts | 8 +-
src/backend/sources/ScrobbleSources.ts | 9 +-
23 files changed, 469 insertions(+), 3 deletions(-)
diff --git a/config/config.json.example b/config/config.json.example
index c141f26e..bc12c14d 100644
--- a/config/config.json.example
+++ b/config/config.json.example
@@ -21,6 +21,7 @@
"sources": [
{
"type": "spotify",
+ "enable": true,
"clients": ["myConfig"],
"name": "mySpotifySource",
"data": {
@@ -33,6 +34,7 @@
"clients": [
{
"type": "maloja",
+ "enable": true,
"name": "myConfig",
"data": {
"url": "http://localhost:42010",
diff --git a/config/deezer.json.example b/config/deezer.json.example
index 2a8ec885..6d2664a8 100644
--- a/config/deezer.json.example
+++ b/config/deezer.json.example
@@ -1,6 +1,7 @@
[
{
"name": "FoxxMDeezer",
+ "enable": true,
"clients": [],
"data": {
"clientId": "a89cba1569901a0671d5a9875fed4be1",
diff --git a/config/jellyfin.json.example b/config/jellyfin.json.example
index 16eeb487..a9fe8b64 100644
--- a/config/jellyfin.json.example
+++ b/config/jellyfin.json.example
@@ -1,6 +1,7 @@
[
{
"name": "MyJellyfin",
+ "enable": true,
"clients": [],
"data": {
"users": ["FoxxMD"],
diff --git a/config/jriver.json.example b/config/jriver.json.example
index 6f6ab3f3..891f2496 100644
--- a/config/jriver.json.example
+++ b/config/jriver.json.example
@@ -1,6 +1,7 @@
[
{
"name": "MyJriver",
+ "enable": true,
"data": {
"url": "0.0.0.0",
"username": "auser",
diff --git a/config/kodi.json.example b/config/kodi.json.example
index 3167eef6..5b7ebf34 100644
--- a/config/kodi.json.example
+++ b/config/kodi.json.example
@@ -1,6 +1,7 @@
[
{
"name": "MyKodi",
+ "enable": true,
"data": {
"url": "http://localhost:8080/jsonrpc",
"username": "myUser",
diff --git a/config/lastfm.json.example b/config/lastfm.json.example
index a63fa977..7bfdcf57 100644
--- a/config/lastfm.json.example
+++ b/config/lastfm.json.example
@@ -1,6 +1,7 @@
[
{
"name": "myLastFm",
+ "enable": true,
"configureAs": "client",
"data": {
"apiKey": "a89cba1569901a0671d5a9875fed4be1",
diff --git a/config/listenbrainz.json.example b/config/listenbrainz.json.example
index 9e0e6781..5bdf8dda 100644
--- a/config/listenbrainz.json.example
+++ b/config/listenbrainz.json.example
@@ -1,6 +1,7 @@
[
{
"name": "brainz",
+ "enable": true,
"configureAs": "client",
"data": {
"token": "029b081ba-9156-4pe7-88e5-3be671f5ea2b",
diff --git a/config/maloja.json.example b/config/maloja.json.example
index 7c7e2a73..62f64295 100644
--- a/config/maloja.json.example
+++ b/config/maloja.json.example
@@ -1,6 +1,7 @@
[
{
"name": "myMaloja",
+ "enable": true,
"data": {
"url": "http://localhost:42010",
"apiKey": "myMalojaKey"
diff --git a/config/mopidy.json.example b/config/mopidy.json.example
index 58c6ff03..b4f9286e 100644
--- a/config/mopidy.json.example
+++ b/config/mopidy.json.example
@@ -1,6 +1,7 @@
[
{
"name": "MyMopidy",
+ "enable": true,
"data": {
"url": "localhost",
"uriBlacklist": [],
diff --git a/config/mpris.json.example b/config/mpris.json.example
index 3026f10c..32d589ff 100644
--- a/config/mpris.json.example
+++ b/config/mpris.json.example
@@ -1,6 +1,7 @@
[
{
"name": "ubuntu",
+ "enable": true,
"data": {
"whitelist": ["vlc", "mpd"],
"blacklist": ["spotify"]
diff --git a/config/plex.json.example b/config/plex.json.example
index 0410a3d1..c8350cc0 100644
--- a/config/plex.json.example
+++ b/config/plex.json.example
@@ -1,6 +1,7 @@
[
{
"name": "MyPlex",
+ "enable": true,
"clients": [],
"data": {
"user": ["username@gmail.com","anotherUser@gmail.com"],
diff --git a/config/spotify.json.example b/config/spotify.json.example
index 4a2594fe..19dea779 100644
--- a/config/spotify.json.example
+++ b/config/spotify.json.example
@@ -1,6 +1,7 @@
[
{
"name": "MySpotify",
+ "enable": true,
"clients": [],
"data": {
"clientId": "a89cba1569901a0671d5a9875fed4be1",
diff --git a/config/subsonic.json.example b/config/subsonic.json.example
index e306478f..4a01fe1d 100644
--- a/config/subsonic.json.example
+++ b/config/subsonic.json.example
@@ -1,6 +1,7 @@
[
{
"name": "MySubsonic",
+ "enable": true,
"data": {
"url": "http://localhost:4040/airsonic",
"user": "yourUser",
diff --git a/config/tautulli.json.example b/config/tautulli.json.example
index eaa6c822..ebb8b233 100644
--- a/config/tautulli.json.example
+++ b/config/tautulli.json.example
@@ -1,6 +1,7 @@
[
{
"name": "MyTautuilli",
+ "enable": true,
"clients": [],
"data": {
"user": ["username@gmail.com","anotherUser@gmail.com"],
diff --git a/config/ytmusic.json.example b/config/ytmusic.json.example
index 4cd545ab..f052449a 100644
--- a/config/ytmusic.json.example
+++ b/config/ytmusic.json.example
@@ -1,6 +1,7 @@
[
{
"name": "MyYTMusic",
+ "enable": true,
"clients": [],
"data": {
"cookie": "VISITOR_INFO1_LIVE=jMDXz2_L8rY; __Secure-3PAPISID=3AxsXpSXGqOInSDn1jEKn; DEVICE_INFO=ChxOekU0TmTBpjek5EWZ0G; YSC=7gZdl3Zdl3; SID=TwhNsaZRXYTAtXxzGyu6rZdpg2HvGROeW8J4Ym_FhkhoZMUYEQ.; __Secure-1PSID=TwhNOsaZRXYTyRBe4rxAtXRIKsIEtk_Qot2VLBNfHQrQ.; __Secure-3PSID=ZRXYTAtXRIKsIEtk_Qot2yRBerZdpg2HvvZRXYTAtXRIKsIEtk_Qot2yRBerkuZICFQ.; HSID=A1UMmELW79; SSID=AKhomOs; APISID=IlHHmuzkPdQzZZDhHn3; SAPISID=3AxsXpy0u75Qb/n1jEKn; __Secure-1PAPISID=3AxsXpQb/AkSDn1jEKn; LOGIN_INFO=AFmP6vFpyVCZZAIgDwbkhWMBBhluaIWAPP:QUQ314UW5NWMjNmd2ZUJnYnJsakdIMjZoaE5zVVMjNmd2ZZUiHRlb3ZlV3ZIcUVyRVIMjNmdjNmd2ZZUivYlNqX2ZNZUiHdUNFNFdaYmJIW1NkJRX3hqdlU2YnFESkFuSS1uTldnZVRmLXNjWFc5OUJuR3dTd3JsZGZYa2EtZFQ2a0k2Ry1KQQ==; PREF=volume=26; SIDCC=AFvI_94PxXwls-ndqpGfPgFX3FWj80y_94PxXwls-ndqfSh15sP; __Secure-1PSIDCC=AFvIBnUbRr96I96UCIp2U4T8HRVk2B0HfKzhzxwsiP; __Secure-3PSIDCC=AFvIB3bINuUN0ETDR9gO91wpwWIVmpGki3BxT3bINuUN0ETDR9gO91wCH",
diff --git a/src/backend/common/infrastructure/config/common.ts b/src/backend/common/infrastructure/config/common.ts
index 5ba98ca7..fe4de683 100644
--- a/src/backend/common/infrastructure/config/common.ts
+++ b/src/backend/common/infrastructure/config/common.ts
@@ -1,6 +1,13 @@
export interface CommonConfig {
name?: string
data?: CommonData
+ /**
+ * Should MS use this client/source? Defaults to true
+ *
+ * @default true
+ * @examples [true]
+ * */
+ enable?: boolean
}
export interface CommonData {
diff --git a/src/backend/common/schema/aio-client.json b/src/backend/common/schema/aio-client.json
index 0cabfea1..c33f967e 100644
--- a/src/backend/common/schema/aio-client.json
+++ b/src/backend/common/schema/aio-client.json
@@ -101,6 +101,15 @@
"description": "Specific data required to configure this client",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this client. Used with sources to restrict where scrobbles are sent.",
"examples": [
@@ -203,6 +212,15 @@
"description": "Specific data required to configure this client",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this client. Used with sources to restrict where scrobbles are sent.",
"examples": [
@@ -319,6 +337,15 @@
"description": "Specific data required to configure this client",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this client. Used with sources to restrict where scrobbles are sent.",
"examples": [
diff --git a/src/backend/common/schema/aio-source.json b/src/backend/common/schema/aio-source.json
index 42163f25..a86c8818 100644
--- a/src/backend/common/schema/aio-source.json
+++ b/src/backend/common/schema/aio-source.json
@@ -139,6 +139,15 @@
"$ref": "#/definitions/DeezerData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -265,6 +274,15 @@
"$ref": "#/definitions/JRiverData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -417,6 +435,15 @@
"$ref": "#/definitions/JellyData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -545,6 +572,15 @@
"$ref": "#/definitions/KodiData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -601,6 +637,15 @@
"$ref": "#/definitions/LastFmSourceData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -751,6 +796,15 @@
"$ref": "#/definitions/ListenBrainzSourceData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -965,6 +1019,15 @@
"$ref": "#/definitions/MPRISData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -1111,6 +1174,15 @@
"$ref": "#/definitions/MopidyData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -1155,6 +1227,15 @@
"$ref": "#/definitions/PlexSourceData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -1424,6 +1505,15 @@
"$ref": "#/definitions/SpotifySourceData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -1644,6 +1734,15 @@
"$ref": "#/definitions/SubsonicData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -1688,6 +1787,15 @@
"$ref": "#/definitions/PlexSourceData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -1808,6 +1916,15 @@
"$ref": "#/definitions/YTMusicData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
diff --git a/src/backend/common/schema/aio.json b/src/backend/common/schema/aio.json
index 6570e563..c70cf9ec 100644
--- a/src/backend/common/schema/aio.json
+++ b/src/backend/common/schema/aio.json
@@ -212,6 +212,15 @@
"$ref": "#/definitions/DeezerData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -387,6 +396,15 @@
"$ref": "#/definitions/JRiverData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -539,6 +557,15 @@
"$ref": "#/definitions/JellyData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -667,6 +694,15 @@
"$ref": "#/definitions/KodiData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -723,6 +759,15 @@
"$ref": "#/definitions/LastFmSourceData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -868,6 +913,15 @@
"description": "Specific data required to configure this client",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this client. Used with sources to restrict where scrobbles are sent.",
"examples": [
@@ -970,6 +1024,15 @@
"description": "Specific data required to configure this client",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this client. Used with sources to restrict where scrobbles are sent.",
"examples": [
@@ -1111,6 +1174,15 @@
"$ref": "#/definitions/ListenBrainzSourceData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -1370,6 +1442,15 @@
"$ref": "#/definitions/MPRISData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -1401,6 +1482,15 @@
"description": "Specific data required to configure this client",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this client. Used with sources to restrict where scrobbles are sent.",
"examples": [
@@ -1660,6 +1750,15 @@
"$ref": "#/definitions/MopidyData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -1760,6 +1859,15 @@
"$ref": "#/definitions/PlexSourceData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -2094,6 +2202,15 @@
"$ref": "#/definitions/SpotifySourceData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -2314,6 +2431,15 @@
"$ref": "#/definitions/SubsonicData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -2358,6 +2484,15 @@
"$ref": "#/definitions/PlexSourceData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -2489,6 +2624,15 @@
"$ref": "#/definitions/YTMusicData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
diff --git a/src/backend/common/schema/client.json b/src/backend/common/schema/client.json
index 53e3f097..29127046 100644
--- a/src/backend/common/schema/client.json
+++ b/src/backend/common/schema/client.json
@@ -98,6 +98,15 @@
"description": "Specific data required to configure this client",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this client. Used with sources to restrict where scrobbles are sent.",
"examples": [
@@ -192,6 +201,15 @@
"description": "Specific data required to configure this client",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this client. Used with sources to restrict where scrobbles are sent.",
"examples": [
@@ -300,6 +318,15 @@
"description": "Specific data required to configure this client",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this client. Used with sources to restrict where scrobbles are sent.",
"examples": [
diff --git a/src/backend/common/schema/source.json b/src/backend/common/schema/source.json
index 0f8d9647..9bd8b494 100644
--- a/src/backend/common/schema/source.json
+++ b/src/backend/common/schema/source.json
@@ -180,6 +180,15 @@
"$ref": "#/definitions/DeezerData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -298,6 +307,15 @@
"$ref": "#/definitions/JRiverData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -442,6 +460,15 @@
"$ref": "#/definitions/JellyData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -562,6 +589,15 @@
"$ref": "#/definitions/KodiData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -704,6 +740,15 @@
"$ref": "#/definitions/LastFmSourceData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -752,6 +797,15 @@
"$ref": "#/definitions/ListenBrainzSourceData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -958,6 +1012,15 @@
"$ref": "#/definitions/MPRISData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -1096,6 +1159,15 @@
"$ref": "#/definitions/MopidyData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -1132,6 +1204,15 @@
"$ref": "#/definitions/PlexSourceData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -1316,6 +1397,15 @@
"$ref": "#/definitions/SpotifySourceData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -1442,6 +1532,15 @@
"$ref": "#/definitions/SubsonicData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -1564,6 +1663,15 @@
"$ref": "#/definitions/PlexSourceData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
@@ -1676,6 +1784,15 @@
"$ref": "#/definitions/YTMusicData",
"title": "data"
},
+ "enable": {
+ "default": true,
+ "description": "Should MS use this client/source? Defaults to true",
+ "examples": [
+ true
+ ],
+ "title": "enable",
+ "type": "boolean"
+ },
"name": {
"description": "Unique identifier for this source.",
"title": "name",
diff --git a/src/backend/scrobblers/ScrobbleClients.ts b/src/backend/scrobblers/ScrobbleClients.ts
index cf23a71c..274fa80c 100644
--- a/src/backend/scrobblers/ScrobbleClients.ts
+++ b/src/backend/scrobblers/ScrobbleClients.ts
@@ -296,7 +296,13 @@ ${sources.join('\n')}`);
if (isValidConfig !== true) {
throw new Error(`Config object from ${clientConfig.source || 'unknown'} with name [${clientConfig.name || 'unnamed'}] of type [${clientConfig.type || 'unknown'}] has errors: ${isValidConfig.join(' | ')}`)
}*/
- const {type, name, data: d = {}} = clientConfig;
+ const {type, name, enable = true, data: d = {}} = clientConfig;
+
+ if(enable === false) {
+ this.logger.warn(`${type} (${name}) client was disabled by config`);
+ return;
+ }
+
// add defaults
const data = {...defaults, ...d};
let newClient;
diff --git a/src/backend/sources/ScrobbleSources.ts b/src/backend/sources/ScrobbleSources.ts
index 2fca2a25..25d2fd5a 100644
--- a/src/backend/sources/ScrobbleSources.ts
+++ b/src/backend/sources/ScrobbleSources.ts
@@ -410,7 +410,7 @@ export default class ScrobbleSources {
}
}
- addSource = async (clientConfig: any, defaults: SourceDefaults = {}) => {
+ addSource = async (clientConfig: ParsedConfig, defaults: SourceDefaults = {}) => {
// const isValidConfig = isValidConfigStructure(clientConfig, {name: true, data: true, type: true});
// if (isValidConfig !== true) {
// throw new Error(`Config object from ${clientConfig.source || 'unknown'} with name [${clientConfig.name || 'unnamed'}] of type [${clientConfig.type || 'unknown'}] has errors: ${isValidConfig.join(' | ')}`)
@@ -422,7 +422,12 @@ export default class ScrobbleSources {
logger: this.logger
};
- const {type, name, data: d = {}, options: clientOptions = {}} = clientConfig;
+ const {type, name, data: d = {}, enable = true, options: clientOptions = {}} = clientConfig;
+
+ if(enable === false) {
+ this.logger.warn(`${type} (${name}) source was disabled by config`);
+ return;
+ }
// add defaults
const {options: defaultOptions = {}, ...restDefaults} = defaults;
--
2.54.0
From bdbbce4cb07b5b551bd8826db9e07b747d6499d6 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Tue, 26 Sep 2023 11:39:34 -0400
Subject: [PATCH 41/82] feat(jellyfin): Initial testing suite
---
src/backend/tests/jellyfin.test.ts | 41 ++++++++++++
.../jellyfin/playbackProgressSample.json | 64 +++++++++++++++++++
2 files changed, 105 insertions(+)
create mode 100644 src/backend/tests/jellyfin.test.ts
create mode 100644 src/backend/tests/jellyfin/playbackProgressSample.json
diff --git a/src/backend/tests/jellyfin.test.ts b/src/backend/tests/jellyfin.test.ts
new file mode 100644
index 00000000..2485e205
--- /dev/null
+++ b/src/backend/tests/jellyfin.test.ts
@@ -0,0 +1,41 @@
+import {describe, it} from 'mocha';
+import {assert} from 'chai';
+import samplePayload from './jellyfin/playbackProgressSample.json';
+
+import JellyfinSource from "../sources/JellyfinSource";
+import EventEmitter from "events";
+import {getLogger} from "../common/logging";
+import {JsonPlayObject, PlayObject} from "../../core/Atomic";
+
+const dataAsFixture = (data: any): TestFixture => {
+ return data as TestFixture;
+}
+
+interface TestFixture {
+ data: any
+ expected: JsonPlayObject
+}
+
+describe('Jellyfin Payload Parsing', function () {
+
+ it('Should parse PlayProgress payload as PlayObject', async function () {
+ const fixture = dataAsFixture(samplePayload[0]);
+ const play = JellyfinSource.formatPlayObj(fixture.data);
+
+ assert.equal(play.data.track, fixture.expected.data.track);
+ assert.equal(play.meta.mediaType, 'Audio');
+ });
+
+ describe('Correctly detects events as valid/invalid', function () {
+
+ const jfSource = new JellyfinSource('Test', {data: {}}, {localUrl: 'test', configDir: 'test', logger: getLogger({}, 'Test')}, new EventEmitter());
+
+ it('Should parse PlayProgress with Audio ItemType as valid event', async function () {
+ const fixture = dataAsFixture(samplePayload[0]);
+ const play = JellyfinSource.formatPlayObj(fixture.data);
+
+ assert.isTrue(jfSource.isValidEvent(play))
+ });
+
+ });
+});
diff --git a/src/backend/tests/jellyfin/playbackProgressSample.json b/src/backend/tests/jellyfin/playbackProgressSample.json
new file mode 100644
index 00000000..3cc5e850
--- /dev/null
+++ b/src/backend/tests/jellyfin/playbackProgressSample.json
@@ -0,0 +1,64 @@
+[
+ {
+ "data": {
+ "ServerId": "6c41d884a9c1490f8fd0863e04d75a69",
+ "ServerName": "maine",
+ "ServerVersion": "10.8.10",
+ "ServerUrl": "",
+ "NotificationType": "PlaybackProgress",
+ "Timestamp": "2023-09-26T11:18:17.8399513-03:00",
+ "UtcTimestamp": "2023-09-26T14:18:17.8399533Z",
+ "Name": "Brother Madonna",
+ "Overview": "",
+ "Tagline": "",
+ "ItemId": "514941104216969e74a8d87fc7893ad5",
+ "ItemType": "Audio",
+ "RunTimeTicks": 926533312,
+ "RunTime": "00:01:32",
+ "Year": 2008,
+ "Album": "I Kind of Really Want to Make Love to You",
+ "Artist": "Kitsune\u00B2",
+ "Provider_musicbrainzalbumartist": "6e659357-5963-4daf-a634-a3596893f3d7",
+ "Provider_musicbrainzartist": "6e659357-5963-4daf-a634-a3596893f3d7",
+ "Provider_musicbrainzalbum": "7c10f8a5-669f-4ff6-bb88-de59846c0e0c",
+ "Provider_musicbrainzreleasegroup": "85c9ae8b-13e4-4fbf-9f09-5e405039772c",
+ "Provider_musicbrainztrack": "3c2c1f40-b22e-360e-a29a-1b1610190a8c",
+ "Audio_0_Title": "FLAC - Stereo",
+ "Audio_0_Type": "Audio",
+ "Audio_0_Language": null,
+ "Audio_0_Codec": "flac",
+ "Audio_0_Channels": 2,
+ "Audio_0_Bitrate": 917349,
+ "Audio_0_SampleRate": 44100,
+ "Audio_0_Default": false,
+ "PlaybackPositionTicks": 640000000,
+ "PlaybackPosition": "00:01:04",
+ "MediaSourceId": "514941104216969e74a8d87fc7893ad5",
+ "IsPaused": false,
+ "IsAutomated": true,
+ "DeviceId": "BC9311B570ABE4BD71EEEC0856A448AD",
+ "DeviceName": "samsung SM-S901E",
+ "ClientName": "Android",
+ "NotificationUsername": "Tabitha",
+ "UserId": "6d23c30c5e9d4ec6a66db6b05b63e516"
+ },
+ "expected": {
+ "data": {
+ "track": "Brother Madonna",
+ "artists": ["Kitsune²"],
+ "album": "I Kind of Really Want to Make Love to You",
+ "duration": 92
+ },
+ "meta": {
+ "source": "Jellyfin",
+ "event": "PlaybackProgress",
+ "mediaType": "Audio",
+ "trackId": "514941104216969e74a8d87fc7893ad5",
+ "user": "Tabitha",
+ "sourceVersion": "10.8.10",
+ "deviceId": "BC9311B570-samsung SM-S901E",
+ "trackProgressPosition": 64
+ }
+ }
+ }
+]
--
2.54.0
From ce5f8d5fca0856c273b473a36418ba59afd39a09 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Tue, 26 Sep 2023 14:03:03 -0400
Subject: [PATCH 42/82] fix(jellyfin): Enable parsing requests with more
content-types and add logging on invalid types
* Add text/* wildcard so any text-prefixed content-type is valid (Jellyfin should always be sending json anyway)
* Add logging WARN when body is empty
Fixes #101
---
src/backend/server/jellyfinRoutes.ts | 19 ++++++++++++++++++-
1 file changed, 18 insertions(+), 1 deletion(-)
diff --git a/src/backend/server/jellyfinRoutes.ts b/src/backend/server/jellyfinRoutes.ts
index 6a580b05..a6728a4e 100644
--- a/src/backend/server/jellyfinRoutes.ts
+++ b/src/backend/server/jellyfinRoutes.ts
@@ -5,11 +5,12 @@ import ScrobbleSources from "../sources/ScrobbleSources";
import bodyParser from "body-parser";
import { JellyfinNotifier } from "../sources/ingressNotifiers/JellyfinNotifier";
import JellyfinSource from "../sources/JellyfinSource";
+import {ExpressRequest} from "../common/infrastructure/Atomic.js";
export const setupJellyfinRoutes = (app: ExpressWithAsync, logger: Logger, scrobbleSources: ScrobbleSources) => {
// webhook plugin sends json with context type text/utf-8 so we need to parse it differently
- const jellyfinJsonParser = bodyParser.json({type: 'text/*'});
+ const jellyfinJsonParser = bodyParser.json({type: ['text/*', 'application/json']});
const jellyIngress = new JellyfinNotifier();
app.postAsync('/jellyfin', async function(req, res) {
res.redirect(307, '/api/jellyfin/ingress');
@@ -26,6 +27,22 @@ export const setupJellyfinRoutes = (app: ExpressWithAsync, logger: Logger, scrob
res.send('OK');
+ const bodyEmpty = req.body === undefined || req.body === null || (typeof req.body === 'object' && Object.keys(req.body).length === 0);
+ if(bodyEmpty) {
+ const length = req.header('content-length') !== undefined ? Number.parseInt(req.header('content-length')) : undefined;
+ // can't think of a way a user would send an empty body for a webhook payload but if they meant to do it don't spam them with errors...
+ if(length === 0) {
+ return;
+ }
+ if(length === undefined) {
+ logger.warn(`Jellyfin is not sending a well-formatted request. It does not have valid headers (application/json - text/*) OR it is missing content-length header: Content-Type => ${req.header('content-type')} | Length => ${length}`);
+ } else {
+ logger.warn(`Jellyfin is not sending a request with valid headers. Content-Type must be either application/json or a text/* wildcard (like text/plain) -- given: Content-Type => ${req.header('content-type')}`);
+ }
+ res.status(400).send('Invalid Content-Type. Must be either application/json or a text wildcard (like text/plain)');
+ return;
+ }
+
const parts = remoteHostIdentifiers(req);
const connectionId = `${parts.host}-${parts.proxy ?? ''}`;
--
2.54.0
From a46389e74535e2446fa7662efb206900302e4e47 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Tue, 26 Sep 2023 14:08:02 -0400
Subject: [PATCH 43/82] Remove unused import
---
src/backend/server/jellyfinRoutes.ts | 1 -
1 file changed, 1 deletion(-)
diff --git a/src/backend/server/jellyfinRoutes.ts b/src/backend/server/jellyfinRoutes.ts
index a6728a4e..4f4201ad 100644
--- a/src/backend/server/jellyfinRoutes.ts
+++ b/src/backend/server/jellyfinRoutes.ts
@@ -5,7 +5,6 @@ import ScrobbleSources from "../sources/ScrobbleSources";
import bodyParser from "body-parser";
import { JellyfinNotifier } from "../sources/ingressNotifiers/JellyfinNotifier";
import JellyfinSource from "../sources/JellyfinSource";
-import {ExpressRequest} from "../common/infrastructure/Atomic.js";
export const setupJellyfinRoutes = (app: ExpressWithAsync, logger: Logger, scrobbleSources: ScrobbleSources) => {
--
2.54.0
From 6e1e2c3534b194b30659be8bdd9737c3008493e2 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Tue, 26 Sep 2023 15:36:17 -0400
Subject: [PATCH 44/82] Add verify example for body parser
---
src/backend/server/jellyfinRoutes.ts | 13 ++++++++++---
1 file changed, 10 insertions(+), 3 deletions(-)
diff --git a/src/backend/server/jellyfinRoutes.ts b/src/backend/server/jellyfinRoutes.ts
index 4f4201ad..dedce837 100644
--- a/src/backend/server/jellyfinRoutes.ts
+++ b/src/backend/server/jellyfinRoutes.ts
@@ -9,7 +9,14 @@ import JellyfinSource from "../sources/JellyfinSource";
export const setupJellyfinRoutes = (app: ExpressWithAsync, logger: Logger, scrobbleSources: ScrobbleSources) => {
// webhook plugin sends json with context type text/utf-8 so we need to parse it differently
- const jellyfinJsonParser = bodyParser.json({type: ['text/*', 'application/json']});
+ const jellyfinJsonParser = bodyParser.json({
+ type: ['text/*', 'application/json'],
+ // verify: function(req, res, buf, encoding) {
+ // // get rawBody
+ // // @ts-ignore
+ // req.rawBody = buf.toString();
+ // }
+ });
const jellyIngress = new JellyfinNotifier();
app.postAsync('/jellyfin', async function(req, res) {
res.redirect(307, '/api/jellyfin/ingress');
@@ -34,9 +41,9 @@ export const setupJellyfinRoutes = (app: ExpressWithAsync, logger: Logger, scrob
return;
}
if(length === undefined) {
- logger.warn(`Jellyfin is not sending a well-formatted request. It does not have valid headers (application/json - text/*) OR it is missing content-length header: Content-Type => ${req.header('content-type')} | Length => ${length}`);
+ logger.warn(`Jellyfin is not sending a well-formatted request. It does not have valid headers (application/json - text/*) OR it is missing content-length header: Content-Type => '${req.header('content-type')}' | Length => ${length}`);
} else {
- logger.warn(`Jellyfin is not sending a request with valid headers. Content-Type must be either application/json or a text/* wildcard (like text/plain) -- given: Content-Type => ${req.header('content-type')}`);
+ logger.warn(`Jellyfin is not sending a request with valid headers. Content-Type must be either application/json or a text/* wildcard (like text/plain) -- given: Content-Type => '${req.header('content-type')}'`);
}
res.status(400).send('Invalid Content-Type. Must be either application/json or a text wildcard (like text/plain)');
return;
--
2.54.0
From 43cbadcd2de876bd498e863740736acd377d6b5f Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Wed, 27 Sep 2023 13:18:20 -0400
Subject: [PATCH 45/82] refactor(scrobble): Clean up base design a little
* Make alreadyScrobbled abstract function
* Make sorting recentScrobbles automatic by refactoring into a getter/setter
---
.../scrobblers/AbstractScrobbleClient.ts | 20 ++++++++++++-------
src/backend/scrobblers/LastfmScrobbler.ts | 2 +-
.../scrobblers/ListenbrainzScrobbler.ts | 2 +-
src/backend/scrobblers/MalojaScrobbler.ts | 2 +-
4 files changed, 16 insertions(+), 10 deletions(-)
diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts
index 58aa92f3..77532831 100644
--- a/src/backend/scrobblers/AbstractScrobbleClient.ts
+++ b/src/backend/scrobblers/AbstractScrobbleClient.ts
@@ -3,7 +3,7 @@ import {
isPlayTemporallyClose,
mergeArr,
playObjDataMatch,
- setIntersection,
+ setIntersection, sortByOldestPlayDate,
} from "../utils";
import {
ClientType,
@@ -36,7 +36,7 @@ export default abstract class AbstractScrobbleClient {
requiresAuthInteraction: boolean = false;
authed: boolean = false;
- recentScrobbles: PlayObject[] = [];
+ #recentScrobblesList: PlayObject[] = [];
scrobbledPlayObjs: FixedSizeList;
newestScrobbleTime?: Dayjs
oldestScrobbleTime?: Dayjs
@@ -97,6 +97,16 @@ export default abstract class AbstractScrobbleClient {
};
}
+ set recentScrobbles(scrobbles: PlayObject[]) {
+ const sorted = [...scrobbles];
+ sorted.sort(sortByOldestPlayDate);
+ this.#recentScrobblesList = sorted;
+ }
+
+ get recentScrobbles() {
+ return this.#recentScrobblesList;
+ }
+
get initialized() {
return this.#initState === INITIALIZED;
}
@@ -136,11 +146,7 @@ export default abstract class AbstractScrobbleClient {
this.logger.debug('Scrobbler does not have refresh function implemented!');
}
- alreadyScrobbled = async (playObj: PlayObject, log = false) => {
- this.logger.debug('Scrobbler does not have alreadyScrobbled check implemented!');
- return false;
- }
-
+ public abstract alreadyScrobbled(playObj: PlayObject, log?: boolean): Promise;
scrobblesLastCheckedAt = () => {
return this.lastScrobbleCheck;
}
diff --git a/src/backend/scrobblers/LastfmScrobbler.ts b/src/backend/scrobblers/LastfmScrobbler.ts
index c623bc2e..69ea0e52 100644
--- a/src/backend/scrobblers/LastfmScrobbler.ts
+++ b/src/backend/scrobblers/LastfmScrobbler.ts
@@ -91,7 +91,7 @@ export default class LastfmScrobbler extends AbstractScrobbleClient {
this.logger.debug(x);
return acc;
}
- }, []).sort(sortByOldestPlayDate);
+ }, []);
this.logger.debug(`Found ${this.recentScrobbles.length} recent scrobbles`);
if (this.recentScrobbles.length > 0) {
const [{data: {playDate: newestScrobbleTime = dayjs()} = {}} = {}] = this.recentScrobbles.slice(-1);
diff --git a/src/backend/scrobblers/ListenbrainzScrobbler.ts b/src/backend/scrobblers/ListenbrainzScrobbler.ts
index 46ccf2bd..75f9cdf6 100644
--- a/src/backend/scrobblers/ListenbrainzScrobbler.ts
+++ b/src/backend/scrobblers/ListenbrainzScrobbler.ts
@@ -68,7 +68,7 @@ export default class ListenbrainzScrobbler extends AbstractScrobbleClient {
this.logger.debug('Refreshing recent scrobbles');
const resp = await this.api.getRecentlyPlayed(this.MAX_STORED_SCROBBLES);
this.logger.debug(`Found ${resp.length} recent scrobbles`);
- this.recentScrobbles = resp.sort(sortByOldestPlayDate);
+ this.recentScrobbles = resp;
if (this.recentScrobbles.length > 0) {
const [{data: {playDate: newestScrobbleTime = dayjs()} = {}} = {}] = this.recentScrobbles.slice(-1);
const [{data: {playDate: oldestScrobbleTime = dayjs()} = {}} = {}] = this.recentScrobbles.slice(0, 1);
diff --git a/src/backend/scrobblers/MalojaScrobbler.ts b/src/backend/scrobblers/MalojaScrobbler.ts
index 3f375658..35e43d32 100644
--- a/src/backend/scrobblers/MalojaScrobbler.ts
+++ b/src/backend/scrobblers/MalojaScrobbler.ts
@@ -324,7 +324,7 @@ export default class MalojaScrobbler extends AbstractScrobbleClient {
} = {},
} = resp;
this.logger.debug(`Found ${list.length} recent scrobbles`);
- this.recentScrobbles = list.map((x: any) => this.formatPlayObj(x)).sort(sortByOldestPlayDate);
+ this.recentScrobbles = list.map((x: any) => this.formatPlayObj(x));
if (this.recentScrobbles.length > 0) {
const [{data: {playDate: newestScrobbleTime = dayjs()} = {}} = {}] = this.recentScrobbles.slice(-1);
const [{data: {playDate: oldestScrobbleTime = dayjs()} = {}} = {}] = this.recentScrobbles.slice(0, 1);
--
2.54.0
From 8371ec3a0135583ae80500f5bff84b181a9c55b9 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Wed, 27 Sep 2023 13:18:58 -0400
Subject: [PATCH 46/82] fix(scrobble): Fix config destructing default value
---
src/backend/scrobblers/AbstractScrobbleClient.ts | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts
index 77532831..b6e01d1e 100644
--- a/src/backend/scrobblers/AbstractScrobbleClient.ts
+++ b/src/backend/scrobblers/AbstractScrobbleClient.ts
@@ -70,7 +70,7 @@ export default abstract class AbstractScrobbleClient {
checkExistingScrobbles = true,
verbose = {},
} = {},
- },
+ } = {},
} = config;
this.config = config;
this.refreshEnabled = refreshEnabled;
--
2.54.0
From 7885c4ca9c5b6567fc41443784705e7b28f45459 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Wed, 27 Sep 2023 13:19:25 -0400
Subject: [PATCH 47/82] fix(scrobble): Fix wrong artists used for comparison
---
src/backend/scrobblers/AbstractScrobbleClient.ts | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts
index b6e01d1e..ad577941 100644
--- a/src/backend/scrobblers/AbstractScrobbleClient.ts
+++ b/src/backend/scrobblers/AbstractScrobbleClient.ts
@@ -271,7 +271,7 @@ export default abstract class AbstractScrobbleClient {
data: {
artists: sourceArtists = [],
} = {},
- } = candidate;
+ } = existing;
const {
data: {
artists = [],
--
2.54.0
From ec4ab8c078bafec2057c40e1217dff9628bff844 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Wed, 27 Sep 2023 13:20:03 -0400
Subject: [PATCH 48/82] test(scrobble): Implement basic scrobble tests
---
package-lock.json | 34 ++++++
package.json | 3 +
src/backend/tests/plays/mixedDuration.json | 70 +++++++++++
src/backend/tests/plays/withDuration.json | 70 +++++++++++
src/backend/tests/scrobbler/TestScrobbler.ts | 23 ++++
.../tests/scrobbler/scrobblers.test.ts | 113 ++++++++++++++++++
src/backend/tests/utils/PlayTestUtils.ts | 95 +++++++++++++++
7 files changed, 408 insertions(+)
create mode 100644 src/backend/tests/plays/mixedDuration.json
create mode 100644 src/backend/tests/plays/withDuration.json
create mode 100644 src/backend/tests/scrobbler/TestScrobbler.ts
create mode 100644 src/backend/tests/scrobbler/scrobblers.test.ts
create mode 100644 src/backend/tests/utils/PlayTestUtils.ts
diff --git a/package-lock.json b/package-lock.json
index 214a4ce3..97f14740 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -69,6 +69,7 @@
},
"devDependencies": {
"@babel/plugin-proposal-private-property-in-object": "^7.21.11",
+ "@faker-js/faker": "^8.1.0",
"@istanbuljs/nyc-config-typescript": "^1.0.2",
"@testing-library/jest-dom": "^5.17.0",
"@testing-library/react": "^13.4.0",
@@ -76,6 +77,7 @@
"@tsconfig/node18": "^1.0.1",
"@types/chai": "^4.3.0",
"@types/chai-as-promised": "^7.1.5",
+ "@types/clone": "^2.1.2",
"@types/common-tags": "^1.8.1",
"@types/concat-stream": "^2.0.0",
"@types/express": "^4.17.13",
@@ -96,6 +98,7 @@
"@types/xml2js": "^0.4.11",
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
+ "clone": "^2.1.2",
"concurrently": "^8.2.0",
"jscodeshift": "^0.15.0",
"mocha": "^9.2.1",
@@ -2690,6 +2693,22 @@
"node": "^12.22.0 || ^14.17.0 || >=16.0.0"
}
},
+ "node_modules/@faker-js/faker": {
+ "version": "8.1.0",
+ "resolved": "https://registry.npmjs.org/@faker-js/faker/-/faker-8.1.0.tgz",
+ "integrity": "sha512-38DT60rumHfBYynif3lmtxMqMqmsOQIxQgEuPZxCk2yUYN0eqWpTACgxi0VpidvsJB8CRxCpvP7B3anK85FjtQ==",
+ "dev": true,
+ "funding": [
+ {
+ "type": "opencollective",
+ "url": "https://opencollective.com/fakerjs"
+ }
+ ],
+ "engines": {
+ "node": "^14.17.0 || ^16.13.0 || >=18.0.0",
+ "npm": ">=6.14.13"
+ }
+ },
"node_modules/@fortawesome/fontawesome-common-types": {
"version": "6.4.2",
"resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-common-types/-/fontawesome-common-types-6.4.2.tgz",
@@ -4630,6 +4649,12 @@
"@types/chai": "*"
}
},
+ "node_modules/@types/clone": {
+ "version": "2.1.2",
+ "resolved": "https://registry.npmjs.org/@types/clone/-/clone-2.1.2.tgz",
+ "integrity": "sha512-fKXTsQvII/8K7xSxQF2OEuN6Pt4/PDasws60s/qeqoIOaBX7Aoevy5CmaQ4fANbvOo04MN3kx3xUIc6ZNXsmaQ==",
+ "dev": true
+ },
"node_modules/@types/common-tags": {
"version": "1.8.1",
"resolved": "https://registry.npmjs.org/@types/common-tags/-/common-tags-1.8.1.tgz",
@@ -7356,6 +7381,15 @@
"node": ">=8"
}
},
+ "node_modules/clone": {
+ "version": "2.1.2",
+ "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz",
+ "integrity": "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==",
+ "dev": true,
+ "engines": {
+ "node": ">=0.8"
+ }
+ },
"node_modules/clone-deep": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz",
diff --git a/package.json b/package.json
index 8d8ebe32..050075a7 100644
--- a/package.json
+++ b/package.json
@@ -106,6 +106,7 @@
},
"devDependencies": {
"@babel/plugin-proposal-private-property-in-object": "^7.21.11",
+ "@faker-js/faker": "^8.1.0",
"@istanbuljs/nyc-config-typescript": "^1.0.2",
"@testing-library/jest-dom": "^5.17.0",
"@testing-library/react": "^13.4.0",
@@ -113,6 +114,7 @@
"@tsconfig/node18": "^1.0.1",
"@types/chai": "^4.3.0",
"@types/chai-as-promised": "^7.1.5",
+ "@types/clone": "^2.1.2",
"@types/common-tags": "^1.8.1",
"@types/concat-stream": "^2.0.0",
"@types/express": "^4.17.13",
@@ -133,6 +135,7 @@
"@types/xml2js": "^0.4.11",
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
+ "clone": "^2.1.2",
"concurrently": "^8.2.0",
"jscodeshift": "^0.15.0",
"mocha": "^9.2.1",
diff --git a/src/backend/tests/plays/mixedDuration.json b/src/backend/tests/plays/mixedDuration.json
new file mode 100644
index 00000000..7cfaadf5
--- /dev/null
+++ b/src/backend/tests/plays/mixedDuration.json
@@ -0,0 +1,70 @@
+[
+ {
+ "data": {
+ "artists": [
+ "Menagerie"
+ ],
+ "track": "Danieda's Dance",
+ "duration": null,
+ "playDate": "2023-09-18T19:00:25.000Z"
+ },
+ "meta": {
+ "source": "Maloja"
+ }
+ },
+ {
+ "data": {
+ "artists": [
+ "Dan Tenor City",
+ "Ricoh",
+ "Tom Funk"
+ ],
+ "track": "Close To The Source",
+ "duration": null,
+ "playDate": "2023-09-18T20:17:58.000Z"
+ },
+ "meta": {
+ "source": "Maloja"
+ }
+ },
+ {
+ "data": {
+ "artists": [
+ "medline"
+ ],
+ "track": "Run For Cover",
+ "duration": 300,
+ "playDate": "2023-09-20T14:02:05.000Z"
+ },
+ "meta": {
+ "source": "Maloja"
+ }
+ },
+ {
+ "data": {
+ "artists": [
+ "King Owusu",
+ "Zeitgeist Freedom Energy Exchange"
+ ],
+ "track": "Obana",
+ "duration": null,
+ "playDate": "2023-09-20T14:07:02.000Z"
+ },
+ "meta": {
+ "source": "Maloja"
+ }
+ },
+ {
+ "data": {
+ "artists": [
+ "Noro Morales"
+ ],
+ "track": "Mississippi Mambo",
+ "duration": 159,
+ "playDate": "2023-09-20T17:49:26.000Z"
+ },
+ "meta": {
+ "source": "Maloja"
+ }
+ }
+]
diff --git a/src/backend/tests/plays/withDuration.json b/src/backend/tests/plays/withDuration.json
new file mode 100644
index 00000000..a9a1724f
--- /dev/null
+++ b/src/backend/tests/plays/withDuration.json
@@ -0,0 +1,70 @@
+[
+ {
+ "data": {
+ "artists": [
+ "Kaidi Tatham"
+ ],
+ "track": "Fricassee",
+ "duration": 305,
+ "playDate": "2023-09-20T14:32:39.000Z"
+ },
+ "meta": {
+ "source": "Maloja"
+ }
+ },
+ {
+ "data": {
+ "artists": [
+ "Kuna Maze"
+ ],
+ "track": "Jimbo",
+ "duration": 255,
+ "playDate": "2023-09-20T14:46:00.000Z"
+ },
+ "meta": {
+ "source": "Maloja"
+ }
+ },
+ {
+ "data": {
+ "artists": [
+ "Norman Person",
+ "Shamek Farrah"
+ ],
+ "track": "Aisha",
+ "duration": 559,
+ "playDate": "2023-09-20T15:10:09.000Z"
+ },
+ "meta": {
+ "source": "Maloja"
+ }
+ },
+ {
+ "data": {
+ "artists": [
+ "Cheo Feliciano"
+ ],
+ "track": "Aprieta (Oye Cómo Va)",
+ "duration": 195,
+ "playDate": "2023-09-20T15:38:12.000Z"
+ },
+ "meta": {
+ "source": "Maloja"
+ }
+ },
+ {
+ "data": {
+ "artists": [
+ "A Collection of Boogie",
+ "Jazz Funk",
+ "Disco with Mehdi El"
+ ],
+ "track": "Aquil",
+ "duration": 3072,
+ "playDate": "2023-09-20T17:55:05.000Z"
+ },
+ "meta": {
+ "source": "Maloja"
+ }
+ }
+]
diff --git a/src/backend/tests/scrobbler/TestScrobbler.ts b/src/backend/tests/scrobbler/TestScrobbler.ts
new file mode 100644
index 00000000..d8f7d410
--- /dev/null
+++ b/src/backend/tests/scrobbler/TestScrobbler.ts
@@ -0,0 +1,23 @@
+import AbstractScrobbleClient from "../../scrobblers/AbstractScrobbleClient";
+import {PlayObject} from "../../../core/Atomic";
+import {getLogger} from "../../common/logging";
+import {Notifiers} from "../../notifier/Notifiers";
+import EventEmitter from "events";
+
+export class TestScrobbler extends AbstractScrobbleClient {
+
+ constructor() {
+ const logger = getLogger({});
+ const notifier = new Notifiers(new EventEmitter(), new EventEmitter(), new EventEmitter());
+ super('test', 'Test', {name: 'test'}, notifier, new EventEmitter(), logger);
+ }
+
+ scrobble(playObj: PlayObject): Promise {
+ return Promise.resolve(true);
+ }
+
+ alreadyScrobbled = async (playObj: PlayObject, log?: boolean): Promise => {
+ return (await this.existingScrobble(playObj)) !== undefined;
+ }
+
+}
diff --git a/src/backend/tests/scrobbler/scrobblers.test.ts b/src/backend/tests/scrobbler/scrobblers.test.ts
new file mode 100644
index 00000000..f017ba58
--- /dev/null
+++ b/src/backend/tests/scrobbler/scrobblers.test.ts
@@ -0,0 +1,113 @@
+import {describe, it} from 'mocha';
+import {assert} from 'chai';
+import clone from 'clone';
+
+import withDuration from '../plays/withDuration.json';
+import mixedDuration from '../plays/mixedDuration.json';
+
+import {TestScrobbler} from "./TestScrobbler";
+import {asPlays, generatePlay, normalizePlays} from "../utils/PlayTestUtils";
+import dayjs from "dayjs";
+
+const firstPlayDate = dayjs().subtract(1, 'hour');
+
+const withDurPlays = asPlays(withDuration);
+const mixedDurPlays = asPlays(mixedDuration);
+const normalizedWithDur = normalizePlays(withDurPlays, {initialDate: firstPlayDate});
+const normalizedWithMixedDur = normalizePlays(mixedDurPlays, {initialDate: firstPlayDate});
+
+const testScrobbler = new TestScrobbler();
+testScrobbler.lastScrobbleCheck = dayjs().subtract(60, 'seconds');
+
+describe('When scrobble is unique', function () {
+
+ it('It is not detected as duplicate when play date is newer than most recent', async function () {
+
+ testScrobbler.recentScrobbles = normalizedWithDur;
+
+ const newScrobble = generatePlay({
+ playDate: normalizedWithDur[normalizedWithDur.length - 1].data.playDate.add(70, 'seconds')
+ });
+
+ assert.isFalse(await testScrobbler.alreadyScrobbled(newScrobble));
+ });
+
+ it('It is not detected as duplicate when play date is close to an existing scrobble', async function () {
+
+ testScrobbler.recentScrobbles = normalizedWithDur;
+
+ const newScrobble = generatePlay({
+ playDate: normalizedWithDur[normalizedWithDur.length - 2].data.playDate.add(3, 'seconds')
+ });
+
+ assert.isFalse(await testScrobbler.alreadyScrobbled(newScrobble));
+ });
+});
+
+describe('When scrobble track/artist/album matches existing but is a new scrobble', function () {
+
+
+ it('Is not detected as duplicate when play date is different by more than 10 seconds (high granularity source)', async function () {
+
+ testScrobbler.recentScrobbles = normalizedWithDur;
+
+ const timeOffPos = clone(normalizedWithDur[normalizedWithDur.length - 1]);
+ timeOffPos.data.playDate = timeOffPos.data.playDate.add(11, 's');
+
+ const timeOffNeg = clone(normalizedWithDur[normalizedWithDur.length - 1]);
+ timeOffNeg.data.playDate = timeOffNeg.data.playDate.subtract(11, 's');
+
+ assert.isFalse(await testScrobbler.alreadyScrobbled(timeOffPos));
+ assert.isFalse(await testScrobbler.alreadyScrobbled(timeOffNeg));
+ });
+
+ it('Is not detected as duplicate when play date is different by more than 60 seconds (low granularity source)', async function () {
+
+ testScrobbler.recentScrobbles = normalizePlays(withDurPlays, {initialDate: firstPlayDate, defaultMeta: {source: 'subsonic'}});
+
+ const timeOffPos = clone(normalizedWithDur[normalizedWithDur.length - 1]);
+ timeOffPos.data.playDate = timeOffPos.data.playDate.add(61, 's');
+
+ const timeOffNeg = clone(normalizedWithDur[normalizedWithDur.length - 1]);
+ timeOffNeg.data.playDate = timeOffNeg.data.playDate.subtract(61, 's');
+
+ assert.isFalse(await testScrobbler.alreadyScrobbled(timeOffPos));
+ assert.isFalse(await testScrobbler.alreadyScrobbled(timeOffNeg));
+ });
+});
+
+describe('When scrobble is a duplicate', function () {
+
+ it('Is detected as duplicate when an exact match', async function () {
+ testScrobbler.recentScrobbles = normalizedWithDur;
+ assert.isTrue(await testScrobbler.alreadyScrobbled(normalizedWithDur[normalizedWithDur.length - 1]));
+ });
+
+ it('Is detected as duplicate when play date is off by less than 10 seconds (high granularity source)', async function () {
+
+ testScrobbler.recentScrobbles = normalizedWithDur;
+
+ const timeOffPos = clone(normalizedWithDur[normalizedWithDur.length - 1]);
+ timeOffPos.data.playDate = timeOffPos.data.playDate.add(10, 's');
+
+ const timeOffNeg = clone(normalizedWithDur[normalizedWithDur.length - 1]);
+ timeOffNeg.data.playDate = timeOffNeg.data.playDate.subtract(10, 's');
+
+ assert.isTrue(await testScrobbler.alreadyScrobbled(timeOffPos));
+ assert.isTrue(await testScrobbler.alreadyScrobbled(timeOffNeg));
+ });
+
+ it('Is detected as duplicate when play date is off by less than 60 seconds (low granularity source)', async function () {
+
+ testScrobbler.recentScrobbles = normalizePlays(withDurPlays, {initialDate: firstPlayDate, defaultMeta: {source: 'subsonic'}});
+
+ const timeOffPos = clone(normalizedWithDur[normalizedWithDur.length - 1]);
+ timeOffPos.data.playDate = timeOffPos.data.playDate.add(59, 's');
+
+ const timeOffNeg = clone(normalizedWithDur[normalizedWithDur.length - 1]);
+ timeOffNeg.data.playDate = timeOffNeg.data.playDate.subtract(59, 's');
+
+ assert.isTrue(await testScrobbler.alreadyScrobbled(timeOffPos));
+ assert.isTrue(await testScrobbler.alreadyScrobbled(timeOffNeg));
+ });
+});
diff --git a/src/backend/tests/utils/PlayTestUtils.ts b/src/backend/tests/utils/PlayTestUtils.ts
new file mode 100644
index 00000000..e4bf1cc7
--- /dev/null
+++ b/src/backend/tests/utils/PlayTestUtils.ts
@@ -0,0 +1,95 @@
+import {JsonPlayObject, ObjectPlayData, PlayData, PlayMeta, PlayObject} from "../../../core/Atomic";
+import dayjs, {Dayjs} from "dayjs";
+import utc from "dayjs/plugin/utc.js";
+import isBetween from "dayjs/plugin/isBetween.js";
+import relativeTime from "dayjs/plugin/relativeTime.js";
+import duration from "dayjs/plugin/duration.js";
+import timezone from "dayjs/plugin/timezone.js";
+import {faker} from '@faker-js/faker';
+
+dayjs.extend(utc)
+dayjs.extend(isBetween);
+dayjs.extend(relativeTime);
+dayjs.extend(duration);
+dayjs.extend(timezone);
+
+export const asPlays = (data: object[]): PlayObject[] => {
+ return data.map(x => {
+ const y = x as JsonPlayObject;
+ return {
+ ...y,
+ data: {
+ ...y.data,
+ playDate: dayjs(y.data.playDate)
+ }
+ }
+ });
+}
+
+export const normalizePlays = (plays: PlayObject[],
+ options?: {
+ initialDate?: Dayjs,
+ defaultDuration?: number,
+ defaultData?: ObjectPlayData,
+ defaultMeta?: PlayMeta
+ }
+): PlayObject[] => {
+ const {
+ initialDate,
+ defaultDuration = 180,
+ defaultData = {},
+ defaultMeta = {}
+ } = options || {};
+ let date = initialDate;
+ if (date === undefined) {
+ const firstDatedPlay = plays.find(x => x.data.playDate !== undefined);
+ if (firstDatedPlay !== undefined) {
+ date = firstDatedPlay.data.playDate;
+ } else {
+ throw new Error('No initial date specified and no play had a defined date');
+ }
+ }
+ let lastTrackEndsAt: Dayjs | undefined = undefined;
+ const normalizedPlays: PlayObject[] = [];
+
+ for (const play of plays) {
+
+ const cleanPlay = {...play};
+
+ if (lastTrackEndsAt === undefined) {
+ // first track
+ cleanPlay.data.playDate = date;
+ } else {
+ cleanPlay.data.playDate = lastTrackEndsAt.add(1, 'second');
+ }
+ cleanPlay.data = {
+ ...cleanPlay.data,
+ ...defaultData
+ }
+ cleanPlay.meta = {
+ ...cleanPlay.meta,
+ ...defaultMeta
+ }
+ lastTrackEndsAt = cleanPlay.data.playDate.add(cleanPlay.data.duration ?? defaultDuration, 'seconds');
+ normalizedPlays.push(cleanPlay);
+ }
+
+ return normalizedPlays;
+}
+
+export const generatePlay = (data: ObjectPlayData = {}, meta: PlayMeta = {}): PlayObject => {
+ return {
+ data: {
+ track: faker.music.songName(),
+ artists: faker.helpers.multiple(faker.person.fullName, {count: {min: 1, max: 2}}),
+ duration: faker.number.int({min: 30, max: 300}),
+ playDate: dayjs().subtract(faker.number.int({min: 1, max: 800})),
+ album: faker.music.songName(),
+ ...data
+ },
+ meta: {
+ source: faker.word.noun(),
+ ...meta,
+ }
+ }
+}
--
2.54.0
From 2bbe2e6ba0c7bb7c2c4764976cc7fa3818847447 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Wed, 27 Sep 2023 14:14:34 -0400
Subject: [PATCH 49/82] refactor(scrobble): Print score breakdown in different
statement
Is more readable
---
src/backend/scrobblers/AbstractScrobbleClient.ts | 14 ++++++++++----
1 file changed, 10 insertions(+), 4 deletions(-)
diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts
index ad577941..ced42e87 100644
--- a/src/backend/scrobblers/AbstractScrobbleClient.ts
+++ b/src/backend/scrobblers/AbstractScrobbleClient.ts
@@ -298,7 +298,7 @@ export default abstract class AbstractScrobbleClient {
}
let existingScrobble;
- let closestMatch: {score: number, breakdowns: string[], scrobble?: PlayObject} = {score: 0, breakdowns: ['None']};
+ let closestMatch: {score: number, breakdowns: string[], confidence: string, scrobble?: PlayObject} = {score: 0, breakdowns: [], confidence: 'None'};
// then check if we have already recorded this
const [existingExactSubmitted, existingDataSubmitted = []] = this.findExistingSubmittedPlayObj(playObj);
@@ -309,7 +309,8 @@ export default abstract class AbstractScrobbleClient {
closestMatch = {
score: 1,
- breakdowns: ['Exact Match found in previously successfully scrobbled']
+ breakdowns: [],
+ confidence: 'Exact Match found in previously successfully scrobbled'
}
}
// if not though then we need to check recent scrobbles from scrobble api.
@@ -363,7 +364,8 @@ export default abstract class AbstractScrobbleClient {
const scoreInfo = {
score,
scrobble: x,
- breakdowns: this.verboseOptions.match.confidenceBreakdown ? scoreBreakdowns : [confidence]
+ confidence,
+ breakdowns: scoreBreakdowns
}
if (closestMatch.score <= score && score > 0) {
@@ -375,8 +377,12 @@ export default abstract class AbstractScrobbleClient {
}
if ((existingScrobble !== undefined && this.verboseOptions.match.onMatch) || (existingScrobble === undefined && this.verboseOptions.match.onNoMatch)) {
- const closestScrobble = closestMatch.scrobble === undefined ? closestMatch.breakdowns.join(' | ') : `Closest Scrobble: ${buildTrackString(closestMatch.scrobble, scoreTrackOpts)} => ${closestMatch.breakdowns.join(' | ')}`;
+ const closestScrobble = `Closest Scrobble: ${buildTrackString(closestMatch.scrobble, scoreTrackOpts)} => ${closestMatch.confidence}`;
this.logger.debug(`(Existing Check) Source: ${buildTrackString(playObj, scoreTrackOpts)} => ${closestScrobble}`);
+ if (this.verboseOptions.match.confidenceBreakdown === true) {
+ this.logger.debug(`Breakdown:
+${closestMatch.breakdowns.join('\n')}`);
+ }
}
return existingScrobble;
}
--
2.54.0
From a953f83d07eebb94003568d8ff40f2fa69ed9218 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Wed, 27 Sep 2023 14:14:49 -0400
Subject: [PATCH 50/82] test(scrobble): Add more tests
---
src/backend/tests/plays/withDuration.json | 2 +-
.../tests/scrobbler/scrobblers.test.ts | 78 ++++++++++++++++++-
2 files changed, 78 insertions(+), 2 deletions(-)
diff --git a/src/backend/tests/plays/withDuration.json b/src/backend/tests/plays/withDuration.json
index a9a1724f..3ceb0541 100644
--- a/src/backend/tests/plays/withDuration.json
+++ b/src/backend/tests/plays/withDuration.json
@@ -17,7 +17,7 @@
"artists": [
"Kuna Maze"
],
- "track": "Jimbo",
+ "track": "Jimbó",
"duration": 255,
"playDate": "2023-09-20T14:46:00.000Z"
},
diff --git a/src/backend/tests/scrobbler/scrobblers.test.ts b/src/backend/tests/scrobbler/scrobblers.test.ts
index f017ba58..bf05853c 100644
--- a/src/backend/tests/scrobbler/scrobblers.test.ts
+++ b/src/backend/tests/scrobbler/scrobblers.test.ts
@@ -17,6 +17,12 @@ const normalizedWithDur = normalizePlays(withDurPlays, {initialDate: firstPlayDa
const normalizedWithMixedDur = normalizePlays(mixedDurPlays, {initialDate: firstPlayDate});
const testScrobbler = new TestScrobbler();
+testScrobbler.verboseOptions = {
+ match: {
+ onMatch: true,
+ onNoMatch: true,
+ confidenceBreakdown: true
+ }};
testScrobbler.lastScrobbleCheck = dayjs().subtract(60, 'seconds');
describe('When scrobble is unique', function () {
@@ -46,6 +52,28 @@ describe('When scrobble is unique', function () {
describe('When scrobble track/artist/album matches existing but is a new scrobble', function () {
+ it('Is not detected as duplicate when artist is same but track is different (similar time)', async function () {
+
+ testScrobbler.recentScrobbles = normalizedWithDur;
+
+ const diffPlay = clone(normalizedWithDur[1]);
+ diffPlay.data.playDate = diffPlay.data.playDate.add(9, 's');
+ diffPlay.data.track = 'A Totally Different Track'
+
+ assert.isFalse(await testScrobbler.alreadyScrobbled(diffPlay));
+ });
+
+ it('Is not detected as duplicate when track is same but artist is different (similar time)', async function () {
+
+ testScrobbler.recentScrobbles = normalizedWithDur;
+
+ const diffPlay = clone(normalizedWithDur[1]);
+ diffPlay.data.playDate = diffPlay.data.playDate.add(9, 's');
+ diffPlay.data.artists = ['A Different Artist'];
+
+ assert.isFalse(await testScrobbler.alreadyScrobbled(diffPlay));
+ });
+
it('Is not detected as duplicate when play date is different by more than 10 seconds (high granularity source)', async function () {
@@ -76,13 +104,47 @@ describe('When scrobble track/artist/album matches existing but is a new scrobbl
});
});
-describe('When scrobble is a duplicate', function () {
+describe('When scrobble is a duplicate (title/artists/album)', function () {
it('Is detected as duplicate when an exact match', async function () {
testScrobbler.recentScrobbles = normalizedWithDur;
assert.isTrue(await testScrobbler.alreadyScrobbled(normalizedWithDur[normalizedWithDur.length - 1]));
});
+ it('Is detected as duplicate when artist/title differences are whitespace or case', async function () {
+ testScrobbler.recentScrobbles = normalizedWithDur;
+ const ref = normalizedWithDur[3];
+
+ const diffPlay = clone(ref);
+ diffPlay.data.playDate = diffPlay.data.playDate.add(9, 's');
+
+
+ diffPlay.data.track = ref.data.track.toUpperCase();
+ assert.isTrue(await testScrobbler.alreadyScrobbled(diffPlay));
+
+ diffPlay.data.track = ` ${ref.data.track} `;
+ assert.isTrue(await testScrobbler.alreadyScrobbled(diffPlay));
+
+ diffPlay.data.track = ref.data.track.replaceAll(' ', ' ');
+ assert.isTrue(await testScrobbler.alreadyScrobbled(diffPlay));
+
+ diffPlay.data.artists = ref.data.artists.map(x => x.toUpperCase());
+ assert.isTrue(await testScrobbler.alreadyScrobbled(diffPlay));
+
+ diffPlay.data.artists = ref.data.artists.map(x => x.replaceAll(' ', ' '));
+ assert.isTrue(await testScrobbler.alreadyScrobbled(diffPlay));
+ });
+
+ it('Is detected as duplicate when artist/title differences are from unicode normalization', async function () {
+ testScrobbler.recentScrobbles = normalizedWithDur;
+ const ref = normalizedWithDur[1];
+
+ const diffPlay = clone(ref);
+ diffPlay.data.playDate = diffPlay.data.playDate.add(9, 's');
+ diffPlay.data.track = 'Jimbo';
+ assert.isTrue(await testScrobbler.alreadyScrobbled(diffPlay));
+ });
+
it('Is detected as duplicate when play date is off by less than 10 seconds (high granularity source)', async function () {
testScrobbler.recentScrobbles = normalizedWithDur;
@@ -110,4 +172,18 @@ describe('When scrobble is a duplicate', function () {
assert.isTrue(await testScrobbler.alreadyScrobbled(timeOffPos));
assert.isTrue(await testScrobbler.alreadyScrobbled(timeOffNeg));
});
+
+ describe('When existing has duration', function () {
+
+ it('Is detected as duplicate when play date is close to the end of an existing scrobble', async function () {
+
+ testScrobbler.recentScrobbles = normalizedWithDur;
+
+ const timeEnd = clone(normalizedWithDur[normalizedWithDur.length - 1]);
+ timeEnd.data.playDate = timeEnd.data.playDate.add(timeEnd.data.duration, 's');
+
+ assert.isTrue(await testScrobbler.alreadyScrobbled(timeEnd));
+ });
+
+ });
});
--
2.54.0
From 9758568ded422ffb902d8c7577f905c4f1d2eef7 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Wed, 27 Sep 2023 15:43:44 -0400
Subject: [PATCH 51/82] feat: Implement new string comparison logic
---
.../common/vendor/ListenbrainzApiClient.ts | 17 +-
src/backend/tests/strings.test.ts | 65 ++++++
src/backend/utils.ts | 132 +-----------
src/backend/utils/StringUtils.ts | 196 ++++++++++++++++++
4 files changed, 271 insertions(+), 139 deletions(-)
create mode 100644 src/backend/tests/strings.test.ts
create mode 100644 src/backend/utils/StringUtils.ts
diff --git a/src/backend/common/vendor/ListenbrainzApiClient.ts b/src/backend/common/vendor/ListenbrainzApiClient.ts
index 173621ca..9e8aa99f 100644
--- a/src/backend/common/vendor/ListenbrainzApiClient.ts
+++ b/src/backend/common/vendor/ListenbrainzApiClient.ts
@@ -1,23 +1,20 @@
import AbstractApiClient from "./AbstractApiClient";
import request, {Request} from 'superagent';
import { ListenBrainzClientData } from "../infrastructure/config/client/listenbrainz";
-import {DEFAULT_RETRY_MULTIPLIER, DELIMITERS, FormatPlayObjectOptions} from "../infrastructure/Atomic";
+import {DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions} from "../infrastructure/Atomic";
import dayjs from "dayjs";
import { stringSameness } from '@foxxmd/string-sameness';
import {
combinePartsToString,
- containsDelimiters,
- findDelimiters,
- normalizeStr,
- parseArtistCredits,
- parseCredits,
- parseStringList,
- parseTrackCredits,
- unique,
- uniqueNormalizedStrArr,
} from "../../utils";
import { PlayObject } from "../../../core/Atomic";
import { slice } from "../../../core/StringUtils";
+import {
+ findDelimiters,
+ normalizeStr, parseArtistCredits, parseCredits,
+ parseTrackCredits,
+ uniqueNormalizedStrArr
+} from "../../utils/StringUtils";
export interface ArtistMBIDMapping {
diff --git a/src/backend/tests/strings.test.ts b/src/backend/tests/strings.test.ts
new file mode 100644
index 00000000..75a44e41
--- /dev/null
+++ b/src/backend/tests/strings.test.ts
@@ -0,0 +1,65 @@
+import {describe, it} from 'mocha';
+import {assert} from 'chai';
+import {compareNormalizedStrings} from "../utils/StringUtils";
+
+
+describe('String Comparisons', function () {
+
+ it('should ignore punctuation', async function () {
+ const result = compareNormalizedStrings('this string! is the. same', 'this string is the same');
+ assert.isAtLeast(result.highScore, 100);
+ });
+
+ it('should ignore whitespace', async function () {
+ const result = compareNormalizedStrings('this string is the same ', 'this string is the same');
+ assert.isAtLeast(result.highScore, 100);
+ });
+
+ it('should ignore case', async function () {
+ const result = compareNormalizedStrings('ThIs STRING iS THe SAMe', 'this string is the same');
+ assert.isAtLeast( result.highScore, 100);
+ });
+
+ it('should normalize unicode', async function () {
+ const tests = [
+ ['Enamorate Bailando','Enamórate Bailando'],
+ ['Dina Ogon', 'Dina Ögon'],
+ ['Nana', 'Nanã'],
+ ['Nana', 'Nanä']
+ ]
+
+ for(const test of tests) {
+ const result = compareNormalizedStrings(test[0], test[1]);
+ assert.equal(100, result.highScore);
+ }
+ });
+
+ it('should score small changes correctly', async function () {
+ const tests = [
+ ['there is change', 'therr is change'],
+ ['laser cannon', 'lazer cannon'],
+ ['Monster', 'Manster'],
+ ['What are you doing', 'hwat are you doing']
+ ]
+
+ for(const test of tests) {
+ const result = compareNormalizedStrings(test[0], test[1]);
+ assert.isAtLeast( result.highScoreWeighted, 80, `Comparing: '${test[0]}' | '${test[1]}'`);
+ }
+ });
+
+ it('should score word bags correctly', async function () {
+ const tests = [
+ ['there change is', 'there is change'],
+ ['laser cannon', 'cannon laser'],
+ ['One two three four', 'One four two threee'],
+ ['you, doin hwat are', 'hwat are you doing'],
+ ['My Track (feat. Tourist1, Artist2)', 'My Track (feat. Artist2, Tourist1)']
+ ]
+
+ for(const test of tests) {
+ const result = compareNormalizedStrings(test[0], test[1]);
+ assert.isAtLeast( result.highScoreWeighted, 80, `Comparing: '${test[0]}' | '${test[1]}'`);
+ }
+ });
+});
diff --git a/src/backend/utils.ts b/src/backend/utils.ts
index b3ce4e95..b54c952b 100644
--- a/src/backend/utils.ts
+++ b/src/backend/utils.ts
@@ -7,8 +7,8 @@ import {TimeoutError, WebapiError} from "spotify-web-api-node/src/response-error
import Ajv, {Schema} from 'ajv';
import {
asPlayerStateData,
- DEFAULT_SCROBBLE_DURATION_THRESHOLD, DEFAULT_SCROBBLE_PERCENT_THRESHOLD,
- DELIMITERS,
+ DEFAULT_SCROBBLE_DURATION_THRESHOLD,
+ DEFAULT_SCROBBLE_PERCENT_THRESHOLD,
lowGranularitySources,
NO_DEVICE,
NO_USER,
@@ -24,7 +24,7 @@ import {Request} from "express";
import pathUtil from "path";
import {ErrorWithCause} from "pony-cause";
import backoffStrategies from '@kenyip/backoff-strategies';
-import {ScrobbleThresholds} from "./common/infrastructure/config/source/index";
+import {ScrobbleThresholds} from "./common/infrastructure/config/source";
import {replaceResultTransformer, stripIndentTransformer, TemplateTag, trimResultTransformer} from 'common-tags';
import {Duration} from "dayjs/plugin/duration.js";
import {ListenRange, PlayObject} from "../core/Atomic";
@@ -149,22 +149,6 @@ export const unique = (arr: T[]): T[] => {
return Array.from(new Set(arr))
}
-export const PUNCTUATION_WHITESPACE_REGEX = new RegExp(/[^\w\d]/g);
-export const uniqueNormalizedStrArr = (arr: string[]): string[] => {
- return arr.reduce((acc: string[], curr) => {
- const normalizedCurr = normalizeStr(curr)
- if (!acc.some(x => normalizeStr(x) === normalizedCurr)) {
- return acc.concat(curr);
- }
- return acc;
- }, []);
-}
-
-// https://stackoverflow.com/a/37511463/1469797
-export const normalizeStr = (str: string): string => {
- return str.normalize('NFD').replace(/[\u0300-\u036f]/g, "").replace(PUNCTUATION_WHITESPACE_REGEX, '').toLocaleLowerCase();
-}
-
export const returnDuplicateStrings = (arr: any) => {
const alreadySeen: any = [];
const dupes: any = [];
@@ -744,99 +728,6 @@ export const pollingBackoff = (attempt: number, scaleFactor: number = 1): number
return Math.round(backoffStrat(attempt + 1) / 1000);
}
-export interface PlayCredits {
- primary: string
- secondary?: string[]
-}
-/**
- * For matching the most common track/artist pattern that has a joiner
- *
- * Primary ft. 2nd Artist, 3rd Artist
- * Primary (2nd Artist)
- * Primary [featuring 2nd Artist]
- *
- * ____
- *
- * => Primary may or may not exist
- * => Primary must not have an opening character ( [
- * => Secondaries may or may not have an opening character ( [
- * => MUST begin with joiner ft. feat. featuring with vs.
- * => May have closing character ) ]
- * */
-export const SECONDARY_ARTISTS_SECTION_REGEX = new RegExp(/^(?[^(\[]*)?(?[(\[]?(?\Wft\.?|\Wfeat\.?|featuring|\Wvs\.?) (?[^)\]]*)(?:[)\]]|\s*)$)/i);
-// export const SECONDARY_ARTISTS_REGEX = new RegExp(//ig);
-export const parseCredits = (str: string, delimiters?: boolean | string[]): PlayCredits => {
- if(str.trim() === '') {
- return undefined;
- }
- let primary: string | undefined;
- let secondary: string[] = [];
- const results = parseRegexSingleOrFail(SECONDARY_ARTISTS_SECTION_REGEX, str);
- if(results !== undefined) {
- primary = results.named.primary !== undefined ? results.named.primary.trim() : undefined;
- let delims: string[] | undefined;
- if(Array.isArray(delimiters)) {
- delims = delimiters;
- } else if(delimiters === false) {
- delims = [];
- }
- secondary = parseStringList(results.named.secondaryArtists as string, delims)
- return {
- primary,
- secondary
- };
- }
- return undefined;
-}
-
-export const parseArtistCredits = (str: string, delimiters?: boolean | string[]): PlayCredits | undefined => {
- if(str.trim() === '') {
- return undefined;
- }
- let delims: string[] | undefined;
- if(Array.isArray(delimiters)) {
- delims = delimiters;
- } else if(delimiters === false) {
- delims = [];
- }
- const withJoiner = parseCredits(str, delimiters);
- if(withJoiner !== undefined) {
- // all this does is make sure and "ft" or parenthesis/brackets are separated --
- // it doesn't also separate primary artists so do that now
- const primaries = parseStringList(withJoiner.primary, delims);
- if(primaries.length > 1) {
- return {
- primary: primaries[0],
- secondary: primaries.slice(1).concat(withJoiner.secondary)
- }
- }
- return withJoiner;
- }
- // likely this is a plain string with just delims
- const artists = parseStringList(str, delims);
- if(artists.length > 1) {
- return {
- primary: artists[0],
- secondary: artists.slice(1)
- }
- }
- return {
- primary: artists[0]
- }
-}
-
-export const parseTrackCredits = (str: string, delimiters?: boolean | string[]): PlayCredits | undefined => parseCredits(str, delimiters);
-
-export const parseStringList = (str: string, delimiters: string[] = [',', '&', '/', '\\']): string[] => {
- if(delimiters.length === 0) {
- return [str];
- }
- return delimiters.reduce((acc: string[], curr: string) => {
- const explodedStrings = acc.map(x => x.split(curr));
- return explodedStrings.flat(1);
- }, [str]).map(x => x.trim());
-}
-
export const parseRegex = (reg: RegExp, val: string): RegExResult[] | undefined => {
if (reg.global) {
@@ -877,23 +768,6 @@ export const parseRegexSingleOrFail = (reg: RegExp, val: string): RegExResult |
return undefined;
}
-export const containsDelimiters = (str: string) => {
- return null !== str.match(/[,&\/\\]+/i);
-}
-
-export const findDelimiters = (str: string) => {
- const found: string[] = [];
- for(const d of DELIMITERS) {
- if(str.indexOf(d) !== -1) {
- found.push(d);
- }
- }
- if(found.length === 0) {
- return undefined;
- }
- return found;
-}
-
export const intersect = (a: Array, b: Array) => {
const setA = new Set(a);
const setB = new Set(b);
diff --git a/src/backend/utils/StringUtils.ts b/src/backend/utils/StringUtils.ts
new file mode 100644
index 00000000..6ca9cf66
--- /dev/null
+++ b/src/backend/utils/StringUtils.ts
@@ -0,0 +1,196 @@
+import {DELIMITERS} from "../common/infrastructure/Atomic";
+import {parseRegexSingleOrFail} from "../utils";
+import {PlayObject} from "../../core/Atomic";
+import {stringSameness, StringSamenessResult} from "@foxxmd/string-sameness";
+// @ts-ignore
+import {levenStrategy} from "@foxxmd/string-sameness/strategies/leven"
+// @ts-ignore
+import {diceStrategy} from "@foxxmd/string-sameness/strategies/deice";
+
+export const PUNCTUATION_WHITESPACE_REGEX = new RegExp(/[^\w\d]/g);
+export const PUNCTUATION_REGEX = new RegExp(/[^\w\d\s]/g);
+
+export const MULTI_WHITESPACE_REGEX = new RegExp(/\s{2,}/g);
+export const uniqueNormalizedStrArr = (arr: string[]): string[] => {
+ return arr.reduce((acc: string[], curr) => {
+ const normalizedCurr = normalizeStr(curr)
+ if (!acc.some(x => normalizeStr(x) === normalizedCurr)) {
+ return acc.concat(curr);
+ }
+ return acc;
+ }, []);
+}
+// https://stackoverflow.com/a/37511463/1469797
+export const normalizeStr = (str: string, options?: {keepSingleWhitespace?: boolean}): string => {
+ const {keepSingleWhitespace = false} = options;
+ const normal = str.normalize('NFD').replace(/[\u0300-\u036f]/g, "");
+ if(!keepSingleWhitespace) {
+ return normal.replace(PUNCTUATION_WHITESPACE_REGEX, '').toLocaleLowerCase();
+ }
+ return normal.replace(PUNCTUATION_REGEX, '').replace(MULTI_WHITESPACE_REGEX, ' ').toLocaleLowerCase().trim();
+}
+
+export interface PlayCredits {
+ primary: string
+ secondary?: string[]
+}
+
+/**
+ * For matching the most common track/artist pattern that has a joiner
+ *
+ * Primary ft. 2nd Artist, 3rd Artist
+ * Primary (2nd Artist)
+ * Primary [featuring 2nd Artist]
+ *
+ * ____
+ *
+ * => Primary may or may not exist
+ * => Primary must not have an opening character ( [
+ * => Secondaries may or may not have an opening character ( [
+ * => MUST begin with joiner ft. feat. featuring with vs.
+ * => May have closing character ) ]
+ * */
+export const SECONDARY_ARTISTS_SECTION_REGEX = new RegExp(/^(?[^(\[]*)?(?[(\[]?(?\Wft\.?|\Wfeat\.?|featuring|\Wvs\.?) (?[^)\]]*)(?:[)\]]|\s*)$)/i);
+// export const SECONDARY_ARTISTS_REGEX = new RegExp(//ig);
+export const parseCredits = (str: string, delimiters?: boolean | string[]): PlayCredits => {
+ if (str.trim() === '') {
+ return undefined;
+ }
+ let primary: string | undefined;
+ let secondary: string[] = [];
+ const results = parseRegexSingleOrFail(SECONDARY_ARTISTS_SECTION_REGEX, str);
+ if (results !== undefined) {
+ primary = results.named.primary !== undefined ? results.named.primary.trim() : undefined;
+ let delims: string[] | undefined;
+ if (Array.isArray(delimiters)) {
+ delims = delimiters;
+ } else if (delimiters === false) {
+ delims = [];
+ }
+ secondary = parseStringList(results.named.secondaryArtists as string, delims)
+ return {
+ primary,
+ secondary
+ };
+ }
+ return undefined;
+}
+export const parseArtistCredits = (str: string, delimiters?: boolean | string[]): PlayCredits | undefined => {
+ if (str.trim() === '') {
+ return undefined;
+ }
+ let delims: string[] | undefined;
+ if (Array.isArray(delimiters)) {
+ delims = delimiters;
+ } else if (delimiters === false) {
+ delims = [];
+ }
+ const withJoiner = parseCredits(str, delimiters);
+ if (withJoiner !== undefined) {
+ // all this does is make sure and "ft" or parenthesis/brackets are separated --
+ // it doesn't also separate primary artists so do that now
+ const primaries = parseStringList(withJoiner.primary, delims);
+ if (primaries.length > 1) {
+ return {
+ primary: primaries[0],
+ secondary: primaries.slice(1).concat(withJoiner.secondary)
+ }
+ }
+ return withJoiner;
+ }
+ // likely this is a plain string with just delims
+ const artists = parseStringList(str, delims);
+ if (artists.length > 1) {
+ return {
+ primary: artists[0],
+ secondary: artists.slice(1)
+ }
+ }
+ return {
+ primary: artists[0]
+ }
+}
+export const parseTrackCredits = (str: string, delimiters?: boolean | string[]): PlayCredits | undefined => parseCredits(str, delimiters);
+export const parseStringList = (str: string, delimiters: string[] = [',', '&', '/', '\\']): string[] => {
+ if (delimiters.length === 0) {
+ return [str];
+ }
+ return delimiters.reduce((acc: string[], curr: string) => {
+ const explodedStrings = acc.map(x => x.split(curr));
+ return explodedStrings.flat(1);
+ }, [str]).map(x => x.trim());
+}
+export const containsDelimiters = (str: string) => {
+ return null !== str.match(/[,&\/\\]+/i);
+}
+export const findDelimiters = (str: string) => {
+ const found: string[] = [];
+ for (const d of DELIMITERS) {
+ if (str.indexOf(d) !== -1) {
+ found.push(d);
+ }
+ }
+ if (found.length === 0) {
+ return undefined;
+ }
+ return found;
+}
+
+export const compareScrobbleTracks = (existing: PlayObject, candidate: PlayObject): number => {
+ const {
+ data: {
+ track: existingTrack,
+ } = {},
+ } = existing;
+
+ const {
+ data: {
+ track: candidateTrack
+ }
+ } = candidate;
+
+ return compareNormalizedStrings(existingTrack, candidateTrack).highScoreWeighted;
+}
+
+export const compareNormalizedStrings = (existing: string, candidate: string): StringSamenessResult => {
+
+ const normalExisting = normalizeStr(existing, {keepSingleWhitespace: true});
+ const normalCandidate = normalizeStr(candidate, {keepSingleWhitespace: true});
+
+ // there may be scenarios where a track differs in *ordering* of ancillary information between sources
+ // EX My Track (feat. Art1, Art2) -- My Track (feat. Art2 Art1)
+ // so instead of naively comparing the entire track string against the candidate we
+ // * first try to match up all white-space separated tokens
+ // * recombine with closest tokens in order
+ // * then check sameness
+ const eTokens = normalExisting.split(' ');
+ const cTokens = normalCandidate.split(' ');
+
+ const orderedCandidateTokens = eTokens.reduce((acc: { ordered: string[], remaining: string[] }, curr) => {
+ let highScore = 0;
+ let highIndex = undefined;
+ let index = 0;
+ for (const token of acc.remaining) {
+ const result = stringSameness(curr, token);
+ if (result.highScoreWeighted > highScore) {
+ highScore = result.highScoreWeighted;
+ highIndex = index;
+ }
+ index++;
+ }
+
+ const splicedRemaining = [...acc.remaining];
+ splicedRemaining.splice(highIndex, 1);
+
+ return {ordered: acc.ordered.concat(acc.remaining[highIndex]), remaining: splicedRemaining};
+ }, {ordered: [], remaining: cTokens});
+
+ const allOrderedCandidateTokens = orderedCandidateTokens.ordered.concat(orderedCandidateTokens.remaining);
+ const orderedCandidateString = allOrderedCandidateTokens.join(' ');
+
+ // since we have already "matched" up words by order we don't want to use cosine strat
+ // bc it only does comparisons between whole words in a sentence (instead of all letters in a string)
+ // which makes it inaccurate for small-n sentences and typos
+
+ return stringSameness(normalExisting, orderedCandidateString, {transforms: [], strategies: [levenStrategy, diceStrategy]});
+}
--
2.54.0
From 159901f79e250d2567f24517416b134c3f630fd5 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Wed, 27 Sep 2023 15:44:22 -0400
Subject: [PATCH 52/82] refactor(tests): Move jellyfin/listenbrainz test into
own folders
---
.../tests/{ => jellyfin}/jellyfin.test.ts | 8 ++++----
.../{ => listenbrainz}/listenbrainz.test.ts | 20 +++++++++----------
2 files changed, 14 insertions(+), 14 deletions(-)
rename src/backend/tests/{ => jellyfin}/jellyfin.test.ts (83%)
rename src/backend/tests/{ => listenbrainz}/listenbrainz.test.ts (81%)
diff --git a/src/backend/tests/jellyfin.test.ts b/src/backend/tests/jellyfin/jellyfin.test.ts
similarity index 83%
rename from src/backend/tests/jellyfin.test.ts
rename to src/backend/tests/jellyfin/jellyfin.test.ts
index 2485e205..71894025 100644
--- a/src/backend/tests/jellyfin.test.ts
+++ b/src/backend/tests/jellyfin/jellyfin.test.ts
@@ -1,11 +1,11 @@
import {describe, it} from 'mocha';
import {assert} from 'chai';
-import samplePayload from './jellyfin/playbackProgressSample.json';
+import samplePayload from './playbackProgressSample.json';
-import JellyfinSource from "../sources/JellyfinSource";
+import JellyfinSource from "../../sources/JellyfinSource";
import EventEmitter from "events";
-import {getLogger} from "../common/logging";
-import {JsonPlayObject, PlayObject} from "../../core/Atomic";
+import {getLogger} from "../../common/logging";
+import {JsonPlayObject, PlayObject} from "../../../core/Atomic";
const dataAsFixture = (data: any): TestFixture => {
return data as TestFixture;
diff --git a/src/backend/tests/listenbrainz.test.ts b/src/backend/tests/listenbrainz/listenbrainz.test.ts
similarity index 81%
rename from src/backend/tests/listenbrainz.test.ts
rename to src/backend/tests/listenbrainz/listenbrainz.test.ts
index 35fb54e0..99479e06 100644
--- a/src/backend/tests/listenbrainz.test.ts
+++ b/src/backend/tests/listenbrainz/listenbrainz.test.ts
@@ -1,19 +1,19 @@
import {describe, it} from 'mocha';
import {assert} from 'chai';
// correct mappings
-import multiArtistInArtistName from './listenbrainz/correctlyMapped/multiArtistInArtistName.json';
-import multiArtistsInTrackName from './listenbrainz/correctlyMapped/multiArtistInTrackName.json';
-import noArtistMapping from './listenbrainz/correctlyMapped/noArtistMapping.json';
-import multiMappedArtistsWithSingleUserArtist from './listenbrainz/correctlyMapped/multiArtistMappingWithSingleRecordedArtist.json';
-import artistWithProperJoiner from './listenbrainz/correctlyMapped/artistProperHasJoinerInName.json';
-import normalizedValues from './listenbrainz/correctlyMapped/normalizedName.json';
-import slightlyDifferentNames from './listenbrainz/correctlyMapped/trackNameSlightlyDifferent.json';
+import multiArtistInArtistName from './correctlyMapped/multiArtistInArtistName.json';
+import multiArtistsInTrackName from './correctlyMapped/multiArtistInTrackName.json';
+import noArtistMapping from './correctlyMapped/noArtistMapping.json';
+import multiMappedArtistsWithSingleUserArtist from './correctlyMapped/multiArtistMappingWithSingleRecordedArtist.json';
+import artistWithProperJoiner from './correctlyMapped/artistProperHasJoinerInName.json';
+import normalizedValues from './correctlyMapped/normalizedName.json';
+import slightlyDifferentNames from './correctlyMapped/trackNameSlightlyDifferent.json';
// incorrect mappings
-import incorrectMultiArtistsTrackName from './listenbrainz/incorrectlyMapped/multiArtistsInTrackName.json';
-import veryWrong from './listenbrainz/incorrectlyMapped/veryWrong.json';
+import incorrectMultiArtistsTrackName from './incorrectlyMapped/multiArtistsInTrackName.json';
+import veryWrong from './incorrectlyMapped/veryWrong.json';
-import {ListenbrainzApiClient, ListenResponse} from "../common/vendor/ListenbrainzApiClient";
+import {ListenbrainzApiClient, ListenResponse} from "../../common/vendor/ListenbrainzApiClient";
interface ExpectedResults {
artists: string[]
--
2.54.0
From 9512ca7e84771288dad914d0d49de1b83cca4027 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Wed, 27 Sep 2023 16:01:09 -0400
Subject: [PATCH 53/82] feat(scrobble): Use new string approach for track
And move weights to constants
---
src/backend/common/infrastructure/Atomic.ts | 6 +++
.../scrobblers/AbstractScrobbleClient.ts | 49 +++++++------------
2 files changed, 23 insertions(+), 32 deletions(-)
diff --git a/src/backend/common/infrastructure/Atomic.ts b/src/backend/common/infrastructure/Atomic.ts
index e551104e..048744f4 100644
--- a/src/backend/common/infrastructure/Atomic.ts
+++ b/src/backend/common/infrastructure/Atomic.ts
@@ -206,3 +206,9 @@ export interface numberFormatOptions {
}
export const DELIMITERS = [',','&','\/','\\'];
+
+export const DUP_SCORE_THRESHOLD = 0.7;
+export const ARTIST_WEIGHT = 0.2;
+export const TITLE_WEIGHT = 0.3;
+export const TIME_WEIGHT = 0.5;
+export const REFERENCE_WEIGHT = 0.5;
diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts
index ced42e87..ffb97ebe 100644
--- a/src/backend/scrobblers/AbstractScrobbleClient.ts
+++ b/src/backend/scrobblers/AbstractScrobbleClient.ts
@@ -6,13 +6,14 @@ import {
setIntersection, sortByOldestPlayDate,
} from "../utils";
import {
- ClientType,
+ ARTIST_WEIGHT,
+ ClientType, DUP_SCORE_THRESHOLD,
FormatPlayObjectOptions,
INITIALIZED,
INITIALIZING,
InitState,
- NOT_INITIALIZED,
- ScrobbledPlayObject,
+ NOT_INITIALIZED, REFERENCE_WEIGHT,
+ ScrobbledPlayObject, TIME_WEIGHT, TITLE_WEIGHT,
} from "../common/infrastructure/Atomic";
import winston, {Logger} from '@foxxmd/winston';
import { CommonClientConfig } from "../common/infrastructure/config/client/index";
@@ -22,6 +23,7 @@ import {FixedSizeList} from 'fixed-size-list';
import { PlayObject, TrackStringOptions } from "../../core/Atomic";
import {buildTrackString, capitalize, truncateStringToLength} from "../../core/StringUtils";
import EventEmitter from "events";
+import {compareScrobbleTracks} from "../utils/StringUtils";
export default abstract class AbstractScrobbleClient {
@@ -246,24 +248,7 @@ export default abstract class AbstractScrobbleClient {
return [closeTime, fuzzyTime];
}
protected compareExistingScrobbleTitle = (existing: PlayObject, candidate: PlayObject): number => {
-
- const {
- data: {
- track: scrobbleTitle,
- } = {},
- } = existing;
-
- let cleanSourceTitle = this.cleanSourceSearchTitle(candidate);
-
- let titleMatch;
- const lowerScrobbleTitle = scrobbleTitle.toLocaleLowerCase().trim();
- // because of all this replacing we need a more position-agnostic way of comparing titles so use intersection on title split by spaces
- // and compare against length of scrobble title
- const sourceTitleTerms = new Set(cleanSourceTitle.split(' ').filter((x: any) => x !== ''));
- const commonTerms = setIntersection(new Set(lowerScrobbleTitle.split(' ')), sourceTitleTerms);
-
- titleMatch = commonTerms.size / sourceTitleTerms.size;
- return titleMatch;
+ return Math.min(compareScrobbleTracks(existing, candidate)/100, 1);
}
protected compareExistingScrobbleArtist = (existing: PlayObject, candidate: PlayObject): number => {
@@ -345,21 +330,21 @@ export default abstract class AbstractScrobbleClient {
const artistMatch = this.compareExistingScrobbleArtist(x, playObj);
- const artistScore = .2 * artistMatch;
- const titleScore = .3 * titleMatch;
- const timeScore = .5 * (closeTime ? 1 : (fuzzyTime ? 0.5 : 0));
- const referenceScore = .5 * (referenceMatch ? 1 : 0);
+ const artistScore = ARTIST_WEIGHT * artistMatch;
+ const titleScore = TITLE_WEIGHT * titleMatch;
+ const timeScore = TIME_WEIGHT * (closeTime ? 1 : (fuzzyTime ? 0.5 : 0));
+ const referenceScore = REFERENCE_WEIGHT * (referenceMatch ? 1 : 0);
const score = artistScore + titleScore + timeScore;
let scoreBreakdowns = [
- `Reference: ${(referenceMatch ? 1 : 0)} * .5 = ${referenceScore.toFixed(2)}`,
- `Artist ${artistMatch.toFixed(2)} * .2 = ${artistScore.toFixed(2)}`,
- `Title: ${titleMatch.toFixed(2)} * .3 = ${titleScore.toFixed(2)}`,
- `Time: ${closeTime ? 1 : 0} * .5 = ${timeScore.toFixed(2)}`,
- `Score ${score.toFixed(2)} => ${score >= .7 ? 'Matched!' : 'No Match'}`
+ `Reference: ${(referenceMatch ? 1 : 0)} * ${REFERENCE_WEIGHT} = ${referenceScore.toFixed(2)}`,
+ `Artist ${artistMatch.toFixed(2)} * ${ARTIST_WEIGHT} = ${artistScore.toFixed(2)}`,
+ `Title: ${titleMatch.toFixed(2)} * ${TITLE_WEIGHT} = ${titleScore.toFixed(2)}`,
+ `Time: ${closeTime ? 1 : (fuzzyTime ? 0.5 : 0)} * ${TIME_WEIGHT} = ${timeScore.toFixed(2)}`,
+ `Score ${score.toFixed(2)} => ${score >= DUP_SCORE_THRESHOLD ? 'Matched!' : 'No Match'}`
];
- const confidence = `Score ${score.toFixed(2)} => ${score >= .7 ? 'Matched!' : 'No Match'}`
+ const confidence = `Score ${score.toFixed(2)} => ${score >= DUP_SCORE_THRESHOLD ? 'Matched!' : 'No Match'}`
const scoreInfo = {
score,
@@ -372,7 +357,7 @@ export default abstract class AbstractScrobbleClient {
closestMatch = scoreInfo
}
- return score >= .7;
+ return score >= DUP_SCORE_THRESHOLD;
});
}
--
2.54.0
From 521130562d97b758760eff148e016050b06a9325 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Thu, 28 Sep 2023 09:55:09 -0400
Subject: [PATCH 54/82] feat(scrobble): Use new string approach for artists
---
.../scrobblers/AbstractScrobbleClient.ts | 19 ++-----------------
src/backend/utils/StringUtils.ts | 16 ++++++++++++++++
2 files changed, 18 insertions(+), 17 deletions(-)
diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts
index ffb97ebe..ff288530 100644
--- a/src/backend/scrobblers/AbstractScrobbleClient.ts
+++ b/src/backend/scrobblers/AbstractScrobbleClient.ts
@@ -23,7 +23,7 @@ import {FixedSizeList} from 'fixed-size-list';
import { PlayObject, TrackStringOptions } from "../../core/Atomic";
import {buildTrackString, capitalize, truncateStringToLength} from "../../core/StringUtils";
import EventEmitter from "events";
-import {compareScrobbleTracks} from "../utils/StringUtils";
+import {compareScrobbleArtists, compareScrobbleTracks} from "../utils/StringUtils";
export default abstract class AbstractScrobbleClient {
@@ -252,22 +252,7 @@ export default abstract class AbstractScrobbleClient {
}
protected compareExistingScrobbleArtist = (existing: PlayObject, candidate: PlayObject): number => {
- const {
- data: {
- artists: sourceArtists = [],
- } = {},
- } = existing;
- const {
- data: {
- artists = [],
- } = {},
- } = candidate;
- let artistMatch;
- const lowerSourceArtists = sourceArtists.map((x: any) => x.toLocaleLowerCase());
- const lowerScrobbleArtists = artists.map(x => x.toLocaleLowerCase());
- artistMatch = setIntersection(new Set(lowerScrobbleArtists), new Set(lowerSourceArtists)).size / artists.length;
-
- return artistMatch;
+ return Math.min(compareScrobbleArtists(existing, candidate)/100, 1)
}
existingScrobble = async (playObj: PlayObject) => {
diff --git a/src/backend/utils/StringUtils.ts b/src/backend/utils/StringUtils.ts
index 6ca9cf66..5937fdb4 100644
--- a/src/backend/utils/StringUtils.ts
+++ b/src/backend/utils/StringUtils.ts
@@ -152,6 +152,22 @@ export const compareScrobbleTracks = (existing: PlayObject, candidate: PlayObjec
return compareNormalizedStrings(existingTrack, candidateTrack).highScoreWeighted;
}
+export const compareScrobbleArtists = (existing: PlayObject, candidate: PlayObject): number => {
+ const {
+ data: {
+ artists: existingArtists = [],
+ } = {},
+ } = existing;
+
+ const {
+ data: {
+ artists: candidateArtists = [],
+ }
+ } = candidate;
+
+ return compareNormalizedStrings(existingArtists.reduce((acc, curr) => `${acc} ${curr}`, ''), candidateArtists.reduce((acc, curr) => `${acc} ${curr}`, '')).highScoreWeighted;
+}
+
export const compareNormalizedStrings = (existing: string, candidate: string): StringSamenessResult => {
const normalExisting = normalizeStr(existing, {keepSingleWhitespace: true});
--
2.54.0
From 77c9977d2706aa2aeb972bd25f212b1cbe2ce124 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Thu, 28 Sep 2023 10:29:34 -0400
Subject: [PATCH 55/82] Use string highScore instead of weighted since cosine
isn't messing things up anymore
---
src/backend/utils/StringUtils.ts | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/backend/utils/StringUtils.ts b/src/backend/utils/StringUtils.ts
index 5937fdb4..9bb09bf3 100644
--- a/src/backend/utils/StringUtils.ts
+++ b/src/backend/utils/StringUtils.ts
@@ -149,7 +149,7 @@ export const compareScrobbleTracks = (existing: PlayObject, candidate: PlayObjec
}
} = candidate;
- return compareNormalizedStrings(existingTrack, candidateTrack).highScoreWeighted;
+ return compareNormalizedStrings(existingTrack, candidateTrack).highScore;
}
export const compareScrobbleArtists = (existing: PlayObject, candidate: PlayObject): number => {
@@ -165,7 +165,7 @@ export const compareScrobbleArtists = (existing: PlayObject, candidate: PlayObje
}
} = candidate;
- return compareNormalizedStrings(existingArtists.reduce((acc, curr) => `${acc} ${curr}`, ''), candidateArtists.reduce((acc, curr) => `${acc} ${curr}`, '')).highScoreWeighted;
+ return compareNormalizedStrings(existingArtists.reduce((acc, curr) => `${acc} ${curr}`, ''), candidateArtists.reduce((acc, curr) => `${acc} ${curr}`, '')).highScore;
}
export const compareNormalizedStrings = (existing: string, candidate: string): StringSamenessResult => {
--
2.54.0
From 1ca7834feb0c5cd55e7fe709b9d3d1bd4394c20a Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Thu, 28 Sep 2023 10:31:06 -0400
Subject: [PATCH 56/82] test(scrobble): Add more tests for similar-but-unique
scrobble detection
---
src/backend/tests/plays/withDuration.json | 54 +++++++++++++++++++
.../tests/scrobbler/scrobblers.test.ts | 31 +++++++++--
2 files changed, 82 insertions(+), 3 deletions(-)
diff --git a/src/backend/tests/plays/withDuration.json b/src/backend/tests/plays/withDuration.json
index 3ceb0541..65a4b085 100644
--- a/src/backend/tests/plays/withDuration.json
+++ b/src/backend/tests/plays/withDuration.json
@@ -39,6 +39,20 @@
"source": "Maloja"
}
},
+ {
+ "data": {
+ "artists": [
+ "Frédéric Chopin",
+ "Krystian Zimerman"
+ ],
+ "track": "Ballade No. 4 in F Minor, Op. 52",
+ "duration": 575,
+ "playDate": "2023-09-20T15:17:12.000Z"
+ },
+ "meta": {
+ "source": "Maloja"
+ }
+ },
{
"data": {
"artists": [
@@ -52,6 +66,33 @@
"source": "Maloja"
}
},
+ {
+ "data": {
+ "artists": [
+ "Pink Floyd"
+ ],
+ "track": "Another Brick in the Wall, Pt. 1",
+ "duration": 192,
+ "playDate": "2023-09-20T15:39:12.000Z"
+ },
+ "meta": {
+ "source": "Maloja"
+ }
+ },
+ {
+ "data": {
+ "artists": [
+ "Frédéric Chopin",
+ "Krystian Zimerman"
+ ],
+ "track": "Ballade No. 1 in G Minor, Op. 23",
+ "duration": 575,
+ "playDate": "2023-09-20T15:41:12.000Z"
+ },
+ "meta": {
+ "source": "Maloja"
+ }
+ },
{
"data": {
"artists": [
@@ -66,5 +107,18 @@
"meta": {
"source": "Maloja"
}
+ },
+ {
+ "data": {
+ "artists": [
+ "Outkast"
+ ],
+ "track": "Da Art of Storytellin' (Pt. 1)",
+ "duration": 422,
+ "playDate": "2023-09-20T15:39:12.000Z"
+ },
+ "meta": {
+ "source": "Maloja"
+ }
}
]
diff --git a/src/backend/tests/scrobbler/scrobblers.test.ts b/src/backend/tests/scrobbler/scrobblers.test.ts
index bf05853c..ffc61a70 100644
--- a/src/backend/tests/scrobbler/scrobblers.test.ts
+++ b/src/backend/tests/scrobbler/scrobblers.test.ts
@@ -43,7 +43,7 @@ describe('When scrobble is unique', function () {
testScrobbler.recentScrobbles = normalizedWithDur;
const newScrobble = generatePlay({
- playDate: normalizedWithDur[normalizedWithDur.length - 2].data.playDate.add(3, 'seconds')
+ playDate: normalizedWithDur[normalizedWithDur.length - 3].data.playDate.add(3, 'seconds')
});
assert.isFalse(await testScrobbler.alreadyScrobbled(newScrobble));
@@ -52,7 +52,7 @@ describe('When scrobble is unique', function () {
describe('When scrobble track/artist/album matches existing but is a new scrobble', function () {
- it('Is not detected as duplicate when artist is same but track is different (similar time)', async function () {
+ it('Is not detected as duplicate when artist is same, time is similar, but track is different', async function () {
testScrobbler.recentScrobbles = normalizedWithDur;
@@ -63,7 +63,7 @@ describe('When scrobble track/artist/album matches existing but is a new scrobbl
assert.isFalse(await testScrobbler.alreadyScrobbled(diffPlay));
});
- it('Is not detected as duplicate when track is same but artist is different (similar time)', async function () {
+ it('Is not detected as duplicate when track is same, time is similar, but artist is different', async function () {
testScrobbler.recentScrobbles = normalizedWithDur;
@@ -102,6 +102,31 @@ describe('When scrobble track/artist/album matches existing but is a new scrobbl
assert.isFalse(await testScrobbler.alreadyScrobbled(timeOffPos));
assert.isFalse(await testScrobbler.alreadyScrobbled(timeOffNeg));
});
+
+ it('A track with continuity to the previous track is not detected as a duplicate', async function () {
+
+ testScrobbler.recentScrobbles = normalizedWithDur;
+
+ const brickPt1 = normalizedWithDur.find(x => x.data.track.includes('Another Brick'));
+ const brickPt2 = clone(brickPt1);
+ brickPt2.data.track = 'Another Brick in the Wall, Pt. 2';
+ brickPt2.data.playDate = brickPt1.data.playDate.add(brickPt1.data.duration + 1, 'seconds');
+ assert.isFalse(await testScrobbler.alreadyScrobbled(brickPt2));
+
+ const story1 = normalizedWithDur.find(x => x.data.track.includes('Da Art of'));
+ const story2 = clone(story1);
+ story2.data.track = `Da Art of Storytellin' (Pt. 2)`;
+ story2.data.playDate = story2.data.playDate.add(story1.data.duration + 1, 'seconds');
+
+ assert.isFalse(await testScrobbler.alreadyScrobbled(story2));
+
+ const ballad1 = normalizedWithDur.find(x => x.data.track.includes('Ballade No. 1'));
+ const ballad2 = clone(ballad1);
+ ballad2.data.track = `Ballade No. 2 in G Minor, Op. 27`;
+ ballad2.data.playDate = ballad2.data.playDate.add(ballad1.data.duration + 1, 'seconds');
+
+ assert.isFalse(await testScrobbler.alreadyScrobbled(ballad2));
+ });
});
describe('When scrobble is a duplicate (title/artists/album)', function () {
--
2.54.0
From c6d38e40d0ce780427c0a92570ca2d1531ac4d80 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Thu, 28 Sep 2023 10:31:45 -0400
Subject: [PATCH 57/82] refactor(scrobble): Adjust dup score weights
---
src/backend/common/infrastructure/Atomic.ts | 6 +++---
src/backend/scrobblers/AbstractScrobbleClient.ts | 2 +-
src/backend/tests/strings.test.ts | 15 +++++++++++++--
3 files changed, 17 insertions(+), 6 deletions(-)
diff --git a/src/backend/common/infrastructure/Atomic.ts b/src/backend/common/infrastructure/Atomic.ts
index 048744f4..01448ffc 100644
--- a/src/backend/common/infrastructure/Atomic.ts
+++ b/src/backend/common/infrastructure/Atomic.ts
@@ -207,8 +207,8 @@ export interface numberFormatOptions {
export const DELIMITERS = [',','&','\/','\\'];
-export const DUP_SCORE_THRESHOLD = 0.7;
-export const ARTIST_WEIGHT = 0.2;
-export const TITLE_WEIGHT = 0.3;
+export const ARTIST_WEIGHT = 0.3;
+export const TITLE_WEIGHT = 0.4;
export const TIME_WEIGHT = 0.5;
export const REFERENCE_WEIGHT = 0.5;
+export const DUP_SCORE_THRESHOLD = 1;
diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts
index ff288530..1e3f5053 100644
--- a/src/backend/scrobblers/AbstractScrobbleClient.ts
+++ b/src/backend/scrobblers/AbstractScrobbleClient.ts
@@ -317,7 +317,7 @@ export default abstract class AbstractScrobbleClient {
const artistScore = ARTIST_WEIGHT * artistMatch;
const titleScore = TITLE_WEIGHT * titleMatch;
- const timeScore = TIME_WEIGHT * (closeTime ? 1 : (fuzzyTime ? 0.5 : 0));
+ const timeScore = TIME_WEIGHT * (closeTime ? 1 : (fuzzyTime ? 0.6 : 0));
const referenceScore = REFERENCE_WEIGHT * (referenceMatch ? 1 : 0);
const score = artistScore + titleScore + timeScore;
diff --git a/src/backend/tests/strings.test.ts b/src/backend/tests/strings.test.ts
index 75a44e41..b4ceb9cc 100644
--- a/src/backend/tests/strings.test.ts
+++ b/src/backend/tests/strings.test.ts
@@ -44,7 +44,7 @@ describe('String Comparisons', function () {
for(const test of tests) {
const result = compareNormalizedStrings(test[0], test[1]);
- assert.isAtLeast( result.highScoreWeighted, 80, `Comparing: '${test[0]}' | '${test[1]}'`);
+ assert.isAtLeast( result.highScore, 75, `Comparing: '${test[0]}' | '${test[1]}'`);
}
});
@@ -59,7 +59,18 @@ describe('String Comparisons', function () {
for(const test of tests) {
const result = compareNormalizedStrings(test[0], test[1]);
- assert.isAtLeast( result.highScoreWeighted, 80, `Comparing: '${test[0]}' | '${test[1]}'`);
+ assert.isAtLeast( result.highScore, 80, `Comparing: '${test[0]}' | '${test[1]}'`);
+ }
+ });
+
+ it('should not score very similar as identical', async function () {
+ const tests = [
+ ['Another Brick in the Wall, Pt. 1', 'Another Brick in the Wall, Pt. 2'],
+ ]
+
+ for(const test of tests) {
+ const result = compareNormalizedStrings(test[0], test[1]);
+ assert.isAtMost( result.highScore, 99, `Comparing: '${test[0]}' | '${test[1]}'`);
}
});
});
--
2.54.0
From 3b624205add569c2562f80a452a329c874c23dcf Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Thu, 28 Sep 2023 10:35:38 -0400
Subject: [PATCH 58/82] fix: Fix missing destructure default value
---
src/backend/utils/StringUtils.ts | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/backend/utils/StringUtils.ts b/src/backend/utils/StringUtils.ts
index 9bb09bf3..77dc6169 100644
--- a/src/backend/utils/StringUtils.ts
+++ b/src/backend/utils/StringUtils.ts
@@ -22,7 +22,7 @@ export const uniqueNormalizedStrArr = (arr: string[]): string[] => {
}
// https://stackoverflow.com/a/37511463/1469797
export const normalizeStr = (str: string, options?: {keepSingleWhitespace?: boolean}): string => {
- const {keepSingleWhitespace = false} = options;
+ const {keepSingleWhitespace = false} = options || {};
const normal = str.normalize('NFD').replace(/[\u0300-\u036f]/g, "");
if(!keepSingleWhitespace) {
return normal.replace(PUNCTUATION_WHITESPACE_REGEX, '').toLocaleLowerCase();
--
2.54.0
From a82aafa6b80757b908dec1b1acb1b896f8d4973e Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Thu, 28 Sep 2023 10:47:07 -0400
Subject: [PATCH 59/82] test(scrobble): Add test for duplicate when exact
title/time
---
.../tests/scrobbler/scrobblers.test.ts | 52 ++++++++++++-------
1 file changed, 34 insertions(+), 18 deletions(-)
diff --git a/src/backend/tests/scrobbler/scrobblers.test.ts b/src/backend/tests/scrobbler/scrobblers.test.ts
index ffc61a70..c4eb2cc0 100644
--- a/src/backend/tests/scrobbler/scrobblers.test.ts
+++ b/src/backend/tests/scrobbler/scrobblers.test.ts
@@ -103,30 +103,33 @@ describe('When scrobble track/artist/album matches existing but is a new scrobbl
assert.isFalse(await testScrobbler.alreadyScrobbled(timeOffNeg));
});
- it('A track with continuity to the previous track is not detected as a duplicate', async function () {
+ describe('When existing has duration', function () {
+ it('A track with continuity to the previous track is not detected as a duplicate', async function () {
- testScrobbler.recentScrobbles = normalizedWithDur;
+ testScrobbler.recentScrobbles = normalizedWithDur;
- const brickPt1 = normalizedWithDur.find(x => x.data.track.includes('Another Brick'));
- const brickPt2 = clone(brickPt1);
- brickPt2.data.track = 'Another Brick in the Wall, Pt. 2';
- brickPt2.data.playDate = brickPt1.data.playDate.add(brickPt1.data.duration + 1, 'seconds');
- assert.isFalse(await testScrobbler.alreadyScrobbled(brickPt2));
+ const brickPt1 = normalizedWithDur.find(x => x.data.track.includes('Another Brick'));
+ const brickPt2 = clone(brickPt1);
+ brickPt2.data.track = 'Another Brick in the Wall, Pt. 2';
+ brickPt2.data.playDate = brickPt1.data.playDate.add(brickPt1.data.duration + 1, 'seconds');
+ assert.isFalse(await testScrobbler.alreadyScrobbled(brickPt2));
- const story1 = normalizedWithDur.find(x => x.data.track.includes('Da Art of'));
- const story2 = clone(story1);
- story2.data.track = `Da Art of Storytellin' (Pt. 2)`;
- story2.data.playDate = story2.data.playDate.add(story1.data.duration + 1, 'seconds');
+ const story1 = normalizedWithDur.find(x => x.data.track.includes('Da Art of'));
+ const story2 = clone(story1);
+ story2.data.track = `Da Art of Storytellin' (Pt. 2)`;
+ story2.data.playDate = story2.data.playDate.add(story1.data.duration + 1, 'seconds');
- assert.isFalse(await testScrobbler.alreadyScrobbled(story2));
+ assert.isFalse(await testScrobbler.alreadyScrobbled(story2));
- const ballad1 = normalizedWithDur.find(x => x.data.track.includes('Ballade No. 1'));
- const ballad2 = clone(ballad1);
- ballad2.data.track = `Ballade No. 2 in G Minor, Op. 27`;
- ballad2.data.playDate = ballad2.data.playDate.add(ballad1.data.duration + 1, 'seconds');
+ const ballad1 = normalizedWithDur.find(x => x.data.track.includes('Ballade No. 1'));
+ const ballad2 = clone(ballad1);
+ ballad2.data.track = `Ballade No. 2 in G Minor, Op. 27`;
+ ballad2.data.playDate = ballad2.data.playDate.add(ballad1.data.duration + 1, 'seconds');
- assert.isFalse(await testScrobbler.alreadyScrobbled(ballad2));
+ assert.isFalse(await testScrobbler.alreadyScrobbled(ballad2));
+ });
});
+
});
describe('When scrobble is a duplicate (title/artists/album)', function () {
@@ -162,7 +165,7 @@ describe('When scrobble is a duplicate (title/artists/album)', function () {
it('Is detected as duplicate when artist/title differences are from unicode normalization', async function () {
testScrobbler.recentScrobbles = normalizedWithDur;
- const ref = normalizedWithDur[1];
+ const ref = normalizedWithDur.find(x => x.data.track === 'Jimbó');
const diffPlay = clone(ref);
diffPlay.data.playDate = diffPlay.data.playDate.add(9, 's');
@@ -198,6 +201,19 @@ describe('When scrobble is a duplicate (title/artists/album)', function () {
assert.isTrue(await testScrobbler.alreadyScrobbled(timeOffNeg));
});
+ it('Is detected as duplicate when title is exact, artist is similar, and time is similar', async function () {
+ testScrobbler.recentScrobbles = normalizedWithDur;
+ const ref = normalizedWithDur[3];
+
+ const diffPlay = clone(ref);
+ diffPlay.data.playDate = diffPlay.data.playDate.add(3, 's');
+ diffPlay.data.artists = [diffPlay.data.artists[0]]
+ assert.isTrue(await testScrobbler.alreadyScrobbled(diffPlay));
+
+ diffPlay.data.artists = [ref.data.artists[1]]
+ assert.isTrue(await testScrobbler.alreadyScrobbled(diffPlay));
+ });
+
describe('When existing has duration', function () {
it('Is detected as duplicate when play date is close to the end of an existing scrobble', async function () {
--
2.54.0
From f0cee72f43df81732c462c1a661e83966afbf279 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Thu, 28 Sep 2023 10:55:17 -0400
Subject: [PATCH 60/82] test(scrobble): Improve test durability by using mixed
duration sources
---
.../scrobblers/AbstractScrobbleClient.ts | 2 +-
src/backend/tests/plays/mixedDuration.json | 96 ++++++++++++++-----
.../tests/scrobbler/scrobblers.test.ts | 58 +++++------
3 files changed, 103 insertions(+), 53 deletions(-)
diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts
index 1e3f5053..5010bdad 100644
--- a/src/backend/scrobblers/AbstractScrobbleClient.ts
+++ b/src/backend/scrobblers/AbstractScrobbleClient.ts
@@ -257,7 +257,7 @@ export default abstract class AbstractScrobbleClient {
existingScrobble = async (playObj: PlayObject) => {
const tr = truncateStringToLength(27);
- const scoreTrackOpts: TrackStringOptions = {include: ['track', 'time'], transformers: {track: (t: any) => tr(t).padEnd(30)}};
+ const scoreTrackOpts: TrackStringOptions = {include: ['track', 'artist', 'time'], transformers: {track: (t: any, data, existing) => `${existing ? '- ': ''}${tr(t)}`}};
// return early if we don't care about checking existing
if (false === this.checkExistingScrobbles) {
diff --git a/src/backend/tests/plays/mixedDuration.json b/src/backend/tests/plays/mixedDuration.json
index 7cfaadf5..00f40cde 100644
--- a/src/backend/tests/plays/mixedDuration.json
+++ b/src/backend/tests/plays/mixedDuration.json
@@ -2,11 +2,11 @@
{
"data": {
"artists": [
- "Menagerie"
+ "Kaidi Tatham"
],
- "track": "Danieda's Dance",
- "duration": null,
- "playDate": "2023-09-18T19:00:25.000Z"
+ "track": "Fricassee",
+ "duration": 305,
+ "playDate": "2023-09-20T14:32:39.000Z"
},
"meta": {
"source": "Maloja"
@@ -15,13 +15,10 @@
{
"data": {
"artists": [
- "Dan Tenor City",
- "Ricoh",
- "Tom Funk"
+ "Kuna Maze"
],
- "track": "Close To The Source",
- "duration": null,
- "playDate": "2023-09-18T20:17:58.000Z"
+ "track": "Jimbó",
+ "playDate": "2023-09-20T14:46:00.000Z"
},
"meta": {
"source": "Maloja"
@@ -30,11 +27,12 @@
{
"data": {
"artists": [
- "medline"
+ "Norman Person",
+ "Shamek Farrah"
],
- "track": "Run For Cover",
- "duration": 300,
- "playDate": "2023-09-20T14:02:05.000Z"
+ "track": "Aisha",
+ "duration": 559,
+ "playDate": "2023-09-20T15:10:09.000Z"
},
"meta": {
"source": "Maloja"
@@ -43,12 +41,12 @@
{
"data": {
"artists": [
- "King Owusu",
- "Zeitgeist Freedom Energy Exchange"
+ "Frédéric Chopin",
+ "Krystian Zimerman"
],
- "track": "Obana",
- "duration": null,
- "playDate": "2023-09-20T14:07:02.000Z"
+ "track": "Ballade No. 4 in F Minor, Op. 52",
+ "duration": 575,
+ "playDate": "2023-09-20T15:17:12.000Z"
},
"meta": {
"source": "Maloja"
@@ -57,11 +55,63 @@
{
"data": {
"artists": [
- "Noro Morales"
+ "Cheo Feliciano"
],
- "track": "Mississippi Mambo",
- "duration": 159,
- "playDate": "2023-09-20T17:49:26.000Z"
+ "track": "Aprieta (Oye Cómo Va)",
+ "playDate": "2023-09-20T15:38:12.000Z"
+ },
+ "meta": {
+ "source": "Maloja"
+ }
+ },
+ {
+ "data": {
+ "artists": [
+ "Pink Floyd"
+ ],
+ "track": "Another Brick in the Wall, Pt. 1",
+ "playDate": "2023-09-20T15:39:12.000Z"
+ },
+ "meta": {
+ "source": "Maloja"
+ }
+ },
+ {
+ "data": {
+ "artists": [
+ "Frédéric Chopin",
+ "Krystian Zimerman"
+ ],
+ "track": "Ballade No. 1 in G Minor, Op. 23",
+ "duration": 575,
+ "playDate": "2023-09-20T15:41:12.000Z"
+ },
+ "meta": {
+ "source": "Maloja"
+ }
+ },
+ {
+ "data": {
+ "artists": [
+ "A Collection of Boogie",
+ "Jazz Funk",
+ "Disco with Mehdi El"
+ ],
+ "track": "Aquil",
+ "playDate": "2023-09-20T17:55:05.000Z"
+ },
+ "meta": {
+ "source": "Maloja"
+ }
+ },
+ {
+ "data": {
+ "artists": [
+ "Outkast"
+ ],
+ "track": "Da Art of Storytellin' (Pt. 1)",
+ "duration": 422,
+ "playDate": "2023-09-20T15:39:12.000Z"
},
"meta": {
"source": "Maloja"
diff --git a/src/backend/tests/scrobbler/scrobblers.test.ts b/src/backend/tests/scrobbler/scrobblers.test.ts
index c4eb2cc0..a79edf64 100644
--- a/src/backend/tests/scrobbler/scrobblers.test.ts
+++ b/src/backend/tests/scrobbler/scrobblers.test.ts
@@ -29,10 +29,10 @@ describe('When scrobble is unique', function () {
it('It is not detected as duplicate when play date is newer than most recent', async function () {
- testScrobbler.recentScrobbles = normalizedWithDur;
+ testScrobbler.recentScrobbles = normalizedWithMixedDur;
const newScrobble = generatePlay({
- playDate: normalizedWithDur[normalizedWithDur.length - 1].data.playDate.add(70, 'seconds')
+ playDate: normalizedWithMixedDur[normalizedWithMixedDur.length - 1].data.playDate.add(70, 'seconds')
});
assert.isFalse(await testScrobbler.alreadyScrobbled(newScrobble));
@@ -40,10 +40,10 @@ describe('When scrobble is unique', function () {
it('It is not detected as duplicate when play date is close to an existing scrobble', async function () {
- testScrobbler.recentScrobbles = normalizedWithDur;
+ testScrobbler.recentScrobbles = normalizedWithMixedDur;
const newScrobble = generatePlay({
- playDate: normalizedWithDur[normalizedWithDur.length - 3].data.playDate.add(3, 'seconds')
+ playDate: normalizedWithMixedDur[normalizedWithMixedDur.length - 3].data.playDate.add(3, 'seconds')
});
assert.isFalse(await testScrobbler.alreadyScrobbled(newScrobble));
@@ -54,9 +54,9 @@ describe('When scrobble track/artist/album matches existing but is a new scrobbl
it('Is not detected as duplicate when artist is same, time is similar, but track is different', async function () {
- testScrobbler.recentScrobbles = normalizedWithDur;
+ testScrobbler.recentScrobbles = normalizedWithMixedDur;
- const diffPlay = clone(normalizedWithDur[1]);
+ const diffPlay = clone(normalizedWithMixedDur[1]);
diffPlay.data.playDate = diffPlay.data.playDate.add(9, 's');
diffPlay.data.track = 'A Totally Different Track'
@@ -65,9 +65,9 @@ describe('When scrobble track/artist/album matches existing but is a new scrobbl
it('Is not detected as duplicate when track is same, time is similar, but artist is different', async function () {
- testScrobbler.recentScrobbles = normalizedWithDur;
+ testScrobbler.recentScrobbles = normalizedWithMixedDur;
- const diffPlay = clone(normalizedWithDur[1]);
+ const diffPlay = clone(normalizedWithMixedDur[1]);
diffPlay.data.playDate = diffPlay.data.playDate.add(9, 's');
diffPlay.data.artists = ['A Different Artist'];
@@ -77,12 +77,12 @@ describe('When scrobble track/artist/album matches existing but is a new scrobbl
it('Is not detected as duplicate when play date is different by more than 10 seconds (high granularity source)', async function () {
- testScrobbler.recentScrobbles = normalizedWithDur;
+ testScrobbler.recentScrobbles = normalizedWithMixedDur;
- const timeOffPos = clone(normalizedWithDur[normalizedWithDur.length - 1]);
+ const timeOffPos = clone(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]);
timeOffPos.data.playDate = timeOffPos.data.playDate.add(11, 's');
- const timeOffNeg = clone(normalizedWithDur[normalizedWithDur.length - 1]);
+ const timeOffNeg = clone(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]);
timeOffNeg.data.playDate = timeOffNeg.data.playDate.subtract(11, 's');
assert.isFalse(await testScrobbler.alreadyScrobbled(timeOffPos));
@@ -91,12 +91,12 @@ describe('When scrobble track/artist/album matches existing but is a new scrobbl
it('Is not detected as duplicate when play date is different by more than 60 seconds (low granularity source)', async function () {
- testScrobbler.recentScrobbles = normalizePlays(withDurPlays, {initialDate: firstPlayDate, defaultMeta: {source: 'subsonic'}});
+ testScrobbler.recentScrobbles = normalizePlays(mixedDurPlays, {initialDate: firstPlayDate, defaultMeta: {source: 'subsonic'}});
- const timeOffPos = clone(normalizedWithDur[normalizedWithDur.length - 1]);
+ const timeOffPos = clone(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]);
timeOffPos.data.playDate = timeOffPos.data.playDate.add(61, 's');
- const timeOffNeg = clone(normalizedWithDur[normalizedWithDur.length - 1]);
+ const timeOffNeg = clone(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]);
timeOffNeg.data.playDate = timeOffNeg.data.playDate.subtract(61, 's');
assert.isFalse(await testScrobbler.alreadyScrobbled(timeOffPos));
@@ -135,13 +135,13 @@ describe('When scrobble track/artist/album matches existing but is a new scrobbl
describe('When scrobble is a duplicate (title/artists/album)', function () {
it('Is detected as duplicate when an exact match', async function () {
- testScrobbler.recentScrobbles = normalizedWithDur;
- assert.isTrue(await testScrobbler.alreadyScrobbled(normalizedWithDur[normalizedWithDur.length - 1]));
+ testScrobbler.recentScrobbles = normalizedWithMixedDur;
+ assert.isTrue(await testScrobbler.alreadyScrobbled(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]));
});
it('Is detected as duplicate when artist/title differences are whitespace or case', async function () {
- testScrobbler.recentScrobbles = normalizedWithDur;
- const ref = normalizedWithDur[3];
+ testScrobbler.recentScrobbles = normalizedWithMixedDur;
+ const ref = normalizedWithMixedDur[3];
const diffPlay = clone(ref);
diffPlay.data.playDate = diffPlay.data.playDate.add(9, 's');
@@ -164,8 +164,8 @@ describe('When scrobble is a duplicate (title/artists/album)', function () {
});
it('Is detected as duplicate when artist/title differences are from unicode normalization', async function () {
- testScrobbler.recentScrobbles = normalizedWithDur;
- const ref = normalizedWithDur.find(x => x.data.track === 'Jimbó');
+ testScrobbler.recentScrobbles = normalizedWithMixedDur;
+ const ref = normalizedWithMixedDur.find(x => x.data.track === 'Jimbó');
const diffPlay = clone(ref);
diffPlay.data.playDate = diffPlay.data.playDate.add(9, 's');
@@ -175,12 +175,12 @@ describe('When scrobble is a duplicate (title/artists/album)', function () {
it('Is detected as duplicate when play date is off by less than 10 seconds (high granularity source)', async function () {
- testScrobbler.recentScrobbles = normalizedWithDur;
+ testScrobbler.recentScrobbles = normalizedWithMixedDur;
- const timeOffPos = clone(normalizedWithDur[normalizedWithDur.length - 1]);
+ const timeOffPos = clone(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]);
timeOffPos.data.playDate = timeOffPos.data.playDate.add(10, 's');
- const timeOffNeg = clone(normalizedWithDur[normalizedWithDur.length - 1]);
+ const timeOffNeg = clone(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]);
timeOffNeg.data.playDate = timeOffNeg.data.playDate.subtract(10, 's');
assert.isTrue(await testScrobbler.alreadyScrobbled(timeOffPos));
@@ -189,12 +189,12 @@ describe('When scrobble is a duplicate (title/artists/album)', function () {
it('Is detected as duplicate when play date is off by less than 60 seconds (low granularity source)', async function () {
- testScrobbler.recentScrobbles = normalizePlays(withDurPlays, {initialDate: firstPlayDate, defaultMeta: {source: 'subsonic'}});
+ testScrobbler.recentScrobbles = normalizePlays(mixedDurPlays, {initialDate: firstPlayDate, defaultMeta: {source: 'subsonic'}});
- const timeOffPos = clone(normalizedWithDur[normalizedWithDur.length - 1]);
+ const timeOffPos = clone(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]);
timeOffPos.data.playDate = timeOffPos.data.playDate.add(59, 's');
- const timeOffNeg = clone(normalizedWithDur[normalizedWithDur.length - 1]);
+ const timeOffNeg = clone(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]);
timeOffNeg.data.playDate = timeOffNeg.data.playDate.subtract(59, 's');
assert.isTrue(await testScrobbler.alreadyScrobbled(timeOffPos));
@@ -202,8 +202,8 @@ describe('When scrobble is a duplicate (title/artists/album)', function () {
});
it('Is detected as duplicate when title is exact, artist is similar, and time is similar', async function () {
- testScrobbler.recentScrobbles = normalizedWithDur;
- const ref = normalizedWithDur[3];
+ testScrobbler.recentScrobbles = normalizedWithMixedDur;
+ const ref = normalizedWithMixedDur[3];
const diffPlay = clone(ref);
diffPlay.data.playDate = diffPlay.data.playDate.add(3, 's');
@@ -220,7 +220,7 @@ describe('When scrobble is a duplicate (title/artists/album)', function () {
testScrobbler.recentScrobbles = normalizedWithDur;
- const timeEnd = clone(normalizedWithDur[normalizedWithDur.length - 1]);
+ const timeEnd = clone(normalizedWithDur[normalizedWithMixedDur.length - 1]);
timeEnd.data.playDate = timeEnd.data.playDate.add(timeEnd.data.duration, 's');
assert.isTrue(await testScrobbler.alreadyScrobbled(timeEnd));
--
2.54.0
From 0ec8d8814d39f46072bb50031ab304f07d62e229 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Thu, 28 Sep 2023 11:25:33 -0400
Subject: [PATCH 61/82] fix(scrobble): Fix not using reference duration for
fuzzy comparison
---
src/backend/tests/scrobbler/scrobblers.test.ts | 12 +++++++++++-
src/backend/utils.ts | 2 +-
2 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/src/backend/tests/scrobbler/scrobblers.test.ts b/src/backend/tests/scrobbler/scrobblers.test.ts
index a79edf64..de757c1c 100644
--- a/src/backend/tests/scrobbler/scrobblers.test.ts
+++ b/src/backend/tests/scrobbler/scrobblers.test.ts
@@ -214,7 +214,7 @@ describe('When scrobble is a duplicate (title/artists/album)', function () {
assert.isTrue(await testScrobbler.alreadyScrobbled(diffPlay));
});
- describe('When existing has duration', function () {
+ describe('When at least one play has duration', function () {
it('Is detected as duplicate when play date is close to the end of an existing scrobble', async function () {
@@ -224,6 +224,16 @@ describe('When scrobble is a duplicate (title/artists/album)', function () {
timeEnd.data.playDate = timeEnd.data.playDate.add(timeEnd.data.duration, 's');
assert.isTrue(await testScrobbler.alreadyScrobbled(timeEnd));
+
+ // only one has duration
+ timeEnd.data.duration = undefined;
+
+ assert.isTrue(await testScrobbler.alreadyScrobbled(timeEnd));
+
+ // only one has duration
+ timeEnd.data.duration = undefined;
+
+ assert.isTrue(await testScrobbler.alreadyScrobbled(timeEnd));
});
});
diff --git a/src/backend/utils.ts b/src/backend/utils.ts
index b54c952b..ad025e69 100644
--- a/src/backend/utils.ts
+++ b/src/backend/utils.ts
@@ -535,7 +535,7 @@ export const comparePlayTemporally = (existingPlay: PlayObject, candidatePlay: P
// if the source has a duration its possible one play was scrobbled at the beginning of the track and the other at the end
// so check if the duration matches the diff between the two play dates
if (result.close === false && referenceDuration !== undefined && fuzzyDuration) {
- const fuzzyDiff = Math.abs(scrobblePlayDiff - newDuration);
+ const fuzzyDiff = Math.abs(scrobblePlayDiff - referenceDuration);
result.date.fuzzyDiff = fuzzyDiff;
if(fuzzyDiff < 10) { // TODO use finer comparison for this?
result.close = true;
--
2.54.0
From 249a606bf563e0272d3b9f84e5cb9fe9a321f2a0 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Thu, 28 Sep 2023 12:19:11 -0400
Subject: [PATCH 62/82] feat(scrobble): Implement artist dup bonus scoring
heuristic
When looking for dups and finding a fuzzy timestamp match, close title, and mismatch (but some matched) multiple artist -- we give a bonus to artist score since its likely a dup.
---
.../scrobblers/AbstractScrobbleClient.ts | 70 ++++++++++++++-----
src/backend/tests/plays/mixedDuration.json | 14 ++++
src/backend/tests/plays/withDuration.json | 14 ++++
.../tests/scrobbler/scrobblers.test.ts | 8 +++
src/backend/utils.ts | 15 ++++
5 files changed, 105 insertions(+), 16 deletions(-)
diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts
index 5010bdad..3f26f0f0 100644
--- a/src/backend/scrobblers/AbstractScrobbleClient.ts
+++ b/src/backend/scrobblers/AbstractScrobbleClient.ts
@@ -1,5 +1,6 @@
import dayjs, {Dayjs} from "dayjs";
import {
+ comparingMultipleArtists,
isPlayTemporallyClose,
mergeArr,
playObjDataMatch,
@@ -23,7 +24,7 @@ import {FixedSizeList} from 'fixed-size-list';
import { PlayObject, TrackStringOptions } from "../../core/Atomic";
import {buildTrackString, capitalize, truncateStringToLength} from "../../core/StringUtils";
import EventEmitter from "events";
-import {compareScrobbleArtists, compareScrobbleTracks} from "../utils/StringUtils";
+import {compareScrobbleArtists, compareScrobbleTracks, normalizeStr} from "../utils/StringUtils";
export default abstract class AbstractScrobbleClient {
@@ -251,8 +252,22 @@ export default abstract class AbstractScrobbleClient {
return Math.min(compareScrobbleTracks(existing, candidate)/100, 1);
}
- protected compareExistingScrobbleArtist = (existing: PlayObject, candidate: PlayObject): number => {
- return Math.min(compareScrobbleArtists(existing, candidate)/100, 1)
+ protected compareExistingScrobbleArtist = (existing: PlayObject, candidate: PlayObject): [number, number] => {
+ const {
+ data: {
+ artists: existingArtists = [],
+ } = {}
+ } = existing;
+ const {
+ data: {
+ artists: candidateArtists = [],
+ } = {}
+ } = candidate;
+ const normExisting = existingArtists.map(x => normalizeStr(x, {keepSingleWhitespace: true}));
+ const candidateExisting = candidateArtists.map(x => normalizeStr(x, {keepSingleWhitespace: true}));
+
+ const wholeMatches = setIntersection(new Set(normExisting), new Set(candidateExisting)).size;
+ return [Math.min(compareScrobbleArtists(existing, candidate)/100, 1), wholeMatches]
}
existingScrobble = async (playObj: PlayObject) => {
@@ -297,35 +312,58 @@ export default abstract class AbstractScrobbleClient {
return undefined;
}
- // we have have found an existing submission but without an exact date
+ // we have found an existing submission but without an exact date
// in which case we can check the scrobble api response against recent scrobbles (also from api) for a more accurate comparison
const referenceApiScrobbleResponse = existingDataSubmitted.length > 0 ? existingDataSubmitted[0].scrobble : undefined;
- // clean source title so it matches title from the scrobble api response as closely as we can get it
- let cleanSourceTitle = this.cleanSourceSearchTitle(playObj);
-
existingScrobble = this.recentScrobbles.find((x) => {
- const referenceMatch = referenceApiScrobbleResponse !== undefined && playObjDataMatch(x, referenceApiScrobbleResponse);
+ //const referenceMatch = referenceApiScrobbleResponse !== undefined && playObjDataMatch(x, referenceApiScrobbleResponse);
const [closeTime, fuzzyTime = false] = this.compareExistingScrobbleTime(x, playObj);
+ const timeMatch = (closeTime ? 1 : (fuzzyTime ? 0.6 : 0));
const titleMatch = this.compareExistingScrobbleTitle(x, playObj);
- const artistMatch = this.compareExistingScrobbleArtist(x, playObj);
+ const [artistMatch, wholeMatches] = this.compareExistingScrobbleArtist(x, playObj);
- const artistScore = ARTIST_WEIGHT * artistMatch;
+ let artistScore = ARTIST_WEIGHT * artistMatch;
const titleScore = TITLE_WEIGHT * titleMatch;
- const timeScore = TIME_WEIGHT * (closeTime ? 1 : (fuzzyTime ? 0.6 : 0));
- const referenceScore = REFERENCE_WEIGHT * (referenceMatch ? 1 : 0);
- const score = artistScore + titleScore + timeScore;
+ const timeScore = TIME_WEIGHT * timeMatch;
+ //const referenceScore = REFERENCE_WEIGHT * (referenceMatch ? 1 : 0);
+ let score = artistScore + titleScore + timeScore;
+
+ let artistWholeMatchBonus = 0;
+ let artistBreakdown = `Artist: ${artistMatch.toFixed(2)} * ${ARTIST_WEIGHT} = ${artistScore.toFixed(2)}`;
+
+ if(score < 1 && timeMatch > 0 && titleMatch > 0.98 && artistMatch > 0.1 && wholeMatches > 0 && comparingMultipleArtists(x, playObj)) {
+ // address scenario where:
+ // * title is very close
+ // * time falls within plausible dup range
+ // * artist is not totally different
+ // * AND score is still not high enough for a dup
+ //
+ // if we detect the plays have multiple artists and we have at least one whole match (stricter comparison than regular score)
+ // then bump artist score a little to see if it gets it over the fence
+ //
+ // EX: Source: The Bongo Hop - Sonora @ 2023-09-28T10:54:06-04:00 => Closest Scrobble: Nidia Gongora / The Bongo Hop - Sonora @ 2023-09-28T10:59:34-04:00 => Score 0.83 => No Match
+ // one play is only returning primary artist, and timestamp is at beginning instead of end of play
+
+ const scoreBonus = artistMatch * 0.5;
+ const scoreGapBonus = (1 - artistMatch) * 0.75;
+ // use the smallest bump or 0.1
+ artistWholeMatchBonus = Math.max(scoreBonus, scoreGapBonus, 0.1);
+ artistScore = (ARTIST_WEIGHT + 0.05) * (artistMatch + artistWholeMatchBonus);
+ score = artistScore + titleScore + timeScore;
+ artistBreakdown = `Artist: (${artistMatch.toFixed(2)} + Whole Match Bonus ${artistWholeMatchBonus.toFixed(2)}) * (${ARTIST_WEIGHT} + Whole Match Bonus 0.05) = ${artistScore.toFixed(2)}`;
+ }
let scoreBreakdowns = [
- `Reference: ${(referenceMatch ? 1 : 0)} * ${REFERENCE_WEIGHT} = ${referenceScore.toFixed(2)}`,
- `Artist ${artistMatch.toFixed(2)} * ${ARTIST_WEIGHT} = ${artistScore.toFixed(2)}`,
+ //`Reference: ${(referenceMatch ? 1 : 0)} * ${REFERENCE_WEIGHT} = ${referenceScore.toFixed(2)}`,
+ artistBreakdown,
`Title: ${titleMatch.toFixed(2)} * ${TITLE_WEIGHT} = ${titleScore.toFixed(2)}`,
- `Time: ${closeTime ? 1 : (fuzzyTime ? 0.5 : 0)} * ${TIME_WEIGHT} = ${timeScore.toFixed(2)}`,
+ `Time: ${timeMatch} * ${TIME_WEIGHT} = ${timeScore.toFixed(2)}`,
`Score ${score.toFixed(2)} => ${score >= DUP_SCORE_THRESHOLD ? 'Matched!' : 'No Match'}`
];
diff --git a/src/backend/tests/plays/mixedDuration.json b/src/backend/tests/plays/mixedDuration.json
index 00f40cde..e0e553e2 100644
--- a/src/backend/tests/plays/mixedDuration.json
+++ b/src/backend/tests/plays/mixedDuration.json
@@ -52,6 +52,20 @@
"source": "Maloja"
}
},
+ {
+ "data": {
+ "artists": [
+ "Nidia Gongora",
+ "The Bongo Hop"
+ ],
+ "track": "Sonora",
+ "duration": 327,
+ "playDate": "2023-09-27T20:02:17.000Z"
+ },
+ "meta": {
+ "source": "Maloja"
+ }
+ },
{
"data": {
"artists": [
diff --git a/src/backend/tests/plays/withDuration.json b/src/backend/tests/plays/withDuration.json
index 65a4b085..0ff0c912 100644
--- a/src/backend/tests/plays/withDuration.json
+++ b/src/backend/tests/plays/withDuration.json
@@ -53,6 +53,20 @@
"source": "Maloja"
}
},
+ {
+ "data": {
+ "artists": [
+ "Nidia Gongora",
+ "The Bongo Hop"
+ ],
+ "track": "Sonora",
+ "duration": 327,
+ "playDate": "2023-09-27T20:02:17.000Z"
+ },
+ "meta": {
+ "source": "Maloja"
+ }
+ },
{
"data": {
"artists": [
diff --git a/src/backend/tests/scrobbler/scrobblers.test.ts b/src/backend/tests/scrobbler/scrobblers.test.ts
index de757c1c..53c448b2 100644
--- a/src/backend/tests/scrobbler/scrobblers.test.ts
+++ b/src/backend/tests/scrobbler/scrobblers.test.ts
@@ -212,6 +212,14 @@ describe('When scrobble is a duplicate (title/artists/album)', function () {
diffPlay.data.artists = [ref.data.artists[1]]
assert.isTrue(await testScrobbler.alreadyScrobbled(diffPlay));
+
+
+ const son = normalizedWithMixedDur.find(x => x.data.track === 'Sonora')
+
+ const sonDiffPlay = clone(son);
+ sonDiffPlay.data.playDate = sonDiffPlay.data.playDate.subtract(son.data.duration + 1, 's');
+ sonDiffPlay.data.artists = [sonDiffPlay.data.artists[1]]
+ assert.isTrue(await testScrobbler.alreadyScrobbled(sonDiffPlay));
});
describe('When at least one play has duration', function () {
diff --git a/src/backend/utils.ts b/src/backend/utils.ts
index ad025e69..f455c5fa 100644
--- a/src/backend/utils.ts
+++ b/src/backend/utils.ts
@@ -900,3 +900,18 @@ export const getAddress = (host = '0.0.0.0', logger?: Logger): { v4?: string, v6
v6
};
}
+
+export const comparingMultipleArtists = (existing: PlayObject, candidate: PlayObject): boolean => {
+ const {
+ data: {
+ artists: eArtists = [],
+ } = {}
+ } = existing;
+ const {
+ data: {
+ artists: cArtists = [],
+ } = {}
+ } = candidate;
+
+ return eArtists.length > 1 || cArtists.length > 1;
+}
--
2.54.0
From a663e9af14a2ed4ed262c32825c3b25a0519ddfd Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Thu, 28 Sep 2023 15:22:22 -0400
Subject: [PATCH 63/82] feat(player): Defer play discovery so listenedFor is
accurate
Wait until player gets a new play or becomes stale before trying to discover the (previous) play so that we store a more accurate amount of time the user actually listened to that play
---
src/backend/sources/MemorySource.ts | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/src/backend/sources/MemorySource.ts b/src/backend/sources/MemorySource.ts
index e203448c..f2a5154d 100644
--- a/src/backend/sources/MemorySource.ts
+++ b/src/backend/sources/MemorySource.ts
@@ -154,8 +154,11 @@ export default class MemorySource extends AbstractSource {
const [currPlay, prevPlay] = asPlayerStateData(incomingData) ? player.setState(incomingData.status, incomingData.play) : player.setState(undefined, incomingData);
const candidate = prevPlay !== undefined ? prevPlay : currPlay;
+ const playChanged = prevPlay !== undefined;
- if (candidate !== undefined) {
+ // wait to discover play until it is stale or current play has changed
+ // so that our discovered track has an accurate "listenedFor" count
+ if (candidate !== undefined && (playChanged || player.isUpdateStale())) {
const thresholdResults = timePassesScrobbleThreshold(scrobbleThresholds, candidate.data.listenedFor, candidate.data.duration);
if (thresholdResults.passes) {
--
2.54.0
From 983b4c7417ba4609bc502aed0c583e40f61a7c0c Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Thu, 28 Sep 2023 15:23:14 -0400
Subject: [PATCH 64/82] feat(maloja): Add listened duration to scrobble payload
---
src/backend/common/vendor/maloja/interfaces.ts | 17 +++++++++++++++--
src/backend/scrobblers/MalojaScrobbler.ts | 6 +++++-
2 files changed, 20 insertions(+), 3 deletions(-)
diff --git a/src/backend/common/vendor/maloja/interfaces.ts b/src/backend/common/vendor/maloja/interfaces.ts
index ba6644ee..0dd30062 100644
--- a/src/backend/common/vendor/maloja/interfaces.ts
+++ b/src/backend/common/vendor/maloja/interfaces.ts
@@ -41,19 +41,32 @@ export interface MalojaV3ScrobbleData {
export type MalojaScrobbleData = MalojaV2ScrobbleData | MalojaV3ScrobbleData;
export interface MalojaScrobbleRequestData {
+ /** The auth key used to scrobble */
key: string
+ /** name of the track */
title: string
- album: string
+ /** name of the album */
+ album?: string
+ /** unix timestamp of when the track was scrobbled*/
time: number
- length: number
+ /** length of the track in seconds */
+ length?: number
+ /** number of seconds the track was listened to */
+ duration?: number
}
export interface MalojaScrobbleV2RequestData extends MalojaScrobbleRequestData {
+ /** comma-separated list of artists for this track */
artist: string
}
export interface MalojaScrobbleV3RequestData extends MalojaScrobbleRequestData {
+ /** a list of artists for this track */
artists: string[]
+ /** a list of artists for the album the track is on */
+ albumartists: string[]
+ /** skip server-side metadata parsing */
+ nofix?: boolean
}
interface MalojaScrobbleWarning {
diff --git a/src/backend/scrobblers/MalojaScrobbler.ts b/src/backend/scrobblers/MalojaScrobbler.ts
index 35e43d32..1666bd48 100644
--- a/src/backend/scrobblers/MalojaScrobbler.ts
+++ b/src/backend/scrobblers/MalojaScrobbler.ts
@@ -373,7 +373,8 @@ export default class MalojaScrobbler extends AbstractScrobbleClient {
album,
track,
duration,
- playDate
+ playDate,
+ listenedFor
} = {},
meta: {
source,
@@ -391,6 +392,9 @@ export default class MalojaScrobbler extends AbstractScrobbleClient {
// https://github.com/FoxxMD/multi-scrobbler/issues/42#issuecomment-1100184135
length: duration,
};
+ if(listenedFor !== undefined && listenedFor > 0) {
+ scrobbleData.duration = listenedFor;
+ }
let responseBody: MalojaScrobbleV3ResponseData;
--
2.54.0
From 8717fad08ae92dad4d7d983ca9b2c6d2bf60c219 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Fri, 29 Sep 2023 11:28:32 -0400
Subject: [PATCH 65/82] feat(source): Control backlogging via config #100
---
.../infrastructure/config/source/index.ts | 13 +++++
src/backend/common/schema/aio-source.json | 10 ++++
src/backend/common/schema/aio.json | 10 ++++
src/backend/common/schema/source.json | 10 ++++
src/backend/sources/AbstractSource.ts | 50 ++++++++++++++++---
src/backend/sources/DeezerSource.ts | 5 ++
src/backend/sources/JellyfinSource.ts | 3 +-
src/backend/sources/LastfmSource.ts | 5 ++
src/backend/sources/ListenbrainzSource.ts | 5 ++
src/backend/sources/PlexSource.ts | 3 +-
src/backend/sources/SpotifySource.ts | 10 ++--
11 files changed, 111 insertions(+), 13 deletions(-)
diff --git a/src/backend/common/infrastructure/config/source/index.ts b/src/backend/common/infrastructure/config/source/index.ts
index d99eec66..45bd1b8c 100644
--- a/src/backend/common/infrastructure/config/source/index.ts
+++ b/src/backend/common/infrastructure/config/source/index.ts
@@ -70,6 +70,19 @@ export interface CommonSourceOptions {
* @examples [false]
* */
logPlayerState?: boolean
+
+ /**
+ * If this source
+ *
+ * * supports fetching a listen history
+ * * and this option is enabled
+ *
+ * then on startup MS will attempt to scrobble the recent listens from that history
+ *
+ * @default true
+ * @examples [true, false]
+ * */
+ scrobbleBacklog?: boolean
}
export interface CommonSourceData extends CommonData, SourceRetryOptions {
diff --git a/src/backend/common/schema/aio-source.json b/src/backend/common/schema/aio-source.json
index a86c8818..3ea6e293 100644
--- a/src/backend/common/schema/aio-source.json
+++ b/src/backend/common/schema/aio-source.json
@@ -33,6 +33,16 @@
],
"title": "logPlayerState",
"type": "boolean"
+ },
+ "scrobbleBacklog": {
+ "default": true,
+ "description": "If this source\n\n* supports fetching a listen history\n* and this option is enabled\n\nthen on startup MS will attempt to scrobble the recent listens from that history",
+ "examples": [
+ true,
+ false
+ ],
+ "title": "scrobbleBacklog",
+ "type": "boolean"
}
},
"title": "CommonSourceOptions",
diff --git a/src/backend/common/schema/aio.json b/src/backend/common/schema/aio.json
index c70cf9ec..c2525220 100644
--- a/src/backend/common/schema/aio.json
+++ b/src/backend/common/schema/aio.json
@@ -106,6 +106,16 @@
],
"title": "logPlayerState",
"type": "boolean"
+ },
+ "scrobbleBacklog": {
+ "default": true,
+ "description": "If this source\n\n* supports fetching a listen history\n* and this option is enabled\n\nthen on startup MS will attempt to scrobble the recent listens from that history",
+ "examples": [
+ true,
+ false
+ ],
+ "title": "scrobbleBacklog",
+ "type": "boolean"
}
},
"title": "CommonSourceOptions",
diff --git a/src/backend/common/schema/source.json b/src/backend/common/schema/source.json
index 9bd8b494..770e69b4 100644
--- a/src/backend/common/schema/source.json
+++ b/src/backend/common/schema/source.json
@@ -74,6 +74,16 @@
],
"title": "logPlayerState",
"type": "boolean"
+ },
+ "scrobbleBacklog": {
+ "default": true,
+ "description": "If this source\n\n* supports fetching a listen history\n* and this option is enabled\n\nthen on startup MS will attempt to scrobble the recent listens from that history",
+ "examples": [
+ true,
+ false
+ ],
+ "title": "scrobbleBacklog",
+ "type": "boolean"
}
},
"title": "CommonSourceOptions",
diff --git a/src/backend/sources/AbstractSource.ts b/src/backend/sources/AbstractSource.ts
index 53278547..f79f678b 100644
--- a/src/backend/sources/AbstractSource.ts
+++ b/src/backend/sources/AbstractSource.ts
@@ -63,6 +63,7 @@ export default abstract class AbstractSource {
canPoll: boolean = false;
polling: boolean = false;
+ canBacklog: boolean = false;
userPollingStopSignal: undefined | any;
pollRetries: number = 0;
tracksDiscovered: number = 0;
@@ -169,9 +170,7 @@ export default abstract class AbstractSource {
return existing !== undefined;
}
-
- protected scrobble = (plays: PlayObject[], options: { forceRefresh?: boolean, checkAll?: boolean } = {}) => {
-
+ discover = (plays: PlayObject[], options: { checkAll?: boolean, [key: string]: any } = {}): PlayObject[] => {
const newDiscoveredPlays: PlayObject[] = [];
for(const play of plays) {
@@ -181,9 +180,16 @@ export default abstract class AbstractSource {
}
}
+ newDiscoveredPlays.sort(sortByOldestPlayDate);
+
+ return newDiscoveredPlays;
+ }
+
+
+ protected scrobble = (newDiscoveredPlays: PlayObject[], options: { forceRefresh?: boolean, [key: string]: any } = {}) => {
+
if(newDiscoveredPlays.length > 0) {
newDiscoveredPlays.sort(sortByOldestPlayDate);
-
this.emitter.emit('discoveredToScrobble', {
data: newDiscoveredPlays,
options: {
@@ -194,8 +200,37 @@ export default abstract class AbstractSource {
}
});
}
+ }
- return newDiscoveredPlays;
+ protected processBacklog = async () => {
+ if (this.canBacklog) {
+ this.logger.info('Discovering backlogged tracks from recently played API...');
+ const backlogPlays = await this.getBackloggedPlays();
+ const discovered = this.discover(backlogPlays);
+
+ const {
+ options: {
+ scrobbleBacklog = true
+ } = {}
+ } = this.config;
+
+ if (scrobbleBacklog) {
+ if (discovered.length > 0) {
+ this.logger.info('Scrobbling backlogged tracks...');
+ this.scrobble(discovered);
+ this.logger.info('Backlog scrobbling complete.');
+ } else {
+ this.logger.info('All tracks already discovered!');
+ }
+ } else {
+ this.logger.info('Backlog scrobbling is disabled by config, skipping...');
+ }
+ }
+ }
+
+ protected getBackloggedPlays = async (): Promise => {
+ this.logger.debug('Backlogging not implemented');
+ return [];
}
protected notify = (payload) => {
@@ -227,6 +262,8 @@ export default abstract class AbstractSource {
if(!(await this.onPollPostAuthCheck())) {
return;
}
+ await this.processBacklog();
+
await this.startPolling();
}
@@ -330,7 +367,8 @@ export default abstract class AbstractSource {
this.logger.info('Potential plays were discovered close to polling interval! Delaying scrobble clients refresh by 10 seconds so other clients have time to scrobble first');
await sleep(10 * 1000);
}
- newDiscovered = this.scrobble(playObjs,
+ newDiscovered = this.discover(playObjs);
+ this.scrobble(newDiscovered,
{
forceRefresh: closeToInterval
});
diff --git a/src/backend/sources/DeezerSource.ts b/src/backend/sources/DeezerSource.ts
index c02dc479..f8658ceb 100644
--- a/src/backend/sources/DeezerSource.ts
+++ b/src/backend/sources/DeezerSource.ts
@@ -45,6 +45,7 @@ export default class DeezerSource extends AbstractSource {
this.workingCredsPath = `${this.configDir}/currentCreds-${name}.json`;
this.canPoll = true;
+ this.canBacklog = true;
}
static formatPlayObj(obj: any, options: FormatPlayObjectOptions = {}): PlayObject {
@@ -231,4 +232,8 @@ export default class DeezerSource extends AbstractSource {
return error;
}
}
+
+ protected getBackloggedPlays = async () => {
+ return await this.getRecentlyPlayed({formatted: true});
+ }
}
diff --git a/src/backend/sources/JellyfinSource.ts b/src/backend/sources/JellyfinSource.ts
index 3fd41b6a..698e40c1 100644
--- a/src/backend/sources/JellyfinSource.ts
+++ b/src/backend/sources/JellyfinSource.ts
@@ -355,7 +355,8 @@ export default class JellyfinSource extends MemorySource {
if(newPlays.length > 0) {
try {
- this.scrobble(newPlays, scrobbleOpts);
+ const discovered = this.discover(newPlays, scrobbleOpts);
+ this.scrobble(discovered);
} catch (e) {
this.logger.error('Encountered error while scrobbling')
this.logger.error(e)
diff --git a/src/backend/sources/LastfmSource.ts b/src/backend/sources/LastfmSource.ts
index 868c05f6..80222ec9 100644
--- a/src/backend/sources/LastfmSource.ts
+++ b/src/backend/sources/LastfmSource.ts
@@ -27,6 +27,7 @@ export default class LastfmSource extends MemorySource {
} = config;
super('lastfm', name, {...config, data: {interval, maxInterval, ...restData}}, internal, emitter);
this.canPoll = true;
+ this.canBacklog = true;
this.api = new LastfmApiClient(name, {...config.data, configDir: internal.configDir, localUrl: internal.localUrl});
this.playerSourceOfTruth = false;
}
@@ -98,4 +99,8 @@ export default class LastfmSource extends MemorySource {
this.processRecentPlays(now);
return history;
}
+
+ protected getBackloggedPlays = async () => {
+ return await this.getRecentlyPlayed({formatted: true});
+ }
}
diff --git a/src/backend/sources/ListenbrainzSource.ts b/src/backend/sources/ListenbrainzSource.ts
index 994b9e72..ad3892a3 100644
--- a/src/backend/sources/ListenbrainzSource.ts
+++ b/src/backend/sources/ListenbrainzSource.ts
@@ -23,6 +23,7 @@ export default class ListenbrainzSource extends MemorySource {
} = config;
super('listenbrainz', name, {...config, data: {interval, maxInterval, ...restData}}, internal, emitter);
this.canPoll = true;
+ this.canBacklog = true;
this.api = new ListenbrainzApiClient(name, config.data);
this.playerSourceOfTruth = false;
}
@@ -65,4 +66,8 @@ export default class ListenbrainzSource extends MemorySource {
this.processRecentPlays(now.listens.map(x => ListenbrainzSource.formatPlayObj(x)));
return await this.api.getRecentlyPlayed(limit);
}
+
+ protected getBackloggedPlays = async () => {
+ return await this.getRecentlyPlayed({formatted: true});
+ }
}
diff --git a/src/backend/sources/PlexSource.ts b/src/backend/sources/PlexSource.ts
index 7b5a03b6..942ae0f9 100644
--- a/src/backend/sources/PlexSource.ts
+++ b/src/backend/sources/PlexSource.ts
@@ -210,7 +210,8 @@ export default class PlexSource extends AbstractSource {
}
try {
- this.scrobble([playObj]);
+ const discovered = this.discover([playObj]);
+ this.scrobble(discovered);
} catch (e) {
this.logger.error('Encountered error while scrobbling')
this.logger.error(e)
diff --git a/src/backend/sources/SpotifySource.ts b/src/backend/sources/SpotifySource.ts
index 5fbfeb35..ea552228 100644
--- a/src/backend/sources/SpotifySource.ts
+++ b/src/backend/sources/SpotifySource.ts
@@ -62,6 +62,7 @@ export default class SpotifySource extends MemorySource {
this.workingCredsPath = `${this.configDir}/currentCreds-${name}.json`;
this.canPoll = true;
+ this.canBacklog = true;
}
static formatPlayObj(obj: object, options: FormatPlayObjectOptions = {}): PlayObject {
@@ -436,13 +437,12 @@ export default class SpotifySource extends MemorySource {
this.logger.warn('multi-scrobbler does not have sufficient permissions to access Spotify API "Get Playback State". MS will continue to work but accuracy for determining if/when a track played from a Spotify Connect device (smart device controlled through Spotify app) may be degraded. To fix this re-authenticate MS with Spotify and restart polling.');
}
- // and record backlogged tracks
- this.logger.info('Checking recently played API for tracks to backlog...');
- const backlogPlays = await this.getPlayHistory({formatted: true});
- this.scrobble(backlogPlays);
- this.logger.info('Backlog complete.');
return true;
}
+
+ protected getBackloggedPlays = async () => {
+ return await this.getPlayHistory({formatted: true});
+ }
}
const asPlayHistoryObject = (obj: object): obj is PlayHistoryObject => {
--
2.54.0
From 9309f7d8cf0df2ba3196cfff3655e6e42314cc03 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Fri, 29 Sep 2023 14:15:32 -0400
Subject: [PATCH 66/82] feat(scrobble): Initial scrobble queue implementation
* Move scrobble checking into client (out of client manager) for single responsibility
* Implement scrobble queue/deadletter
* Less chance of duplicate processing since sources can't scrobble concurrently
* Better throttling since multiple sources can't scrobble concurrently
* When scrobbling fails we can retry on delay just like polling and *keep* the failed scrobbles for later retry rather than dropping them
* Deadletter (to be fully implemented) for scrobbles that fail multiple times or are signalled as not upstream api issues by client
---
package-lock.json | 2 +-
package.json | 1 +
src/backend/common/errors/UpstreamError.ts | 12 ++
src/backend/common/infrastructure/Atomic.ts | 5 +
src/backend/index.ts | 5 +
.../scrobblers/AbstractScrobbleClient.ts | 204 +++++++++++++++++-
src/backend/scrobblers/LastfmScrobbler.ts | 16 +-
.../scrobblers/ListenbrainzScrobbler.ts | 11 +-
src/backend/scrobblers/MalojaScrobbler.ts | 7 +-
src/backend/scrobblers/ScrobbleClients.ts | 66 +-----
10 files changed, 240 insertions(+), 89 deletions(-)
create mode 100644 src/backend/common/errors/UpstreamError.ts
diff --git a/package-lock.json b/package-lock.json
index 97f14740..4f5055ea 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -40,6 +40,7 @@
"kodi-api": "^0.2.1",
"lastfm-node-client": "^2.2.0",
"mopidy": "^1.3.0",
+ "nanoid": "^3.3.1",
"normalize-url": "^6.1.0",
"ntfy": "1.0.5",
"object-hash": "^3.0.0",
@@ -16549,7 +16550,6 @@
"version": "3.3.1",
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.1.tgz",
"integrity": "sha512-n6Vs/3KGyxPQd6uO0eH4Bv0ojGSUvuLlIHtC3Y0kEO23YRge8H9x1GCzLn28YX0H66pMkxuaeESFq4tKISKwdw==",
- "dev": true,
"bin": {
"nanoid": "bin/nanoid.cjs"
},
diff --git a/package.json b/package.json
index 050075a7..10f952dd 100644
--- a/package.json
+++ b/package.json
@@ -77,6 +77,7 @@
"kodi-api": "^0.2.1",
"lastfm-node-client": "^2.2.0",
"mopidy": "^1.3.0",
+ "nanoid": "^3.3.1",
"normalize-url": "^6.1.0",
"ntfy": "1.0.5",
"object-hash": "^3.0.0",
diff --git a/src/backend/common/errors/UpstreamError.ts b/src/backend/common/errors/UpstreamError.ts
new file mode 100644
index 00000000..b6ba42a6
--- /dev/null
+++ b/src/backend/common/errors/UpstreamError.ts
@@ -0,0 +1,12 @@
+import {ErrorWithCause} from "pony-cause";
+
+export class UpstreamError extends ErrorWithCause {
+
+ showStopper: boolean = false;
+
+ constructor(message: string, options?: { cause?: T, showStopper?: boolean } | undefined) {
+ super(message, options);
+ const {showStopper = false} = options;
+ this.showStopper = showStopper;
+ }
+}
diff --git a/src/backend/common/infrastructure/Atomic.ts b/src/backend/common/infrastructure/Atomic.ts
index 01448ffc..032900c1 100644
--- a/src/backend/common/infrastructure/Atomic.ts
+++ b/src/backend/common/infrastructure/Atomic.ts
@@ -212,3 +212,8 @@ export const TITLE_WEIGHT = 0.4;
export const TIME_WEIGHT = 0.5;
export const REFERENCE_WEIGHT = 0.5;
export const DUP_SCORE_THRESHOLD = 1;
+
+export interface SourceScrobble {
+ source: string
+ play: PlayObject
+}
diff --git a/src/backend/index.ts b/src/backend/index.ts
index 7164eb45..359f9eb2 100644
--- a/src/backend/index.ts
+++ b/src/backend/index.ts
@@ -135,6 +135,11 @@ const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`)
}
}
}
+ for(const client of scrobbleClients.clients) {
+ if((await client.isReady())) {
+ client.initScrobbleMonitoring();
+ }
+ }
if (anyNotReady) {
logger.info(`Some sources are not ready, open the dashboard to continue`);
}
diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts
index 3f26f0f0..60158452 100644
--- a/src/backend/scrobblers/AbstractScrobbleClient.ts
+++ b/src/backend/scrobblers/AbstractScrobbleClient.ts
@@ -3,18 +3,18 @@ import {
comparingMultipleArtists,
isPlayTemporallyClose,
mergeArr,
- playObjDataMatch,
- setIntersection, sortByOldestPlayDate,
+ playObjDataMatch, pollingBackoff,
+ setIntersection, sleep, sortByOldestPlayDate,
} from "../utils";
import {
ARTIST_WEIGHT,
- ClientType, DUP_SCORE_THRESHOLD,
+ ClientType, DEFAULT_RETRY_MULTIPLIER, DUP_SCORE_THRESHOLD,
FormatPlayObjectOptions,
INITIALIZED,
INITIALIZING,
InitState,
NOT_INITIALIZED, REFERENCE_WEIGHT,
- ScrobbledPlayObject, TIME_WEIGHT, TITLE_WEIGHT,
+ ScrobbledPlayObject, SourceScrobble, TIME_WEIGHT, TITLE_WEIGHT,
} from "../common/infrastructure/Atomic";
import winston, {Logger} from '@foxxmd/winston';
import { CommonClientConfig } from "../common/infrastructure/config/client/index";
@@ -25,11 +25,15 @@ import { PlayObject, TrackStringOptions } from "../../core/Atomic";
import {buildTrackString, capitalize, truncateStringToLength} from "../../core/StringUtils";
import EventEmitter from "events";
import {compareScrobbleArtists, compareScrobbleTracks, normalizeStr} from "../utils/StringUtils";
+import {UpstreamError} from "../common/errors/UpstreamError";
+import {nanoid} from "nanoid";
+import {ErrorWithCause} from "pony-cause";
export default abstract class AbstractScrobbleClient {
name: string;
type: ClientType;
+ identifier: string;
#initState: InitState = NOT_INITIALIZED;
@@ -50,6 +54,12 @@ export default abstract class AbstractScrobbleClient {
checkExistingScrobbles: boolean;
verboseOptions;
+ scrobbleRetries: number = 0;
+ scrobbling: boolean = false;
+ userScrobblingStopSignal: undefined | any;
+ queuedScrobbles: (SourceScrobble & {id: string})[] = [];
+ deadLetterScrobbles: (SourceScrobble & {id: string, retries: number})[] = [];
+
config: CommonClientConfig;
logger: Logger;
@@ -59,8 +69,8 @@ export default abstract class AbstractScrobbleClient {
constructor(type: any, name: any, config: CommonClientConfig, notifier: Notifiers, emitter: EventEmitter, logger: Logger) {
this.type = type;
this.name = name;
- const identifier = `${capitalize(this.type)} - ${name}`;
- this.logger = logger.child({labels: [identifier]}, mergeArr);
+ this.identifier = `${capitalize(this.type)} - ${name}`;
+ this.logger = logger.child({labels: [this.identifier]}, mergeArr);
this.notifier = notifier;
this.emitter = emitter;
@@ -183,6 +193,7 @@ export default abstract class AbstractScrobbleClient {
addScrobbledTrack = (playObj: PlayObject, scrobbledPlay: PlayObject) => {
this.scrobbledPlayObjs.add({play: playObj, scrobble: scrobbledPlay});
+ this.tracksScrobbled++;
}
filterScrobbledTracks = () => {
@@ -395,7 +406,186 @@ ${closestMatch.breakdowns.join('\n')}`);
return existingScrobble;
}
- public abstract scrobble(playObj: PlayObject): Promise
+ public abstract scrobble(playObj: PlayObject): Promise
+
+ initScrobbleMonitoring = async () => {
+ if(!this.initialized) {
+ if(this.initializing) {
+ this.logger.warn(`Cannot start scrobble processing because client is still initializing`);
+ return;
+ }
+ if(!(await this.initialize())) {
+ this.logger.warn(`Cannot start scrobble processing because client could not be initialized`);
+ return;
+ }
+ }
+
+ if(this.requiresAuth && !this.authed) {
+ if (this.requiresAuthInteraction) {
+ this.logger.warn(`Cannot start scrobble processing because user interaction is required for authentication`);
+ return;
+ } else if (!(await this.testAuth())) {
+ this.logger.warn(`Cannot start scrobble processing because auth test failed`);
+ return;
+ }
+ }
+
+ if(!(await this.isReady())) {
+ this.logger.warn(`Cannot start scrobble processing because client is not ready`);
+ return;
+ }
+
+ await this.startScrobbling();
+ }
+
+ startScrobbling = async () => {
+ // reset poll attempts if already previously run
+ this.scrobbleRetries = 0;
+
+ const {
+ data: {
+ maxPollRetries = 5,
+ retryMultiplier = DEFAULT_RETRY_MULTIPLIER,
+ } = {},
+ } = this.config;
+
+ // can't have negative retries!
+ const maxRetries = Math.max(0, maxPollRetries);
+
+ if(this.scrobbling === true) {
+ this.logger.warn(`Already scrobble processing! Processing needs to be stopped before it can be started`);
+ return;
+ }
+
+ let pollRes: boolean | undefined = undefined;
+ while (pollRes === undefined && this.scrobbleRetries <= maxRetries) {
+ try {
+ pollRes = await this.doProcessing();
+ if(pollRes === true) {
+ break;
+ }
+ } catch (e) {
+ if(!this.initialized) {
+ this.logger.warn('Stopping scrobble processing due to client no longer being initialized.');
+ await this.notifier.notify({title: `Client - ${this.identifier} - Processing Error`, message: `Encountered error while scrobble processing and client is no longer initialized, stopping processing!. | Error: ${e.message}`, priority: 'error'});
+ break;
+ } else if (this.requiresAuth && !this.authed) {
+ this.logger.warn('Stopping scrobble processing due to client no longer being authenticated.');
+ await this.notifier.notify({title: `Client - ${this.identifier} - Processing Error`, message: `Encountered error while scrobble processing and client is no longer authenticated, stopping processing!. | Error: ${e.message}`, priority: 'error'});
+ break;
+ } else if (this.scrobbleRetries < maxRetries) {
+ const delayFor = pollingBackoff(this.scrobbleRetries + 1, retryMultiplier);
+ this.logger.info(`Scrobble processing retries (${this.scrobbleRetries}) less than max processing retries (${maxRetries}), restarting processing after ${delayFor} second delay...`);
+ await this.notifier.notify({title: `Client - ${this.name} - Processing Retry`, message: `Encountered error while polling but retries (${this.scrobbleRetries}) are less than max poll retries (${maxRetries}), restarting processing after ${delayFor} second delay. | Error: ${e.message}`, priority: 'warn'});
+ await sleep((delayFor) * 1000);
+ } else {
+ this.logger.warn(`Scrobble processing retries (${this.scrobbleRetries}) equal to max processing retries (${maxRetries}), stopping processing!`);
+ await this.notifier.notify({title: `Client - ${this.identifier} - Processing Error`, message: `Encountered error while scrobble processing and retries (${this.scrobbleRetries}) are equal to max processing retries (${maxRetries}), stopping processing!. | Error: ${e.message}`, priority: 'error'});
+ }
+ this.scrobbleRetries++;
+ }
+ }
+ }
+
+ tryStopScrobbling = async () => {
+ if(this.scrobbling === false) {
+ this.logger.warn(`Polling is already stopped!`);
+ return;
+ }
+ this.userScrobblingStopSignal = true;
+ let secsPassed = 0;
+ while(this.userScrobblingStopSignal !== undefined && secsPassed < 10) {
+ await sleep(2000);
+ secsPassed += 2;
+ this.logger.verbose(`Waiting for scrobble processing stop signal to be acknowledged (waited ${secsPassed}s)`);
+ }
+ if(this.userScrobblingStopSignal !== undefined) {
+ this.logger.warn('Could not stop scrobble processing! Or signal was lost :(');
+ return false;
+ }
+ return true;
+ }
+
+ protected doStopScrobbling = (reason: string = 'system') => {
+ this.scrobbling = false;
+ this.userScrobblingStopSignal = undefined;
+ this.logger.info(`Stopped scrobble processing due to: ${reason}`);
+ }
+
+ protected shouldStopScrobbleProcessing = () => this.scrobbling === false || this.userScrobblingStopSignal !== undefined;
+
+ protected doProcessing = async (): Promise => {
+ if (this.scrobbling === true) {
+ return true;
+ }
+ this.logger.info('Scrobble processing started');
+
+ try {
+ this.scrobbling = true;
+ while (!this.shouldStopScrobbleProcessing()) {
+ while (this.queuedScrobbles.length > 0) {
+ if (this.lastScrobbleCheck.unix() < this.getLatestQueuePlayDate().unix()) {
+ await this.refreshScrobbles();
+ }
+ const currQueuedPlay = this.queuedScrobbles[0];
+ const [timeFrameValid, timeFrameValidLog] = this.timeFrameIsValid(currQueuedPlay.play);
+ if (timeFrameValid && !(await this.alreadyScrobbled(currQueuedPlay.play))) {
+ try {
+ const scrobbledPlay = await this.scrobble(currQueuedPlay.play);
+ this.emitEvent('scrobble', {play: currQueuedPlay.play});
+ this.addScrobbledTrack(currQueuedPlay.play, scrobbledPlay);
+ } catch (e) {
+ if (e instanceof UpstreamError && e.showStopper === false) {
+ this.addDeadLetterScrobble(currQueuedPlay);
+ this.logger.warn(`Could not scrobble ${buildTrackString(currQueuedPlay.play)} from Source '${currQueuedPlay.source}' but error was not show stopping. Adding scrobble to Dead Letter Queue and will retry on next heartbeat.`);
+ } else {
+ const processError = new ErrorWithCause('Error occurred while trying to scrobble', {cause: e});
+ //this.logger.error(processError);
+ throw processError;
+ }
+ } finally {
+ await sleep(1000);
+ }
+ } else if (!timeFrameValid) {
+ this.logger.debug(`Will not scrobble ${buildTrackString(currQueuedPlay.play)} from Source '${currQueuedPlay.source}' because it ${timeFrameValidLog}`);
+ }
+ // processing play may have changed index while we were scrobbling
+ const pIndex = this.queuedScrobbles.findIndex(x => x.id === currQueuedPlay.id);
+ if (pIndex !== -1) {
+ this.queuedScrobbles.splice(pIndex, 1);
+ }
+ }
+ await sleep(2000);
+ }
+ if (this.shouldStopScrobbleProcessing()) {
+ this.doStopScrobbling(this.userScrobblingStopSignal !== undefined ? 'user input' : undefined);
+ return true;
+ }
+ } catch (e) {
+ this.logger.error('Scrobble processing interrupted');
+ this.logger.error(e);
+ this.scrobbling = false;
+ throw e;
+ }
+ }
+
+ protected getLatestQueuePlayDate = () => {
+ if (this.queuedScrobbles.length === 0) {
+ return undefined;
+ }
+ return this.queuedScrobbles[this.queuedScrobbles.length - 1].play.data.playDate;
+ }
+
+ queueScrobble = (play: PlayObject, source: string) => {
+ this.queuedScrobbles.push({id: nanoid(), source, play});
+ this.queuedScrobbles.sort((a, b) => sortByOldestPlayDate(a.play, b.play));
+ }
+
+ protected addDeadLetterScrobble = (data: SourceScrobble & { id?: string, retries?: number }) => {
+ this.deadLetterScrobbles.push({id: nanoid(), retries: 0, ...data});
+ this.deadLetterScrobbles.sort((a, b) => sortByOldestPlayDate(a.play, b.play));
+ }
+
public emitEvent = (eventName: string, payload: object) => {
this.emitter.emit(eventName, {
diff --git a/src/backend/scrobblers/LastfmScrobbler.ts b/src/backend/scrobblers/LastfmScrobbler.ts
index 69ea0e52..3a07cc83 100644
--- a/src/backend/scrobblers/LastfmScrobbler.ts
+++ b/src/backend/scrobblers/LastfmScrobbler.ts
@@ -17,6 +17,7 @@ import {Logger} from '@foxxmd/winston';
import { PlayObject, TrackStringOptions } from "../../core/Atomic";
import {buildTrackString, capitalize} from "../../core/StringUtils";
import EventEmitter from "events";
+import {UpstreamError} from "../common/errors/UpstreamError";
export default class LastfmScrobbler extends AbstractScrobbleClient {
@@ -185,9 +186,8 @@ export default class LastfmScrobbler extends AbstractScrobbleClient {
} = response;
if(code === 5) {
this.initialized = false;
- throw new Error('Service reported daily scrobble limit exceeded! 😬 Disabling client');
+ throw new UpstreamError('LastFM API reported daily scrobble limit exceeded! 😬 Disabling client', {showStopper: true});
}
- this.addScrobbledTrack(playObj, this.formatPlayObj({...rest, date: { uts: timestamp}, name: trackName}));
if (newFromSource) {
this.logger.info(`Scrobbled (New) => (${source}) ${buildTrackString(playObj)}`);
} else {
@@ -196,18 +196,22 @@ export default class LastfmScrobbler extends AbstractScrobbleClient {
if(ignored > 0) {
await this.notifier.notify({title: `Client - ${capitalize(this.type)} - ${this.name} - Scrobble Error`, message: `Failed to scrobble => ${buildTrackString(playObj)} | Error: Service ignored this scrobble 😬 => (Code ${ignoreCode}) ${(ignoreMsg === '' ? '(No error message returned)' : ignoreMsg)}`, priority: 'warn'});
this.logger.warn(`Service ignored this scrobble 😬 => (Code ${ignoreCode}) ${(ignoreMsg === '' ? '(No error message returned)' : ignoreMsg)} -- See https://www.last.fm/api/errorcodes for more information`, {payload: scrobblePayload});
+ throw new UpstreamError('LastFM ignored scrobble', {showStopper: false});
}
+ return this.formatPlayObj({...rest, date: { uts: timestamp}, name: trackName});
// last fm has rate limits but i can't find a specific example of what that limit is. going to default to 1 scrobble/sec to be safe
- await sleep(1000);
+ //await sleep(1000);
} catch (e) {
await this.notifier.notify({title: `Client - ${capitalize(this.type)} - ${this.name} - Scrobble Error`, message: `Failed to scrobble => ${buildTrackString(playObj)} | Error: ${e.message}`, priority: 'error'});
this.logger.error(`Scrobble Error (${sType})`, {playInfo: buildTrackString(playObj), payload: scrobblePayload});
- throw e;
+ if(!(e instanceof UpstreamError)) {
+ throw new UpstreamError('Error received from LastFM API', {cause: e, showStopper: true});
+ } else {
+ throw e;
+ }
} finally {
this.logger.debug('Raw Payload: ', rawPayload);
}
-
- return true;
}
}
diff --git a/src/backend/scrobblers/ListenbrainzScrobbler.ts b/src/backend/scrobblers/ListenbrainzScrobbler.ts
index 75f9cdf6..f587d926 100644
--- a/src/backend/scrobblers/ListenbrainzScrobbler.ts
+++ b/src/backend/scrobblers/ListenbrainzScrobbler.ts
@@ -17,6 +17,7 @@ import { ListenbrainzApiClient } from "../common/vendor/ListenbrainzApiClient";
import { PlayObject, TrackStringOptions } from "../../core/Atomic";
import {buildTrackString, capitalize} from "../../core/StringUtils";
import EventEmitter from "events";
+import {UpstreamError} from "../common/errors/UpstreamError";
export default class ListenbrainzScrobbler extends AbstractScrobbleClient {
@@ -98,15 +99,15 @@ export default class ListenbrainzScrobbler extends AbstractScrobbleClient {
try {
const resp = await this.api.submitListen(playObj);
rawPayload = resp;
- this.addScrobbledTrack(playObj, playObj);
+
if (newFromSource) {
this.logger.info(`Scrobbled (New) => (${source}) ${buildTrackString(playObj)}`);
} else {
this.logger.info(`Scrobbled (Backlog) => (${source}) ${buildTrackString(playObj)}`);
}
-
// last fm has rate limits but i can't find a specific example of what that limit is. going to default to 1 scrobble/sec to be safe
- await sleep(1000);
+ //await sleep(1000);
+ return playObj;
} catch (e) {
let message = e.message;
if(e.response !== undefined) {
@@ -118,11 +119,9 @@ export default class ListenbrainzScrobbler extends AbstractScrobbleClient {
}
await this.notifier.notify({title: `Client - ${capitalize(this.type)} - ${this.name} - Scrobble Error`, message: `Failed to scrobble => ${buildTrackString(playObj)} | Error: ${message}`, priority: 'error'});
this.logger.error(`Failed to scrobble => ${message}`, {payload: rawPayload});
- throw e;
+ throw new UpstreamError(`Error received from Listenbrainz API: ${message}`, {cause: e, showStopper: true});
} finally {
this.logger.debug(`Raw Payload:`, {rawPayload});
}
-
- return true;
}
}
diff --git a/src/backend/scrobblers/MalojaScrobbler.ts b/src/backend/scrobblers/MalojaScrobbler.ts
index 1666bd48..401332c1 100644
--- a/src/backend/scrobblers/MalojaScrobbler.ts
+++ b/src/backend/scrobblers/MalojaScrobbler.ts
@@ -24,6 +24,7 @@ import { PlayObject, TrackStringOptions } from "../../core/Atomic";
import {buildTrackString, capitalize} from "../../core/StringUtils";
import EventEmitter from "events";
import normalizeUrl from "normalize-url";
+import {UpstreamError} from "../common/errors/UpstreamError";
const feat = ["ft.", "ft", "feat.", "feat", "featuring", "Ft.", "Ft", "Feat.", "Feat", "Featuring"];
@@ -447,7 +448,7 @@ export default class MalojaScrobbler extends AbstractScrobbleClient {
}
}
} else {
- throw new Error(buildErrorString(response));
+ throw new UpstreamError(buildErrorString(response), {showStopper: true});
}
} else {
const {
@@ -469,7 +470,6 @@ export default class MalojaScrobbler extends AbstractScrobbleClient {
} else {
scrobbledPlay = this.formatPlayObj(scrobbleResponse)
}
- this.addScrobbledTrack(playObj, scrobbledPlay);
const scrobbleInfo = `Scrobbled (${newFromSource ? 'New' : 'Backlog'}) => (${source}) ${buildTrackString(playObj)}`;
if(warning !== '') {
this.logger.warn(`${scrobbleInfo} | ${warning}`);
@@ -477,6 +477,7 @@ export default class MalojaScrobbler extends AbstractScrobbleClient {
} else {
this.logger.info(scrobbleInfo);
}
+ return scrobbledPlay;
} catch (e) {
await this.notifier.notify({title: `Client - ${capitalize(this.type)} - ${this.name} - Scrobble Error`, message: `Failed to scrobble => ${buildTrackString(playObj)} | Error: ${e.message}`, priority: 'error'});
this.logger.error(`Scrobble Error (${sType})`, {playInfo: buildTrackString(playObj), payload: scrobbleData});
@@ -487,8 +488,6 @@ export default class MalojaScrobbler extends AbstractScrobbleClient {
} finally {
this.logger.debug('Raw Payload:', scrobbleData);
}
-
- return true;
}
}
diff --git a/src/backend/scrobblers/ScrobbleClients.ts b/src/backend/scrobblers/ScrobbleClients.ts
index 274fa80c..f48ab39e 100644
--- a/src/backend/scrobblers/ScrobbleClients.ts
+++ b/src/backend/scrobblers/ScrobbleClients.ts
@@ -364,8 +364,6 @@ ${sources.join('\n')}`);
scrobbleFrom = 'source',
} = options;
- const tracksScrobbled: any = [];
-
if (this.clients.length === 0) {
this.logger.warn('Cannot scrobble! No clients are configured.');
}
@@ -375,71 +373,9 @@ ${sources.join('\n')}`);
client.logger.debug(`Client was filtered out by Source '${scrobbleFrom}'`);
continue;
}
- if(!client.initialized) {
- if(client.initializing) {
- client.logger.warn(`Cannot scrobble because it is still initializing`);
- continue;
- }
- if(!(await client.initialize())) {
- client.logger.warn(`Cannot scrobble because it could not be initialized`);
- continue;
- }
- }
-
- if(client.requiresAuth && !client.authed) {
- if (client.requiresAuthInteraction) {
- client.logger.warn(`Cannot scrobble because user interaction is required for authentication`);
- continue;
- } else if (!(await client.testAuth())) {
- client.logger.warn(`Cannot scrobble because auth test failed`);
- continue;
- }
- }
-
- if(!(await client.isReady())) {
- client.logger.warn(`Cannot scrobble because it is not ready`);
- continue;
- }
-
- if (forceRefresh || client.scrobblesLastCheckedAt().unix() < checkTime.unix()) {
- try {
- await client.refreshScrobbles();
- } catch(e) {
- client.logger.error(`Encountered error while refreshing scrobbles`);
- this.logger.error(e);
- }
- }
for (const playObj of playObjs) {
- try {
- const {
- meta: {
- newFromSource = false,
- } = {}
- } = playObj;
- const [timeFrameValid, timeFrameValidLog] = client.timeFrameIsValid(playObj);
- if (timeFrameValid && !(await client.alreadyScrobbled(playObj))) {
- client.emitEvent('scrobble', {play: playObj});
- await client.scrobble(playObj)
- client.tracksScrobbled++;
- // since this is what we return to the source only add to tracksScrobbled if not already in array
- // (source should only know that a track was scrobbled (binary) -- doesn't care if it was scrobbled more than once
- if(!tracksScrobbled.some(x => playObjDataMatch(x, playObj) && x.data.playDate === playObj.data.playDate)) {
- tracksScrobbled.push(playObj);
- }
- } else {
- if(!timeFrameValid) {
- client.logger.debug(`Will not scrobble ${buildTrackString(playObj)} from Source '${scrobbleFrom}' because it ${timeFrameValidLog}`);
- }
- }
- } catch(e) {
- client.logger.error(new ErrorWithCause(`Encountered error while in scrobble loop`, {cause: e}));
- // for now just stop scrobbling plays for this client and move on. the client should deal with logging the issue
- if(e.continueScrobbling !== true) {
- break;
- }
- }
+ client.queueScrobble(playObj, scrobbleFrom);
}
}
- return tracksScrobbled;
}
}
--
2.54.0
From 1c54ae64656ed23ea06b1c7403f4f7e12d88ff2d Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Fri, 29 Sep 2023 15:07:08 -0400
Subject: [PATCH 67/82] feat(scrobble): Implement dead letter queue processing
* Implemented logic for processing and removing scrobbles from dead letter queue
* Only retry scrobbles under X retries based on config or argument
* Add removal-only function (for later use with UI)
---
.../infrastructure/config/client/index.ts | 8 ++
src/backend/common/schema/aio-client.json | 27 ++++++
src/backend/common/schema/aio.json | 27 ++++++
src/backend/common/schema/client.json | 27 ++++++
.../scrobblers/AbstractScrobbleClient.ts | 86 ++++++++++++++++---
5 files changed, 165 insertions(+), 10 deletions(-)
diff --git a/src/backend/common/infrastructure/config/client/index.ts b/src/backend/common/infrastructure/config/client/index.ts
index 667f897c..a04af080 100644
--- a/src/backend/common/infrastructure/config/client/index.ts
+++ b/src/backend/common/infrastructure/config/client/index.ts
@@ -63,6 +63,14 @@ export interface CommonClientData extends CommonData {
match?: MatchLoggingOptions
}
+
+ /**
+ * Number of times MS should automatically retry scrobbles in dead letter queue
+ *
+ * @default 1
+ * @examples [1]
+ * */
+ deadLetterRetries?: boolean
}
}
diff --git a/src/backend/common/schema/aio-client.json b/src/backend/common/schema/aio-client.json
index c33f967e..ef66c4fd 100644
--- a/src/backend/common/schema/aio-client.json
+++ b/src/backend/common/schema/aio-client.json
@@ -37,6 +37,15 @@
"title": "checkExistingScrobbles",
"type": "boolean"
},
+ "deadLetterRetries": {
+ "default": 1,
+ "description": "Number of times MS should automatically retry scrobbles in dead letter queue",
+ "examples": [
+ 1
+ ],
+ "title": "deadLetterRetries",
+ "type": "boolean"
+ },
"refreshEnabled": {
"default": true,
"description": "Try to get fresh scrobble history from client when tracks to be scrobbled are newer than the last scrobble found in client history",
@@ -267,6 +276,15 @@
"title": "checkExistingScrobbles",
"type": "boolean"
},
+ "deadLetterRetries": {
+ "default": 1,
+ "description": "Number of times MS should automatically retry scrobbles in dead letter queue",
+ "examples": [
+ 1
+ ],
+ "title": "deadLetterRetries",
+ "type": "boolean"
+ },
"refreshEnabled": {
"default": true,
"description": "Try to get fresh scrobble history from client when tracks to be scrobbled are newer than the last scrobble found in client history",
@@ -400,6 +418,15 @@
"title": "checkExistingScrobbles",
"type": "boolean"
},
+ "deadLetterRetries": {
+ "default": 1,
+ "description": "Number of times MS should automatically retry scrobbles in dead letter queue",
+ "examples": [
+ 1
+ ],
+ "title": "deadLetterRetries",
+ "type": "boolean"
+ },
"refreshEnabled": {
"default": true,
"description": "Try to get fresh scrobble history from client when tracks to be scrobbled are newer than the last scrobble found in client history",
diff --git a/src/backend/common/schema/aio.json b/src/backend/common/schema/aio.json
index c2525220..f38bed05 100644
--- a/src/backend/common/schema/aio.json
+++ b/src/backend/common/schema/aio.json
@@ -37,6 +37,15 @@
"title": "checkExistingScrobbles",
"type": "boolean"
},
+ "deadLetterRetries": {
+ "default": 1,
+ "description": "Number of times MS should automatically retry scrobbles in dead letter queue",
+ "examples": [
+ 1
+ ],
+ "title": "deadLetterRetries",
+ "type": "boolean"
+ },
"refreshEnabled": {
"default": true,
"description": "Try to get fresh scrobble history from client when tracks to be scrobbled are newer than the last scrobble found in client history",
@@ -1089,6 +1098,15 @@
"title": "checkExistingScrobbles",
"type": "boolean"
},
+ "deadLetterRetries": {
+ "default": 1,
+ "description": "Number of times MS should automatically retry scrobbles in dead letter queue",
+ "examples": [
+ 1
+ ],
+ "title": "deadLetterRetries",
+ "type": "boolean"
+ },
"refreshEnabled": {
"default": true,
"description": "Try to get fresh scrobble history from client when tracks to be scrobbled are newer than the last scrobble found in client history",
@@ -1555,6 +1573,15 @@
"title": "checkExistingScrobbles",
"type": "boolean"
},
+ "deadLetterRetries": {
+ "default": 1,
+ "description": "Number of times MS should automatically retry scrobbles in dead letter queue",
+ "examples": [
+ 1
+ ],
+ "title": "deadLetterRetries",
+ "type": "boolean"
+ },
"refreshEnabled": {
"default": true,
"description": "Try to get fresh scrobble history from client when tracks to be scrobbled are newer than the last scrobble found in client history",
diff --git a/src/backend/common/schema/client.json b/src/backend/common/schema/client.json
index 29127046..1178af0a 100644
--- a/src/backend/common/schema/client.json
+++ b/src/backend/common/schema/client.json
@@ -34,6 +34,15 @@
"title": "checkExistingScrobbles",
"type": "boolean"
},
+ "deadLetterRetries": {
+ "default": 1,
+ "description": "Number of times MS should automatically retry scrobbles in dead letter queue",
+ "examples": [
+ 1
+ ],
+ "title": "deadLetterRetries",
+ "type": "boolean"
+ },
"refreshEnabled": {
"default": true,
"description": "Try to get fresh scrobble history from client when tracks to be scrobbled are newer than the last scrobble found in client history",
@@ -248,6 +257,15 @@
"title": "checkExistingScrobbles",
"type": "boolean"
},
+ "deadLetterRetries": {
+ "default": 1,
+ "description": "Number of times MS should automatically retry scrobbles in dead letter queue",
+ "examples": [
+ 1
+ ],
+ "title": "deadLetterRetries",
+ "type": "boolean"
+ },
"refreshEnabled": {
"default": true,
"description": "Try to get fresh scrobble history from client when tracks to be scrobbled are newer than the last scrobble found in client history",
@@ -373,6 +391,15 @@
"title": "checkExistingScrobbles",
"type": "boolean"
},
+ "deadLetterRetries": {
+ "default": 1,
+ "description": "Number of times MS should automatically retry scrobbles in dead letter queue",
+ "examples": [
+ 1
+ ],
+ "title": "deadLetterRetries",
+ "type": "boolean"
+ },
"refreshEnabled": {
"default": true,
"description": "Try to get fresh scrobble history from client when tracks to be scrobbled are newer than the last scrobble found in client history",
diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts
index 60158452..27614df2 100644
--- a/src/backend/scrobblers/AbstractScrobbleClient.ts
+++ b/src/backend/scrobblers/AbstractScrobbleClient.ts
@@ -28,6 +28,7 @@ import {compareScrobbleArtists, compareScrobbleTracks, normalizeStr} from "../ut
import {UpstreamError} from "../common/errors/UpstreamError";
import {nanoid} from "nanoid";
import {ErrorWithCause} from "pony-cause";
+import {de} from "@faker-js/faker";
export default abstract class AbstractScrobbleClient {
@@ -204,16 +205,6 @@ export default abstract class AbstractScrobbleClient {
return this.scrobbledPlayObjs.data.map(x => x.scrobble);
}
- cleanSourceSearchTitle = (playObj: PlayObject) => {
- const {
- data: {
- track,
- } = {},
- } = playObj;
-
- return track.toLocaleLowerCase().trim();
- };
-
findExistingSubmittedPlayObj = (playObj: PlayObject): ([undefined, undefined] | [ScrobbledPlayObject, ScrobbledPlayObject[]]) => {
const {
data: {
@@ -569,6 +560,81 @@ ${closestMatch.breakdowns.join('\n')}`);
}
}
+ processDeadLetterQueue = async (attemptWithRetries?: number) => {
+
+ if (this.deadLetterScrobbles.length === 0) {
+ return;
+ }
+
+ const {
+ options: {
+ deadLetterRetries = 1
+ } = {}
+ } = this.config.data;
+
+ const retries = attemptWithRetries ?? deadLetterRetries;
+
+ const processable = this.deadLetterScrobbles.filter(x => x.retries < retries);
+ this.logger.info(`${processable.length} of ${this.deadLetterScrobbles.length} dead scrobbles have less than ${retries} retries, ${processable.length === 0 ? 'will skip processing.': 'processing now...'}`, {leaf: 'Dead Letter'});
+ if (processable.length === 0) {
+ return;
+ }
+
+ const idsToRemove = [];
+ for (const deadScrobble of this.deadLetterScrobbles) {
+ if (deadScrobble.retries < retries) {
+ const scrobbled = await this.processDeadLetterScrobble(deadScrobble.id);
+ if (scrobbled) {
+ idsToRemove.push(deadScrobble.id);
+ }
+ }
+ }
+ if (idsToRemove.length > 0) {
+ this.deadLetterScrobbles = this.deadLetterScrobbles.filter(x => !idsToRemove.includes(x.id));
+ this.logger.info(`Removed ${idsToRemove.length} scrobbles from dead letter queue`, {leaf: 'Dead Letter'});
+ }
+ }
+
+ processDeadLetterScrobble = async (id: string) => {
+ if (!(await this.isReady())) {
+ this.logger.warn('Cannot process dead letter scrobble because client is not ready.', {leaf: 'Dead Letter'});
+ return;
+ }
+ const deadScrobbleIndex = this.deadLetterScrobbles.findIndex(x => x.id === id);
+ const deadScrobble = this.deadLetterScrobbles[deadScrobbleIndex];
+ if (this.lastScrobbleCheck.unix() < this.getLatestQueuePlayDate().unix()) {
+ await this.refreshScrobbles();
+ }
+ const [timeFrameValid, timeFrameValidLog] = this.timeFrameIsValid(deadScrobble.play);
+ if (timeFrameValid && !(await this.alreadyScrobbled(deadScrobble.play))) {
+ try {
+ const scrobbledPlay = await this.scrobble(deadScrobble.play);
+ this.emitEvent('scrobble', {play: deadScrobble.play});
+ this.addScrobbledTrack(deadScrobble.play, scrobbledPlay);
+ } catch (e) {
+ deadScrobble.retries++;
+ this.logger.error(`Could not scrobble ${buildTrackString(deadScrobble.play)} from Source '${deadScrobble.source}' due to error`, {leaf: 'Dead Letter'});
+ this.logger.error(e);
+ this.deadLetterScrobbles[deadScrobbleIndex] = deadScrobble;
+ return false;
+ } finally {
+ await sleep(1000);
+ }
+ } else if (!timeFrameValid) {
+ this.logger.debug(`Will not scrobble ${buildTrackString(deadScrobble.play)} from Source '${deadScrobble.source}' because it ${timeFrameValidLog}`, {leaf: 'Dead Letter'});
+ }
+ return true;
+ }
+
+ removeDeadLetterScrobble = (id: string) => {
+ const index = this.deadLetterScrobbles.findIndex(x => x.id === id);
+ if (index === -1) {
+ this.logger.warn(`No scrobble found with ID ${id}`, {leaf: 'Dead Letter'});
+ }
+ this.logger.debug(`Removed scrobble ${buildTrackString(this.deadLetterScrobbles[index].play)} from queue`, {leaf: 'Dead Letter'});
+ this.deadLetterScrobbles.splice(index, 1);
+ }
+
protected getLatestQueuePlayDate = () => {
if (this.queuedScrobbles.length === 0) {
return undefined;
--
2.54.0
From adfb166320a4e04fdc910e5a57166250c48cf1c3 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Fri, 29 Sep 2023 15:07:44 -0400
Subject: [PATCH 68/82] feat(scrobble): Implement heartbeat task for scrobblers
Task does scrobble processing restart and dead letter queue processing
---
src/backend/index.ts | 9 +++-
src/backend/tasks/heartbeatClients.ts | 44 +++++++++++++++++++
.../{heartbeat.ts => heartbeatSources.ts} | 4 +-
3 files changed, 53 insertions(+), 4 deletions(-)
create mode 100644 src/backend/tasks/heartbeatClients.ts
rename src/backend/tasks/{heartbeat.ts => heartbeatSources.ts} (89%)
diff --git a/src/backend/index.ts b/src/backend/index.ts
index 359f9eb2..95b77e83 100644
--- a/src/backend/index.ts
+++ b/src/backend/index.ts
@@ -20,7 +20,8 @@ import {getLogger} from "./common/logging";
import {LogInfo} from "../core/Atomic";
import {initServer} from "./server/index";
import {SimpleIntervalJob, ToadScheduler} from "toad-scheduler";
-import {createHeartbeatTask} from "./tasks/heartbeat";
+import {createHeartbeatSourcesTask} from "./tasks/heartbeatSources";
+import {createHeartbeatClientsTask} from "./tasks/heartbeatClients";
dayjs.extend(utc)
@@ -147,7 +148,11 @@ const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`)
scheduler.addSimpleIntervalJob(new SimpleIntervalJob({
minutes: 20,
runImmediately: false
- }, createHeartbeatTask(scrobbleSources, logger)));
+ }, createHeartbeatSourcesTask(scrobbleSources, logger)));
+ scheduler.addSimpleIntervalJob(new SimpleIntervalJob({
+ minutes: 20,
+ runImmediately: false
+ }, createHeartbeatClientsTask(scrobbleClients, logger)));
logger.info('Scheduler started.');
} catch (e) {
diff --git a/src/backend/tasks/heartbeatClients.ts b/src/backend/tasks/heartbeatClients.ts
new file mode 100644
index 00000000..90619d0c
--- /dev/null
+++ b/src/backend/tasks/heartbeatClients.ts
@@ -0,0 +1,44 @@
+import {config, Logger} from "@foxxmd/winston";
+import {mergeArr} from "../utils";
+import {AsyncTask} from "toad-scheduler";
+import {PromisePool} from "@supercharge/promise-pool";
+import ScrobbleClients from "../scrobblers/ScrobbleClients";
+
+export const createHeartbeatClientsTask = (clients: ScrobbleClients, parentLogger: Logger) => {
+ const logger = parentLogger.child({labels: ['Heartbeat', 'Clients']}, mergeArr);
+
+ return new AsyncTask(
+ 'Heartbeat',
+ (): Promise => {
+ return PromisePool
+ .withConcurrency(1)
+ .for(clients.clients)
+ .process(async (client) => {
+ if(await client.isReady()) {
+ await client.processDeadLetterQueue();
+ if(!client.scrobbling) {
+ client.logger.info('Should be processing scrobbles! Attempting to restart scrobbling...', {leaf: 'Heartbeat'});
+ client.initScrobbleMonitoring();
+ return 1;
+ }
+ return 0;
+ }
+ }).then(({results, errors}) => {
+ logger.info(`Checked Dead letter queue for ${clients.clients.length} clients.`);
+ const restarted = results.reduce((acc, curr) => acc += curr, 0);
+ if (restarted > 0) {
+ logger.info(`Attempted to restart ${restarted} clients that were not processing scrobbles.`);
+ }
+ if (errors.length > 0) {
+ logger.error(`Encountered errors!`);
+ for (const err of errors) {
+ logger.error(err);
+ }
+ }
+ });
+ },
+ (err: Error) => {
+ logger.error(err);
+ }
+ );
+}
diff --git a/src/backend/tasks/heartbeat.ts b/src/backend/tasks/heartbeatSources.ts
similarity index 89%
rename from src/backend/tasks/heartbeat.ts
rename to src/backend/tasks/heartbeatSources.ts
index 6093ea2e..07eb4d1c 100644
--- a/src/backend/tasks/heartbeat.ts
+++ b/src/backend/tasks/heartbeatSources.ts
@@ -4,8 +4,8 @@ import {AsyncTask} from "toad-scheduler";
import {PromisePool} from "@supercharge/promise-pool";
import ScrobbleSources from "../sources/ScrobbleSources";
-export const createHeartbeatTask = (sources: ScrobbleSources, parentLogger: Logger) => {
- const logger = parentLogger.child({labels: ['Heartbeat']}, mergeArr);
+export const createHeartbeatSourcesTask = (sources: ScrobbleSources, parentLogger: Logger) => {
+ const logger = parentLogger.child({labels: ['Heartbeat', 'Sources']}, mergeArr);
return new AsyncTask(
'Heartbeat',
--
2.54.0
From 43ebdcb10c7b33c12e7ddb191f54452f22afb7b3 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Mon, 2 Oct 2023 13:38:08 -0400
Subject: [PATCH 69/82] refactor(scrobble): Refactor scrobble delay and
scrobble processing sleep behavior
* Delay scrobble based on last scrobble attempt TS
* rename abstract scrobble func to doScrobble
* Implement scrobble() in abstract client and track last scrobble attempt time, delay based on arg or client property
* This way first scrobble isn't delayed and we only delay if subsequent scrobble attempts are close in time
* Make scrobble processing sleep time on empty queue configurable
---
.../scrobblers/AbstractScrobbleClient.ts | 35 +++++++++++++++----
src/backend/scrobblers/LastfmScrobbler.ts | 2 +-
.../scrobblers/ListenbrainzScrobbler.ts | 2 +-
src/backend/scrobblers/MalojaScrobbler.ts | 2 +-
src/backend/tests/scrobbler/TestScrobbler.ts | 4 +--
5 files changed, 34 insertions(+), 11 deletions(-)
diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts
index 27614df2..09a7f619 100644
--- a/src/backend/scrobblers/AbstractScrobbleClient.ts
+++ b/src/backend/scrobblers/AbstractScrobbleClient.ts
@@ -29,6 +29,7 @@ import {UpstreamError} from "../common/errors/UpstreamError";
import {nanoid} from "nanoid";
import {ErrorWithCause} from "pony-cause";
import {de} from "@faker-js/faker";
+import {del} from "superagent";
export default abstract class AbstractScrobbleClient {
@@ -51,10 +52,13 @@ export default abstract class AbstractScrobbleClient {
tracksScrobbled: number = 0;
lastScrobbleCheck: Dayjs = dayjs(0)
+ lastScrobbleAttempt: Dayjs = dayjs(0)
refreshEnabled: boolean;
checkExistingScrobbles: boolean;
verboseOptions;
+ scrobbleDelay: number = 1000;
+ scrobbleSleep: number = 2000;
scrobbleRetries: number = 0;
scrobbling: boolean = false;
userScrobblingStopSignal: undefined | any;
@@ -397,7 +401,25 @@ ${closestMatch.breakdowns.join('\n')}`);
return existingScrobble;
}
- public abstract scrobble(playObj: PlayObject): Promise
+ public scrobble = async (playObj: PlayObject, opts?: { delay?: number | false }): Promise => {
+ const {delay} = opts || {};
+ const scrobbleDelay = delay === undefined ? this.scrobbleDelay : (delay === false ? 0 : delay);
+ if (scrobbleDelay !== 0) {
+ const lastScrobbleDiff = dayjs().diff(this.lastScrobbleAttempt, 'ms');
+ const remainingDelay = scrobbleDelay - lastScrobbleDiff;
+ if (remainingDelay > 0) {
+ this.logger.debug(`Waiting ${remainingDelay}ms to scrobble so time passed since previous scrobble is at least ${scrobbleDelay}ms`);
+ await sleep(scrobbleDelay);
+ }
+ }
+ try {
+ return this.doScrobble(playObj);
+ } finally {
+ this.lastScrobbleAttempt = dayjs();
+ }
+ }
+
+ protected abstract doScrobble(playObj: PlayObject): Promise
initScrobbleMonitoring = async () => {
if(!this.initialized) {
@@ -534,8 +556,6 @@ ${closestMatch.breakdowns.join('\n')}`);
//this.logger.error(processError);
throw processError;
}
- } finally {
- await sleep(1000);
}
} else if (!timeFrameValid) {
this.logger.debug(`Will not scrobble ${buildTrackString(currQueuedPlay.play)} from Source '${currQueuedPlay.source}' because it ${timeFrameValidLog}`);
@@ -546,7 +566,7 @@ ${closestMatch.breakdowns.join('\n')}`);
this.queuedScrobbles.splice(pIndex, 1);
}
}
- await sleep(2000);
+ await sleep(this.scrobbleSleep);
}
if (this.shouldStopScrobbleProcessing()) {
this.doStopScrobbling(this.userScrobblingStopSignal !== undefined ? 'user input' : undefined);
@@ -642,8 +662,11 @@ ${closestMatch.breakdowns.join('\n')}`);
return this.queuedScrobbles[this.queuedScrobbles.length - 1].play.data.playDate;
}
- queueScrobble = (play: PlayObject, source: string) => {
- this.queuedScrobbles.push({id: nanoid(), source, play});
+ queueScrobble = (data: PlayObject | PlayObject[], source: string) => {
+ const plays = Array.isArray(data) ? data : [data];
+ for(const p of plays) {
+ this.queuedScrobbles.push({id: nanoid(), source, play: p});
+ }
this.queuedScrobbles.sort((a, b) => sortByOldestPlayDate(a.play, b.play));
}
diff --git a/src/backend/scrobblers/LastfmScrobbler.ts b/src/backend/scrobblers/LastfmScrobbler.ts
index 3a07cc83..65bc6151 100644
--- a/src/backend/scrobblers/LastfmScrobbler.ts
+++ b/src/backend/scrobblers/LastfmScrobbler.ts
@@ -119,7 +119,7 @@ export default class LastfmScrobbler extends AbstractScrobbleClient {
return (await this.existingScrobble(playObj)) !== undefined;
}
- scrobble = async (playObj: PlayObject) => {
+ doScrobble = async (playObj: PlayObject) => {
const {
data: {
artists,
diff --git a/src/backend/scrobblers/ListenbrainzScrobbler.ts b/src/backend/scrobblers/ListenbrainzScrobbler.ts
index f587d926..fbe8dec9 100644
--- a/src/backend/scrobblers/ListenbrainzScrobbler.ts
+++ b/src/backend/scrobblers/ListenbrainzScrobbler.ts
@@ -86,7 +86,7 @@ export default class ListenbrainzScrobbler extends AbstractScrobbleClient {
return (await this.existingScrobble(playObj)) !== undefined;
}
- scrobble = async (playObj: PlayObject) => {
+ doScrobble = async (playObj: PlayObject) => {
const {
meta: {
source,
diff --git a/src/backend/scrobblers/MalojaScrobbler.ts b/src/backend/scrobblers/MalojaScrobbler.ts
index 401332c1..9cff926c 100644
--- a/src/backend/scrobblers/MalojaScrobbler.ts
+++ b/src/backend/scrobblers/MalojaScrobbler.ts
@@ -365,7 +365,7 @@ export default class MalojaScrobbler extends AbstractScrobbleClient {
return (await this.existingScrobble(playObj)) !== undefined;
}
- scrobble = async (playObj: PlayObject) => {
+ doScrobble = async (playObj: PlayObject) => {
const {url, apiKey} = this.config.data;
const {
diff --git a/src/backend/tests/scrobbler/TestScrobbler.ts b/src/backend/tests/scrobbler/TestScrobbler.ts
index d8f7d410..9d3858fb 100644
--- a/src/backend/tests/scrobbler/TestScrobbler.ts
+++ b/src/backend/tests/scrobbler/TestScrobbler.ts
@@ -12,8 +12,8 @@ export class TestScrobbler extends AbstractScrobbleClient {
super('test', 'Test', {name: 'test'}, notifier, new EventEmitter(), logger);
}
- scrobble(playObj: PlayObject): Promise {
- return Promise.resolve(true);
+ doScrobble(playObj: PlayObject): Promise {
+ return Promise.resolve(playObj);
}
alreadyScrobbled = async (playObj: PlayObject, log?: boolean): Promise => {
--
2.54.0
From fefc82c521c96c8b83d2b8f7bade6479a396e9fd Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Mon, 2 Oct 2023 13:39:14 -0400
Subject: [PATCH 70/82] feat(scrobble): Improve existing scrobble comparison
logic
* Use existing time comparison function with "close" matching
* Fix wording and return value
---
.../scrobblers/AbstractScrobbleClient.ts | 22 +++++--------------
1 file changed, 5 insertions(+), 17 deletions(-)
diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts
index 09a7f619..8a82d54f 100644
--- a/src/backend/scrobblers/AbstractScrobbleClient.ts
+++ b/src/backend/scrobblers/AbstractScrobbleClient.ts
@@ -222,25 +222,12 @@ export default abstract class AbstractScrobbleClient {
const dtInvariantMatches = this.scrobbledPlayObjs.data.filter(x => playObjDataMatch(playObj, x.play));
if (dtInvariantMatches.length === 0) {
- return [undefined, undefined];
+ return [undefined, []];
}
const matchPlayDate = dtInvariantMatches.find((x: ScrobbledPlayObject) => {
- const {
- play: {
- data: {
- playDate: sPlayDate
- } = {},
- meta: {
- source: playSource
- } = {},
- } = {},
- } = x;
- // need to account for inaccurate DT from subsonic
- if(source === 'Subsonic' && playSource === 'Subsonic') {
- return playDate.isSame(sPlayDate) || playDate.diff(sPlayDate, 'minute') <= 1;
- }
- return playDate.isSame(sPlayDate);
+ const [closeTime, fuzzyTime = false] = this.compareExistingScrobbleTime(x.play, playObj);
+ return closeTime;
});
return [matchPlayDate, dtInvariantMatches];
@@ -300,8 +287,9 @@ export default abstract class AbstractScrobbleClient {
closestMatch = {
score: 1,
+ scrobble: existingScrobble,
breakdowns: [],
- confidence: 'Exact Match found in previously successfully scrobbled'
+ confidence: 'Exact Match found in previously successfully scrobbled plays'
}
}
// if not though then we need to check recent scrobbles from scrobble api.
--
2.54.0
From 9aae57f96627acce6e9a966a7b19a4b5cc030913 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Mon, 2 Oct 2023 13:39:33 -0400
Subject: [PATCH 71/82] test(scrobble): Add tests for scrobble processing queue
and existing comparisons
---
.../tests/scrobbler/scrobblers.test.ts | 418 ++++++++++++------
1 file changed, 281 insertions(+), 137 deletions(-)
diff --git a/src/backend/tests/scrobbler/scrobblers.test.ts b/src/backend/tests/scrobbler/scrobblers.test.ts
index 53c448b2..24207d9c 100644
--- a/src/backend/tests/scrobbler/scrobblers.test.ts
+++ b/src/backend/tests/scrobbler/scrobblers.test.ts
@@ -1,6 +1,7 @@
import {describe, it} from 'mocha';
import {assert} from 'chai';
import clone from 'clone';
+import pEvent from 'p-event';
import withDuration from '../plays/withDuration.json';
import mixedDuration from '../plays/mixedDuration.json';
@@ -8,6 +9,7 @@ import mixedDuration from '../plays/mixedDuration.json';
import {TestScrobbler} from "./TestScrobbler";
import {asPlays, generatePlay, normalizePlays} from "../utils/PlayTestUtils";
import dayjs from "dayjs";
+import {sleep} from "../../utils";
const firstPlayDate = dayjs().subtract(1, 'hour');
@@ -22,227 +24,369 @@ testScrobbler.verboseOptions = {
onMatch: true,
onNoMatch: true,
confidenceBreakdown: true
- }};
+ }
+};
testScrobbler.lastScrobbleCheck = dayjs().subtract(60, 'seconds');
-describe('When scrobble is unique', function () {
+describe('Detects duplicate and unique scrobbles from client recent history', function () {
- it('It is not detected as duplicate when play date is newer than most recent', async function () {
+ describe('When scrobble is unique', function () {
- testScrobbler.recentScrobbles = normalizedWithMixedDur;
+ it('It is not detected as duplicate when play date is newer than most recent', async function () {
- const newScrobble = generatePlay({
- playDate: normalizedWithMixedDur[normalizedWithMixedDur.length - 1].data.playDate.add(70, 'seconds')
+ testScrobbler.recentScrobbles = normalizedWithMixedDur;
+
+ const newScrobble = generatePlay({
+ playDate: normalizedWithMixedDur[normalizedWithMixedDur.length - 1].data.playDate.add(70, 'seconds')
+ });
+
+ assert.isFalse(await testScrobbler.alreadyScrobbled(newScrobble));
});
- assert.isFalse(await testScrobbler.alreadyScrobbled(newScrobble));
+ it('It is not detected as duplicate when play date is close to an existing scrobble', async function () {
+
+ testScrobbler.recentScrobbles = normalizedWithMixedDur;
+
+ const newScrobble = generatePlay({
+ playDate: normalizedWithMixedDur[normalizedWithMixedDur.length - 3].data.playDate.add(3, 'seconds')
+ });
+
+ assert.isFalse(await testScrobbler.alreadyScrobbled(newScrobble));
+ });
});
- it('It is not detected as duplicate when play date is close to an existing scrobble', async function () {
+ describe('When scrobble track/artist/album matches existing but is a new scrobble', function () {
- testScrobbler.recentScrobbles = normalizedWithMixedDur;
+ it('Is not detected as duplicate when artist is same, time is similar, but track is different', async function () {
- const newScrobble = generatePlay({
- playDate: normalizedWithMixedDur[normalizedWithMixedDur.length - 3].data.playDate.add(3, 'seconds')
+ testScrobbler.recentScrobbles = normalizedWithMixedDur;
+
+ const diffPlay = clone(normalizedWithMixedDur[1]);
+ diffPlay.data.playDate = diffPlay.data.playDate.add(9, 's');
+ diffPlay.data.track = 'A Totally Different Track'
+
+ assert.isFalse(await testScrobbler.alreadyScrobbled(diffPlay));
});
- assert.isFalse(await testScrobbler.alreadyScrobbled(newScrobble));
- });
-});
+ it('Is not detected as duplicate when track is same, time is similar, but artist is different', async function () {
-describe('When scrobble track/artist/album matches existing but is a new scrobble', function () {
+ testScrobbler.recentScrobbles = normalizedWithMixedDur;
- it('Is not detected as duplicate when artist is same, time is similar, but track is different', async function () {
+ const diffPlay = clone(normalizedWithMixedDur[1]);
+ diffPlay.data.playDate = diffPlay.data.playDate.add(9, 's');
+ diffPlay.data.artists = ['A Different Artist'];
- testScrobbler.recentScrobbles = normalizedWithMixedDur;
+ assert.isFalse(await testScrobbler.alreadyScrobbled(diffPlay));
+ });
- const diffPlay = clone(normalizedWithMixedDur[1]);
- diffPlay.data.playDate = diffPlay.data.playDate.add(9, 's');
- diffPlay.data.track = 'A Totally Different Track'
- assert.isFalse(await testScrobbler.alreadyScrobbled(diffPlay));
- });
+ it('Is not detected as duplicate when play date is different by more than 10 seconds (high granularity source)', async function () {
- it('Is not detected as duplicate when track is same, time is similar, but artist is different', async function () {
+ testScrobbler.recentScrobbles = normalizedWithMixedDur;
- testScrobbler.recentScrobbles = normalizedWithMixedDur;
+ const timeOffPos = clone(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]);
+ timeOffPos.data.playDate = timeOffPos.data.playDate.add(11, 's');
- const diffPlay = clone(normalizedWithMixedDur[1]);
- diffPlay.data.playDate = diffPlay.data.playDate.add(9, 's');
- diffPlay.data.artists = ['A Different Artist'];
+ const timeOffNeg = clone(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]);
+ timeOffNeg.data.playDate = timeOffNeg.data.playDate.subtract(11, 's');
- assert.isFalse(await testScrobbler.alreadyScrobbled(diffPlay));
- });
+ assert.isFalse(await testScrobbler.alreadyScrobbled(timeOffPos));
+ assert.isFalse(await testScrobbler.alreadyScrobbled(timeOffNeg));
+ });
+ it('Is not detected as duplicate when play date is different by more than 60 seconds (low granularity source)', async function () {
- it('Is not detected as duplicate when play date is different by more than 10 seconds (high granularity source)', async function () {
+ testScrobbler.recentScrobbles = normalizePlays(mixedDurPlays, {
+ initialDate: firstPlayDate,
+ defaultMeta: {source: 'subsonic'}
+ });
- testScrobbler.recentScrobbles = normalizedWithMixedDur;
+ const timeOffPos = clone(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]);
+ timeOffPos.data.playDate = timeOffPos.data.playDate.add(61, 's');
- const timeOffPos = clone(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]);
- timeOffPos.data.playDate = timeOffPos.data.playDate.add(11, 's');
+ const timeOffNeg = clone(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]);
+ timeOffNeg.data.playDate = timeOffNeg.data.playDate.subtract(61, 's');
- const timeOffNeg = clone(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]);
- timeOffNeg.data.playDate = timeOffNeg.data.playDate.subtract(11, 's');
+ assert.isFalse(await testScrobbler.alreadyScrobbled(timeOffPos));
+ assert.isFalse(await testScrobbler.alreadyScrobbled(timeOffNeg));
+ });
- assert.isFalse(await testScrobbler.alreadyScrobbled(timeOffPos));
- assert.isFalse(await testScrobbler.alreadyScrobbled(timeOffNeg));
- });
+ describe('When existing has duration', function () {
+ it('A track with continuity to the previous track is not detected as a duplicate', async function () {
+
+ testScrobbler.recentScrobbles = normalizedWithDur;
- it('Is not detected as duplicate when play date is different by more than 60 seconds (low granularity source)', async function () {
+ const brickPt1 = normalizedWithDur.find(x => x.data.track.includes('Another Brick'));
+ const brickPt2 = clone(brickPt1);
+ brickPt2.data.track = 'Another Brick in the Wall, Pt. 2';
+ brickPt2.data.playDate = brickPt1.data.playDate.add(brickPt1.data.duration + 1, 'seconds');
+ assert.isFalse(await testScrobbler.alreadyScrobbled(brickPt2));
- testScrobbler.recentScrobbles = normalizePlays(mixedDurPlays, {initialDate: firstPlayDate, defaultMeta: {source: 'subsonic'}});
+ const story1 = normalizedWithDur.find(x => x.data.track.includes('Da Art of'));
+ const story2 = clone(story1);
+ story2.data.track = `Da Art of Storytellin' (Pt. 2)`;
+ story2.data.playDate = story2.data.playDate.add(story1.data.duration + 1, 'seconds');
- const timeOffPos = clone(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]);
- timeOffPos.data.playDate = timeOffPos.data.playDate.add(61, 's');
+ assert.isFalse(await testScrobbler.alreadyScrobbled(story2));
- const timeOffNeg = clone(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]);
- timeOffNeg.data.playDate = timeOffNeg.data.playDate.subtract(61, 's');
+ const ballad1 = normalizedWithDur.find(x => x.data.track.includes('Ballade No. 1'));
+ const ballad2 = clone(ballad1);
+ ballad2.data.track = `Ballade No. 2 in G Minor, Op. 27`;
+ ballad2.data.playDate = ballad2.data.playDate.add(ballad1.data.duration + 1, 'seconds');
+
+ assert.isFalse(await testScrobbler.alreadyScrobbled(ballad2));
+ });
+ });
- assert.isFalse(await testScrobbler.alreadyScrobbled(timeOffPos));
- assert.isFalse(await testScrobbler.alreadyScrobbled(timeOffNeg));
});
- describe('When existing has duration', function () {
- it('A track with continuity to the previous track is not detected as a duplicate', async function () {
+ describe('When scrobble is a duplicate (title/artists/album)', function () {
- testScrobbler.recentScrobbles = normalizedWithDur;
+ it('Is detected as duplicate when an exact match', async function () {
+ testScrobbler.recentScrobbles = normalizedWithMixedDur;
+ assert.isTrue(await testScrobbler.alreadyScrobbled(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]));
+ });
+
+ it('Is detected as duplicate when artist/title differences are whitespace or case', async function () {
+ testScrobbler.recentScrobbles = normalizedWithMixedDur;
+ const ref = normalizedWithMixedDur[3];
+
+ const diffPlay = clone(ref);
+ diffPlay.data.playDate = diffPlay.data.playDate.add(9, 's');
- const brickPt1 = normalizedWithDur.find(x => x.data.track.includes('Another Brick'));
- const brickPt2 = clone(brickPt1);
- brickPt2.data.track = 'Another Brick in the Wall, Pt. 2';
- brickPt2.data.playDate = brickPt1.data.playDate.add(brickPt1.data.duration + 1, 'seconds');
- assert.isFalse(await testScrobbler.alreadyScrobbled(brickPt2));
- const story1 = normalizedWithDur.find(x => x.data.track.includes('Da Art of'));
- const story2 = clone(story1);
- story2.data.track = `Da Art of Storytellin' (Pt. 2)`;
- story2.data.playDate = story2.data.playDate.add(story1.data.duration + 1, 'seconds');
+ diffPlay.data.track = ref.data.track.toUpperCase();
+ assert.isTrue(await testScrobbler.alreadyScrobbled(diffPlay));
- assert.isFalse(await testScrobbler.alreadyScrobbled(story2));
+ diffPlay.data.track = ` ${ref.data.track} `;
+ assert.isTrue(await testScrobbler.alreadyScrobbled(diffPlay));
- const ballad1 = normalizedWithDur.find(x => x.data.track.includes('Ballade No. 1'));
- const ballad2 = clone(ballad1);
- ballad2.data.track = `Ballade No. 2 in G Minor, Op. 27`;
- ballad2.data.playDate = ballad2.data.playDate.add(ballad1.data.duration + 1, 'seconds');
+ diffPlay.data.track = ref.data.track.replaceAll(' ', ' ');
+ assert.isTrue(await testScrobbler.alreadyScrobbled(diffPlay));
- assert.isFalse(await testScrobbler.alreadyScrobbled(ballad2));
+ diffPlay.data.artists = ref.data.artists.map(x => x.toUpperCase());
+ assert.isTrue(await testScrobbler.alreadyScrobbled(diffPlay));
+
+ diffPlay.data.artists = ref.data.artists.map(x => x.replaceAll(' ', ' '));
+ assert.isTrue(await testScrobbler.alreadyScrobbled(diffPlay));
});
- });
-});
+ it('Is detected as duplicate when artist/title differences are from unicode normalization', async function () {
+ testScrobbler.recentScrobbles = normalizedWithMixedDur;
+ const ref = normalizedWithMixedDur.find(x => x.data.track === 'Jimbó');
-describe('When scrobble is a duplicate (title/artists/album)', function () {
+ const diffPlay = clone(ref);
+ diffPlay.data.playDate = diffPlay.data.playDate.add(9, 's');
+ diffPlay.data.track = 'Jimbo';
+ assert.isTrue(await testScrobbler.alreadyScrobbled(diffPlay));
+ });
- it('Is detected as duplicate when an exact match', async function () {
- testScrobbler.recentScrobbles = normalizedWithMixedDur;
- assert.isTrue(await testScrobbler.alreadyScrobbled(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]));
- });
+ it('Is detected as duplicate when play date is off by less than 10 seconds (high granularity source)', async function () {
- it('Is detected as duplicate when artist/title differences are whitespace or case', async function () {
- testScrobbler.recentScrobbles = normalizedWithMixedDur;
- const ref = normalizedWithMixedDur[3];
+ testScrobbler.recentScrobbles = normalizedWithMixedDur;
- const diffPlay = clone(ref);
- diffPlay.data.playDate = diffPlay.data.playDate.add(9, 's');
+ const timeOffPos = clone(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]);
+ timeOffPos.data.playDate = timeOffPos.data.playDate.add(10, 's');
+ const timeOffNeg = clone(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]);
+ timeOffNeg.data.playDate = timeOffNeg.data.playDate.subtract(10, 's');
- diffPlay.data.track = ref.data.track.toUpperCase();
- assert.isTrue(await testScrobbler.alreadyScrobbled(diffPlay));
+ assert.isTrue(await testScrobbler.alreadyScrobbled(timeOffPos));
+ assert.isTrue(await testScrobbler.alreadyScrobbled(timeOffNeg));
+ });
- diffPlay.data.track = ` ${ref.data.track} `;
- assert.isTrue(await testScrobbler.alreadyScrobbled(diffPlay));
+ it('Is detected as duplicate when play date is off by less than 60 seconds (low granularity source)', async function () {
- diffPlay.data.track = ref.data.track.replaceAll(' ', ' ');
- assert.isTrue(await testScrobbler.alreadyScrobbled(diffPlay));
+ testScrobbler.recentScrobbles = normalizePlays(mixedDurPlays, {
+ initialDate: firstPlayDate,
+ defaultMeta: {source: 'subsonic'}
+ });
- diffPlay.data.artists = ref.data.artists.map(x => x.toUpperCase());
- assert.isTrue(await testScrobbler.alreadyScrobbled(diffPlay));
+ const timeOffPos = clone(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]);
+ timeOffPos.data.playDate = timeOffPos.data.playDate.add(59, 's');
- diffPlay.data.artists = ref.data.artists.map(x => x.replaceAll(' ', ' '));
- assert.isTrue(await testScrobbler.alreadyScrobbled(diffPlay));
- });
+ const timeOffNeg = clone(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]);
+ timeOffNeg.data.playDate = timeOffNeg.data.playDate.subtract(59, 's');
- it('Is detected as duplicate when artist/title differences are from unicode normalization', async function () {
- testScrobbler.recentScrobbles = normalizedWithMixedDur;
- const ref = normalizedWithMixedDur.find(x => x.data.track === 'Jimbó');
+ assert.isTrue(await testScrobbler.alreadyScrobbled(timeOffPos));
+ assert.isTrue(await testScrobbler.alreadyScrobbled(timeOffNeg));
+ });
- const diffPlay = clone(ref);
- diffPlay.data.playDate = diffPlay.data.playDate.add(9, 's');
- diffPlay.data.track = 'Jimbo';
- assert.isTrue(await testScrobbler.alreadyScrobbled(diffPlay));
- });
+ it('Is detected as duplicate when title is exact, artist is similar, and time is similar', async function () {
+ testScrobbler.recentScrobbles = normalizedWithMixedDur;
+ const ref = normalizedWithMixedDur[3];
+
+ const diffPlay = clone(ref);
+ diffPlay.data.playDate = diffPlay.data.playDate.add(3, 's');
+ diffPlay.data.artists = [diffPlay.data.artists[0]]
+ assert.isTrue(await testScrobbler.alreadyScrobbled(diffPlay));
+
+ diffPlay.data.artists = [ref.data.artists[1]]
+ assert.isTrue(await testScrobbler.alreadyScrobbled(diffPlay));
+
+
+ const son = normalizedWithMixedDur.find(x => x.data.track === 'Sonora')
+
+ const sonDiffPlay = clone(son);
+ sonDiffPlay.data.playDate = sonDiffPlay.data.playDate.subtract(son.data.duration + 1, 's');
+ sonDiffPlay.data.artists = [sonDiffPlay.data.artists[1]]
+ assert.isTrue(await testScrobbler.alreadyScrobbled(sonDiffPlay));
+ });
+
+ describe('When at least one play has duration', function () {
+
+ it('Is detected as duplicate when play date is close to the end of an existing scrobble', async function () {
+
+ testScrobbler.recentScrobbles = normalizedWithDur;
- it('Is detected as duplicate when play date is off by less than 10 seconds (high granularity source)', async function () {
+ const timeEnd = clone(normalizedWithDur[normalizedWithMixedDur.length - 1]);
+ timeEnd.data.playDate = timeEnd.data.playDate.add(timeEnd.data.duration, 's');
- testScrobbler.recentScrobbles = normalizedWithMixedDur;
+ assert.isTrue(await testScrobbler.alreadyScrobbled(timeEnd));
- const timeOffPos = clone(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]);
- timeOffPos.data.playDate = timeOffPos.data.playDate.add(10, 's');
+ // only one has duration
+ timeEnd.data.duration = undefined;
- const timeOffNeg = clone(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]);
- timeOffNeg.data.playDate = timeOffNeg.data.playDate.subtract(10, 's');
+ assert.isTrue(await testScrobbler.alreadyScrobbled(timeEnd));
- assert.isTrue(await testScrobbler.alreadyScrobbled(timeOffPos));
- assert.isTrue(await testScrobbler.alreadyScrobbled(timeOffNeg));
+ // only one has duration
+ timeEnd.data.duration = undefined;
+
+ assert.isTrue(await testScrobbler.alreadyScrobbled(timeEnd));
+ });
+
+ });
});
+});
- it('Is detected as duplicate when play date is off by less than 60 seconds (low granularity source)', async function () {
+describe('Detects duplicate and unique scrobbles using actively tracked scrobbles', function() {
+ testScrobbler.recentScrobbles = normalizedWithMixedDur;
+ testScrobbler.lastScrobbleCheck = dayjs().subtract(60, 'seconds');
- testScrobbler.recentScrobbles = normalizePlays(mixedDurPlays, {initialDate: firstPlayDate, defaultMeta: {source: 'subsonic'}});
+ it('Detects a unique play', async function() {
+ const newScrobble = generatePlay({
+ playDate: normalizedWithMixedDur[normalizedWithMixedDur.length - 3].data.playDate.add(3, 'seconds')
+ });
- const timeOffPos = clone(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]);
- timeOffPos.data.playDate = timeOffPos.data.playDate.add(59, 's');
+ const [matchedPlay, matchedData] = testScrobbler.findExistingSubmittedPlayObj(newScrobble);
- const timeOffNeg = clone(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]);
- timeOffNeg.data.playDate = timeOffNeg.data.playDate.subtract(59, 's');
+ assert.isUndefined(matchedPlay);
+ assert.isEmpty(matchedData);
+ });
- assert.isTrue(await testScrobbler.alreadyScrobbled(timeOffPos));
- assert.isTrue(await testScrobbler.alreadyScrobbled(timeOffNeg));
+ it('Detects an exact duplicate', async function() {
+ const newScrobble = generatePlay({
+ playDate: normalizedWithMixedDur[normalizedWithMixedDur.length - 3].data.playDate.add(3, 'seconds')
+ });
+ testScrobbler.addScrobbledTrack(newScrobble, newScrobble);
+
+ const [matchedPlay, matchedData] = testScrobbler.findExistingSubmittedPlayObj(newScrobble);
+
+ assert.isDefined(matchedPlay);
+ assert.isNotEmpty(matchedData);
});
- it('Is detected as duplicate when title is exact, artist is similar, and time is similar', async function () {
- testScrobbler.recentScrobbles = normalizedWithMixedDur;
- const ref = normalizedWithMixedDur[3];
+ it('Detects a duplicate with close time', async function() {
+ const newScrobble = generatePlay({
+ playDate: normalizedWithMixedDur[normalizedWithMixedDur.length - 3].data.playDate.add(3, 'seconds')
+ });
+ testScrobbler.addScrobbledTrack(newScrobble, newScrobble);
+
+ const dupScrobble = clone(newScrobble);
+ dupScrobble.data.playDate = newScrobble.data.playDate.add(2, 'seconds');
- const diffPlay = clone(ref);
- diffPlay.data.playDate = diffPlay.data.playDate.add(3, 's');
- diffPlay.data.artists = [diffPlay.data.artists[0]]
- assert.isTrue(await testScrobbler.alreadyScrobbled(diffPlay));
+ const [matchedPlay, matchedData] = testScrobbler.findExistingSubmittedPlayObj(dupScrobble);
- diffPlay.data.artists = [ref.data.artists[1]]
- assert.isTrue(await testScrobbler.alreadyScrobbled(diffPlay));
+ assert.isDefined(matchedPlay);
+ assert.isNotEmpty(matchedData);
+ });
+});
+describe('Manages scrobble queue', function() {
- const son = normalizedWithMixedDur.find(x => x.data.track === 'Sonora')
+ testScrobbler.recentScrobbles = normalizedWithMixedDur;
+ testScrobbler.scrobbleSleep = 500;
+ testScrobbler.scrobbleDelay = 0;
+ testScrobbler.lastScrobbleCheck = dayjs().subtract(60, 'seconds');
+ testScrobbler.initScrobbleMonitoring();
+
+ it('Scrobbles a uniquely queued play', async function() {
+ const newScrobble = generatePlay({
+ playDate: normalizedWithMixedDur[normalizedWithMixedDur.length - 3].data.playDate.add(3, 'seconds')
+ });
+ testScrobbler.queueScrobble(newScrobble, 'test');
+ const res = await Promise.race([pEvent(testScrobbler.emitter, 'scrobble'), sleep(3000)]);
- const sonDiffPlay = clone(son);
- sonDiffPlay.data.playDate = sonDiffPlay.data.playDate.subtract(son.data.duration + 1, 's');
- sonDiffPlay.data.artists = [sonDiffPlay.data.artists[1]]
- assert.isTrue(await testScrobbler.alreadyScrobbled(sonDiffPlay));
+ assert.isDefined(res);
+ assert.isDefined(res.data);
});
- describe('When at least one play has duration', function () {
+ it('Does not Scrobble a duplicate play queued after original is scrobbled', async function() {
+ this.timeout(3500);
- it('Is detected as duplicate when play date is close to the end of an existing scrobble', async function () {
+ const newScrobble = generatePlay({
+ playDate: normalizedWithMixedDur[normalizedWithMixedDur.length - 3].data.playDate.add(3, 'seconds')
+ });
+ const dupScrobble = clone(newScrobble);
+ dupScrobble.data.playDate = newScrobble.data.playDate.add(2, 'seconds');
- testScrobbler.recentScrobbles = normalizedWithDur;
+ testScrobbler.queueScrobble(newScrobble, 'test');
+ const res = await Promise.race([pEvent(testScrobbler.emitter, 'scrobble'), sleep(1500)]);
- const timeEnd = clone(normalizedWithDur[normalizedWithMixedDur.length - 1]);
- timeEnd.data.playDate = timeEnd.data.playDate.add(timeEnd.data.duration, 's');
+ assert.isDefined(res);
+ assert.isDefined(res.data);
- assert.isTrue(await testScrobbler.alreadyScrobbled(timeEnd));
+ testScrobbler.queueScrobble(dupScrobble, 'test');
+ const resDup = await Promise.race([pEvent(testScrobbler.emitter, 'scrobble'), sleep(1100)]);
- // only one has duration
- timeEnd.data.duration = undefined;
+ assert.isUndefined(resDup);
+ });
+
+ it('Does not Scrobble a duplicate play queued before original is scrobbled', async function() {
+ this.timeout(3500);
+
+ const newScrobble = generatePlay({
+ playDate: normalizedWithMixedDur[normalizedWithMixedDur.length - 3].data.playDate.add(3, 'seconds')
+ });
+ const dupScrobble = clone(newScrobble);
+ dupScrobble.data.playDate = newScrobble.data.playDate.add(2, 'seconds');
- assert.isTrue(await testScrobbler.alreadyScrobbled(timeEnd));
+ testScrobbler.queueScrobble(newScrobble, 'test');
+ testScrobbler.queueScrobble(dupScrobble, 'test');
+ const res = await Promise.race([pEvent(testScrobbler.emitter, 'scrobble'), sleep(1500)]);
- // only one has duration
- timeEnd.data.duration = undefined;
+ assert.isDefined(res);
+ assert.isDefined(res.data);
- assert.isTrue(await testScrobbler.alreadyScrobbled(timeEnd));
+ const resDup = await Promise.race([pEvent(testScrobbler.emitter, 'scrobble'), sleep(1100)]);
+
+ assert.isUndefined(resDup);
+ });
+
+ it('Delays scrobbles when many are queued', async function () {
+ this.timeout(3500);
+
+ const newScrobble1 = generatePlay({
+ playDate: normalizedWithMixedDur[normalizedWithMixedDur.length - 3].data.playDate.add(3, 'seconds')
});
+ const newScrobble2 = generatePlay({
+ playDate: normalizedWithMixedDur[normalizedWithMixedDur.length - 3].data.playDate.add(80, 'seconds')
+ });
+ const newScrobble3 = generatePlay({
+ playDate: normalizedWithMixedDur[normalizedWithMixedDur.length - 3].data.playDate.add(160, 'seconds')
+ });
+
+ testScrobbler.scrobbleDelay = 600;
+
+ testScrobbler.queueScrobble([newScrobble1, newScrobble2, newScrobble3], 'test');
+ await pEvent(testScrobbler.emitter, 'scrobble');
+ const initial = dayjs();
+ await pEvent(testScrobbler.emitter, 'scrobble');
+ await pEvent(testScrobbler.emitter, 'scrobble');
+
+ const end = dayjs();
+ // roughly...
+ assert.closeTo(end.diff(initial, 'ms'), 1200, 200);
});
});
--
2.54.0
From fb5d9fb7b0bda11281be68aacb6a8c9d9f30ecaf Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Mon, 2 Oct 2023 13:54:55 -0400
Subject: [PATCH 72/82] feat(player): Log when play fails discovery on player
change
---
src/backend/sources/MemorySource.ts | 4 +++-
src/backend/utils.ts | 2 +-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/src/backend/sources/MemorySource.ts b/src/backend/sources/MemorySource.ts
index f2a5154d..5ee8d839 100644
--- a/src/backend/sources/MemorySource.ts
+++ b/src/backend/sources/MemorySource.ts
@@ -159,11 +159,11 @@ export default class MemorySource extends AbstractSource {
// wait to discover play until it is stale or current play has changed
// so that our discovered track has an accurate "listenedFor" count
if (candidate !== undefined && (playChanged || player.isUpdateStale())) {
+ let stPrefix = `${buildTrackString(candidate, {include: ['trackId', 'artist', 'track']})}`;
const thresholdResults = timePassesScrobbleThreshold(scrobbleThresholds, candidate.data.listenedFor, candidate.data.duration);
if (thresholdResults.passes) {
const matchingRecent = this.existingDiscovered(candidate); //sRecentlyPlayed.find(x => playObjDataMatch(x, candidate));
- let stPrefix = `${buildTrackString(candidate, {include: ['trackId', 'artist', 'track']})}`;
if (matchingRecent === undefined) {
if(this.playerSourceOfTruth) {
player.logger.debug(`${stPrefix} added after ${thresholdResultSummary(thresholdResults)} and not matching any prior plays`);
@@ -192,6 +192,8 @@ export default class MemorySource extends AbstractSource {
}
}
}
+ } else if(playChanged) {
+ player.logger.verbose(`${stPrefix} not added because ${thresholdResultSummary(thresholdResults)}.`);
}
}
diff --git a/src/backend/utils.ts b/src/backend/utils.ts
index f455c5fa..f12020e1 100644
--- a/src/backend/utils.ts
+++ b/src/backend/utils.ts
@@ -643,7 +643,7 @@ export const thresholdResultSummary = (result: ScrobbleThresholdResult) => {
parts.push(`tracked percent of ${(result.percent.value).toFixed(2)}% (wanted ${result.percent.threshold}%)`)
}
- return `${result.passes ? 'met' : 'did not meet'} thresholds with ${parts.join(' and')}`;
+ return `${result.passes ? 'met' : 'did not meet'} thresholds with ${parts.join(' and ')}`;
}
export function parseBool(value: any, prev: any = false): boolean {
--
2.54.0
From 1ab12355f3049beb19c5fff2466a98df08124732 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Mon, 2 Oct 2023 17:22:59 -0400
Subject: [PATCH 73/82] feat(player): Implement seek and repeat detection
* Start new listening ranges if player has seeked forward/backwards
* Treat track repeats as new plays
---
.../PlayerState/AbstractPlayerState.ts | 102 +++++++++++++++++-
src/backend/utils.ts | 2 +-
2 files changed, 102 insertions(+), 2 deletions(-)
diff --git a/src/backend/sources/PlayerState/AbstractPlayerState.ts b/src/backend/sources/PlayerState/AbstractPlayerState.ts
index 8c270772..452e3ea0 100644
--- a/src/backend/sources/PlayerState/AbstractPlayerState.ts
+++ b/src/backend/sources/PlayerState/AbstractPlayerState.ts
@@ -133,7 +133,20 @@ export abstract class AbstractPlayerState {
} else if (status !== undefined && !AbstractPlayerState.isProgressStatus(status)) {
this.currentListenSessionEnd();
this.calculatedStatus = this.reportedStatus;
+ } else if (this.isSessionRepeat(play.meta.trackProgressPosition)) {
+ // if we detect the track has been restarted end listen session and treat as a new play
+ this.currentListenSessionEnd();
+ const played = this.getPlayedObject();
+ this.currentPlay.data.playDate = dayjs();
+ this.currentListenSessionContinue(play.meta.trackProgressPosition);
+ return [this.getPlayedObject(), played];
} else {
+ const seekedPos = this.isPositionSeeked(play.meta.trackProgressPosition);
+ if (seekedPos !== false) {
+ this.logger.debug(`Detected player was seeked ${seekedPos.toFixed(2)}s, starting new listen range`);
+ // if player has been seeked start a new listen range so our numbers don't get all screwy
+ this.currentListenSessionEnd();
+ }
this.currentListenSessionContinue(play.meta.trackProgressPosition);
}
} else {
@@ -181,7 +194,7 @@ export abstract class AbstractPlayerState {
return undefined;
}
- getListenDuration() {
+ getListenDuration(){
let listenDur: number = 0;
let ranges = [...this.listenRanges];
if (this.currentListenRange !== undefined) {
@@ -232,6 +245,93 @@ export abstract class AbstractPlayerState {
this.currentListenRange = undefined;
}
+ isCurrentRangeInitial() {
+ if(this.currentListenRange === undefined) {
+ return true;
+ }
+ if(this.currentListenRange[0].position !== undefined && this.currentListenRange[1].position !== undefined) {
+ return this.currentListenRange[0].position === this.currentListenRange[1].position;
+ }
+ return this.currentListenRange[0].timestamp.isSame(this.currentListenRange[1].timestamp);
+ }
+
+ isCurrentRangePositional() {
+ if(this.currentListenRange === undefined) {
+ return false;
+ }
+ return this.currentListenRange[0].position !== undefined && this.currentListenRange[1].position !== undefined;
+ }
+
+ isPositionSeeked(position?: number) {
+ if (position === undefined || this.isCurrentRangeInitial() || !this.isCurrentRangePositional()) {
+ return false;
+ }
+ // if (new) position is earlier than last stored position then the user has seeked backwards on the player
+ if (position < this.currentListenRange[1].position) {
+ return position - this.currentListenRange[1].position;
+ }
+ // if (new) position is more than a reasonable number of ms ahead of real time than they have seeked forwards on the player
+ const realTimeDiff = dayjs().diff(this.currentListenRange[1].timestamp, 'ms');
+ const positionDiff = (position - this.currentListenRange[1].position) * 1000;
+ // if user is more than 2.5 seconds ahead of real time
+ if (positionDiff - realTimeDiff > 2500) {
+ return position - this.currentListenRange[1].position;
+ }
+
+ return false;
+ }
+
+ isSessionRepeat(position?: number) {
+ const seekPos = this.isPositionSeeked(position);
+ if (seekPos === false || seekPos > 0) {
+ return false;
+ }
+ let repeatHint = `New Position (${position})`;
+ const trackDur = this.currentPlay.data.duration;
+ // user is within 10 seconds or 10% of start of track
+ const closeStartNum = position <= 12;
+ if(closeStartNum) {
+ repeatHint = `${repeatHint} is within 12 seconds of track start`;
+ }
+ const closeStartPer = (trackDur !== undefined && ((position / trackDur) <= 0.15));
+ if(!closeStartNum && closeStartPer) {
+ repeatHint = `${repeatHint} is within 15% of track start (${formatNumber((position/trackDur)*100)}%).`;
+ }
+ if (closeStartNum || closeStartPer) {
+ // user has played at least 2 minutes or 50% of track
+ const playerDur = this.getListenDuration();
+ const closeDurNum = playerDur >= 120;
+ if(closeDurNum) {
+ repeatHint = `${repeatHint} and listened to more than 120s (${playerDur}s)`
+ }
+ const closeDurPer = (trackDur !== undefined && (playerDur / trackDur) >= 0.5);
+ if(!closeDurNum && closeDurPer) {
+ repeatHint = `${repeatHint} and listened to more than 50% (${formatNumber((playerDur/trackDur)*100)}%).`
+ }
+ if (closeDurNum || closeDurPer) {
+ this.logger.debug(repeatHint);
+ return true;
+ }
+ if (trackDur !== undefined) {
+ const lastPos = this.currentListenRange[1].position;
+ // or last position is within 10 seconds (or 10%) of end of track
+ const nearEndNum = (trackDur - lastPos < 12);
+ if(nearEndNum) {
+ repeatHint = `${repeatHint} and previous position was within 12 seconds of track end.`;
+ }
+ const nearEndPos = ((lastPos / trackDur) < 0.15);
+ if(!nearEndNum && nearEndPos) {
+ repeatHint = `${repeatHint} and previous position was within 15% of track end (${formatNumber((lastPos/trackDur)*100)}%)`;
+ }
+ if(nearEndNum || nearEndPos) {
+ this.logger.debug(repeatHint);
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
setCurrentPlay(play: PlayObject, status?: ReportedPlayerStatus) {
this.currentPlay = play;
this.playFirstSeenAt = dayjs();
diff --git a/src/backend/utils.ts b/src/backend/utils.ts
index f12020e1..6b6f6a6a 100644
--- a/src/backend/utils.ts
+++ b/src/backend/utils.ts
@@ -637,7 +637,7 @@ export const timePassesScrobbleThreshold = (thresholds: ScrobbleThresholds, seco
export const thresholdResultSummary = (result: ScrobbleThresholdResult) => {
const parts: string[] = [];
if(result.duration.passes !== undefined) {
- parts.push(`tracked time of ${result.duration.value}s (wanted ${result.duration.threshold}s)`);
+ parts.push(`tracked time of ${result.duration.value.toFixed(2)}s (wanted ${result.duration.threshold}s)`);
}
if(result.percent.passes !== undefined) {
parts.push(`tracked percent of ${(result.percent.value).toFixed(2)}% (wanted ${result.percent.threshold}%)`)
--
2.54.0
From d04bc038d49efd68f71920ed8aeec4bfd447070e Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Tue, 3 Oct 2023 10:13:45 -0400
Subject: [PATCH 74/82] feat(ui): Implement status indicator
---
src/client/components/StatusIndicator.tsx | 26 +++++++++++++++++++++++
1 file changed, 26 insertions(+)
create mode 100644 src/client/components/StatusIndicator.tsx
diff --git a/src/client/components/StatusIndicator.tsx b/src/client/components/StatusIndicator.tsx
new file mode 100644
index 00000000..af638fb3
--- /dev/null
+++ b/src/client/components/StatusIndicator.tsx
@@ -0,0 +1,26 @@
+import React, {PropsWithChildren} from 'react';
+
+export type StatusType = 'active' | 'warn' | 'error' | 'inactive' | string;
+
+// must use full class names for tailwind to avoid purging
+// https://github.com/tailwindlabs/tailwindcss/discussions/7745#discussioncomment-3304940
+const statusToColor = (status: StatusType) => {
+ switch (status) {
+ case 'active':
+ return 'bg-green-500';
+ case 'warn':
+ return 'bg-yellow-500';
+ case 'error':
+ return 'bg-red-500';
+ case 'inactive':
+ default:
+ return 'bg-gray-500';
+ }
+}
+
+const StatusIndicator = (props: PropsWithChildren<{ type: StatusType }>) => {
+ const cn = `flex w-3 h-3 ${statusToColor(props.type)} rounded-full ml-1.5`
+ return {props.children} ;
+}
+
+export default StatusIndicator;
--
2.54.0
From 336970c1f76747b42809d464fda4f165d2d3d813 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Tue, 3 Oct 2023 10:15:18 -0400
Subject: [PATCH 75/82] feat(ui): Improve status display
* Use running/idle for scrobble client to indicate processing status
* Change polling sources status to polling/idle to better reflect behavior
* Add status indicator color to statuses
---
src/backend/server/api.ts | 5 +++--
.../components/statusCard/ClientStatusCard.tsx | 13 ++++++++++++-
.../components/statusCard/SourceStatusCard.tsx | 13 ++++++++++++-
.../components/statusCard/StatusCardSkeleton.tsx | 5 ++++-
4 files changed, 31 insertions(+), 5 deletions(-)
diff --git a/src/backend/server/api.ts b/src/backend/server/api.ts
index f985d6a4..dfe53be5 100644
--- a/src/backend/server/api.ts
+++ b/src/backend/server/api.ts
@@ -157,7 +157,7 @@ export const setupApi = (app: ExpressWithAsync, logger: Logger, initialLogOutput
} else if (requiresAuth && !authed) {
base.status = requiresAuthInteraction ? 'Auth Interaction Required' : 'Authentication Failed Or Not Attempted'
} else if (canPoll) {
- base.status = polling ? 'Running' : 'Idle';
+ base.status = polling ? 'Polling' : 'Idle';
} else {
base.status = tracksDiscovered > 0 ? 'Received Data' : 'Awaiting Data'
}
@@ -173,6 +173,7 @@ export const setupApi = (app: ExpressWithAsync, logger: Logger, initialLogOutput
requiresAuth = false,
requiresAuthInteraction = false,
authed = false,
+ scrobbling = false,
} = x;
const base: ClientStatusData = {
status: '',
@@ -190,7 +191,7 @@ export const setupApi = (app: ExpressWithAsync, logger: Logger, initialLogOutput
} else if (requiresAuth && !authed) {
base.status = requiresAuthInteraction ? 'Auth Interaction Required' : 'Authentication Failed Or Not Attempted'
} else {
- base.status = tracksScrobbled > 0 ? 'Received Data' : 'Awaiting Data';
+ base.status = scrobbling ? 'Running' : 'Idle';
}
return base;
});
diff --git a/src/client/components/statusCard/ClientStatusCard.tsx b/src/client/components/statusCard/ClientStatusCard.tsx
index e4719dac..d2c45594 100644
--- a/src/client/components/statusCard/ClientStatusCard.tsx
+++ b/src/client/components/statusCard/ClientStatusCard.tsx
@@ -11,6 +11,17 @@ export interface ClientStatusCardData extends StatusCardSkeletonData, PropsFromR
key: any
}
+const statusToStatusType = (status: string) => {
+ const lower = status.toLowerCase();
+ if(lower.includes('running') || lower.includes('data')) {
+ return 'active';
+ }
+ if(lower.includes('idle')) {
+ return 'warn';
+ }
+ return 'error';
+}
+
const ClientStatusCard = (props: ClientStatusCardData) => {
const {
loading = false,
@@ -44,7 +55,7 @@ const ClientStatusCard = (props: ClientStatusCardData) => {
);
}
return (
-
+
{body}
);
diff --git a/src/client/components/statusCard/SourceStatusCard.tsx b/src/client/components/statusCard/SourceStatusCard.tsx
index 087d5d60..5afb7124 100644
--- a/src/client/components/statusCard/SourceStatusCard.tsx
+++ b/src/client/components/statusCard/SourceStatusCard.tsx
@@ -12,6 +12,17 @@ export interface SourceStatusCardData extends StatusCardSkeletonData, PropsFromR
loading?: boolean
}
+const statusToStatusType = (status: string) => {
+ const lower = status.toLowerCase();
+ if(lower.includes('running') || lower.includes('polling') || lower.includes('data')) {
+ return 'active';
+ }
+ if(lower.includes('idle')) {
+ return 'warn';
+ }
+ return 'error';
+}
+
const SourceStatusCard = (props: SourceStatusCardData) => {
const {
loading = false,
@@ -61,7 +72,7 @@ const SourceStatusCard = (props: SourceStatusCardData) => {
);
}
return (
-
+
{body}
);
diff --git a/src/client/components/statusCard/StatusCardSkeleton.tsx b/src/client/components/statusCard/StatusCardSkeleton.tsx
index b0df1b46..0e95b259 100644
--- a/src/client/components/statusCard/StatusCardSkeleton.tsx
+++ b/src/client/components/statusCard/StatusCardSkeleton.tsx
@@ -1,6 +1,7 @@
import React, {PropsWithChildren} from 'react';
import SkeletonTitle from "../skeleton/SkeletonTitle";
import SkeletonParagraph from "../skeleton/SkeletonParagraph";
+import StatusIndicator, {StatusType} from "../StatusIndicator";
export interface StatusCardSkeletonData {
loading?: boolean
@@ -8,6 +9,7 @@ export interface StatusCardSkeletonData {
title?: string
subtitle?: string
status?: string
+ statusType?: StatusType
}
const StatusCardSkeleton = (props: PropsWithChildren) => {
@@ -15,6 +17,7 @@ const StatusCardSkeleton = (props: PropsWithChildren) =>
loading,
header,
status = null,
+ statusType,
title,
subtitle,
children
@@ -31,7 +34,7 @@ const StatusCardSkeleton = (props: PropsWithChildren) =>
{title ? {title}
: null}
{subtitle ? {subtitle}
: null}
-
{status}
+
{status}
--
2.54.0
From f2a171c596d083f4fadb4d1a116244e113ade586 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Tue, 3 Oct 2023 11:06:44 -0400
Subject: [PATCH 76/82] refactor(player): Move listen progress/range
functionality into own class (SRP)
---
.../PlayerState/AbstractPlayerState.ts | 88 +++++++------------
.../sources/PlayerState/ListenProgress.ts | 23 +++--
.../sources/PlayerState/ListenRange.ts | 74 ++++++++++++++++
src/backend/utils.ts | 6 +-
src/core/Atomic.ts | 8 +-
5 files changed, 130 insertions(+), 69 deletions(-)
create mode 100644 src/backend/sources/PlayerState/ListenRange.ts
diff --git a/src/backend/sources/PlayerState/AbstractPlayerState.ts b/src/backend/sources/PlayerState/AbstractPlayerState.ts
index 452e3ea0..94d8f75d 100644
--- a/src/backend/sources/PlayerState/AbstractPlayerState.ts
+++ b/src/backend/sources/PlayerState/AbstractPlayerState.ts
@@ -9,8 +9,10 @@ import dayjs, {Dayjs} from "dayjs";
import { formatNumber, genGroupIdStr, playObjDataMatch, progressBar } from "../../utils";
import {Logger} from "@foxxmd/winston";
import { ListenProgress } from "./ListenProgress";
-import {ListenRange, PlayData, PlayObject, SourcePlayerObj} from "../../../core/Atomic";
+import {PlayObject, SourcePlayerObj} from "../../../core/Atomic";
import { buildTrackString } from "../../../core/StringUtils";
+import {ListenRange} from "./ListenRange";
+import {start} from "repl";
export interface PlayerStateIntervals {
staleInterval?: number
@@ -29,8 +31,8 @@ export abstract class AbstractPlayerState {
currentPlay?: PlayObject
playFirstSeenAt?: Dayjs
playLastUpdatedAt?: Dayjs
- currentListenRange?: [ListenProgress, ListenProgress]
- listenRanges: [ListenProgress, ListenProgress][] = [];
+ currentListenRange?: ListenRange
+ listenRanges: ListenRange[]
createdAt: Dayjs = dayjs();
stateLastUpdatedAt: Dayjs = dayjs();
@@ -141,12 +143,15 @@ export abstract class AbstractPlayerState {
this.currentListenSessionContinue(play.meta.trackProgressPosition);
return [this.getPlayedObject(), played];
} else {
- const seekedPos = this.isPositionSeeked(play.meta.trackProgressPosition);
- if (seekedPos !== false) {
- this.logger.debug(`Detected player was seeked ${seekedPos.toFixed(2)}s, starting new listen range`);
- // if player has been seeked start a new listen range so our numbers don't get all screwy
- this.currentListenSessionEnd();
+ if(this.currentListenRange !== undefined) {
+ const [isSeeked, seekedPos] = this.currentListenRange.seeked(play.meta.trackProgressPosition);
+ if (isSeeked !== false) {
+ this.logger.debug(`Detected player was seeked ${seekedPos.toFixed(2)}s, starting new listen range`);
+ // if player has been seeked start a new listen range so our numbers don't get all screwy
+ this.currentListenSessionEnd();
+ }
}
+
this.currentListenSessionContinue(play.meta.trackProgressPosition);
}
} else {
@@ -200,8 +205,8 @@ export abstract class AbstractPlayerState {
if (this.currentListenRange !== undefined) {
ranges.push(this.currentListenRange);
}
- for (const [start, end] of ranges) {
- listenDur += start.getDuration(end);
+ for (const range of ranges) {
+ listenDur += range.getDuration();
}
return listenDur;
}
@@ -210,11 +215,11 @@ export abstract class AbstractPlayerState {
const now = dayjs();
if (this.currentListenRange === undefined) {
this.logger.debug('Started new Player listen range.');
- this.currentListenRange = [new ListenProgress(now, position), new ListenProgress(now, position)];
+ this.currentListenRange = new ListenRange(new ListenProgress(undefined, position));
} else {
- const newEndProgress = new ListenProgress(now, position);
- if (position !== undefined && this.currentListenRange[1].position !== undefined) {
- const oldEndProgress = this.currentListenRange[1];
+ const oldEndProgress = this.currentListenRange.end;
+ const newEndProgress = new ListenProgress(undefined, position);
+ if (position !== undefined && oldEndProgress !== undefined) {
if (position === oldEndProgress.position && !['paused', 'stopped'].includes(this.calculatedStatus)) {
this.calculatedStatus = this.reportedStatus === 'stopped' ? CALCULATED_PLAYER_STATUSES.stopped : CALCULATED_PLAYER_STATUSES.paused;
if (this.reportedStatus !== this.calculatedStatus) {
@@ -233,57 +238,24 @@ export abstract class AbstractPlayerState {
} else {
this.calculatedStatus = CALCULATED_PLAYER_STATUSES.playing;
}
- this.currentListenRange = [this.currentListenRange[0], newEndProgress];
+ this.currentListenRange.setRangeEnd(newEndProgress);
}
}
currentListenSessionEnd() {
- if (this.currentListenRange !== undefined && this.currentListenRange[0].getDuration(this.currentListenRange[1]) !== 0) {
+ if (this.currentListenRange !== undefined && this.currentListenRange.getDuration() !== 0) {
this.logger.debug('Ended current Player listen range.')
this.listenRanges.push(this.currentListenRange);
}
this.currentListenRange = undefined;
}
- isCurrentRangeInitial() {
- if(this.currentListenRange === undefined) {
- return true;
- }
- if(this.currentListenRange[0].position !== undefined && this.currentListenRange[1].position !== undefined) {
- return this.currentListenRange[0].position === this.currentListenRange[1].position;
- }
- return this.currentListenRange[0].timestamp.isSame(this.currentListenRange[1].timestamp);
- }
-
- isCurrentRangePositional() {
+ isSessionRepeat(position?: number) {
if(this.currentListenRange === undefined) {
return false;
}
- return this.currentListenRange[0].position !== undefined && this.currentListenRange[1].position !== undefined;
- }
-
- isPositionSeeked(position?: number) {
- if (position === undefined || this.isCurrentRangeInitial() || !this.isCurrentRangePositional()) {
- return false;
- }
- // if (new) position is earlier than last stored position then the user has seeked backwards on the player
- if (position < this.currentListenRange[1].position) {
- return position - this.currentListenRange[1].position;
- }
- // if (new) position is more than a reasonable number of ms ahead of real time than they have seeked forwards on the player
- const realTimeDiff = dayjs().diff(this.currentListenRange[1].timestamp, 'ms');
- const positionDiff = (position - this.currentListenRange[1].position) * 1000;
- // if user is more than 2.5 seconds ahead of real time
- if (positionDiff - realTimeDiff > 2500) {
- return position - this.currentListenRange[1].position;
- }
-
- return false;
- }
-
- isSessionRepeat(position?: number) {
- const seekPos = this.isPositionSeeked(position);
- if (seekPos === false || seekPos > 0) {
+ const [isSeeked, seekPos] = this.currentListenRange.seeked(position);
+ if (isSeeked === false || seekPos > 0) {
return false;
}
let repeatHint = `New Position (${position})`;
@@ -313,7 +285,7 @@ export abstract class AbstractPlayerState {
return true;
}
if (trackDur !== undefined) {
- const lastPos = this.currentListenRange[1].position;
+ const lastPos = this.currentListenRange.end.position;
// or last position is within 10 seconds (or 10%) of end of track
const nearEndNum = (trackDur - lastPos < 12);
if(nearEndNum) {
@@ -357,8 +329,8 @@ export abstract class AbstractPlayerState {
}
parts.push(`Reported: ${this.reportedStatus.toUpperCase()} | Calculated: ${this.calculatedStatus.toUpperCase()} | Stale: ${this.isUpdateStale() ? 'Yes' : 'No'} | Orphaned: ${this.isOrphaned() ? 'Yes' : 'No'} | Last Update: ${this.stateLastUpdatedAt.toISOString()}`);
let progress = '';
- if (this.currentListenRange !== undefined && this.currentListenRange[1].position !== undefined && this.currentPlay.data.duration !== undefined) {
- progress = `${progressBar(this.currentListenRange[1].position / this.currentPlay.data.duration, 1, 15)} ${formatNumber(this.currentListenRange[1].position, {toFixed: 0})}/${formatNumber(this.currentPlay.data.duration, {toFixed: 0})}s | `;
+ if (this.currentListenRange !== undefined && this.currentListenRange.end.position !== undefined && this.currentPlay.data.duration !== undefined) {
+ progress = `${progressBar(this.currentListenRange.end.position / this.currentPlay.data.duration, 1, 15)} ${formatNumber(this.currentListenRange.end.position, {toFixed: 0})}/${formatNumber(this.currentPlay.data.duration, {toFixed: 0})}s | `;
}
let listenedPercent = '';
if (this.currentPlay !== undefined && this.currentPlay.data.duration !== undefined) {
@@ -379,16 +351,16 @@ export abstract class AbstractPlayerState {
if(this.calculatedStatus === 'stopped') {
return undefined;
}
- let lastRange: [ListenProgress, ListenProgress] | undefined;
+ let lastRange: ListenRange | undefined;
if(this.currentListenRange !== undefined) {
lastRange = this.currentListenRange;
} else if(this.listenRanges.length > 0) {
lastRange = this.listenRanges[this.listenRanges.length - 1];
}
- if(lastRange === undefined || lastRange[1] === undefined || lastRange[1].position === undefined) {
+ if(lastRange === undefined || lastRange.end === undefined || lastRange.end.position === undefined) {
return undefined;
}
- return lastRange[1].position;
+ return lastRange.end.position;
}
getApiState(): SourcePlayerObj {
diff --git a/src/backend/sources/PlayerState/ListenProgress.ts b/src/backend/sources/PlayerState/ListenProgress.ts
index 00b96f66..6675e73c 100644
--- a/src/backend/sources/PlayerState/ListenProgress.ts
+++ b/src/backend/sources/PlayerState/ListenProgress.ts
@@ -1,14 +1,17 @@
import dayjs, {Dayjs} from "dayjs";
-import { PlayProgress } from "../../../core/Atomic";
+import {PlayProgress} from "../../../core/Atomic";
export class ListenProgress implements PlayProgress {
- constructor(
- public timestamp: Dayjs = dayjs(),
- public position?: number,
- public positionPercent?: number
- ) {
+ public timestamp: Dayjs;
+ public position?: number;
+ public positionPercent?: number;
+
+ constructor(timestamp?: Dayjs, position?: number, positionPercent?: number) {
+ this.timestamp = timestamp ?? dayjs();
+ this.position = position;
+ this.positionPercent = positionPercent;
}
getDuration(end: ListenProgress) {
@@ -18,4 +21,12 @@ export class ListenProgress implements PlayProgress {
return end.timestamp.diff(this.timestamp, 'seconds');
}
}
+
+ toJSON() {
+ return {
+ timestamp: this.timestamp.toISOString(),
+ position: this.position,
+ positionPercent: this.positionPercent
+ }
+ }
}
diff --git a/src/backend/sources/PlayerState/ListenRange.ts b/src/backend/sources/PlayerState/ListenRange.ts
new file mode 100644
index 00000000..8335939a
--- /dev/null
+++ b/src/backend/sources/PlayerState/ListenRange.ts
@@ -0,0 +1,74 @@
+import {ListenProgress} from "./ListenProgress";
+import dayjs, {Dayjs} from "dayjs";
+import {ListenRangeData} from "../../../core/Atomic";
+
+export class ListenRange implements ListenRangeData {
+
+ public start: ListenProgress;
+ public end: ListenProgress;
+
+ constructor(start?: ListenProgress, end?: ListenProgress) {
+ const s = start ?? new ListenProgress();
+ const e = end ?? s;
+
+ this.start = s;
+ this.end = e;
+ }
+
+ isPositional() {
+ return this.start.position !== undefined && this.end.position !== undefined;
+ }
+
+ isInitial() {
+ if (this.isPositional()) {
+ return this.start.position === this.end.position;
+ }
+ return this.start.timestamp.isSame(this.end.timestamp);
+ }
+
+ seeked(position?: number): [boolean, number?] {
+ if (position === undefined || this.isInitial() || !this.isPositional()) {
+ return [false];
+ }
+ // if (new) position is earlier than last stored position then the user has seeked backwards on the player
+ if (position < this.end.position) {
+ return [true, position - this.end.position];
+ }
+
+ // if (new) position is more than a reasonable number of ms ahead of real time than they have seeked forwards on the player
+ const realTimeDiff = dayjs().diff(this.end.timestamp, 'ms');
+ const positionDiff = (position - this.end.position) * 1000;
+ // if user is more than 2.5 seconds ahead of real time
+ if (positionDiff - realTimeDiff > 2500) {
+ return [true, position - this.end.position];
+ }
+
+ return [false];
+ }
+
+ setRangeStart(data: ListenProgress | { position?: number, timestamp?: Dayjs, positionPercent?: number }) {
+ if (data instanceof ListenProgress) {
+ this.start = data;
+ } else {
+ const d = data || {};
+ this.start = new ListenProgress(d.timestamp, d.position, d.positionPercent)
+ }
+ }
+
+ setRangeEnd(data: ListenProgress | { position?: number, timestamp?: Dayjs, positionPercent?: number }) {
+ if (data instanceof ListenProgress) {
+ this.end = data;
+ } else {
+ const d = data || {};
+ this.end = new ListenProgress(d.timestamp, d.position, d.positionPercent)
+ }
+ }
+
+ getDuration() {
+ return this.start.getDuration(this.end);
+ }
+
+ toJSON() {
+ return [this.start, this.end];
+ }
+}
diff --git a/src/backend/utils.ts b/src/backend/utils.ts
index 6b6f6a6a..61320ea4 100644
--- a/src/backend/utils.ts
+++ b/src/backend/utils.ts
@@ -27,7 +27,7 @@ import backoffStrategies from '@kenyip/backoff-strategies';
import {ScrobbleThresholds} from "./common/infrastructure/config/source";
import {replaceResultTransformer, stripIndentTransformer, TemplateTag, trimResultTransformer} from 'common-tags';
import {Duration} from "dayjs/plugin/duration.js";
-import {ListenRange, PlayObject} from "../core/Atomic";
+import {ListenRangeData, PlayObject} from "../core/Atomic";
import address from "address";
dayjs.extend(utc);
@@ -434,7 +434,7 @@ export interface TemporalPlayComparison {
diff: number
fuzzyDiff?: number
}
- range?: false | ListenRange
+ range?: false | ListenRangeData
}
export const temporalPlayComparisonSummary = (data: TemporalPlayComparison, existingPlay?: PlayObject, candidatePlay?: PlayObject) => {
@@ -521,7 +521,7 @@ export const comparePlayTemporally = (existingPlay: PlayObject, candidatePlay: P
// we can check if the new track play date took place while the existing one was being listened to
// which would indicate (assuming same source) the new track is a duplicate
for(const range of existingRanges) {
- if(newPlayDate.isBetween(range[0].timestamp, range[1].timestamp)) {
+ if(newPlayDate.isBetween(range.start.timestamp, range.end.timestamp)) {
result.range = range;
result.close = true;
break;
diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts
index 1ebf8b8d..b6f44a0d 100644
--- a/src/core/Atomic.ts
+++ b/src/core/Atomic.ts
@@ -1,5 +1,6 @@
import {Dayjs} from "dayjs";
import {MESSAGE} from "triple-beam";
+import {ListenProgress} from "../backend/sources/PlayerState/ListenProgress";
export interface SourceStatusData {
status: string;
@@ -46,7 +47,10 @@ export interface PlayProgress {
positionPercent?: number
}
-export type ListenRange = [PlayProgress, PlayProgress]
+export interface ListenRangeData {
+ start: ListenProgress
+ end: ListenProgress
+}
export interface TrackData {
artists?: string[]
@@ -75,7 +79,7 @@ export interface PlayData extends TrackData {
playDate?: Dayjs | string
/** Number of seconds the track was listened to */
listenedFor?: number
- listenRanges?: ListenRange[]
+ listenRanges?: ListenRangeData[]
}
export interface PlayMeta {
--
2.54.0
From 5dff617ed672864bf8181ea41d6b21dc1c6de1b2 Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Tue, 3 Oct 2023 12:35:48 -0400
Subject: [PATCH 77/82] fix(player): Fix some bugs and add param for easier
testing
* Fix repeat end position percent test (should be 85% not 15%)
* Fix current play/range not being reset correctly on repeat
* Add reportedTS param to facilitate testing
---
.../PlayerState/AbstractPlayerState.ts | 42 +++++++++----------
.../sources/PlayerState/ListenRange.ts | 4 +-
2 files changed, 23 insertions(+), 23 deletions(-)
diff --git a/src/backend/sources/PlayerState/AbstractPlayerState.ts b/src/backend/sources/PlayerState/AbstractPlayerState.ts
index 94d8f75d..3f8c58b9 100644
--- a/src/backend/sources/PlayerState/AbstractPlayerState.ts
+++ b/src/backend/sources/PlayerState/AbstractPlayerState.ts
@@ -98,10 +98,10 @@ export abstract class AbstractPlayerState {
return status !== 'paused' && status !== 'stopped';
}
- setState(status?: ReportedPlayerStatus, play?: PlayObject) {
+ setState(status?: ReportedPlayerStatus, play?: PlayObject, reportedTS?: Dayjs) {
this.stateLastUpdatedAt = dayjs();
if (play !== undefined) {
- return this.setPlay(play, status);
+ return this.setPlay(play, status, reportedTS);
} else if (status !== undefined) {
if (status === 'stopped' && this.reportedStatus !== 'stopped' && this.currentPlay !== undefined) {
this.stopPlayer();
@@ -116,7 +116,7 @@ export abstract class AbstractPlayerState {
return [];
}
- setPlay(play: PlayObject, status?: ReportedPlayerStatus): [PlayObject, PlayObject?] {
+ protected setPlay(play: PlayObject, status?: ReportedPlayerStatus, reportedTS?: Dayjs): [PlayObject, PlayObject?] {
this.playLastUpdatedAt = dayjs();
if (status !== undefined) {
this.reportedStatus = status;
@@ -127,7 +127,7 @@ export abstract class AbstractPlayerState {
this.logger.debug(`Incoming play state (${buildTrackString(play, {include: ['trackId', 'artist', 'track']})}) does not match existing state, removing existing: ${buildTrackString(this.currentPlay, {include: ['trackId', 'artist', 'track']})}`)
this.currentListenSessionEnd();
const played = this.getPlayedObject();
- this.setCurrentPlay(play);
+ this.setCurrentPlay(play, undefined, reportedTS);
if (this.calculatedStatus !== CALCULATED_PLAYER_STATUSES.playing) {
this.calculatedStatus = CALCULATED_PLAYER_STATUSES.unknown;
}
@@ -135,16 +135,16 @@ export abstract class AbstractPlayerState {
} else if (status !== undefined && !AbstractPlayerState.isProgressStatus(status)) {
this.currentListenSessionEnd();
this.calculatedStatus = this.reportedStatus;
- } else if (this.isSessionRepeat(play.meta.trackProgressPosition)) {
+ } else if (this.isSessionRepeat(play.meta.trackProgressPosition, reportedTS)) {
// if we detect the track has been restarted end listen session and treat as a new play
this.currentListenSessionEnd();
const played = this.getPlayedObject();
- this.currentPlay.data.playDate = dayjs();
- this.currentListenSessionContinue(play.meta.trackProgressPosition);
+ play.data.playDate = dayjs();
+ this.setCurrentPlay(play, undefined, reportedTS);
return [this.getPlayedObject(), played];
} else {
if(this.currentListenRange !== undefined) {
- const [isSeeked, seekedPos] = this.currentListenRange.seeked(play.meta.trackProgressPosition);
+ const [isSeeked, seekedPos] = this.currentListenRange.seeked(play.meta.trackProgressPosition, reportedTS);
if (isSeeked !== false) {
this.logger.debug(`Detected player was seeked ${seekedPos.toFixed(2)}s, starting new listen range`);
// if player has been seeked start a new listen range so our numbers don't get all screwy
@@ -152,7 +152,7 @@ export abstract class AbstractPlayerState {
}
}
- this.currentListenSessionContinue(play.meta.trackProgressPosition);
+ this.currentListenSessionContinue(play.meta.trackProgressPosition, reportedTS);
}
} else {
this.setCurrentPlay(play);
@@ -166,7 +166,7 @@ export abstract class AbstractPlayerState {
return [this.getPlayedObject(), undefined];
}
- clearPlayer() {
+ protected clearPlayer() {
this.currentPlay = undefined;
this.playLastUpdatedAt = undefined;
this.playFirstSeenAt = undefined;
@@ -174,7 +174,7 @@ export abstract class AbstractPlayerState {
this.currentListenRange = undefined;
}
- stopPlayer() {
+ protected stopPlayer() {
this.reportedStatus = 'stopped';
this.playLastUpdatedAt = dayjs();
this.currentListenSessionEnd();
@@ -211,14 +211,14 @@ export abstract class AbstractPlayerState {
return listenDur;
}
- currentListenSessionContinue(position?: number) {
+ protected currentListenSessionContinue(position?: number, timestamp?: Dayjs) {
const now = dayjs();
if (this.currentListenRange === undefined) {
this.logger.debug('Started new Player listen range.');
- this.currentListenRange = new ListenRange(new ListenProgress(undefined, position));
+ this.currentListenRange = new ListenRange(new ListenProgress(timestamp, position));
} else {
const oldEndProgress = this.currentListenRange.end;
- const newEndProgress = new ListenProgress(undefined, position);
+ const newEndProgress = new ListenProgress(timestamp, position);
if (position !== undefined && oldEndProgress !== undefined) {
if (position === oldEndProgress.position && !['paused', 'stopped'].includes(this.calculatedStatus)) {
this.calculatedStatus = this.reportedStatus === 'stopped' ? CALCULATED_PLAYER_STATUSES.stopped : CALCULATED_PLAYER_STATUSES.paused;
@@ -242,7 +242,7 @@ export abstract class AbstractPlayerState {
}
}
- currentListenSessionEnd() {
+ protected currentListenSessionEnd() {
if (this.currentListenRange !== undefined && this.currentListenRange.getDuration() !== 0) {
this.logger.debug('Ended current Player listen range.')
this.listenRanges.push(this.currentListenRange);
@@ -250,11 +250,11 @@ export abstract class AbstractPlayerState {
this.currentListenRange = undefined;
}
- isSessionRepeat(position?: number) {
+ protected isSessionRepeat(position?: number, reportedTS?: Dayjs) {
if(this.currentListenRange === undefined) {
return false;
}
- const [isSeeked, seekPos] = this.currentListenRange.seeked(position);
+ const [isSeeked, seekPos] = this.currentListenRange.seeked(position, reportedTS);
if (isSeeked === false || seekPos > 0) {
return false;
}
@@ -291,7 +291,7 @@ export abstract class AbstractPlayerState {
if(nearEndNum) {
repeatHint = `${repeatHint} and previous position was within 12 seconds of track end.`;
}
- const nearEndPos = ((lastPos / trackDur) < 0.15);
+ const nearEndPos = ((lastPos / trackDur) > 0.85);
if(!nearEndNum && nearEndPos) {
repeatHint = `${repeatHint} and previous position was within 15% of track end (${formatNumber((lastPos/trackDur)*100)}%)`;
}
@@ -304,7 +304,7 @@ export abstract class AbstractPlayerState {
return false;
}
- setCurrentPlay(play: PlayObject, status?: ReportedPlayerStatus) {
+ protected setCurrentPlay(play: PlayObject, status?: ReportedPlayerStatus, reportedTS?: Dayjs) {
this.currentPlay = play;
this.playFirstSeenAt = dayjs();
this.listenRanges = [];
@@ -317,7 +317,7 @@ export abstract class AbstractPlayerState {
}
if (!['stopped'].includes(this.reportedStatus)) {
- this.currentListenSessionContinue(play.meta.trackProgressPosition);
+ this.currentListenSessionContinue(play.meta.trackProgressPosition, reportedTS);
}
}
@@ -347,7 +347,7 @@ export abstract class AbstractPlayerState {
this.logger.debug(this.textSummary());
}
- getPosition() {
+ protected getPosition() {
if(this.calculatedStatus === 'stopped') {
return undefined;
}
diff --git a/src/backend/sources/PlayerState/ListenRange.ts b/src/backend/sources/PlayerState/ListenRange.ts
index 8335939a..4a3209cf 100644
--- a/src/backend/sources/PlayerState/ListenRange.ts
+++ b/src/backend/sources/PlayerState/ListenRange.ts
@@ -26,7 +26,7 @@ export class ListenRange implements ListenRangeData {
return this.start.timestamp.isSame(this.end.timestamp);
}
- seeked(position?: number): [boolean, number?] {
+ seeked(position?: number, reportedTS: Dayjs = dayjs()): [boolean, number?] {
if (position === undefined || this.isInitial() || !this.isPositional()) {
return [false];
}
@@ -36,7 +36,7 @@ export class ListenRange implements ListenRangeData {
}
// if (new) position is more than a reasonable number of ms ahead of real time than they have seeked forwards on the player
- const realTimeDiff = dayjs().diff(this.end.timestamp, 'ms');
+ const realTimeDiff = Math.max(0, reportedTS.diff(this.end.timestamp, 'ms')); // 0 max used so TS from testing doesn't cause "backward" diff
const positionDiff = (position - this.end.position) * 1000;
// if user is more than 2.5 seconds ahead of real time
if (positionDiff - realTimeDiff > 2500) {
--
2.54.0
From fc87b86f5228e7af2d0f398fa0c36d9b0535e2dc Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Tue, 3 Oct 2023 12:36:00 -0400
Subject: [PATCH 78/82] test(player): Add player tests
---
src/backend/tests/player/player.test.ts | 357 ++++++++++++++++++++++++
1 file changed, 357 insertions(+)
create mode 100644 src/backend/tests/player/player.test.ts
diff --git a/src/backend/tests/player/player.test.ts b/src/backend/tests/player/player.test.ts
new file mode 100644
index 00000000..e7bbff91
--- /dev/null
+++ b/src/backend/tests/player/player.test.ts
@@ -0,0 +1,357 @@
+import {describe, it} from 'mocha';
+import {assert} from 'chai';
+import {generatePlay} from "../utils/PlayTestUtils";
+import {GenericPlayerState} from "../../sources/PlayerState/GenericPlayerState";
+import {getLogger} from "../../common/logging";
+import {
+ CALCULATED_PLAYER_STATUSES,
+ NO_DEVICE,
+ NO_USER,
+ REPORTED_PLAYER_STATUSES
+} from "../../common/infrastructure/Atomic";
+import {playObjDataMatch} from "../../utils";
+import dayjs from "dayjs";
+import clone from "clone";
+
+const logger = getLogger({});
+
+const newPlay = generatePlay({duration: 300});
+
+describe('Basic player state', function () {
+
+ it('Creates new play state when new', function () {
+ const player = new GenericPlayerState(logger, [NO_DEVICE, NO_USER]);
+
+ assert.isUndefined(player.currentListenRange);
+ assert.isUndefined(player.currentPlay);
+
+ player.setState(undefined, newPlay);
+
+ assert.isDefined(player.currentListenRange);
+ assert.isDefined(player.currentPlay);
+ });
+
+ it('Creates new play state in unknown status', function () {
+ const player = new GenericPlayerState(logger, [NO_DEVICE, NO_USER]);
+
+ assert.isUndefined(player.currentListenRange);
+ assert.isUndefined(player.currentPlay);
+
+ player.setState(undefined, newPlay);
+
+ assert.isDefined(player.currentListenRange);
+ assert.isDefined(player.currentPlay);
+ assert.equal(CALCULATED_PLAYER_STATUSES.unknown, player.calculatedStatus);
+ });
+
+ it('Creates new play state when incoming play is not the same as stored play', function () {
+ const player = new GenericPlayerState(logger, [NO_DEVICE, NO_USER]);
+
+ player.setState(undefined, newPlay);
+
+ assert.isTrue(playObjDataMatch(player.currentPlay, newPlay));
+
+ const nextPlay = generatePlay({playDate: newPlay.data.playDate.add(2, 'seconds')});
+ const [returnedPlay, prevPlay] = player.setState(undefined, nextPlay);
+
+ assert.isTrue(playObjDataMatch(prevPlay, newPlay));
+ assert.isTrue(playObjDataMatch(player.currentPlay, nextPlay));
+ });
+});
+
+describe('Player status', function () {
+
+ it('New player transitions from unknown to playing on n+1 states', function () {
+ const player = new GenericPlayerState(logger, [NO_DEVICE, NO_USER]);
+
+ player.setState(undefined, newPlay);
+ assert.equal(CALCULATED_PLAYER_STATUSES.unknown, player.calculatedStatus);
+
+ player.setState(undefined, newPlay, dayjs().add(10, 'seconds'));
+ assert.equal(CALCULATED_PLAYER_STATUSES.playing, player.calculatedStatus);
+ });
+
+ describe('When source provides reported status', function () {
+
+ it('Calculated state is playing when source reports playing', function () {
+ const player = new GenericPlayerState(logger, [NO_DEVICE, NO_USER]);
+
+ player.setState(REPORTED_PLAYER_STATUSES.playing, newPlay);
+ player.setState(REPORTED_PLAYER_STATUSES.playing, newPlay, dayjs().add(10, 'seconds'));
+ assert.equal(CALCULATED_PLAYER_STATUSES.playing, player.calculatedStatus);
+ });
+
+
+ it('Calculated state is paused when source reports paused', function () {
+ const player = new GenericPlayerState(logger, [NO_DEVICE, NO_USER]);
+
+ player.setState(REPORTED_PLAYER_STATUSES.playing, newPlay);
+ player.setState(REPORTED_PLAYER_STATUSES.playing, newPlay, dayjs().add(10, 'seconds'));
+ player.setState(REPORTED_PLAYER_STATUSES.paused, newPlay, dayjs().add(20, 'seconds'));
+ assert.equal(CALCULATED_PLAYER_STATUSES.paused, player.calculatedStatus);
+ });
+
+ it('Calculated state is stopped when source reports stopped', function () {
+ const player = new GenericPlayerState(logger, [NO_DEVICE, NO_USER]);
+
+ player.setState(REPORTED_PLAYER_STATUSES.playing, newPlay);
+ player.setState(REPORTED_PLAYER_STATUSES.playing, newPlay, dayjs().add(10, 'seconds'));
+ player.setState(REPORTED_PLAYER_STATUSES.stopped, newPlay, dayjs().add(20, 'seconds'));
+ assert.equal(CALCULATED_PLAYER_STATUSES.stopped, player.calculatedStatus);
+ });
+
+ });
+
+ describe('When source provides playback position', function () {
+
+ it('Calculated state is playing when position moves forward', function () {
+ const player = new GenericPlayerState(logger, [NO_DEVICE, NO_USER]);
+
+ const positioned = clone(newPlay);
+ positioned.meta.trackProgressPosition = 3;
+
+ player.setState(undefined, positioned);
+
+ positioned.meta.trackProgressPosition = 13;
+ player.setState(undefined, positioned, dayjs().add(10, 'seconds'));
+
+ assert.equal(CALCULATED_PLAYER_STATUSES.playing, player.calculatedStatus);
+ });
+
+ it('Calculated state is paused when position does not change', function () {
+ const player = new GenericPlayerState(logger, [NO_DEVICE, NO_USER]);
+
+ const positioned = clone(newPlay);
+ positioned.meta.trackProgressPosition = 3;
+
+ player.setState(undefined, positioned);
+
+ positioned.meta.trackProgressPosition = 13;
+ player.setState(undefined, positioned, dayjs().add(10, 'seconds'));
+
+ player.setState(undefined, positioned, dayjs().add(20, 'seconds'));
+
+ assert.equal(CALCULATED_PLAYER_STATUSES.paused, player.calculatedStatus);
+ });
+
+ // TODO playback position reported and conflicts with player reported status
+ });
+});
+
+describe('Player listen ranges', function () {
+ describe('When source does not provide playback position', function () {
+
+ it('Duration is timestamp based for unknown/playing reported players', function () {
+ const player = new GenericPlayerState(logger, [NO_DEVICE, NO_USER]);
+
+ player.setState(REPORTED_PLAYER_STATUSES.playing, newPlay);
+ player.setState(REPORTED_PLAYER_STATUSES.playing, newPlay, dayjs().add(10, 'seconds'));
+ player.setState(REPORTED_PLAYER_STATUSES.playing, newPlay, dayjs().add(20, 'seconds'));
+
+ assert.equal(player.getListenDuration(), 20);
+
+ const uplayer = new GenericPlayerState(logger, [NO_DEVICE, NO_USER]);
+
+ uplayer.setState(undefined, newPlay);
+ uplayer.setState(undefined, newPlay, dayjs().add(10, 'seconds'));
+ uplayer.setState(undefined, newPlay, dayjs().add(20, 'seconds'));
+
+ assert.equal(uplayer.getListenDuration(), 20);
+ });
+
+ it('Range ends if player reports paused', function () {
+ const player = new GenericPlayerState(logger, [NO_DEVICE, NO_USER]);
+
+ player.setState(REPORTED_PLAYER_STATUSES.playing, newPlay);
+ player.setState(REPORTED_PLAYER_STATUSES.playing, newPlay, dayjs().add(10, 'seconds'));
+ player.setState(REPORTED_PLAYER_STATUSES.playing, newPlay, dayjs().add(20, 'seconds'));
+ player.setState(REPORTED_PLAYER_STATUSES.paused, newPlay, dayjs().add(30, 'seconds'));
+ player.setState(REPORTED_PLAYER_STATUSES.paused, newPlay, dayjs().add(40, 'seconds'));
+
+ assert.equal(player.getListenDuration(), 20);
+ });
+
+ it('Listen duration continues when player resumes', function () {
+ const player = new GenericPlayerState(logger, [NO_DEVICE, NO_USER]);
+
+ player.setState(REPORTED_PLAYER_STATUSES.playing, newPlay);
+ player.setState(REPORTED_PLAYER_STATUSES.playing, newPlay, dayjs().add(10, 'seconds'));
+ player.setState(REPORTED_PLAYER_STATUSES.playing, newPlay, dayjs().add(20, 'seconds'));
+ player.setState(REPORTED_PLAYER_STATUSES.paused, newPlay, dayjs().add(30, 'seconds'));
+ player.setState(REPORTED_PLAYER_STATUSES.paused, newPlay, dayjs().add(40, 'seconds'));
+ // For TS-only players the player must see two consecutive playing states to count the duration between them
+ // so it does NOT count above paused ^^ to below playing -- only playing-to-playing
+ player.setState(REPORTED_PLAYER_STATUSES.playing, newPlay, dayjs().add(50, 'seconds'));
+ player.setState(REPORTED_PLAYER_STATUSES.playing, newPlay, dayjs().add(60, 'seconds'));
+
+ assert.equal(player.getListenDuration(), 30);
+ });
+ });
+
+ describe('When source does provide playback position', function () {
+
+ it('Duration is position based', function () {
+ const player = new GenericPlayerState(logger, [NO_DEVICE, NO_USER]);
+
+ const positioned = clone(newPlay);
+ positioned.meta.trackProgressPosition = 3;
+ player.setState(undefined, positioned);
+
+ positioned.meta.trackProgressPosition = 10;
+ player.setState(undefined, positioned, dayjs().add(10, 'seconds'));
+
+ assert.equal(player.getListenDuration(), 7);
+ });
+
+ it('Range ends if position does not move', function () {
+ const player = new GenericPlayerState(logger, [NO_DEVICE, NO_USER]);
+
+ const positioned = clone(newPlay);
+ positioned.meta.trackProgressPosition = 3;
+ player.setState(undefined, positioned);
+
+ positioned.meta.trackProgressPosition = 3;
+ player.setState(undefined, positioned, dayjs().add(10, 'seconds'));
+
+ assert.equal(player.getListenDuration(), 0);
+ });
+
+ it('Range continues when position continues moving forward', function () {
+ const player = new GenericPlayerState(logger, [NO_DEVICE, NO_USER]);
+
+ const positioned = clone(newPlay);
+ positioned.meta.trackProgressPosition = 3;
+ player.setState(undefined, positioned);
+
+ positioned.meta.trackProgressPosition = 7;
+ player.setState(undefined, positioned, dayjs().add(10, 'seconds'));
+
+ player.setState(undefined, positioned, dayjs().add(20, 'seconds'));
+
+ positioned.meta.trackProgressPosition = 17;
+ player.setState(undefined, positioned, dayjs().add(30, 'seconds'));
+
+ positioned.meta.trackProgressPosition = 27;
+ player.setState(undefined, positioned, dayjs().add(40, 'seconds'));
+
+ assert.equal(player.getListenDuration(), 24);
+ });
+
+ describe('Detects seeking', function () {
+
+ it('Detects seeking forward', function () {
+ const player = new GenericPlayerState(logger, [NO_DEVICE, NO_USER]);
+
+ const positioned = clone(newPlay);
+ positioned.meta.trackProgressPosition = 3;
+ player.setState(undefined, positioned);
+
+ positioned.meta.trackProgressPosition = 13;
+ player.setState(undefined, positioned, dayjs().add(10, 'seconds'));
+
+ positioned.meta.trackProgressPosition = 30;
+ player.setState(undefined, positioned, dayjs().add(20, 'seconds'));
+
+ assert.equal(player.currentListenRange.start.timestamp, player.currentListenRange.end.timestamp);
+
+ positioned.meta.trackProgressPosition = 40;
+ player.setState(undefined, positioned, dayjs().add(30, 'seconds'));
+
+ assert.equal(player.getListenDuration(), 20);
+ });
+
+ it('Detects seeking backwards', function () {
+ const player = new GenericPlayerState(logger, [NO_DEVICE, NO_USER]);
+
+ const positioned = clone(newPlay);
+ positioned.meta.trackProgressPosition = 30;
+ player.setState(undefined, positioned);
+
+ positioned.meta.trackProgressPosition = 40;
+ player.setState(undefined, positioned, dayjs().add(10, 'seconds'));
+
+ positioned.meta.trackProgressPosition = 20;
+ player.setState(undefined, positioned, dayjs().add(20, 'seconds'));
+
+ assert.equal(player.currentListenRange.start.timestamp, player.currentListenRange.end.timestamp);
+
+ positioned.meta.trackProgressPosition = 30;
+ player.setState(undefined, positioned, dayjs().add(30, 'seconds'));
+
+ assert.equal(player.getListenDuration(), 20);
+ });
+
+ });
+
+ describe('Detects repeating', function () {
+ it('Detects repeat when player was within 12 seconds of ending and seeked back to within 12 seconds of start', function () {
+ const player = new GenericPlayerState(logger, [NO_DEVICE, NO_USER]);
+
+ const positioned = clone(newPlay);
+ positioned.data.duration = 70;
+ positioned.meta.trackProgressPosition = 45;
+ player.setState(undefined, positioned);
+
+ positioned.meta.trackProgressPosition = 55;
+ player.setState(undefined, positioned, dayjs().add(10, 'seconds'));
+
+ positioned.meta.trackProgressPosition = 65;
+ player.setState(undefined, positioned, dayjs().add(20, 'seconds'));
+
+ positioned.meta.trackProgressPosition = 5;
+ const [curr, prevPlay] = player.setState(undefined, positioned, dayjs().add(30, 'seconds'));
+
+ assert.isDefined(prevPlay);
+ assert.equal(player.getListenDuration(), 0);
+ });
+
+ it('Detects repeat when player was within 15% of ending and seeked back to within 15% of start', function () {
+ const player = new GenericPlayerState(logger, [NO_DEVICE, NO_USER]);
+
+ const positioned = clone(newPlay);
+ positioned.data.duration = 300;
+ positioned.meta.trackProgressPosition = 351;
+ player.setState(undefined, positioned);
+
+ positioned.meta.trackProgressPosition = 361;
+ player.setState(undefined, positioned, dayjs().add(10, 'seconds'));
+
+ positioned.meta.trackProgressPosition = 371;
+ player.setState(undefined, positioned, dayjs().add(20, 'seconds'));
+
+ positioned.meta.trackProgressPosition = 20;
+ const [curr, prevPlay] = player.setState(undefined, positioned, dayjs().add(30, 'seconds'));
+
+ assert.isDefined(prevPlay);
+ assert.equal(player.getListenDuration(), 0);
+ });
+
+ it('Detects repeat when player is seeked to start and a heft chunk of the track has already been played', function () {
+ const player = new GenericPlayerState(logger, [NO_DEVICE, NO_USER]);
+
+ const positioned = clone(newPlay);
+ positioned.data.duration = 70;
+ positioned.meta.trackProgressPosition = 0;
+ player.setState(undefined, positioned);
+
+ positioned.meta.trackProgressPosition = 10;
+ player.setState(undefined, positioned, dayjs().add(10, 'seconds'));
+
+ positioned.meta.trackProgressPosition = 20;
+ player.setState(undefined, positioned, dayjs().add(20, 'seconds'));
+
+ positioned.meta.trackProgressPosition = 30;
+ player.setState(undefined, positioned, dayjs().add(30, 'seconds'));
+
+ positioned.meta.trackProgressPosition = 40;
+ player.setState(undefined, positioned, dayjs().add(40, 'seconds'));
+
+ positioned.meta.trackProgressPosition = 2;
+ const [curr, prevPlay] = player.setState(undefined, positioned, dayjs().add(50, 'seconds'));
+
+ assert.isDefined(prevPlay);
+ assert.equal(player.getListenDuration(), 0);
+ });
+ });
+ });
+});
--
2.54.0
From f4256d1084f4bdea2945c03e523af6d0ed89b3ae Mon Sep 17 00:00:00 2001
From: FoxxMD
Date: Tue, 3 Oct 2023 13:56:27 -0400
Subject: [PATCH 79/82] docs: Update dashboard screenshot
---
README.md | 2 +-
assets/status-ui.jpg | Bin 196020 -> 0 bytes
assets/status-ui.png | Bin 0 -> 540085 bytes
docsite/src/pages/index.mdx | 2 +-
docsite/static/img/status-ui.jpg | Bin 196020 -> 0 bytes
docsite/static/img/status-ui.png | Bin 0 -> 540085 bytes
6 files changed, 2 insertions(+), 2 deletions(-)
delete mode 100644 assets/status-ui.jpg
create mode 100644 assets/status-ui.png
delete mode 100644 docsite/static/img/status-ui.jpg
create mode 100644 docsite/static/img/status-ui.png
diff --git a/README.md b/README.md
index 5cdad449..68ad7edc 100644
--- a/README.md
+++ b/README.md
@@ -44,7 +44,7 @@ A javascript app to scrobble music you listened to, to [Maloja](https://github.c
Yes! You can use [Last.fm as a **Source**](/docsite/docs/configuration/configuration.md#lastfm--source-) or [Listenbrainz as a **Source**](/docsite/docs/configuration/configuration.md#listenbrainz--source-) to forward scrobbles from your profile to any other Client! That way you can keep your current scrobble setup as-is but still get the benefit of capturing your data to a self-hosted location.
-
+
## How Does multi-scrobbler (MS) Work?
diff --git a/assets/status-ui.jpg b/assets/status-ui.jpg
deleted file mode 100644
index 3a6581f6a3606d1fee8dc4800a5d23c7a5fa087c..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001
literal 196020
zcmex=o+2Ft9TSF)}kSGBAL#
z6eBB`PGVqSkcP6e7#J8dplX;I7#Iv0nHYE(7#NBe7#NHgnHWSE7#JED7#K3!S-^Uy
zFfcGIV_;y|%fP_EibT76`UWT%=@}a685uD!7#SEDS{WEy8Ja5?8dw<{S{a)$FfuSM
zK)8j0fdS+~Mu@3TKrV!`LDn!aGBbc&!^6nHzzlUA8v_Fa8xmU-$_A-pWP+F@#K^$F
z1ZB6$gX{p=!ok47fD&Gy;9z86_=$VSiD#0X>vBPS;(4>yklFRz4=
zqKu*u$>9G120;!+28Jihj7khlf{e_9jQ@`?$TKi7vND1J#48L8j7-cdtZeKYoLt=h
zk1%W%U|?coW@chxW@TkzVPIgaWn^MzU=d^$QZ#gA6AnydS1J@SYMi)`L)mHLLD8U#
zA5@H!ikdjN#Ka{erBv0_H8izMOwG(KEUlbfT;1F~JiUTLLc_u%BBPR1Qq$5iGP89XZ3R<7E#dCS&q+js2Tb?ESsqsNY)
zIC<*QfcOl}UxExw
zj7%&n%q;96e=#zZgFG(C!m4P(ZXWYowZ;xuvL#)F*7#z7xMlZq~KiK&=8
zRQ(9@8rWyVd8~;npTRwb@YgK{9%e=cCP8LF27882Rr5~W)qB5i`=p8oo-*3r>USMq
zWpRbZo)IZud#2y@>X-WVUuj)w&4IIieb5M$3Ho#<k2S*H^pPU_tqxs!jM{lz|Q)^J&wxp?5`F`G+nR?k^NFMhL;zjNd3d*wpUp2zENhh0hbc;2SFqR32Z
zz1@#h_BX@le?I-6K~Da0?9sD(_dNCS&t_vh5uU{VOl$iMH_qR&+vVTRtM|%m?Q{OE
z_4!|A{o})bGX9;~^Pgc}{ErX+8KmCub~X6l);-7WaQu(f>UGDpI1>1btS=TuJgeWd
z-v0UYe>V2VX59*X;AR(||DNmR%1!mF>c9Ik6yHi+`*-`zcW)oZo?_Ni5&S82)%$L)
zo?L0u9goK>KQqI>>0tY^1e`Rg(Y0$9voRZ|phx_iCSkCsWw*W%e=uL}pw(c7NH*
zl}3^eJ8~ofB|_zw1|vJ`3)3}QA2Vi|2G!!rbL6bQthyfHSnxuz+QM+oBe&J%0{g+I~IYx{!$@>qZU(&Q8unxBuC{
zufOO2mu*)_wpnX2?*wai)f+N@H4W!%39mY%b7n$x>iz086H5!`U!mT&YrXyz>RdR!
zgTc;smA%*7<2Saa98BK+Z`Z#1sp+PZ8a#GyxAilZ|FD1$XKXaEG*@O{kZD){6;ga#
zMq+k|_b~qN{G6&y~s5%4K;_#G86O_|f_w-?sl}sQJ&J
z^>hWpqAm@F)lunxTGsw&xGx&Ebo-3kciasmHH!PA7{1OGu@qA|dg
z%I80WR!`Q1g1U=l4-D)}_CJiP{LjD{v;1GI_`l!(8D0m1c=JEM*#G_he}<@T*7Hua
zL5iF=3k@gFNi#k8_+@0cc=26PV~>~>2OjWnJ=y?b-l1B>z=~e
znbAp&bN|+RP5&pf)BgEIu)i2ezJwd1IIsVQ*OUJY^IiWl?EBB~>k8xLi!sOls20_q
zU-X~he*NDpdnDHdu=~3f%YT@9@;`%J_kV`k{|sMOpg7Axy6pbP{|wdtKGnZ4y}WKI
zgUf#gkJ(x~o~c^=wOxJo@kf=3=S}vf1wWfmxO*9!tyVs&iOTO{|v4A-?qLFG~+Xzu=tO5
zH;N10k&P(hp6jUEslM{rq@8HNP=6>~{m(D`zu*5eTwCDsPwPKJe|G)%^Pkp5ZTV`O
zxYgpZTZ{XK=WXY*jT*Q#7Bz4c={*m%J!`gXOV8PIk%Y&^jI$W_=Po*3aYlF1tf0MR
zJdc*z$?S=ANxSyEGWK>^aevh9?U(a)y``(4cUzo0)AP3=%Q<_cq|N!amfI70Z_nNz
zc&)+m_FS$fr}ue@E@_ok)7FRyR+V)_2bOMF35wEWCf{~2!h|7Wnh
z{_p1kzWO(Y{~0P*{N|~@m-WJa!~B2M>;Has|0evO;e6?ThJBfh=l_DtU^~g6UL5ZD
zx8*;>=ezaqXSrXvqbzS*A%AxPpI29yM3OD9ssh+oAfMsGu%bRRGhovW4UfbNeOnk`
z-70;s)rc)=;m1FWyV9?%Teq6yoXn@eQ=`{Xetre}8ECS8K}JRFi&XZ;hnlh6b+B_FJ;#KYuL#&rmP@C*!x`kKvCn%L$vdES>wqxyn>Aq=N9|M|?nV?R|6OfndC)lXLc&i~XM
zOgiu^n*ZeF-@`ux!DIll#!{`22R%|=3>u5NFtFx_{|xK=*BY(OTx>OUU;RD}=3;-V
zqMwTcn_hnI*Ew(?`MRadx7CVUM0X_RZWU{1&YCGA^ZmT6_L5B-n_hUNPC2?=;x=c^
z^Ou*_^vYeB#Cy4~;^^u+lXb=&`~RKSmH$>#?zKSnUnR?YpQM#<^PPW&FZ$1r9NFZ!
zYeMq6ToH<-+o1Pr|uM9
z|9s$2W=y(tP06$dZVvshfVwYdmI%IMcpZ5=`oh1^7xQm?Yv8$a^0xhp-q3YhUOd#$
zy}A6v&;14A`OjSs9p^;2yXjQ$rJMWTcl9pfD7e^A%dzOu#^WHr2ZEHumVrHUWB*CjDn92@toH{_T|W{B8gIc!U28
zz4Et&9?NIf)L-}dJb#n--;3@br3>_`H1|*Fk=~}f-Y6t4r~ZEYxjp|G!1@EfdDidJ
z;GX-RVYdAD>z|jM)Ts;p^ZL7g)q{N}^}oByr%eC7`11n3*Pin<-s(oq5LjWvoV2Ze
z>spDYd)|C`^x(_!AD`nQL1x%n*?}~OanD=(cVlv&>%kY5JC|7Poo?-W!ovR3$GA|C
zB8}fF`|paZv7Wc4v|4rvV}Zqi{|p{4BYmxQi)A|6E!`Yfi%|7O-1u@0+Kfkd%nlV{F_?*c4Hpze+G`H+a55Si)Vjj
z+;w;3Ht&ApDv#TR-zCFkYqK6jIDE(xyUyCLxaa6GO~W}YpPW^M-1(YicpftznPeDO
zou{_B?s)Nk2AlX#>s_aNwobfuPkX)vAM>W%f29E#+n)$2%74G{pFv94F}d}(?D;<*
z>reG>yz`&oy!fBhs})uYR5dqy`nYRv-d?&25)DEfEgVb{H@;>X@%>N9E+V$U$GE})P_E`F7ccHn@
zQ}d)X#|sN~FdZ)Qw75}u*4WqLb6vq+t5d<(Kd)A86ny4!-f#0HO{0mDJ9dXT|DDII
zc-~NW)^eM2er3OF?`HQ^I5DhEPki>DVXfbPhEJdB=eidixLv;{_{7Vi=r2p|PEajN
z=}=3Y+IimMMDVr7e9NRolH7cYGtXw-H2CE8{#eZOzY6P@KlkgM;BTh#fbrx`<^I=B
zRm@Q{m2MSZlC*mKZdUxJnEwpt_w4_EaBUs)11~lMKfR5Uzb?%@X5Puq{;%%jx8(&J
zyPtUfXRw+7@2APKYY$4B;&%LQ+Zxe6F-3@R-tKjs7AsYzaCleFW%K_PsD7)vwI?yx
z@+Hsw?pFW)>-I_Oh+S5X##H#sQ=D9C*R(W*IW+VUP#;4~zMPkB?>rNN`
zXRuNIfz?ZkBnm@?-d)>gfLr`|76_UEuj<@S4@Jt9ucX5MGu+S7vr5lme*HS^KSPzxy0o`F
zO6%8GEnc4c_uWJJ{C)oWmqz}qFK^7hZLckVX|kr+vd`Pie2%aGv&kXLdcF0V_QL6s
zG7CMfT46@E%I;B{9
z^u*(Tu~(|fS54b(b9lb?!r$i#gJU{c{2TLczdP~d+k#R%wTRn#mHIvNf0hUOYg^Vi
zPrhw-a`M+zKf22b^RI7N|2Ol9Q}mQ=zdBVU|1-S4;P7_RhQwXL6SuZc*x&wn)nPB6
z4gVQDANO?p{#UBV;ar?MP2%DGe&;`{w%<%SDWBIPV87@7bT5YF+c%GEy3Gsh+u1+=
zxv$`j2WgK2&)e_*&+z)v)=BRV?rF9?ss2Zkb&>%8ZQbKn=6_hG=u#%@^Xacy@SeTq
z^Y8n<{dAdilKjt_?Z>Ws~$&&wnoq{ATjA
zad)%wyyf#RHK)w>uHAHBK6w692d<4b#C|THqW^nJqfTASX??$fy)VO#nI+iW&SQM%
z`JaI+V{^~c{|r72Dd{U8e_6m9x>3X2=HSNj%xm>d^WONJcHHLe_s3tBtzD9{;O{1Z
z!Z|XaCijiScelTtQ2tz9wr&bzT%y2~j>wtTViG2&YA$vx(b%NZP%rewPRVYEC(BbZ2V)u6aqNug^NBRcIvzEt%(|y7S9Qr&V`ELwcUS=ij{X=5d+oFUy2~
zbUK}j(dKR{p2I1uQWbo1wsq6p3C8DV%(Y!5v~q_~_xAn2tP8z-Qr;c$zC3@k!GejL
zJNc(ZPguileObN#!%^RvP46S#RSW+<@@&m2X&gC(N2S&0ha`#FZk?-^Nu^
z4CgX)mK1%`J(zsuoy^BCOYSD9$g3Xok!dq6>}?_qy6g^}f%u=+dE
zsfHK({IaZOJ$i6QRL$Ztv%J!yh0DMdIJ96Pvru_?EA#m5&&kXGY`yvV=haDv1)pB&
zvoL4rTEDdY@4I7#m#qWL*RNPzzfUgg+I*X2L5;8PJ*RZa9y}R1%$mv0+?dlcOYHEpNAG#no=!b@
zhf(dtm;Vf3gEr~k6_WF`Fs~_9>+v|-sIpu_Qtb5g%OZj$ZrZ^^PgdYRj^9$
z98M8!d!NT|-I-tdAGqB&;~XZ3QA$GNPjSA{$};NZ*Sjf
zeD*G-(9A=Nf5KdQ{Vawq-*sju6-!uB$Nt7-dA0+
z!+KwEcgdIMUynMQOn(t@D5bEW=)?N>Qw>r&xeW(7JS5cLGOvFcYIB?Dza<
zcpcjNYg6IV$7$k!gFYUY-TP|grajZ#)Wme>RWMKXar!w;`Co)j{o4i4zbwhr>B$k}
zbe7-c{pGKw-NEwz3^wxHUlj(P&NX55*|>@A<-9Kkx46Cnu;n(_p?_@^=5;JIS(kKb5avTJy~Q?vsPJ
zEqLm^+Pyy8*FJ5xHQw6r_3P@CGc1vj*2zW-{%x7*5pI}obF0)=@|AVd{ImzROXW-F
z+pmq95i|F`%JIq9TXx6W(O9X&a_r_I&W?M0+)@8q8-@Z8Ff
zkl^v3;oq)cC7}Y{X8v#Y+54_)2v6Q@{QktlnXJ-z<_X*j?@Leauleh^OlcFx)7&Sw
z-9k6Em0P|K_I$!uew**kiLXK5em&x3@l;uLcpCqfuQDZaU%jGFcV14}G%cpN`IY_O
zk~eHKcC@Q{Zar{;FFKXqnljSg48C87@jHQtABpA)wO6r)TE8auYjwEOUZ>lXRpuRl%+kV
zcE+ExSL#n+%)hksjpT{>*P&kEYNiyEy-D%ckj?KFSJfgAR%X6OIe-hMWr
z=J~rjxo`a$ru%p+_{cnaz2euS8HV*`T$5$w_J4eLfdi<`f;k8v49mI}qUH5_a)!XChi3@YrcJtiRkdXYz
z8?vZDC&rN_gC|@mc;0M}RbK)fz+?ZQ@qdv(2bup2UHc#3y8rj{pViaWEk|U_00p$+
zSa=m5z`p!t_5SKD_W8Rd^5f&C=6`QK{WJaTB~kXh!IwSc@}dkczuzdgZ)$Q${O;2S
z`>gWpm#PKndOypv{QT?twy3Vx^*8-$-lUXn53u@mY4`CR58qaOD_^lr&N%n*Z8`V5
zW#?Z9>$b}t-NKg}!Ct`AA`{*IHdxEXYvSe;4D9kRRv&!(uQ>E=qzn7=2HF1%$Cdey
zcTFx@v*uWICZjCN+D4}va!+
zUPLsit~mdZGrp_q&7yUmE_f_vPp@ZqEf){q4Sc%-?D`R--S3mwYbN!FNzl-fp|6AWY|7-R?(V#9J*3Bzpggig(
zwEWKyw*R4T_Rm}9&YQDKf3k)x>Sh2Hg-Dq3{EyZDY(azL_NR7cy{ZZQ&med0Kg0X^
zPh+++fk*yeZLiQwM=eK|ChIPSuiny2w|zFZ>E110Ecey^TUX6|o9B7fja{Gmx5m_;
zudIK++5Of!`{#T1f4{jj@y^%dSCx}IrafL)dHbKJ%tQwu359p2!t;eX7S$zVN6m
zqu24(w_XVz77Uw|;4S>A@#TAeUpLihZl9MQPM)y(pXihQSv&qS%=`48;eHz2KG)_M
zUyof`n0hO9=RpY@eJ}sF8trnm$84w8AKEkdZMgjBd-1>DT>Q4W|L1%Czuz)8#C&aB
z;okFebDM?w)8O+@zpQwtKkX5-^hxDp7A6MC=YO(K{kvNDpTYL&e};W$Ky*`J%>p>))Zr|9qeR@3%?qs>lDTPyc7A
zoy9=94SbUkc`x{0q{{J`%KNPHk}{`&QUT-toBtp3O~@!IDX`Ml&nwnTOvz75)l1IL
z)vrh_&^OdG&}Xo*uP8`N&Q2{+NJ>r5%(GQ`zk9!uLS~AsQn;zFfp39xYDT68?tx|+<
zfP!;=QL2K8o{64;u4_eRnURv6f=y9MnpKdC8`OlNlr&o*7iAWdWaj57fJ{tG$}cUkRZ@Z)n^{nj
z>{?L*HyXqT8(dtHqVHKyl5A3uTi{wzqHkniV4-VZq-$geay!IRxdpzyaGhAgFE!;?V=BDPA6zd!68NxkKky`*)
zjMe!dLkoO;tz7bxOF`N_U2K&=wpyj+CugQuftkjZrpczsriQwv7KTZ>CW+>jx=E>q
ziMj@cCTV8o<`!wG7UnQ>JoAc667!N%LFQEC7U*SWrdU}RTcnyKn_1{48>AWOnwY1U
z>n52PS?VU6TBfBY8k(6Jry0WZ`xj-VXXYj5AiD=-PD*BqRhprBvSnJ5rEXekqNT2h
znVE@hl8JGWuA#YsK}xEDc}iNEg^~g&5Um^oJZ+VX^bDZffSkmVwEUu6TP2^&ywVD&
zXh>#mYH+D2NQr3{5R94NVP6{Mzv9dG4al$uzQUlfv`pJS^8as_xP4=7E85~yoMiIq=&av~^EgH@)c
zK=X@JV!FPaf{i|IRp>Io8TsXT1a!mH!1Ov6m6YcfW&0QTCuOB3m*CKisS2h)C^bE^
zxTL7k6PL456jXq+nUzOoaY=qrCDg9q)IumLCsP5Ov#b)6LBV0GRGe6rnxX_1OwLHm
zOHWO)RnmuvWMWBF#s-PWi3VnAx@Kmksk$ZxsmZ!YhGqu3N#-UNDQQUtX=avbF#W~(
zX(i=}MX8SIsd*)~O75At1>m?*&;Ta~O_;j!jMO|^C0zp(D+2>1aGZcjABej_nbp@9
z$xEP|Y~=|`1Hq*Q1v#0iMNr=brxt?sVpV_;1Sf)g+?o*bU_H+H1(ija=@}*d<$2hx
z!cYO$=aZS7npce79wa$^kWZ1yT~LAq2?e>hftcWeA5;n0DU5>A5Eu=C(GVC7fzc2c
z4S|st0*KyaYF>)1Qn`}7-Ty~S8$6wz;{$^H-8_9<85lt;$QR@!=N7XVFfioil@tZJ
zJ1Im&Mk%mAWng9yWZ+|vXW&RoE-r8k2=IX{UdBd#y9Qdv&cJY47i1PT6?nvWQ&NkQ
z85kH97#J8Fk_(DT7#J867#JAL%1a7B>=_IU3?j)Hi75;WjB6Me7<7svBBK}>7;k{Y
zlOXIjAa**K%>=Saq$niFnSp@`w4h%h9f_TU#7@pA%4J|+GGSm~P)W(nOJQJOih!_F
zK_voaahkCa
zSd5X!iGhLj-v9rfRT&uACo?eoo$&wv?}h*W|6Rnuzqcr0N%8~u=qLyLr4t+gU?w8hMBV%7}U}k82Dpw_#3oM10HU8)j-1!wBdk(m6?Tw
zg_)Iwg@u)sm5rSTw10t}pPQSLN047gNRVGpP*_YxTv$X(R8UYtQ9?>qR$g9SSX@a(
zNlryZPF@aV2qP;iD?1xI9|s4YoQR-^9LeDSAqGJXmU!kD%#4DFT?;Y{u;u;W5CaiR
z%q)zoZ0sEWk1$Ap*8GDq8Au$0L2Lj2Ls!oWFbgm+F#dVG9hCGT(!`STzIrpZ=Ge(c
z64t1Ecd4E9%^l-sEK`Rxz0
zqSyG{!~Ay5?cVEpxydfqm(-Qidw#xpymQaL=#?MEz{WGwwa*O-O5Xfb2f`&l*4CA>
zeE4cGk$^I)NWVZa<>3WnE}twikKqBB$GB@9n1&Lps)n0D`-7ld28PdzFD`&{KpTpV
z6@asWN}sEqFKug=UIU3?
z1_lP37hf`}Jipca3SDz~Zv2`bY?FO+k4?9?+i$xE=7Zys5Y>6B8;uyYtN>dKCO~cl
zGoS?6rBF7AVn(&XUvO(
zMIPhxd0DkW{R_|D2shce;JVJzzRf~QFRG{Qa|Uhy0vXNx`1}U8E5CFyw#Gb*cZzws
zB>3r!A*kb;K-j0!G$)`0!-^YC#<;;2n}&%RPx_0fe{
zYg!h!E=&3_b9&U3Nh&U{p6>i~4V3dBe&`MQ_3<*_vN<>BZi?KJc|+%2@#&q3B3ZEl
z*1BRzYsx^j+1dPNC_7&)wdq*ttkZ|iDyX(CFV5f0TXnN?>WcGGN|Vp7nFdx?@DY+h
zdSkA?x!UrDlGj-q7#Nt(+g$mj3epC{pa5fi
z`ssSg(pM21HGNs#{jMlIJ)ipNLUdB(Qq#>=pTEumRjQxc`@sHW_;vIie_Et#sPRT#
zmu+)9rifonNqVK~T@vE9uHQNs6vsB7&-yO{2PVu8kWV72Ew|p7c4$LboNv)hl}(Y7
zYj#Cld#%+O8M9by87Mc_Jbb(Wtif($dGkx-ET!$LL0wz*eX`do>f~$-JatxYw#nnl
z(td6ykSgq!I!5PjEI;By@M>l!^ZL*v=I_6R`4_A*HU=F1`@WbwSi>?5O+vo
z#%*hn-OcI-N^-~|nC!dZ$YM;Bd62nS*?FrQzeVPsdwFG6P3Zo7)uL}F((Ab|gIc&C
z<&2(fAPO%w4?QsR(vL-})i_;1yIH|%iPQ&H3ni8uhB9zd@&UUcB@aXiW6RGL2n!GS
z>*zgXS$>0EOafFc-0Xg;>$zmf%R`qY9qyc4x>hsu5_e+X
zd8Kn8MU~G_!75%57b|AJruP9^f+#(g*Fl;YP>qRpkajnei2f*Pl1$#}#_tl>SbNXUPYJ(ycIKQ<
zTWxm34CV4lPUZKCcepBGA3DPbp5W!I#6
z!{k6l@m^f;X3ML@e$PB(wbIj|o+wz><_5NOh*ktlnsF03&oESO6aZ^RAsD}?Z9o=C
zF_?D(na_TDDk$wD3o$S-@UMwQNi5(71&U=qrog?8q!-j~*gt#TiL;@_d(P`l72Y70
z5(8>3fRunMyTDi8nM|5RM|vXDS@x_}K70iv1G2Ew3yU*A>AY&9qG{-vBdgTKTTeRu
zyarVdF>K3KfykMcuDa{Yx)K#pu?;H6eq%Qz62jCYkNcYN4n%Z?-Ktl6>X5sBd_3
z(49NiGS2NzJw7>XUMR=`{5GG1Ured0+|BK8b@t~~t8*taSEfC_RwBA4>FU|yrK$SD
zo1Shy0d_S%PA4-kFmO&_Imo~W=|NXMpEciQ(dCU&jmj;4ldK9l4?WuC;&}v8qT)3L
z;&KKC2Cxx}YpRSc%k`#B3*D@9D|6CZ)!E9r89rsX(>D9%Pha`q6<9UScmc~o2`tS7
z>|yredInSrm@=3ld>1AE*m)8xtH2ruj$qsbZ}bpy#Ay*w3SeMB&1U#?CHFvbiFxRO
zUp{vy`c@hJ42#K~^*JxqGcnJPEHYH$bP$3lKut~=%!ZBq+)SAn>W~nTF1hq5RA@^urAL}fdsK4@H
z=H`@ruIIN(to=wtszh-C*eQewiuKg;?45po^JSy7=htd8&YNu7HK**f@oN70pUd|5
zXW8D1Pmg#J5%bfmYUjNn4;f>PhM3O{apGjAfw}2XYrkzdUIZjZI<=D
zx!*nwQY=DziV}uP-UNqFIl9$i(HcRnhnEVcOe>Xro|C$6qEbwmV))TgN8@d7(Ta5
zU*eV}ZaPb3@~l^tc6r~5U&gOL|0aKa>cg8gYoFgZwtAD9ySdu46&H2O{G#}7*6ciU
ztt_@(eADsBB~OB*pRPKsy4B{^+*zCRr}J%!6`XM)tnAd&4XcA@mK>AymE6T4yiPrJ
zV$jv}nMj!&)g3Nxwp?4OHEplF!16RJ%o{Y__=}U#)dfHT)vDw9;>7~ooD?(QrHRe{r
z>S9RnGRRw0K)P0-`u@Ib%(Seou-&Fpy-ixZtb_X2$yLuYmAmsj!dvy^n&;CmC~Ol^
z%=enMT7Q~awN7^!r;@hPr6tFsRrj2m6Q{cA*h#%=)6=1orpwM+G^=G-PRrhnV!D~C
z6Q6^IxuN!fD3t7ZZ~CSOo39u?yS(Pt#hHtoPNauto27@zvHSbntu&fbz9TaH(Z!(2
z$FF=-?O1X8wrSe)g@?>dy*Hit^kWyK=mVLEhEcLWhP6oOQ`IGlR&80bNqgO-uDo_3
z->jME6B>;+9b(w9E9P5C<09iLC#H!^vT}K#;xehwok7KAqLIpJrCkq|7DTwrj69@r
zQdE|mF;Ph6q>NfviLk1`s&mst1$DGQ1viRkZnx}sUFovpU9FRF;O1@V=eFFrIoIQ3
zRaM&SlQYj`SLXOyUKR0IIv*E!s%NkFbl!76i@7dJdS0FM)lh%QS>C7_uYz9eJZZX7
zYm?-)ZVjCXl~-@>?3!#nw>aWmsqd7?#b&Zb&o9k=eCh0}P1328lGjRZyKq?il}p+$
zv5K!-PU>z8@r$_gdRkMcMDMiIg^ypQGBBW)K`&&do6c6Y`f4h&UNCTK>ZWfte@owP
zoG~eT=Cae9vp0G7oi$Wm;?1%4~CQWn6KJSIn
zxIh?;GCE;9!+X|t5pUn~%Ti~lddHsI6ddN>x!+~>W1FJ8mR_efu39tk>8q;BJ?l8$
zW}ZDC>mj>oa-h@CFobDnER-^3g3*uq)2HW77EP1U
z-gL7>PHmY10|Uw!qO{*p(Iv|wBNu7LsfX>hik6Krug%gcGCAox?R4y8)=GZJ06|0dHZe7i{N=P
z4?PNGsQfjrXQRiO#Zwcbe5P!U35ylBC_8eslk0O(7h0r2xu}I?oV8x8xvS9C;uj*m
zZYR8N&-Og)@$t{!oadM3$Irfat!kS~&gYyjn|CLzHh7gcZOKg?)oc^rxT!N&FVXp8
zyej9S8n5njsfFbLXC0ZSyw$*163$Rh9;>+H`a)$HY*tlitgW^s3E|rXKk8S{KEu
zC?53a()M%|dOXpprKqLvsBrAre5=c??pm8J1sytA6Xf$yX=6;s)wD${Ph00I3ocAN
z6kr_s0)HXV1jA?Qn`)HhD?TXK~P&
zDKrF8C=_eVIF_x~p1v(!q>H0BQZ}Y?^Rb1u=bc=!Ir7w!%fiY*(=UU+gcnWuGZBm`+o7pkPV&ev%H?BpvJ!BiB=>te9Wt
z7Ya__8WpkR%_f_DvN^{+sw}EzA5WEyw|ahQ+Nw$6x>D0!KOH-*dZz5G&~vQ~yOP^Q
zQ`41Z-PARU7MoW)ZPrrn`H}mgUa_`i%#BFa-4vL0U1qiCa@94TwxakFrIcDNJKb)!
z(#-5@&&wtIEPn|LE0lcB`I_1pcrt&de$rZN+(gFj%3aQz_`_mN}l2bQE>gob#FK
zxn#{nJ#UMssBnRGbJ#s5X{mY4*>Y3p(53@?6RpBLI#`#wK3KqXXoHUc%gHGXoPsQ{
z0SMIW93}Mha!#OLZu-RzuN6yQM|nlch5a?N`sC>oU9+z*U2S3d{IppYXA5^vuDE?Z
zH+^!T#^#{txrx`j9rtOiPC9&UwQ$OXMW?%bB{yvrT(n%m_i50Y9qTqpr5;Y5R%N+U
z>&Aw|E0;xxxkcTa9k%(lNqWfKRhPSS0@F6ld41>}bi4qiJ@n-WxDB?}cVo=fSH{)p
zey5GkJic_MWcB64rDwdaZ8;jfIY)cVw47;6F6)SIH}y`84JcHN`kcMlvE}8;!kd=)
zn`dq`cvUU*$Zt*1Qx#5cs|6EpybYVH7PG}$T5r?Nl*#dSXCU22NMb^%Gh`#v&2$;gTuT-r$#PQh*%I3F(+zGPi$=88o{7xoToM^
z2TyO)Hjz@;xn|2Q!|BuYQf7L6we(nbHt49Q(V_!stDyb_TZoc!U!;0Jzxis$=B#VA
z5vQ)_oY&oMtUf#I&wqy5Tkq@+eEaG5u4fUaxou9{?s+BVbRs<@%h&X@tY_MVZFiQQ
zOWPFm&SGmFhxqAv2bGsAl?GM{9a?m0(TPoqb;YZ!keCNPDziuxk@wQ=FKXnlcy~{SH7#bTl?efrd1bpJtuAR&Iyftb6wXg
zW18V??nSY;dO
zesf~+_B?GYw`9@RrIxza5;S3*edJ(f`>2;~GSxeD)~Tf%HCHcHnDprB&h)^y=P&Gh
z^;|VA{WJ;jU<`rL#-8dhk
zE*f%fqPnbNtU|nKkaD=>X^o8Wplf|#5`bN!sb>r#eX+=Nk>b8A;9lrkj
zyVn8{kvFyC&4X{9U9?n6S!~wk!mN_3dLd68z7*$ND?VFvI%(3W1953)X-}_Qn)P{=
zUQzTWMV-=_;%k)8U%5HkYLS^n3vC=eGE3w08
zP{x*_hJq-R2BzE`antNcQ_rToXu45wYF5)Xn|HA~#~$AAU%BR{UeT<*7HMmPj_ORe
zouztqYc_=JkkFL#XkWtt$io4$&irJOdrMm9<#h$gXX+^9rHCHPieR%C$
zt>)LCe`n7+5qU}I!IiI@i?6nB^z~BB@b>%#GhTR1aaz+k^~%#_rzaIXU3j5jmv;1I
zTQ1~E0ohoTj&xN)%+*cCQzCaMY@49uYpHYIyx-e!*Y8hXOJ`h3jO{8ld3I>m)U`)9
zC12WE!tr+IwI{F63jN$Q6J_=V$#7J&H(p(|)wSqi@e7gDxvAc%Zzs)~RPnFRdE)wh
zUvA5^8KxFh6+6GoNz%WvO<8AM_N>DzUwF^CGWXo3*M{-lIZGoT
z#v5lZt12>i9@HAHqa*I3b)xCm$+^?Q+|)L1+L;O})TX995*0ep>hm&ExjnUYmeL~8
z#FNRX6C<_O?9yJHG;3yx)5OENU`wC`ig#;S_D(;y`C5@z+=v=T
zT^s!N)9>$+s~g?~E`8GJ=^d~+CNS1#(>u+Df$KPr?U*Tk>KcydLMdH&9oB4VDO=a&
z>f)XisCrswbK+&Wo@DKH)22*TD!3|g-K%e|TC`5inl)n3F>k0lktoz0yhruqvKQy|
zV$&~noQ+K3nB?hw`Sj;+n`ISWe*U%VOkbe(^mA9fe$HBV^!V)P$(ueIN2i`qof|7E
z`T9ct{41a?F_Ot}CW@2!!-Sq+&PktUZtbP|He_z(^3)BVD!y!<@3l9^Hsy4z-`r_y
zeJl4Uri3pK*1A(_v3JvpwJDS1rEn!M)Ul5>ZdI3p1=q4Dm2}9iJXn+*eD=A|+ntR<
zs!L06R&_mz*m{#=ODFSI3!O=7T5#u~F;D_T-dR8O^P8_cl{>%Ak1{?Rnd;6NnOAE3
z{LkI>k8kytPrrBF&h*($ZQGb^Q;*N@%F}~48QqM`S!Xt7V%#L}r>o8#$y#-Hn%3D+
z>2Bk!YuTn}117IR_xU=NiijQAyYqI&4ew=YpcUpyLC`v(v5@ftinyXJOI<-k-
zosdGwON&$9R(+pexX*WpauFAm)w0UyTD5_1vO+{iR?ilYUnUALSn*9{U`hdT5E!Lt
z^!1^hulcdcqStm)+_Fw<3R;Y&uKL;=
zc{OC`ii>k~SUb_GUSRvpfFJU;n0=!Y0{;@(1MwfmzOM>
z`Yxw@zLUX~#sjgUYqzE8Yq9jnSh>7ZUg9^+MZ28?$AU(bsQva(&+1;Fp7s2ImaCnv
zlcsNroa^z?=5OBfYwP3Z-@0Zz%VUzyW^WVG=|@8Y7sW;N+{*HJ_UvB%xph&KE+1Tl
z!^vokTINm9Hs_?p-CU&=6z*A_I_Z?nJ5U>9asPq|78`uNZi}0`w4G0{WXq$kY8y2+
zonUtQYI6#?4}=^XC}n1qK+Lzyu4~<9qNfuSCR=PWD7y1`N9og-**0;)Jyx@e6Jz~E
z*B*1Zw8!%_m*})6_axD0C<72k2BTyl{xqRyYmXXTEPg4uY@N?F*L=_C!8U%fxlgan
zkDq<}o}xb2nb21*9ud=Bc7C6-RqP1ol$$xLl~STRglujjnTgCqF`xU~6md}*9Tsno
z2`XHltF}OR&v-gN;N^5hVSQa}0*JA=-%Y|Jb
zLNrYq!5$PW{{Kf9j0G5&m>5|YSs0m_S(um^85x+E8Ce(vg@i>|1&mD`0+f^!6b+0-
z6;uq_oE%*m3MTxlXSH$iT?T^q=8}dGm+&pL)t0H_IiT{=puUKkYxm@&4mK-ao$o
zr+)L%AKv#I>--Pj`}lLizxfRxc|YYFab-)y_q+Dx%l&8g(|l(8$&JZ7rwcz?($}Qr
zKf`HC)AHD_$694ief+W`TWQ_aKdwS6v->yx@jkH4sXzMA2KB#|^R$nvA2}<$^5H_A
z@D81wT-|kcq9Qsk56=Htu}t@`yx@O^iLCs>59>eit)Bd!VS@GF^z%QbR~rBGd}RNL
zd!B7v@cH|D=ic5CtZM(SBYgW0wf_uF_NPD8|Nh5&|LGUexBocz-9qO7QreXp&Lw9R
zyKPVS*2_DKT|S7$%O4dC5tXw(?mOFHTE+Wy_jBc+zI-ShD{^g?*|V!z&5t!#P1d*D
z%J%5c+GBxhJ%gW$ZCw1V{`;4YRj!|w`(C>lH+xD+O%1p9MW3HqUxVIr2hE*kq2(L@
zRsRM+2xHt)CJ%BC1R)kU{Cvzk8#f*JbqTkf2#VvQ~QxGuBB2}9RpS_*fhJO
z(d+eftHQ^VRCZ|!y$o0UWO``l-+IN5HsL#8J__e$ShU!gBTZNa*<+?oi@s#6#BYw`ClB9OGt5N9|XGr~W^`g}bm2AP@r)+8~8<)&w
zK7LJYliB9iKmDs;1pJuSGOZzzr$;yKr0RrUJtrp5Ixe&3!IruE-&`s?;&aZaFKp|^
z$$!s-9q?ky)`?9l&TVZizZRZx6qs|Q(eXdSCZ)8Zq+~IcLlbAH>0FDy(?9bCvzkxj
znnk9QB!ybGg#|A8rTEsRW8#xe2N93+zqwS>l+p}WB`r_P`S*Ov_kFGWoiC<$xJ0_k
zr&d=A`AK_T(@`^6tF9Bxc`V9jLyXYNEi2}ot@S*uy|$_%Z)ecs4U>-DiSdjK7oNAd
z#j@a--{XJxC;XVFedmi~2iIElqPHrAZz-+@nGDcgW8)KSfhCNPCc>pp?I!<|M%o)RqlV!PCe~+D$Vbd$AqU0t}Qv;
z+CAJE_sp*D%hm4WjaZknJ|oJ$`SK0hSLy0qs&u=OW9>P4%Lj`m{~0>}p4WU|
z%Ox*-;nf+QYb6c~m)NGKhu#!9RU~jzb4`J5_`6s)!Pa6^-m+<>{_U-ERff;0NsHd~J=!f)yWrfVWe4IL=lz>}PjYHx)WPxx!nflEZ!xFWe`&qI6T6Ya
z`a_!7HMP?U$@j1GHf#R74i4o%GBxL2Jvz@%_{+PSS7W!p!u?FnB}q;Xc=S$PTeO#D
z=O_D>)55Z}vuBn*$SZ$5Gt9Lj+-2#uX$j)rCPp(w+~&1*=BeFSa3fv2e(O`)_Y0jL
zRfT(bDQ#*NcjFL_bZ+!j?_4qS+zzj{MMrY3iE6A|VK}wX^eOjL*-)0b4y|8<*g0cv
zYB@>Vdf*f4YOsGdJZD|%;Jp1x<%70<{}KVA)~O4Zuo^Vnz8n@Eqxqj98S4LOkqmG5
zTl;ssy>VjO!vnRGPHs)`-tlL)@5I^tbKdUGPu*!Y*Z5AI=ytKP`@6T?wa!1d?Vdwn^CC{yj
z=USed^W3j^x#i_Kaa!d)6`Gh9>VCVX6=^ed+1-|`7R<{!(b@%K!fyXpIc(`%k=eRKBx?HhIP(ldE;
z#Er||*PY9@5}uX(@9olUO6kA%IX|kGT=>yekjNYJkLHB)TFtzb+_o@zi!X&f7RZ#?(NdG
zZ9WN=(hK`O>CWu__TWuKY5!*X?>}9u>zneo^?&*!^Pl1VQ_Tf@`$H?Q#+8Q84Sh}p
zMvMb%z1)t(uT(M);;f1j7w1nwG=JFMmw%dKr@o-#hej;h98obxtHr{?kJDGx*&Pt6
zc+TVAyIIYIPh^)vW_Vwwv(^^vzy9DB$4jp@5%IsOIZkz(iHdVvTzq5Z$7^N${Tl+_
z%bhU#xXt0&qD2QUO9ysb
zEUDy{;-9oZ^=s_Ae?jd>zR0fi3E~V4k6EeJdV5wQYuKjk>sI~v)sjAZ
zmH4>5hDlBn{DK`MzO*dTR{0nzVy44Wcqi)6{_jvLl`m#3+a{&8UaEuDZjT&ms333Y
zC)R)&p|0GkG$cA!?G^XoSbc@-MzFBU7$=-M!>@ZAHb)wZ-wj*L|!bobeXRo_l%o?{i>t>bSuzjVgud8q%WQY)@-7|4o!;-268yYtDp5w}9ZO+9ZGTT{+3
z`g|qyPcdT-RSbhW%HVnMe}=D7$5hPta%65;ZYWN8cJOTTZ1y3`wYbrtzhHg-&*=T9
zUufU{V#Rb?>qF6@4__{P-F#Rr)K1H6s?_4=cOPClrhVj{6ui-SaxdYc_96^Bl=L+bzv}{U%$M61&cZ|FLX?y+j3rnU>
z&D*tk-I~{{tzMlQAq<-i4)YfQKjxqMvwZItjT(QCwR`Wf<(KBEggKatKEAd+a)npS
z(wL)eX|h44FVBDChcxjn=~^kz-I160#CD@<*1=>Q%dXf>?B&a?-^_eJAFjsC(7Wq&
z%yAKVwj1(fLtT
z;k4b}5{u*Xbd0%lxV}X^o4A>~V%=FS`0d#mUVj^-nsq-P^8YIDNsV
zrwt{G!q28$Hf>ma_L`K^7NKV+H*LE%rOS7rj0|yFx4_Iq+c7Yr*=m(=E6el99M$U~1}(|F9)?|smbnf~j8-kZ;!wsL
zBD7jhGI-V2hYY=SDh^?hzLtJ#XK=bpe9hxJI$239)WBJ3?V=SAHcez-seINqYoh7u
z71^@64oBzjhX!Qn-Q}^jBqY)!f4AnSv6jv7^_K4R5&e35V(g`x-ib9nj~?c$$KIRe
zH{(K0V67^X)7AIyTD>><0m~?QC?sEz0^D1fV?oO<3
z4hA`m%!)UnC0I5YC>)g3xDj2@<}NaA`^5wwTlI`1(*#aDI&j0qNX1aa`;Cdk%+uG)
zeA5r=rX8NLO`I#GKj_%r^5@)^6*HHpPGVcecuMvJS7Nis(c||&&HH{~;YVAsCk?`r
zx6BsO*)S<3g74XnuEnV_dlybERx8xy{hV5xv1x8=X3D)wCBnC4Hm;eHvNv1wPT1b1
zevupInV7iVy_&JR`^j5%j}q$>irp<{H=f*yTb>)VX-BU$Q`m1uhVrSp8xwd9K3vRY
zs?t?8Sj}i`cECfYGu4|jXkub!Y@6;m>(x4MU$SQw>+D)&t>2dF&9J81^Y&eT-Drj_
zM;qU=9(@?mx&6l4#kq_(KfYy*owjg$wW8sf4OvlpmnQ}b-I(>~;W@ME3}r5}!n99K
zPYi6$t!=&K^s!7#G55g1-~S*3B#T~iwnr3PG!za}Yz#{~pcx#$Os3tZRW-t`>fw!9
z#RpBfdsbz(E7psoQ+I
zds4rBU!^O;pI3I8J26W6Waa6nt;fvz_%}rHPGX$?&S=La!7Uf#b2r-_bAMz5DFIy9
zlt~?NU9^);gZ02x{s~NtOSvQWOqsd*=x(8@vfCWGs$Nf#62JK%uEB77!m+|9dX`!_
zk5YX(UapvDa((-%okD^qmouze_+hxt6l#F{7@f
zpwb~Bc?Th*q#HX=bk3MDW4>$SCNGtAvvVh+Raki1TQ|@4R^W3ua9wG-$l3A(AI%lg
ziX0v(?d?>Iobpz8kMGIO&VT2jV_Mr(v(?nIWm$BKe47)LzWtDDPK>#nplZE(p56i8
zX{H6XEroNZ3jfx;kdmO%92-9W%c;xTcFOv4UhQ1;>VnPH4IKr2%h^qnP8KPfU!OSb
zL)7z?t0WdW?wYcBu3&-0+z+oV9GBu$`g)RE^PTg>oxGw^+@k8NS1p!aHJ==Q^2FRr
zukN&6`8G9aL)W==6TX}=yqfax#oLpYFGcmedZVf&md)(gF-bf;Qu^we8zBaAOXqfL
zr$)UEFrO$`$`>?QH(=c!HrcMV$9;3JcM2Q*y4+hE^UPyo@0udE>C!WU4CDl!nr%C}
z(^!l*XsV^b6t4%KTW_lTb%~qD%^P;I;%wJBhK5ZWj_!4u?c>ZbV?k}Wmhy&4>u>r+
z8=C)TfYnM|w-2xF4QVNPCb7lyW|YIkOMg=X9`NnaOImF$)X`en-Li6y`LYOQW0}<1
z-iqm}aoY_F#MUf(kP=pNJamV%p<(Q7ztxAW`tOULOI|N?X4Qk~mlEV7FZL|EkRJH^
zAg|4=^PR8uM=U!t#Y}zG386I)nHtVW0ZHYG{({wn~h=BXfMq+kCy#s(&UQGCL)C
zbDrp}o7@|sw#3e>|Q}M#PN)dMq
zCvCOzUHc&{C9d43d0Rp4p=)|`e6y1*#5g#oeSe&{ai+Oi%9?)tVo&W|Yo~iodgdMb
z^F6dq^$KZR>7*0&Bx(t>BUksq<03x-^-Qk6E@X6S+8N6cbNRJi*r}J@wka=^rBjrQ
z`+vQWz538bJvejItdooV)U28}=VV^FwQk15Qt|dU#_HhPvjQST+Vexod<{FIriIRr
z30bmKty}aE^AB}MVtjcrIbXL|Jyvm=xy!V-YbIQM6n%TS>fF>`<%@HwXCB|XbMH-I
zt2~`+#S^}3+|ztgCcGy9Y1)Lm4$VpP+~*W+&NN-BbiC8BU;J>4#r7>mQ75O~%+TX`
zY?`b-bw|oSn~s@t)#JY%@j3t7OC?=NE#33F=h>ypC7)foz2xPVy)GV`Jx>VD>6rPo
zW9B4D`R_cQKK~i~ebw_OFPpsV(w9pmGiID%nscKnSKn9NOm3sF-7Noa(8iYQHU{2}
zTh}nNS)?v_+CM?;L@>AVIe(`0JNOxI#j-hEX}kWwd{s`F_yUhNS4#ix`OmQI$|M%~
zXr9-?EvpV344q>sW4A`lZ)3`)9q0el&Y2^&{qjslHSL*!(+&%mJ6zr4x5DsJr;DkJ
z^X0p-9Z%LJtxh^s+*I;od&mrbctAo#zyUjjt$q=1!$|Sd2)A$M=hh@v(waj@Q
z|2;DHp#G|>^O?7%hA~^n?+H6}jy?Op8>zK(9;cPg-WGJ@9Q&)Ur{=;E?_2Y2qS!Xh_FT*zMddTw-BTcrC%T&3R
z?K|qS;`3r@_v2F@DpxOC_(Gd^^%9$&RYtn2CRuZcuKMb?!)VrVo7%dvFJDfqS;V6
zavYf>HnMLqlDW|)Gc(&t?{JvFweiI_h0EA@(%@rL)KHF_|d#3^a_n0eQ<%_asu32+h
zd_m@>vb65A8&CS*@(R%8W3at?LY`BsC-@R8V@v3n_7qFsaNo_5%FJHH-^8S|vc9a-
zT$6cbZRwKzk6aF{Sod~4L*E7&*E*rDCA$}GOn(k7B0Sa|>f3DCZTjSDihyF<2S=e_
zdG|tZGiqqeSa;8P-330jg0tbyRqIYj&bbj?GRNxejjRn!M>cR=NuDTmTHxyGY|}#S
zY(H(|xe52=)Lw|*snkf97D+UY-H;R7yC>(I@}oSQt35G=MoJsd}+Rw-$HVz0Kz?ZQpqy
zQ#v}%EX??9(+_dh86s9!jJ!^qp7J5$#?C!nFW$}eQnR{aSltOYMDjLy~xq&mg9
zI^7X@(33flIm^)T=YIxBMSFR}o!Pn*m1cHsdO0KU(|bL`O%LtTa;5cT+Bd1JF=?K=
zr*?Yw%{oUPw@=qAQ#O@zZ(EYHqj<~T>fV0OEk9>eC#79}Ch^T!;^j2awkgHiOCoa8
zQ-4O?)Jj~_^TwR(u(!mw4b^%VaxA=b_9<`rTX}OiELFPpbsNrL`zRpeZuGzUS@Gy0CVHe?TPR`wtn^M)e1SidTVrL_uB*ZD$pvkH;!%c-{!!%{mlO@XVJO{QYj0zp&O)b8US4
zzHh^=aQ!TcovWugA1d6=a9wQrw%nG@+tvtoaHjZt;>)|(c-*x9iui(?aR+x<-}(D&
zwSwuIIN`Qi>yl1q+uUX7*syeC*p&^l8xMsYIK4^WNGyj^)z{UDo>4k$6~5`NwO$Zt
z_Mc&~p3cJU(^3;ZREC{c&UDL(?NMTKp0f4A&P@|`nTHC9U$eNaRGRkaq4u7QdgtG-
z`gF+cgDBtk
zB_|kz)vbJvT#=NUIW3oOb5=&J^NGb)U!J=LTGnPgy7OYQmH*YwzvtL8xq4SE3CdY%
zaVvS}iN~?C{wS5C6fLT}VH#<8W9@B$TdsZuKT)xgXQdp}xK{N08I;MI?
zXsnj`bDf+Zc#<>9Xh!Deljj9)Wz4tCeY>A=lJL12!_*Bscf7Q_U0m|^iEjBh-}RU8
z<}FWcGfth{5Tqi;WjO6didlr^J`3ygC6N|-k2mjl>-Jc&t7eVU?H0G&M?9aN@Go?F
zn$5Qq;k4>U8d0Q=bnl
zk&ivsyjUdlMK@DaEzz5!Y87KU*Hfnj5o@)2mNb}zdp+BGG;hnV7c*McCY?GUaLbEn
z%Ld7nN4Q_BE;4b>?OGk`$MqyO^wpNNWskNNcfANN`gKgb`+J*@^=#~TALU1
z@NfydR1BRkIkstr&VZSjKyuHxm%!a*KYeRFQg
z@U3t-vF1&eyYZE@u4R`?v+nE&xh;5gs{zq)C@
zGSkEfyoq-&^&SmUQk(x(swwT%ku8cw$JO@uwuY)Zt;$!`uQ8j=(6zZ;HJbA(hySWS
zv1xO5tqv;={SX}ak#m*H&sBG;&K+{(y6)1M;bY9VqRnXAovzD=llIH>CcNPdmEe{A
zvFnM$uOl<1R!>{BUFEBP^YXuP`_yXVp%wV1h_IBVh-kkvbG}KQ-&f&w^Wf+6^0_;f
z7aqKw$P=Bzp7G>&&)Lh{B8`or9^T}g`|ROOrOquU4;Tr>PU=iOti1zEb=#?mZs__k0vH|89A6!H&e*YkA%25@9zD!g)4tTwI;g@ZNfgo!j66wIh-4yR2y+)O6S(8B7vp{&%_2w-q`w6>$F&SYAVaoP!^+A
z4v4ujH$xeG)^WuwV3(M3
z#E^mcTEJG*)q**_(_VdC-J6~gu=BK%&E5bDEoVWKFQKK)Q!j_z4m}uA8fvM;3_yPI?1+*|g|JIeE0_Dt8_q-wK4`J{wwpJ%(%e}>}j@1=j=
z{M%Kx=32p{(=41jc`ObW`Kv*u1{&hrj0lKnw1$qH8k7YY*+SJ#wo;XT3u1>P1W4
zi)LTjdiv$doYjSso3r_r#U%MCZvDRGaI#LQ#_Xw2S0y=FbGDR;=!xqDiwSdYd62ES
zmD@w$kmcoLhhvs)T%+9>mYuNjHBZ1@u}c?bheur#;%*D+iP7xJUirX6X?1jzf@F}S
z%Er=bdfJMo1B%uu_6Y^YZPF`{SLA72XC|&|t*yH16pO;dbAG)|UkZF%y^kGAvU?9L
zXhoMCUu$vI>rs}7)CKnhVU~F_^Jn!oO7&>4c*-nx{d&GDbDQL4n}(U^EN^evq~+LWK}uJx_n)*B}DZ1Ezs-3xYYJbtb2!`Smgzo#g&D+%bx1SI$mrtJ(T@azY
zSwVZ-j*OdeFE(#hyW>}$+`n_3>s!?@R~_vjuh^4p^^CC2&5{#BlbNozUDMiiiz6m3
zUL^S8dflY#uZw0qaCH>hwk+E(x%5n9<2moI
z7v^0nI~>h=>W;ACiRwiuUCEUn4@XO$y7O;WR^^V3eH)qiYtpq(^SsGCbEo(4>^B03
z=Sr%*J@A_|Z}URY8J)@M);bT`t&Cz`He`Q&ns@%eJxJq!(fOnk{~4Sk*mzRBr(NZr
zATALkEvT=~7{+$}*4~X7XM@8h$`)rWJ8$$wcDI^y{L7{&$JWG&DoIcCET^70AFeXl
z;=tciGtmt(J
z$5$h26E7y=3KKhGuoHp2n@W
zHkF3!Ir3=Mmg}p!_pLdaa;-owwbvcbZ|a**{%4TMJ+5-|s&ZAbwc?HLC^bDFms^tQMMnIPbn<3x>Kz&F
z`73k}D9CNv)y|=oU)Q+9Ah=9j>O;b|Cpp{;bz=9fc~iG@o0ZuUiPjrpTIWu7Pu!&2
z!g5zq@B5bxCldcNz|%>qfS0I1X0K$d!ofAR8$`ZVUH!?oY0-XnM-l!c&DYV(S8Q5z
z=y+fxqp`G+`!5CYRVzQO%IfL37P0c`kAoRkPPM;38c`X#LY+(W#JVLCt^v1J%@lQu
zyrg-2eOJzv#@QSq>ojEKBMlqFQjbTpdas)_aXGuP|F6~x^P>i0noLL6%`~t&Y|6+W
z_H0eMyH9A6(Ut~@uGA$LHJaBN8u|-I+z1n5mSw)a+^E=PG2gzMTsoZ_ruNCo#+W`U
z%6h3Bpd=Z@n!6zS%^ETLCTPO9);qcBrKIMJxAqZ-|0oJb~tZ(qi)cV
zW8pn_hnbW`UgDA48;$jI4qCRTryg%iT-fqV#CN)1v3YsoE@_{pT)UJxhdQ_XG|Xn_
zJKSeoX6`%vgy?h;qr;JifvZJl_B(f`uoa%!6i~@fxo(Pe^_gp>;ilW#Uv6JzHEU7!
zoOL-T!QJ$fBMEb!i{@_Hwb5YL9YzK@uN9q&%@NLB+jO*?S=@{^y`Iz{$nj|XHm4<`
z3q$yM;~YYB0>W%A+O~dOcq)q9Y{iD@-hJOr@IR1)}My;anVL{0`?D@ak?%(P7~S7*kIgMm}(nV^xy`jSC?ne*%~qEm&f?LTcl
z9+9g*efw|al9Kkt^K_3W>GoaPwkbWgLVbaEa>*RO;uQvYYRbFzyz5E46{#U9-Sz52
z?)t>Lk_R^}i*$AnHoLrg*Iu_5opT#ZdY3vy=TDfN>aaE1X_{CFbLwg1_brkycmLpr
zRtLLYo1QIAT>Q&7)Y)nKpQTM!mqTTGRv0ua7j9^Lb2R?OtGb(^G4{*$2wYRGzntfq
z>hV=_x$i7qvss6>@E%pJ>T7(Q`ESEYLo;oz4_SV`+=2~FI$Av)vvPV~Zg0q%*c05@
zdSxc(k|IZkhyNK4Ku6I04i~I3XzB7&JMC5JKfAK9cRJ^1xqFwtyxCrEUGeB}V(s#i
zWpigQ_kFtV<;ks9-{wwEy%d}{J3KA%b){}<>QceXi<2{N86BJrGOhsi$M6q-$lo<*jx8!7Uf(KY*@~Saj83Q^=cqk!JDVr(4`72;cNb@O-AFbzs%+
zu%@?87q2|_xo*08vvypw;BjTiZN>#$t(q-}uID~@M~TC;LrYye
ziq$aAytPEkH{oqlcJqu4uQi;6Uae%|=v)@E&u*1@=@e#BQG=QP8G63xuGM-WR;ZD)
zF{7o+En(#udRbt}q$%L35wf!X-<+##
z2H8u_ua0tW;MywS*;^RU+!S{I5VR7VTb_Bntc^|bTHS+S-p1Hv!a>I#1+x|`^I?k0
zd2G7}u@b^_)4{JnCzo&8CL!SIEX=@CE<0`dCTwSj=f1YoD>(_SBzZ1WxaYrclRW1&(r>59g#8Z
zg@&!g9$Ri)-ZA%arjzzI0k2Ivw(MTmv2B9s?U3}+n(|oQf2`0-KySm0(so|qC09BU
zLT7H^3R%2Nrro1e(Wi@1P_JvL)~r^BP?K_h1=BwJ%bSW<$<(GLy_~ny=b_%Rq?~m}
z`#B~nE#_icnQ^Q5)Dy4krl+@<_IoZ@og6#i+`3g=eKTLbxVYl1*S;IC&anGlo?9t(
z|3D37I^FML$}J@(BhXp^KD$cGN76aX&*#qFo+);E@}_$lTW-YM&Y54hb643XzulX+
zGOMO^Sod7ZGjdsP(Y~p=@WO+39Vbs^pvgjTZQ<7o}39X6OQYbRxFV`w&Y=`+tr4Nu348GT9>%(Ek6Hm=6ZLd
zNt?3O#DivRwN2nUw`x)5wWw#BO1>*uFR5s*QPGk!RZd=|7{n~O&4|sYn5{Ga@8$TRnx3wS#-pwTOq-pMOJ-b=9VAdHkV6p{?E{>?qj&6aLt2*Vj0mq
z9z0%dKG7$)e31)w+dL=tuJo6=huhgyq8r*cHf-;#TiJhAP@yU}Wy^wgmTrzS-V-)Y
z*j)4E-2S=H@=ol?+LR-2lNYr1dTz*JiMwYQprbx3hh>Sba>m)bxc&+McmMBVoz~i3CuKG6h4P7s
z+=$?~x;N?JYjMtCi+sN#rZkJ^d52D}{GGGnqFHE)xAu$&jFLvP-68`Xg-1^EU9HWx
zZGotan`WYa=&!Tk2foJdShXQ}WpaULi(%|#!^q2X)8A_O^|-1TssH=W&;V^9&Yf$p
z<{e-1t+MUYQz!lNd?=mD^X9yG;jFU#WxCr$#ig0ePWkVM-2LUWTAFvv?CQeu2*Eou
zr^nvWF6{i?$6C(AzavVv(Nyi?YPOes+(`=SM2_6CJQi!_6RmOXX2xzIfw1oh(G&GP
zZ|i*|)7`rKgFGaHFM5S^LT9p@k!G@A9d&i!a>?nsl<`_Gtn{*2)yWHPDk{pJ5Pl&M>yEr3qe#t}*Odp~YME77*eaGFe|u%XSlD^VBcg}pp?PWI`psGH}?-cGmj
zpSp9;r>L79njOP4l!3T{172k~!+3!3Chw`V+GHKIjZ`0`YimKOl96xZm
zR^xm3$C~|xOJ&PvZO>nOHmpwg@~^$eR_x(YU2n*z&ptg`YEkZu`bj=M6ZZ=(Gyl8D
zrkDKl3pD6;nzW4v;e|e+ZNZeqX*x?Otug_flr?;-y?@Np0Bz>lE18YIvp3eH&
zU0DH!3s)#)$+y+~4ZFuaTSD%6{NCty>#9B*82JRgG2f{^;oW}*h3zJf`m-MOtF2z8
z%(Q69Y&m2xL!mZy^1OzT|-&6jyDvu*A>!nHxQ?noG$X~mlAARbSVCXOXx
z97_Z}L|@7soOM?A&LvOb#;D1W(vwp10`0C&yz8hk$z!IX=AE1+H=0=-*;XsA?vDx
zMWSVvO*L?sdSsiB^0Db3@9}{B9Y3}B`pm{Ip1?OYSN}8Ax@0
zlrV6Vzsllwq{7ZG`)rCIr;4AZ{p?AZDwjO>Oj3XFMlhLqBh%~WF1vnQd}CU-`@|C7
zJk!E=u7*Elt-C%bzKwYO*kzsKjV8UKuOF8Pdc1y=dH+Ad=7mumC-2{=Ty<+j$?@--
zdOMVVD3u9K>=n>8(tEEjb@s_D|D^ao@4AX6WbT+UQKjNh#u`tR9~X0DLi4ZJ8tpo`
z!t_2qJ
zb|1ZS=AfCv<*holGfo7%Ze)LHr2qU6*E5N?*LP$|3#`1SRW4!dmJ)sScb0XbU%2pF
zjx}Y;wDQr?$S#rDnOYKAdspr&|t`1q{d*o=|o)t6Aoy4@&>mwx9
zuFOj7oiuT2*U`2oQ`G$9m9lQkJ>qFw<92Reb4kfT+fKDj+J?q*SDSb|#k(peB)KPP
zGC4im>lN@r?6YZRRg9r#-xUGZB^3%?N23&61aD+b^ib$ZXe^WM*x9Mjbx`i!th4Ex
zRk}N_%?rN#D$h4+=bX-q7K@ABPO4Nax#YR#ny1<(PnAi&YLh+R_^NI8RKI=0pl%^|
z5|32-ZUeVN7BRXubIzZsPrtZu>;B*FcSFA3EBxW@;?TD1hhU+m)!!FWrJD4a?=*32
zn7FL_s8nz@_l(K$!eM(~ZeLlq&UT^PiGv+#U!on=c%AQFG>mvyXp(vHvuR;P@n5r-
zl`?!~4l{le@np3>*8ZWa^Y?K-eBsZ2iUu>7N0o{ev^l5h4G@+a=T)n9bH
zRHfW)^(G&oWuD7Ys*TFm$9ws{n5nVh{8jh8V#i!J-qS99k}!8gubE-kJM}jlh8}yu
zc=(P4aah!duQk)({QBgJ((h6~s8sGs?P)UWr1FltDwF-C
zr~5dreybn6%kJyy3X40wRu40m#~)NG@D*NM663~U
zkP_0{THJc*^64{wj(@AQQi~7j-*r!YrNti8MGDTZpQ^-odAxq^vf_$hg(SPCn2^cE
zip73w3prx^)Lw2ZG*uNUthlwK@88;Q!XN(e2>%U<7D_m4;I*MoY%O>DXSsbc{~26L
z+5hpk|DB(8`SdBpub+%GPF{cSx>Yh$?)4h3%^O~3MI`?Tj^R%Z_?CRf`9`7H;yWAf
zx?K2Xn&*2)Y*OY-x%2Bj<-NXsY33rGh_sCFi*B53-V^d}k;#nJcj69xR{9vRB#gtP
zU~9H-cjMXzPp;m{y13f^kqb|kd!E+Sq)uPuL!UfV?#*+V;u
z+vk37y
z61qDMN^?q8WQWa6e(LD998IYeK}ECq*AI$IV4cP@%l9zR@+Xo*UF>vO
zpw}c>?veaMxkq*a#-45r6-p|sM
zo$*yb)po_3)xX;FPHzLq}y#E>^iQDbsv$$`W;^jmkTf)Ynh6?aR4tWZNyQa;aL_M?&`Gw=;stEO9d?
z@klC7P*vw|){39;@6;E6`)K*H>js4uH4_ViIFprH~~f;MlpQ^rO;|im+_c
zS!YvXVm*r%U(TLsYcIU#ap7JS&P$$eJWXeLD({};x#W_k`XO+p&Pu4h%(_O|>GDba
zwQXB&yYK&L(Wx8IYawQJV8Z=)#do^xSqT-d_jNd*y#C%T-^F3qPmYkQ+hdO{5^Q4K
z&?Eck)=W{szTHQ6E-%b=e_(NZWwYP()rpz;8#i33>iE-A;g#8;_T`9B(qvCzi<;Es
z?)rPKep_obckT5(zmK&&vzq)iYI66<#y2N-F1r}1cER__fxAyqCKpCcE{y6uE3CeL
zqUofpWA8joCT{Yy_f)xPnD6q^wuW=HtaOkz$1MMNVP%!eS^k?AGz+=D>{K{1v0{xt
z=EGu5Th={iqvEFX9AuHc#ImIGjcMZQUcn-cbt+kpf@3&+6{>j!t5iLfaD8diTq2-q
za^Q_AOB}}%p8o8!X`5XmRbt$x$juHpxjOjD(@O;&Yu%naQ{SjE$@kf0&n4Gcx~|Of
zwC!BAHMq-Dsqn@m&n3UJ+4!$0=K4qqY}3u#r0mzL@3L>3Jiq^g@BbOph5hwbh{qe5
zx&P(#j|=g5=Okvww1hYEYvYsVfBzXKX-{UmTiGe)qGI>OjaQ}glfqG*U!RxA)*X?(
z()h-BPk84iu~#!Bz6ne{6CC!uKJtT%OrPb-u6^gOD}NHaS@xlg$I*PzElxwG@`)0{
zf{Q!0y!`y1!F}0fyKswHldoz-ERv6Re5bg{bHfLZxpJP1cN*_h+A1t8@J)KR+LyD^
zCzl;#b+0=l+!&F()5Pyc|LqM42gSP%c1kHtaM+O_G^L82drwtV!E4b*O{Ix$ZRGvA46)DOSvpv
zSMvOls)an2$|rfQ<5ZbcZRD#m%dSU@=or;q`o8IK`3>9d5280E^YbwpJCxgi(N#v`y)9yHx5KyP
zl_s;VTCGx-kAKz?Qhm{3|3t2BOZzKYw6d0QzJ8?TtDd(>{jHD0Bz@5)_9RUfsfzrt
z;|pq5KBCv%Y-VAu#pD8&@Hv
zj0+P$6|cuE_pa1~R{t3UZs*l{uYGAUpZ~$ZUoE;T+I_^fM+z={8D;fJZmsa?_kV95
z>C5JpOCJDDnDC)-#Sp%Q%)u<$ElvbCJGma-;iTr_u+N
zB=;nJHV&np?pbS>xpgdwIUakZKgT!y%84(_cb?d;7?f!`>ulO)Pn*sqmukItn$Pky
zoqbk)qaEYH=C2&lfjP!H(PA5VW%f*eo%pGazy4F9P}CZo(uoqwLX8qi+~)1uQYmZo
z>v~W~U2M#IxqFh`$5%E_o!9&6bDfuswaSd9eAg{PzG{=yG9P_<*|))di(dZBBUjI^
za#43L%9wlPq)NynPebN%fjL6fz5RziBpei9rqaZga*)Ywp19G(54z8?E}wef5~jD3
zC5UC|#7P{YCj>f{RHS9wE%PfEsLBmiTk9V@J@Soz637?oha*5usw~BsY249kIxa5v
z2RB6jGpxV=XO6G7-wHRY_vWw995qe3(_xfYI&s~;B*E)fD&&sGJ0>*dB(HnK)8S_I
zNlMnh_2|ZpEP?O(9_FYj?PQz(`A^6DH`=o@{xHh!Sg!3=TT;=@rZD?rE=%V(6`zYo
zpI*-W&v5a3E2xB+wl=6mEobFaPqo#FlM*JHy7`_|nfN1)!%!to^O+;_Mv+Ekf8JL?
z?FY8cTu3R$7
z_sLPUO&)jVT%YBs@*^2sI=Zx_&QMO2GRdq`{kq{tR6YBiN10ErGTmPHNmr%xv*@q)
za)$#{ub=Vq`@G}G#Bz__5)+xCyAyA8|Csfm=v7~Su$(%i&8U>zY~thATU7!o>6{+rBmgcy8*xzU0HhFHJqTgq$DoY(Y4$2>xGC`$WF|WS=
zgKA0T*NrR!j!X(i`VI#?NUzxAuJin=vC22~-|66KEzn^~Rx(>bCzDxBV$qv@f{$?y}ik
zSsU#gJNJlw{$qb~*_q$m@BDq`Cbu&MKMP;h?em|Zqc`Hp{YNp}KR3Tf&3ts*#@(XO
zBJ26rzx#f!Uhwf(ebOGu6Sq&ht*h$3|DR#~yVBoZXVn}2XRy|8uV1lq*2ZP?uYUd8
z^)SA^VdsB_1OL|L?9{mw(?4Ub@cS2fbH51xezwo`S=_!>%a?!rl!G&kSIpJ*@Sk|p
z2P0`_jGlm8SRBSFtN*B~|PW`V%xa(aS8TbIR_Zx87TIZg)Qr
zc&Ny^DWXO3kg3~B4krOGAq`c@!yn@W6EDfAb8Jn8$KfpL@U2pXh}NH}2Sc%a{GB)ctU;hm(?oB8QVeiz0_p
zO`JfBBIk!^ii!L)etv&qIJf?v_Z+ip>-U)}-1Ad3zp`WTRoV
zo$L;?{uHL;gbnJr|)0r_Q7b%4vXEta=n6{?^h1jpY{HZ
z{@I?)dw)46ubffp^g?&q-?EzBUrft?cIWN?oLWEoLe+vhf4wdHLrix2DSee&5&rAX
zOZKIvZ0jy(ZgKg~@NQSo+oMgrhr7hYpL=e(rBo`lMDg(F#m`Fqe0#k$ZC&h7k(l|l
zzHXV3F{b|>Z@KjI&x4u0AK!$Y*=twqacSX&SCwJS@93!NgUgRxoiF_1SHg#_wo$^>F>{Xp{?{I~ao^rAa{PLKwk=9}
zviq~fJ-)rarhdOa^ZE9enUgR7$@w_%m$2%I<0sa{969yu{QPIZE9UXWKmIP=9-{O8
zN9UCp$Ks7o)Ep_1x4mp7wNth%UUc`J*mYlaU+Rv^Z>{Itx9@o5_uqSOO<8(-W}QCY
zqQh!x_m)^aczR-XL9d+p%^zRWr~j3I8*|?Neey1>j{>3T0f8|o&Ei0VYUHGKN;rpYI
z*Q;09UUYYGw|G|YicPUAedFxr>KWE9EwoqmTe>nNP+WTZq$xfxg8HRnC(LwjnzGdD
zpwaZ1ULiqm-5%Vz)p==NyI|SUPZn*@k1Lwbn0<)9!tsYy+&aBKt>0=jUGF=)wJ!E?
zQ9QgUpheLvt2gjT%PoP7DJ_a-EsC6zTUu`Anl-6i&TtZ#FH>DAZv4yrd{(KzX1={Y
zMW5u~y7X!HEA_@#t>^1z$DI4w*?H+v$5OrOFB+$3*iZZT=IWo&`43k9vA+H;WZ&K2
zTlEh{PF}I$tM*t5kK$oD18|
z-Mpz^CXk)C$myACSe9Ieiu05)fxBRQsEc;8@5)k5
zmlrc#S4~+OZR@J(rCt{@+bOxik<&?F-lc$tQcK?NKb$$sQNxF$Me*>7rU;J~spWw>
z0!=@|SL~enp+75TX0!EQ?lbwfE}!3Tq29P7?mvU>y7ZbOM_#%FckFzBVR7mi`O_7@
z=h^PO;Bt5U^tijOXXd?;H|&-5e0T7()%Pdv6W6ECs$BPLX4~ou=j(lUeSfAX3G%$G
zZ`KDZJxTd|bGhC8omXNyJ4%&X&fQhKTK>dGb(`OR
zhCo@Kp7Zs6x2ohX@AHj0e&SUn-}fJL{SDVEKDnZo?z^}-_HP|y`HtN|Q+DjWQdn&g
zsi$}S)}j-0Z)PpM_2z4Cx!<3aW-Z4)v>dr5Fz?dA!_7fU-hcYLZRQf8mIED5GB%wo
z8zyp?`Yno3WbL1-dBtzj@5Q!UrP|HUw+q}{f4r*HWa@v0!~VBSZGFGI{2A=F&(_90
zXnNjqfv>!WPnTnxZJ81uIO-a5@`9Bb~n>@hVMhOmVYew
z^6kv8#GHKBKjYe6*At(tzCTG<{#KgTn(&{Y*#9&4gr9c5=K5^Be0krp>OW>>JpL2*
zPdV|c=jwT;pJDF{%a-1I>t!$O^{jsV^3Io^ZoMsEYRTJQdXDpc()+T!YR*g3Z{}5N
z-hElNRI{+QEA->cE#NHvmZM<8)dz*ydp|Y(Hum1HQ4wWfqHAs9Bv6%UU}D<7DWFNn
zqL4pGIJ1dUaMo0*#UT-jSG<&RISS^S@Nq0YIZ?w&;Ikr!QHsZ`VSFO328_$QFn>
zw!x_Ut3}}!b*1tzRx%b+yPX$Yyf8y?b>@fFo6^NJzAX=W)U#aBH_qz9t)$=nd;auV
zU0WJ3cI7pnj>ztq-m@#O_&mC~Y03_bJ9>VKJ4*Dt8+Sy+
zbUpB!6Zcl{iqGEa&p)=PeHN~8e9vLe?S1t~{q~EB%bhOm2;{sn`*167>_QGF-XP)3
zEsA_wT5jcX?om6h9MU4SMA3>jcG-m%MUKvo>5$O={;KGbT{Q{+PF`Q4!ulP?a^#AprfmQD=_vHN-
z^8d`*HLLBcozGsknDSS_E6yyO^7lVO(Y)#y?iu?XOJvV{l~*(TdXsWtgAZHwS_!4k*x76668qQnM7ENPn(N74UjB)ky*mYa`BjwPf0U9B
zEEam=!!xI)WtY@ctKO2Ep>GV=^agU86!gqoB7Qvc%F?5fUZqVHdRLaMm-m}K5mxzx
z?$x`JC!Q5F$5oFjB&hf7N-uLwuTtTq8guooi!8eO?(VuT%e}YWdYl!w>Cs!S*cEqf
zO<6AJ`eEmuBlmNVYc&Hp|0FR+q|rtgfpr-UCmmg6wO-XJba(sQTY*l8(b#OyOKX+Q}F%^>y^{?
ziw9l1Ht$2`$F1S(&&w~h(XFWxTRU6-r{3AaXZt7ov|n*_(iQb7>L+$AU;gkPSL{00
zGnd|Z8O`;+e`V2W*QDoW>$4}H-`5`OmRRUzW;Cnl$j|a8g}iUNyGmQXXUw=~Z@hBp
z@9o(qK3o6gik)s~^v784Y}`Hm(#?13-fy;@wtjPeJ{wPU#O=#6w+__t|Y~J*0
zOV>T;uXlWFDKO{v3)7F*dyn3lvO9LiMBg~Gx%Ox3cU(Sl-9^`PRrgjx&HHvFD&adERhr;&CJur85DN}JxES^|5@Ya(XBrh$yh~O)R@0Z@KhD-Ps;6G*tYH1<&1stuk=#B
zC;aT?sde2R_ow5NNp_s_b@2+5mmh_y_T2h+y!GgT9XtD1PMotz**bmy@o(bJ;qk}+
zbn|$w`_CXd`H7xKnDX+|+pD{`YL@K|>OXpM`Oi7_cQPJjpV+SYP5S7q>Dhmzg8pna
zYdLaHV1^=x;}7w8(RYruv-E0DR8(1*tW}?Jk;C-e#yKjARcIIqG%Vf
zJkaX&yPnrvvFmkrw4Jr%zOrJE-m>+4T6V4{-Og=ry4uM5sJEZ>jGoz}TW`HeG*_No
zcU^wo*>mYZ55DN?Y5on~moBF9RW@O+#YVs7f?kTR4lcJ^Fpn!LhWqF4&%*l0)IL{k
z`KGY-RfPMmSvzI&r(Mq66YZ35)}qKClxsHo?6cP_M{GShN*5gzneCJ>_{#Sohky6p
zUEMSPDnIj`A6R75xA&*yiL>>u3b!to|B?APJbup07Zv`&7v?SeDr@!TKf~0X+JDP~
zZU0GIU$}bz<+gblm$%n1I}yHr-SzW(xE{p(i`=n0=zZBziwS!|zS`PF$4vhrE`EEz
z-r0qQW?vP}S}fdp>(d`ykFRL@Z5&no$wpU3N^J6(Di75YvrVi2uyx+QGDGe{l}G8K
zK%T4#UNROtc4?cd?bXqu$hP5I%Prx|3oUAAg9PU57|nG(
zbDra8Px31@XjkAXv?Nwr^J9MdW%kyUuP)5C>X-c`>ylYm{-mg5?_a*=n%^%sC)Ynu
z(fiL3y#GpV^(T#=bF05<+^?T-N4|dM{qMrU$G%G!YwMl<8DHyf_lx&w@8K}6tS?{w
zGcbR#TJ(0_e+J&I{~7x4B$YodukPOZ?WOBlvoC8m>}~H)dtaB>9;k7$bNfoRoio+$
zD7$xltFsOL^q*nz-_