import backoffStrategies from '@kenyip/backoff-strategies'; import { replaceResultTransformer, stripIndentTransformer, TemplateTag, trimResultTransformer } from 'common-tags'; import dayjs, { type Dayjs } from "dayjs"; import type {Duration} from "dayjs/plugin/duration.js"; import utc from 'dayjs/plugin/utc.js'; import type {Request} from "express"; // https://github.com/jfromaniello/url-join#in-nodejs import { TimeoutError, WebapiError } from "spotify-web-api-node/src/response-error.js"; import { DEFAULT_MISSING_MBIDS_TYPES, type MissingMbidType, type PlayObject } from "../core/Atomic.ts"; import { asPlayerStateDataMaybePlay, type PlayerStateDataMaybePlay, type ProgressAwarePlayObject, type RemoteIdentityParts, type ScrobbleThresholdResult, } from "./common/infrastructure/Atomic.ts"; import { NO_USER } from '../core/Atomic.ts'; import { NO_DEVICE } from '../core/Atomic.ts'; import type {PlayPlatformId} from '../core/Atomic.ts'; import { genGroupIdStr } from '../core/PlayUtils.ts'; import { durationToNormalizedTime } from '../core/TimeUtils.ts'; import { setTimeout as delay } from 'node:timers/promises' dayjs.extend(utc); export const sleep = (ms: number, opts?: Parameters[2]) => delay(ms, undefined, opts); /** sorts playObj formatted objects by playDate in ascending (oldest first) order */ export const sortByOldestPlayDate = (a: PlayObject, b: PlayObject) => { const { data: { playDate: aPlayDate } = {} } = a; const { data: { playDate: bPlayDate } = {} } = b; if(aPlayDate === undefined && bPlayDate === undefined) { return 0; } if(aPlayDate === undefined) { return 1; } if(bPlayDate === undefined) { return -1; } return aPlayDate.isAfter(bPlayDate) ? 1 : -1 }; export const setIntersection = (setA: any, setB: any) => { const _intersection = new Set() for (const elem of setB) { if (setA.has(elem)) { _intersection.add(elem) } } return _intersection } export const unique = (arr: T[]): T[] => { return Array.from(new Set(arr)) } export const returnDuplicateStrings = (arr: any) => { const alreadySeen: any = []; const dupes: any = []; arr.forEach((str: any) => alreadySeen[str] ? dupes.push(str) : alreadySeen[str] = true); return dupes; } const sentenceLengthWeight = (length: number) => { // thanks jordan :') // constants are black magic return (Math.log(length) / 0.20) - 5; } /** * Check if two play objects are the same by comparing non time-related data using most-to-least specific/confidence * * Checks sources and source ID's (unique identifiers) first then * Checks track, album, and artists in that order * */ export const playObjDataMatch = (a: PlayObject, b: PlayObject) => { const { data: { artists: aArtists = [], album: aAlbum, track: aTrack, } = {}, meta: { source: aSource, trackId: atrackId, } = {}, } = a; const { data: { artists: bArtists = [], album: bAlbum, track: bTrack, } = {}, meta: { source: bSource, trackId: btrackId, } = {}, } = b; // if sources are the same and both plays have source ids then we can just compare by id if(aSource === bSource && atrackId !== undefined && btrackId !== undefined) { if(atrackId !== btrackId) { return false; } } if (aTrack !== bTrack) { return false; } if (aAlbum !== bAlbum) { return false; } if (aArtists.length !== bArtists.length) { return false; } // check if every artist from either playObj matches (one way or another) with the artists from the other play obj if (!aArtists.every((x: any) => bArtists.includes(x)) && bArtists.every((x: any) => aArtists.includes(x))) { return false } return true; } export const parseRetryAfterSecsFromObj = (err: any) => { let raVal; if (err instanceof TimeoutError) { return undefined; } if (err instanceof WebapiError || 'headers' in err) { const {headers = {}} = err; raVal = headers['retry-after'] } // if (err instanceof Response) { // const {headers = {}} = err; // raVal = headers['retry-after'] // } const { response: { headers, // returned in superagent error } = {}, retryAfter: ra // possible custom property we have set } = err; if (ra !== undefined) { raVal = ra; } else if (headers !== null && typeof headers === 'object') { raVal = headers['retry-after']; } if (raVal === undefined || raVal === null) { return raVal; } // first try to parse as float let retryAfter: number | Dayjs = Number.parseFloat(raVal); if (!isNaN(retryAfter)) { return retryAfter; // got a number! } // try to parse as date retryAfter = dayjs(retryAfter); if (!dayjs.isDayjs(retryAfter)) { return undefined; // could not parse string if not in ISO 8601 format } // otherwise we got a date! now get the difference the specified retry-after date and now in seconds const diff = retryAfter.diff(dayjs(), 'second'); if (diff <= 0) { // if diff is in the past returned undefined as its irrelevant now return undefined; } return diff; } export const spreadDelay = (retries: any, multiplier: any) => { if(retries === 0) { return []; } let r; const s = []; for(r = 0; r < retries; r++) { s.push(((r+1) * multiplier) * 1000); } return s; } export const removeEmptyArrays = >(obj: T): T => { const newObj: any = {}; Object.keys(obj).forEach((key) => { if(Array.isArray(obj[key])) { if(obj[key].length !== 0) { newObj[key] = obj[key]; } } else { newObj[key] = obj[key]; } }); return newObj; } export const remoteHostIdentifiers = (req: Request): RemoteIdentityParts => { const remote = req.connection.remoteAddress; const proxyRemote = Array.isArray(req.headers["x-forwarded-for"]) ? req.headers["x-forwarded-for"][0] : req.headers["x-forwarded-for"]; const ua = req.headers["user-agent"]; return {host: remote, proxy: proxyRemote, agent: ua}; } export const remoteHostStr = (req: Request): string => { const {host, proxy, agent} = remoteHostIdentifiers(req); return `${host}${proxy !== undefined ? ` (${proxy})` : ''}${agent !== undefined ? ` (UA: ${agent})` : ''}`; } /** * Remove duplicates based on trackId, deviceId, and play date * */ export const removeDuplicates = (plays: PlayObject[]): PlayObject[] => { return plays.reduce((acc: PlayObject[], currPlay: PlayObject) => { if(currPlay.meta.trackId !== undefined && currPlay.meta.deviceId !== undefined && currPlay.data.playDate !== undefined) { if(acc.some((x: PlayObject) => x.meta.trackId === currPlay.meta.trackId && x.meta.deviceId === currPlay.meta.deviceId && x.data.playDate.isSame(currPlay.data.playDate, 'minute'))) { // don't add current play to list if we find an existing that matches track, device, and play date return acc; } } return acc.concat(currPlay); }, []); } export const toProgressAwarePlayObject = (play: PlayObject): ProgressAwarePlayObject => { return {...play, meta: {...play.meta, initialTrackProgressPosition: play.meta.trackProgressPosition}}; } export const getProgress = (initial: ProgressAwarePlayObject, curr: PlayObject): number | undefined => { if(initial.meta.initialTrackProgressPosition !== undefined && curr.meta.trackProgressPosition !== undefined) { return Math.round(Math.abs(curr.meta.trackProgressPosition - initial.meta.initialTrackProgressPosition)); } return undefined; } export const thresholdResultSummary = (result: ScrobbleThresholdResult) => { const parts: string[] = []; if(result.duration.passes !== undefined) { parts.push(`tracked time of ${result.duration.value.toFixed(2)}s (wanted ${result.duration.threshold}s)`); } if(result.percent.passes !== undefined) { parts.push(`tracked percent of ${(result.percent.value).toFixed(2)}% (wanted ${result.percent.threshold}%)`) } return `${result.passes ? 'met' : 'did not meet'} thresholds with ${parts.join(' and ')}`; } export function parseBool(value: any, prev: any = false): boolean { let usedVal = value; if (value === undefined || value === '') { usedVal = prev; } if(usedVal === undefined || usedVal === '') { return false; } if (typeof usedVal === 'string') { return ['1','true','yes'].includes(usedVal.toLocaleLowerCase().trim()); } else if (typeof usedVal === 'boolean') { return usedVal; } throw new Error(`'${value.toString()}' is not a boolean value.`); } export function parseBoolStrict(value: string | boolean): boolean { if(typeof value === 'boolean') { return value; } const strTrue = ['1', 'true', 'yes'].includes(value.toLocaleLowerCase().trim()); if (strTrue) { return strTrue; } const strFalse = ['0', 'false', 'no'].includes(value.toLocaleLowerCase().trim()); if (strFalse) { return false; } throw new Error(`'${value.toString()}' is not a strict boolean value.`); } export const genGroupIdStrFromPlay = (play: PlayObject) => { const groupId = genGroupId(play); return genGroupIdStr(groupId); }; export const genGroupId = (play: PlayObject): PlayPlatformId => [play.meta.deviceId ?? NO_DEVICE, play.meta.user ?? NO_USER]; export const getPlatformIdFromData = (data: PlayObject | PlayerStateDataMaybePlay) => { if(asPlayerStateDataMaybePlay(data)) { return data.platformId; } return genGroupId(data); } export const mergeArr = (objValue: [], srcValue: []): (any[] | undefined) => { if (Array.isArray(objValue)) { return objValue.concat(srcValue); } } export const pollingBackoff = (attempt: number, scaleFactor: number = 1): number => { const backoffStrat = backoffStrategies({ delay: 1000, strategy: "exponential", jitter: true, minimumDelay: 1000, scaleFactor }); // first attempt delay is never enough so always add + 1 return Math.round(backoffStrat(attempt + 1) / 1000); } export const intersect = (a: Array, b: Array) => { const setA = new Set(a); const setB = new Set(b); const intersection = new Set([...setA].filter(x => setB.has(x))); return Array.from(intersection); } /** Return an array of elements from array a (first arg) that are not in array b (second arg) */ export const difference = (a: Array, b: Array) => { const setA = new Set(a); const setB = new Set(b); const diff = new Set([...setA].filter(x => !setB.has(x))); return Array.from(diff); } /** * https://github.com/Mw3y/Text-ProgressBar/blob/master/ProgressBar.js * */ export const progressBar = (value: number, maxValue: number, size: number) => { const percentage = value / maxValue; // Calculate the percentage of the bar const progress = Math.round((size * percentage)); // Calculate the number of square caracters to fill the progress side. const emptyProgress = size - progress; // Calculate the number of dash caracters to fill the empty progress side. const progressText = '▇'.repeat(progress); // Repeat is creating a string with progress * caracters in it const emptyProgressText = '—'.repeat(emptyProgress); // Repeat is creating a string with empty progress * caracters in it const percentageText = Math.round(percentage * 100) + '%'; // Displaying the percentage of the bar const bar = `[${progressText}${emptyProgressText}]${percentageText}`; return bar; }; // https://github.com/zspecza/common-tags/issues/176#issuecomment-1650242734 export const doubleReturnNewline = new TemplateTag( stripIndentTransformer('all'), // remove instances of single line breaks replaceResultTransformer(/(?<=.)\n(?!\n+)/g, ''), // replace instances of two or more line breaks with one line break replaceResultTransformer(/(?<=.)\n{2,}/g, '\n'), trimResultTransformer(), ); export const durationToTimestamp = (dur: Duration): string => { const nTime = durationToNormalizedTime(dur); const parts: string[] = []; if (nTime.hours !== 0) { parts.push(nTime.hours.toString().padStart(2, "0")); } parts.push(nTime.minutes.toString().padStart(2, "0")); parts.push(nTime.seconds.toString().padStart(2, "0")); return parts.join(':'); } export const comparingMultipleArtists = (existing: PlayObject, candidate: PlayObject): boolean => { const { data: { artists: eArtists = [], } = {} } = existing; const { data: { artists: cArtists = [], } = {} } = candidate; return eArtists.length > 1 || cArtists.length > 1; } export const missingMbidTypes = (play: PlayObject): MissingMbidType[] => { let missing: MissingMbidType[] = []; if(play.data.duration === undefined) { missing.push('duration'); } if(play.data.meta?.brainz === undefined) { missing = missing.concat(DEFAULT_MISSING_MBIDS_TYPES); return missing; } const { recording: track, album, artist } = play.data.meta.brainz; if(track === undefined) { missing.push('title'); } if(album === undefined) { missing.push('album'); } if(artist === undefined || (artist ?? []).length !== (play.data.artists ?? []).length) { missing.push('artists') } return missing; } export interface NonEmptyOptions { ofType?: string, test?: (val: T) => boolean } export const getFirstNonEmptyVal = (values: unknown[], options: NonEmptyOptions = {}): NonNullable | undefined => { for(const v of values) { const nonEmptyVal = getNonEmptyVal(v, options); if(nonEmptyVal !== undefined) { return nonEmptyVal as T; } } return undefined; } export const getNonEmptyVal = (value: unknown, options: NonEmptyOptions = {}): NonNullable | undefined => { if (value === undefined || value === null) { return undefined; } if (options.ofType !== undefined && typeof value !== options.ofType) { return undefined; } if (options.test !== undefined && options.test(value as T) === false) { return undefined; } return value as T; } const nonEmptyStringOpts: NonEmptyOptions = { ofType: 'string', test: (v) => v.trim() !== '' }; export const getFirstNonEmptyString = (values: unknown[]) => getFirstNonEmptyVal(values, nonEmptyStringOpts); export const getNonEmptyString = (value: unknown) => getNonEmptyVal(value, nonEmptyStringOpts); const nonEmptyValueOpts: NonEmptyOptions = { test: (v) => v !== undefined && v !== null && typeof v !== 'string' || v.trim() !== '' }; export const isEmptyArrayOrUndefined = (arr: unknown[] | undefined, options: NonEmptyOptions = {}): boolean => { if(arr === undefined || arr === null || arr.length === 0) { return true; } const opts = {...nonEmptyValueOpts, ...options}; for(const v of arr) { const nonEmptyVal = getNonEmptyVal(v, opts); if(nonEmptyVal !== undefined) { return false; } } return true; } export const nonEmptyObj = (obj: object): boolean => { return Object.keys(obj).length > 0; } /** * Runs the function `fn` * and retries automatically if it fails. * * Tries max `1 + retries` times * with `retryIntervalMs` milliseconds between retries. * * From https://mtsknn.fi/blog/js-retry-on-fail/ */ export const retry = async ( fn: () => Promise | T, { retries, retryIntervalMs }: { retries: number; retryIntervalMs: number } ): Promise => { try { return await fn() } catch (error) { if (retries <= 0) { throw error } await sleep(retryIntervalMs) return retry(fn, { retries: retries - 1, retryIntervalMs }) } } export const isDebugMode = (): boolean => process.env.DEBUG_MODE === 'true';