diff --git a/docsite/docs/configuration/configuration.mdx b/docsite/docs/configuration/configuration.mdx
index 9ffb4941..6687ff27 100644
--- a/docsite/docs/configuration/configuration.mdx
+++ b/docsite/docs/configuration/configuration.mdx
@@ -440,6 +440,273 @@ Example
+
+
+## Database
+
+Multi-scrobbler depends on a SQLite database (`ms.db`) that is created on first run and stored in the [`CONFIG_DIR`](/installation/?dockerSetting=storage#recommended-settings). When upgrading Multi-scrobbler version, if there are any required database changes than this database is [automatically backed up and migrated.](/updating#database)
+
+The database stores *all* Plays for your Sources/Clients as well as metadata and debugging information to help troubleshoot issues. Each Play is associated with a Source/Config in the database based on your configuration.
+
+You **should set IDs for each Source/Client** so that the database can identify these even when the configuration is changed.
+
+### Retention
+
+
+
+The amount of data stored for each Play can widely vary based on a few factors:
+
+* how much data the Source service exposes
+* how many clients each Source is scrobble to (Plays are duplicated for each Client a Source sends a scrobble to)
+* if you are using any [Transforms](/configuration/transforms) the diff of each step is stored, along with any external request/response data (like [Musicbrainz queries](/configuration/transforms/musicbrainz))
+
+The [MS repository contains a benchmark](https://github.com/FoxxMD/multi-scrobbler/blob/master/src/backend/tests/database/drizzle.test.ts#L551) to measure an average database size in a few common scenarios.
+
+
+
+
+
+Assuming your Sources send a minimal amount of data or you have compacted all plays:
+
+| Play Count | DB Size |
+| ---------- | -------- |
+| 100 | `160kb` |
+| 1000 | `1MB` |
+| 10k | `10.2MB` |
+
+
+
+
+
+Assuming your Sources have a non-trivial amount of input data (like Spotify or Listenbrainz) that is not compacted:
+
+| Play Count | DB Size |
+| ---------- | -------- |
+| 100 | `356kb` |
+| 1000 | `3MB` |
+| 10k | `29.6MB` |
+
+
+
+
+
+Assuming your Sources/Clients:
+
+* have a non-trivial amount of input data (like Spotify or Listenbrainz)
+* and has many [Transforms](/configuration/transforms) steps that include requests
+* nothing is compacted
+
+| Play Count | DB Size |
+| ---------- | -------- |
+| 100 | `684kb` |
+| 1000 | `6.28MB` |
+| 10k | `61.3MB` |
+
+
+
+
+
+
+
+A retention policy can be configured to delete Plays, or unused debug data, from the database after a certain amount of time. If no configuration is provided then a default policy is used that should be reasonable for most users.
+
+
+
+
+
+The **Compaction** Retention Policy is used to delete different types of debug data from your stored Plays.
+
+This is a useful way to reduce used storage space when you are not having problems with your Plays, or iterating on a configuration, that requires referencing all this extra data.
+
+There are two types of data that can be compacted (deleted from the Play):
+
+* `input` - this is the untouched data retrieved by Multi-scrobbler, from a Source, and used to generate a Play/scrobble. This can be used to reconstruct and replay a Play, when used from troubleshooting or reporting an issue
+* `transform` - this is all of the steps generated by [transforms](/configuration/transforms), the diff of the Play resulting from the step, and any request/responses used to complete the step
+
+:::note[Defaults]
+
+When no Compact configuration is provided, Multi-scrobbler uses this policy:
+
+* Compact (delete) `input` and `transform` data on all Plays after 3 days
+
+:::
+
+#### Configuring Compaction Policy
+
+
+
+Details
+
+Each value in the configuration properties below can be either
+
+* a number of seconds EX `3600` = 10 minutes
+* a unit of a common duration with the pattern `X unit` EX
+ * `30 minutes`
+ * `5 hours`
+ * `2 days`
+
+
+
+
+
+* `COMPACT_PROPERTIES` - which properties to compact
+* `RETENTION_COMPACT_AFTER` - Default to use for all Plays
+* `RETENTION_COMPACT_COMPLETED_AFTER` - Compact only completed Plays after...
+* `RETENTION_COMPACT_FAILED_AFTER` - Compact only failed Plays after...
+* `RETENTION_COMPACT_DUPED_AFTER` - Compact only duped/discard Plays after...
+
+Example
+
+```ini
+# only delete input when compacting
+COMPACT_PROPERTIES=input
+# compact all plays after 3 days
+RETENTION_COMPACT_AFTER=3 days
+# specifically compact completed plays after 30 minutes
+RETENTION_COMPACT_COMPLETED_AFTER=30 minutes
+```
+
+
+
+
+Compacting all Play types and deleting both input and transform:
+
+```json title="config.json"
+{
+ "database": {
+ "retention": {
+ "compactAfter": "3 days",
+ "compact": [
+ "input",
+ "transform"
+ ]
+ }
+ }
+}
+```
+
+* Delete only input during compacting
+* Compact all after 3 days except completed which compacts after 30 minutes
+
+```json title="config.json"
+{
+ "database": {
+ "retention": {
+ "compactAfter": {
+ "completed": "30 minutes",
+ "duped": "3 days",
+ "failed": "3 days"
+ },
+ "compact": [
+ "input"
+ ]
+ }
+ }
+}
+```
+
+
+
+
+
+
+
+
+
+
+The **Deletion** Retention Policy is used to delete different types of stored Plays from Multi-scrobbler database.
+
+**This does not delete Plays from your Clients.** It's only deleting the "in-flight" data MS used to create the scrobble that was eventually sent to your clients.
+
+:::warning[Plays Should be Ephemeral]
+
+**Multi-Scrobbler is not designed to store Plays/Scrobbles indefinitely.**
+
+It should scale fine for thousands of scrobbles but it not meant to store 10's of thousands of scrobbles forever. It is not a scrobbler server.
+
+You **should** set a reasonable deletion policy so that MS stores less than 1000 scrobbles at a time, ideally less.
+
+:::
+
+:::note[Defaults]
+
+When no Deletion policy configuration is provided, Multi-scrobbler uses this policy:
+
+* Delete all Plays after 7 days
+
+:::
+
+#### Configuring Deletion Policy
+
+
+
+Details
+
+Each value in the configuration properties below can be either
+
+* a number of seconds EX `3600` = 10 minutes
+* a unit of a common duration with the pattern `X unit` EX
+ * `30 minutes`
+ * `5 hours`
+ * `2 days`
+
+
+
+
+
+* `RETENTION_DELETE_AFTER` - Default to use for all Plays
+* `RETENTION_DELETE_COMPLETED_AFTER` - Delete only completed Plays after...
+* `RETENTION_DELETE_FAILED_AFTER` - Delete only failed Plays after...
+* `RETENTION_DELETE_DUPED_AFTER` - Delete only duped/discard Plays after...
+
+Example
+
+```ini
+# delete all plays after 3 days
+RETENTION_DELETE_AFTER=3 days
+# specifically, delete completed plays after 30 minutes
+RETENTIOND_DELETE_COMPLETED_AFTER=30 minutes
+```
+
+
+
+
+Deleting all Play types after 3 days:
+
+```json title="config.json"
+{
+ "database": {
+ "retention": {
+ "deleteAfter": "3 days"
+ }
+ }
+}
+```
+
+* Delete all after 3 days except completed which are deleted after 30 minutes
+
+```json title="config.json"
+{
+ "database": {
+ "retention": {
+ "deleteAfter": {
+ "completed": "30 minutes",
+ "duped": "3 days",
+ "failed": "3 days"
+ },
+ }
+ }
+}
+```
+
+
+
+
+
+
+
+
+
## Debug Mode
diff --git a/src/backend/common/AbstractComponent.ts b/src/backend/common/AbstractComponent.ts
index 514bd622..3cbfcb6c 100644
--- a/src/backend/common/AbstractComponent.ts
+++ b/src/backend/common/AbstractComponent.ts
@@ -58,7 +58,7 @@ export default abstract class AbstractComponent extends AbstractInitializable {
super(config);
this.transformManager = config.transformManager ?? getRoot().items.transformerManager;
this.cache = getRoot().items.cache();
- const cProps = config.options?.retention?.compact ?? parseArrayFromMaybeString(process.env.COMPACT_PROPERTIES, {lower: true});
+ const cProps = config.options?.retention?.compact ?? parseArrayFromMaybeString(process.env.COMPACT_PROPERTIES ?? 'input,transform', {lower: true});
if(!cProps.every(isCompactableProperty)) {
throw new SimpleError(`Compactable properties must be one of 'transform' or 'input'. Given: ${cProps.join(',')}`);
}
diff --git a/src/backend/common/database/Database.ts b/src/backend/common/database/Database.ts
index 069504d9..00e89a82 100644
--- a/src/backend/common/database/Database.ts
+++ b/src/backend/common/database/Database.ts
@@ -4,7 +4,7 @@ import { promises as fs } from 'fs'
import { childLogger, Logger } from '@foxxmd/logging';
import { loggerNoop } from '../MaybeLogger.js';
import { fileExists, fileOrDirectoryIsWriteable } from '../../utils/FSUtils.js';
-import { COMPACTABLE, compactableProperties, CompactableProperty, DEFAULT_RETENTION_DELETE_AFTER, RententionGranular, RetentionConfig, RetentionConfigValue, RetentionOption, RetentionValue, RetentionValueUnparsed } from '../infrastructure/config/database.js';
+import { COMPACTABLE, compactableProperties, CompactableProperty, DEFAULT_RETENTION_COMPACT_AFTER, DEFAULT_RETENTION_DELETE_AFTER, RententionGranular, RetentionConfig, RetentionConfigValue, RetentionOption, RetentionValue, RetentionValueUnparsed } from '../infrastructure/config/database.js';
import { DurationValue } from '../infrastructure/Atomic.js';
import { Duration } from 'dayjs/plugin/duration.js';
import dayjs from 'dayjs';
@@ -71,11 +71,11 @@ const parseRetentionValue = (val: RetentionValueUnparsed): RetentionValue => {
throw new SimpleError('retention value be of one: false, number, or string');
}
-const parseRetentionFromEnv = (): RetentionOption => {
- const deleteAfterEnv = process.env.RETENTION_DELETE_AFTER ?? DEFAULT_RETENTION_DELETE_AFTER,
- deleteCompletedEnv = process.env.RETENTION_DELETE_COMPLETED_AFTER ?? deleteAfterEnv,
- deleteFailedEnv = process.env.RETENTION_DELETE_FAILED_AFTER ?? deleteAfterEnv,
- deleteDupedEnv = process.env.RETENTION_DELETE_DUPED_AFTER ?? deleteAfterEnv;
+const parseRetentionFromEnv = (type: string, defaultVal: number = DEFAULT_RETENTION_DELETE_AFTER): RetentionOption => {
+ const deleteAfterEnv = process.env[`RETENTION_${type}_AFTER`] ?? defaultVal,
+ deleteCompletedEnv = process.env[`RETENTION_${type}_COMPLETED_AFTER`] ?? deleteAfterEnv,
+ deleteFailedEnv = process.env[`RETENTION_${type}_FAILED_AFTER`] ?? deleteAfterEnv,
+ deleteDupedEnv = process.env[`RETENTION_${type}_DUPED_AFTER`] ?? deleteAfterEnv;
return {
completed: parseRetentionValue(deleteCompletedEnv),
@@ -95,7 +95,7 @@ retentionCompactAfterFromEnv: RetentionOption;
export const getRetentionDeleteAfterFromEnv = () => {
if (retentionDeleteAfterFromEnv === undefined) {
- const deleteEnv = parseRetentionFromEnv();
+ const deleteEnv = parseRetentionFromEnv('DELETE');
if(isRetentionOptionDurations(deleteEnv)) {
retentionDeleteAfterFromEnv = deleteEnv;
} else {
@@ -106,7 +106,7 @@ export const getRetentionDeleteAfterFromEnv = () => {
}
export const getRetentionCompactAfterFromEnv = () => {
if (retentionCompactAfterFromEnv === undefined) {
- const compactEnv = parseRetentionFromEnv();
+ const compactEnv = parseRetentionFromEnv('COMPACT', DEFAULT_RETENTION_COMPACT_AFTER);
retentionCompactAfterFromEnv = compactEnv;
}
return retentionCompactAfterFromEnv;
diff --git a/src/backend/common/infrastructure/config/database.ts b/src/backend/common/infrastructure/config/database.ts
index 92085e1e..abcfab19 100644
--- a/src/backend/common/infrastructure/config/database.ts
+++ b/src/backend/common/infrastructure/config/database.ts
@@ -33,4 +33,5 @@ export interface RetentionOptions {
compact: CompactableProperty[]
}
-export const DEFAULT_RETENTION_DELETE_AFTER = 604800; // 7 days
\ No newline at end of file
+export const DEFAULT_RETENTION_DELETE_AFTER = 604800; // 7 days
+export const DEFAULT_RETENTION_COMPACT_AFTER = 259200; // 3 days
\ No newline at end of file