From 2dca03879b66eaa5c6408ef9fcc357cdaaf21d20 Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Sat, 10 Jan 2026 16:44:13 +0300 Subject: [PATCH] Add unique constraints and genres schema Add genres and artist_genres tables and create unique indexes on join tables (album_tracks, artist_albums, artist_tracks, loved_tracks, user_albums/user_artists/user_tracks). Update migration SQL, snapshot and journal metadata, add PAGE_SIZE and imports in CLI sync, and tweak agent logging. --- ...robertson.sql => 0000_thankful_leader.sql} | 37 +- apps/cli/drizzle/meta/0000_snapshot.json | 259 ++++++++- apps/cli/drizzle/meta/_journal.json | 4 +- apps/cli/src/cmd/sync.ts | 495 ++++++++++++++---- apps/cli/src/lib/agent.ts | 6 +- apps/cli/src/schema/album-tracks.ts | 37 +- apps/cli/src/schema/artist-albums.ts | 36 +- apps/cli/src/schema/artist-genres.ts | 17 + apps/cli/src/schema/artist-tracks.ts | 36 +- apps/cli/src/schema/genres.ts | 18 + apps/cli/src/schema/index.ts | 4 + apps/cli/src/schema/loved-tracks.ts | 32 +- apps/cli/src/schema/user-albums.ts | 40 +- apps/cli/src/schema/user-artists.ts | 41 +- apps/cli/src/schema/user-tracks.ts | 40 +- 15 files changed, 859 insertions(+), 243 deletions(-) rename apps/cli/drizzle/{0000_parched_robbie_robertson.sql => 0000_thankful_leader.sql} (79%) create mode 100644 apps/cli/src/schema/artist-genres.ts create mode 100644 apps/cli/src/schema/genres.ts diff --git a/apps/cli/drizzle/0000_parched_robbie_robertson.sql b/apps/cli/drizzle/0000_thankful_leader.sql similarity index 79% rename from apps/cli/drizzle/0000_parched_robbie_robertson.sql rename to apps/cli/drizzle/0000_thankful_leader.sql index e759dbc6..c4a4c1bc 100644 --- a/apps/cli/drizzle/0000_parched_robbie_robertson.sql +++ b/apps/cli/drizzle/0000_thankful_leader.sql @@ -8,6 +8,7 @@ CREATE TABLE `album_tracks` ( FOREIGN KEY (`track_id`) REFERENCES `tracks`(`id`) ON UPDATE no action ON DELETE no action ); --> statement-breakpoint +CREATE UNIQUE INDEX `album_tracks_unique_index` ON `album_tracks` (`album_id`,`track_id`);--> statement-breakpoint CREATE TABLE `albums` ( `id` text PRIMARY KEY NOT NULL, `title` text NOT NULL, @@ -42,6 +43,14 @@ CREATE TABLE `artist_albums` ( FOREIGN KEY (`album_id`) REFERENCES `albums`(`id`) ON UPDATE no action ON DELETE no action ); --> statement-breakpoint +CREATE UNIQUE INDEX `artist_albums_unique_index` ON `artist_albums` (`artist_id`,`album_id`);--> statement-breakpoint +CREATE TABLE `artist_genres ` ( + `id` text PRIMARY KEY NOT NULL, + `artist_id` text NOT NULL, + `genre_id` text NOT NULL +); +--> statement-breakpoint +CREATE UNIQUE INDEX `artist_genre_unique_index` ON `artist_genres ` (`artist_id`,`genre_id`);--> statement-breakpoint CREATE TABLE `artist_tracks` ( `id` text PRIMARY KEY NOT NULL, `artist_id` text NOT NULL, @@ -52,6 +61,7 @@ CREATE TABLE `artist_tracks` ( FOREIGN KEY (`track_id`) REFERENCES `tracks`(`id`) ON UPDATE no action ON DELETE no action ); --> statement-breakpoint +CREATE UNIQUE INDEX `artist_tracks_unique_index` ON `artist_tracks` (`artist_id`,`track_id`);--> statement-breakpoint CREATE TABLE `artists` ( `id` text PRIMARY KEY NOT NULL, `name` text NOT NULL, @@ -80,6 +90,14 @@ CREATE TABLE `auth_sessions` ( `updated_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL ); --> statement-breakpoint +CREATE TABLE `genres` ( + `id` text PRIMARY KEY NOT NULL, + `name` text NOT NULL, + `created_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL, + `updated_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL +); +--> statement-breakpoint +CREATE UNIQUE INDEX `genres_name_unique` ON `genres` (`name`);--> statement-breakpoint CREATE TABLE `loved_tracks` ( `id` text PRIMARY KEY NOT NULL, `user_id` text NOT NULL, @@ -91,6 +109,7 @@ CREATE TABLE `loved_tracks` ( ); --> statement-breakpoint CREATE UNIQUE INDEX `loved_tracks_uri_unique` ON `loved_tracks` (`uri`);--> statement-breakpoint +CREATE UNIQUE INDEX `loved_tracks_unique_index` ON `loved_tracks` (`user_id`,`track_id`);--> statement-breakpoint CREATE TABLE `scrobbles` ( `xata_id` text PRIMARY KEY NOT NULL, `user_id` text, @@ -146,6 +165,20 @@ CREATE UNIQUE INDEX `tracks_tidal_link_unique` ON `tracks` (`tidal_link`);--> st CREATE UNIQUE INDEX `tracks_uri_unique` ON `tracks` (`uri`);--> statement-breakpoint CREATE UNIQUE INDEX `tracks_cid_unique` ON `tracks` (`cid`);--> statement-breakpoint CREATE TABLE `user_albums` ( + `id` text PRIMARY KEY NOT NULL, + `user_id` text NOT NULL, + `album_id` text NOT NULL, + `created_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL, + `updated_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL, + `scrobbles` integer, + `uri` text NOT NULL, + FOREIGN KEY (`user_id`) REFERENCES `users`(`id`) ON UPDATE no action ON DELETE no action, + FOREIGN KEY (`album_id`) REFERENCES `albums`(`id`) ON UPDATE no action ON DELETE no action +); +--> statement-breakpoint +CREATE UNIQUE INDEX `user_albums_uri_unique` ON `user_albums` (`uri`);--> statement-breakpoint +CREATE UNIQUE INDEX `user_albums_unique_index` ON `user_albums` (`user_id`,`album_id`);--> statement-breakpoint +CREATE TABLE `user_artists` ( `id` text PRIMARY KEY NOT NULL, `user_id` text NOT NULL, `artist_id` text NOT NULL, @@ -157,7 +190,8 @@ CREATE TABLE `user_albums` ( FOREIGN KEY (`artist_id`) REFERENCES `artists`(`id`) ON UPDATE no action ON DELETE no action ); --> statement-breakpoint -CREATE UNIQUE INDEX `user_albums_uri_unique` ON `user_albums` (`uri`);--> statement-breakpoint +CREATE UNIQUE INDEX `user_artists_uri_unique` ON `user_artists` (`uri`);--> statement-breakpoint +CREATE UNIQUE INDEX `user_artists_unique_index` ON `user_artists` (`user_id`,`artist_id`);--> statement-breakpoint CREATE TABLE `user_tracks` ( `id` text PRIMARY KEY NOT NULL, `user_id` text NOT NULL, @@ -171,6 +205,7 @@ CREATE TABLE `user_tracks` ( ); --> statement-breakpoint CREATE UNIQUE INDEX `user_tracks_uri_unique` ON `user_tracks` (`uri`);--> statement-breakpoint +CREATE UNIQUE INDEX `user_tracks_unique_index` ON `user_tracks` (`user_id`,`track_id`);--> statement-breakpoint CREATE TABLE `users` ( `id` text PRIMARY KEY NOT NULL, `did` text NOT NULL, diff --git a/apps/cli/drizzle/meta/0000_snapshot.json b/apps/cli/drizzle/meta/0000_snapshot.json index bacb6ec3..40c843ee 100644 --- a/apps/cli/drizzle/meta/0000_snapshot.json +++ b/apps/cli/drizzle/meta/0000_snapshot.json @@ -1,7 +1,7 @@ { "version": "6", "dialect": "sqlite", - "id": "571a287d-ea60-4ac4-847a-307da78c375c", + "id": "0542b673-3d07-4b88-91a8-591a3a265288", "prevId": "00000000-0000-0000-0000-000000000000", "tables": { "album_tracks": { @@ -45,7 +45,16 @@ "default": "CURRENT_TIMESTAMP" } }, - "indexes": {}, + "indexes": { + "album_tracks_unique_index": { + "name": "album_tracks_unique_index", + "columns": [ + "album_id", + "track_id" + ], + "isUnique": true + } + }, "foreignKeys": { "album_tracks_album_id_albums_id_fk": { "name": "album_tracks_album_id_albums_id_fk", @@ -279,7 +288,16 @@ "default": "CURRENT_TIMESTAMP" } }, - "indexes": {}, + "indexes": { + "artist_albums_unique_index": { + "name": "artist_albums_unique_index", + "columns": [ + "artist_id", + "album_id" + ], + "isUnique": true + } + }, "foreignKeys": { "artist_albums_artist_id_artists_id_fk": { "name": "artist_albums_artist_id_artists_id_fk", @@ -312,6 +330,46 @@ "uniqueConstraints": {}, "checkConstraints": {} }, + "artist_genres ": { + "name": "artist_genres ", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "artist_id": { + "name": "artist_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "genre_id": { + "name": "genre_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "artist_genre_unique_index": { + "name": "artist_genre_unique_index", + "columns": [ + "artist_id", + "genre_id" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, "artist_tracks": { "name": "artist_tracks", "columns": { @@ -353,7 +411,16 @@ "default": "CURRENT_TIMESTAMP" } }, - "indexes": {}, + "indexes": { + "artist_tracks_unique_index": { + "name": "artist_tracks_unique_index", + "columns": [ + "artist_id", + "track_id" + ], + "isUnique": true + } + }, "foreignKeys": { "artist_tracks_artist_id_artists_id_fk": { "name": "artist_tracks_artist_id_artists_id_fk", @@ -565,6 +632,54 @@ "uniqueConstraints": {}, "checkConstraints": {} }, + "genres": { + "name": "genres", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "genres_name_unique": { + "name": "genres_name_unique", + "columns": [ + "name" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, "loved_tracks": { "name": "loved_tracks", "columns": { @@ -612,6 +727,14 @@ "uri" ], "isUnique": true + }, + "loved_tracks_unique_index": { + "name": "loved_tracks_unique_index", + "columns": [ + "user_id", + "track_id" + ], + "isUnique": true } }, "foreignKeys": { @@ -1051,8 +1174,8 @@ "notNull": true, "autoincrement": false }, - "artist_id": { - "name": "artist_id", + "album_id": { + "name": "album_id", "type": "text", "primaryKey": false, "notNull": true, @@ -1096,6 +1219,14 @@ "uri" ], "isUnique": true + }, + "user_albums_unique_index": { + "name": "user_albums_unique_index", + "columns": [ + "user_id", + "album_id" + ], + "isUnique": true } }, "foreignKeys": { @@ -1112,9 +1243,113 @@ "onDelete": "no action", "onUpdate": "no action" }, - "user_albums_artist_id_artists_id_fk": { - "name": "user_albums_artist_id_artists_id_fk", + "user_albums_album_id_albums_id_fk": { + "name": "user_albums_album_id_albums_id_fk", "tableFrom": "user_albums", + "tableTo": "albums", + "columnsFrom": [ + "album_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user_artists": { + "name": "user_artists", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "artist_id": { + "name": "artist_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "scrobbles": { + "name": "scrobbles", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "uri": { + "name": "uri", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "user_artists_uri_unique": { + "name": "user_artists_uri_unique", + "columns": [ + "uri" + ], + "isUnique": true + }, + "user_artists_unique_index": { + "name": "user_artists_unique_index", + "columns": [ + "user_id", + "artist_id" + ], + "isUnique": true + } + }, + "foreignKeys": { + "user_artists_user_id_users_id_fk": { + "name": "user_artists_user_id_users_id_fk", + "tableFrom": "user_artists", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "user_artists_artist_id_artists_id_fk": { + "name": "user_artists_artist_id_artists_id_fk", + "tableFrom": "user_artists", "tableTo": "artists", "columnsFrom": [ "artist_id" @@ -1192,6 +1427,14 @@ "uri" ], "isUnique": true + }, + "user_tracks_unique_index": { + "name": "user_tracks_unique_index", + "columns": [ + "user_id", + "track_id" + ], + "isUnique": true } }, "foreignKeys": { diff --git a/apps/cli/drizzle/meta/_journal.json b/apps/cli/drizzle/meta/_journal.json index d13f04ed..cd3c709d 100644 --- a/apps/cli/drizzle/meta/_journal.json +++ b/apps/cli/drizzle/meta/_journal.json @@ -5,8 +5,8 @@ { "idx": 0, "version": "6", - "when": 1768034891092, - "tag": "0000_parched_robbie_robertson", + "when": 1768051808522, + "tag": "0000_thankful_leader", "breakpoints": true } ] diff --git a/apps/cli/src/cmd/sync.ts b/apps/cli/src/cmd/sync.ts index 7260c5d6..f097a061 100644 --- a/apps/cli/src/cmd/sync.ts +++ b/apps/cli/src/cmd/sync.ts @@ -14,6 +14,10 @@ import { SelectUser } from "schema/users"; import schema from "schema"; import { createId } from "@paralleldrive/cuid2"; import _ from "lodash"; +import { and, eq } from "drizzle-orm"; +import { indexBy } from "ramda"; + +const PAGE_SIZE = 100; type Artists = { value: Artist.Record; uri: string; cid: string }[]; type Albums = { value: Album.Record; uri: string; cid: string }[]; @@ -118,120 +122,385 @@ const createUser = async ( return user; }; -const createArtists = async (artists: Artists, _user: SelectUser) => { +const createArtists = async (artists: Artists, user: SelectUser) => { if (artists.length === 0) return; - await ctx.db - .insert(schema.artists) - .values( - artists.map((artist) => ({ - id: createId(), - name: artist.value.name, - cid: artist.cid, - uri: artist.uri, - biography: artist.value.bio, - born: artist.value.born ? new Date(artist.value.born) : null, - bornIn: artist.value.bornIn, - died: artist.value.died ? new Date(artist.value.died) : null, - picture: artist.value.pictureUrl, - sha256: artist.value.sha256 as string, - genres: (artist.value.genres as string[]).join(", "), - })), - ) - .onConflictDoNothing({ - target: schema.artists.cid, - }) - .returning() - .execute(); + const tags = artists.map((artist) => artist.value.tags || []); + + // Batch genre inserts to avoid stack overflow + const uniqueTags = tags + .flat() + .filter((tag) => tag) + .map((tag) => ({ + id: createId(), + name: tag, + })); + + const BATCH_SIZE = 500; + for (let i = 0; i < uniqueTags.length; i += BATCH_SIZE) { + const batch = uniqueTags.slice(i, i + BATCH_SIZE); + await ctx.db + .insert(schema.genres) + .values(batch) + .onConflictDoNothing({ + target: schema.genres.name, + }) + .execute(); + } + + const genres = await ctx.db.select().from(schema.genres).execute(); + + const genreMap = indexBy((genre) => genre.name, genres); + + // Process artists in batches + let totalArtistsImported = 0; + + for (let i = 0; i < artists.length; i += BATCH_SIZE) { + const batch = artists.slice(i, i + BATCH_SIZE); + + ctx.db.transaction((tx) => { + const newArtists = tx + .insert(schema.artists) + .values( + batch.map((artist) => ({ + id: createId(), + name: artist.value.name, + cid: artist.cid, + uri: artist.uri, + biography: artist.value.bio, + born: artist.value.born ? new Date(artist.value.born) : null, + bornIn: artist.value.bornIn, + died: artist.value.died ? new Date(artist.value.died) : null, + picture: artist.value.pictureUrl, + genres: artist.value.tags?.join(", "), + })), + ) + .onConflictDoNothing({ + target: schema.artists.cid, + }) + .returning() + .all(); + + if (newArtists.length === 0) return; + + const artistGenres = newArtists + .map( + (artist) => + artist.genres + ?.split(", ") + .filter((tag) => !!tag && !!genreMap[tag]) + .map((tag) => ({ + id: createId(), + artistId: artist.id, + genreId: genreMap[tag].id, + })) || [], + ) + .flat(); + + if (artistGenres.length > 0) { + tx.insert(schema.artistGenres) + .values(artistGenres) + .onConflictDoNothing({ + target: [schema.artistGenres.artistId, schema.artistGenres.genreId], + }) + .returning() + .run(); + } + + tx.insert(schema.userArtists) + .values( + newArtists.map((artist) => ({ + id: createId(), + userId: user.id, + artistId: artist.id, + uri: artist.uri, + })), + ) + .run(); + + totalArtistsImported += newArtists.length; + }); + } + + logger.info`👤 ${totalArtistsImported} Artists imported`; }; const createAlbums = async (albums: Albums, user: SelectUser) => { if (albums.length === 0) return; - await ctx.db - .insert(schema.albums) - .values( - albums.map((album) => ({ - id: createId(), - cid: album.cid, - title: "", - artist: "", - sha256: "", - uri: album.uri, - mbid: "", - description: "", - imageUrl: "", - spotifyId: "", - appleMusicId: "", - genres: "", - releaseDate: "", - year: undefined, - })), - ) - .onConflictDoNothing({ - target: schema.albums.cid, - }) - .returning() - .execute(); + const artists = await Promise.all( + albums.map(async (album) => + ctx.db + .select() + .from(schema.artists) + .where(eq(schema.artists.name, album.value.artist)) + .execute() + .then(([artist]) => artist), + ), + ); + + const validAlbumData = albums + .map((album, index) => ({ album, artist: artists[index] })) + .filter(({ artist }) => artist); + + // Process albums in batches + const BATCH_SIZE = 500; + let totalAlbumsImported = 0; + + for (let i = 0; i < validAlbumData.length; i += BATCH_SIZE) { + const batch = validAlbumData.slice(i, i + BATCH_SIZE); + + ctx.db.transaction((tx) => { + const newAlbums = tx + .insert(schema.albums) + .values( + batch.map(({ album, artist }) => ({ + id: createId(), + cid: album.cid, + uri: album.uri, + title: album.value.title, + artist: album.value.artist, + releaseDate: album.value.releaseDate, + year: album.value.year, + albumArt: album.value.albumArtUrl, + artistUri: artist.uri, + appleMusicLink: album.value.appleMusicLink, + spotifyLink: album.value.spotifyLink, + tidalLink: album.value.tidalLink, + youtubeLink: album.value.youtubeLink, + })), + ) + .onConflictDoNothing({ + target: schema.albums.cid, + }) + .returning() + .all(); + + if (newAlbums.length === 0) return; + + tx.insert(schema.userAlbums) + .values( + newAlbums.map((album) => ({ + id: createId(), + userId: user.id, + albumId: album.id, + uri: album.uri, + })), + ) + .run(); + + totalAlbumsImported += newAlbums.length; + }); + } + + logger.info`💿 ${totalAlbumsImported} Albums imported`; }; const createSongs = async (songs: Songs, user: SelectUser) => { if (songs.length === 0) return; - await ctx.db - .insert(schema.tracks) - .values( - songs.map((song) => ({ - id: createId(), - cid: song.cid, - uri: song.uri, - title: song.value.title, - artist: song.value.artist, - albumArtist: song.value.albumArtist, - albumArt: song.value.albumArtUrl, - album: song.value.album, - trackNumber: song.value.trackNumber, - duration: song.value.duration, - mbId: song.value.mbid, - youtubeLink: song.value.youtubeLink, - spotifyLink: song.value.spotifyLink, - appleMusicLink: song.value.appleMusicLink, - tidalLink: song.value.tidalLink, - discNumber: song.value.discNumber, - lyrics: song.value.lyrics, - composer: song.value.composer, - genre: song.value.genre, - label: song.value.label, - copyrightMessage: song.value.copyrightMessage, - albumUri: "", - artistUri: "", - })), - ) - .onConflictDoNothing({ - target: schema.tracks.cid, - }) - .returning() - .execute(); + const albums = await Promise.all( + songs.map((song) => + ctx.db + .select() + .from(schema.albums) + .where( + and( + eq(schema.albums.artist, song.value.albumArtist), + eq(schema.albums.title, song.value.album), + ), + ) + .execute() + .then((result) => result[0]), + ), + ); + + const artists = await Promise.all( + songs.map((song) => + ctx.db + .select() + .from(schema.artists) + .where(eq(schema.artists.name, song.value.albumArtist)) + .execute() + .then((result) => result[0]), + ), + ); + + const validSongData = songs + .map((song, index) => ({ + song, + artist: artists[index], + album: albums[index], + })) + .filter(({ artist, album }) => artist && album); + + // Process in batches to avoid stack overflow with large datasets + const BATCH_SIZE = 500; + let totalTracksImported = 0; + + for (let i = 0; i < validSongData.length; i += BATCH_SIZE) { + const batch = validSongData.slice(i, i + BATCH_SIZE); + + ctx.db.transaction((tx) => { + const tracks = tx + .insert(schema.tracks) + .values( + batch.map(({ song, artist, album }) => ({ + id: createId(), + cid: song.cid, + uri: song.uri, + title: song.value.title, + artist: song.value.artist, + albumArtist: song.value.albumArtist, + albumArt: song.value.albumArtUrl, + album: song.value.album, + trackNumber: song.value.trackNumber, + duration: song.value.duration, + mbId: song.value.mbid, + youtubeLink: song.value.youtubeLink, + spotifyLink: song.value.spotifyLink, + appleMusicLink: song.value.appleMusicLink, + tidalLink: song.value.tidalLink, + discNumber: song.value.discNumber, + lyrics: song.value.lyrics, + composer: song.value.composer, + genre: song.value.genre, + label: song.value.label, + copyrightMessage: song.value.copyrightMessage, + albumUri: album.uri, + artistUri: artist.uri, + })), + ) + .onConflictDoNothing({ + target: schema.tracks.cid, + }) + .returning() + .all(); + + if (tracks.length === 0) return; + + tx.insert(schema.albumTracks) + .values( + tracks.map((track, index) => ({ + id: createId(), + albumId: batch[index].album.id, + trackId: track.id, + })), + ) + .onConflictDoNothing({ + target: [schema.albumTracks.albumId, schema.albumTracks.trackId], + }) + .run(); + + tx.insert(schema.userTracks) + .values( + tracks.map((track) => ({ + id: createId(), + userId: user.id, + trackId: track.id, + uri: track.uri, + })), + ) + .onConflictDoNothing({ + target: [schema.userTracks.userId, schema.userTracks.trackId], + }) + .run(); + + totalTracksImported += tracks.length; + }); + } + + logger.info`▶️ ${totalTracksImported} Tracks imported`; }; const createScrobbles = async (scrobbles: Scrobbles, user: SelectUser) => { if (!scrobbles.length) return; - await ctx.db - .insert(schema.scrobbles) - .values( - scrobbles.map((scrobble) => ({ - id: createId(), - trackId: "", - userId: user.id, - timestamp: new Date(), - })), - ) - .onConflictDoNothing({ - target: schema.scrobbles.cid, - }) - .returning() - .execute(); + const tracks = await Promise.all( + scrobbles.map((scrobble) => + ctx.db + .select() + .from(schema.tracks) + .where( + and( + eq(schema.tracks.title, scrobble.value.title), + eq(schema.tracks.artist, scrobble.value.artist), + eq(schema.tracks.album, scrobble.value.album), + eq(schema.tracks.albumArtist, scrobble.value.albumArtist), + ), + ) + .execute() + .then(([track]) => track), + ), + ); + + const albums = await Promise.all( + scrobbles.map((scrobble) => + ctx.db + .select() + .from(schema.albums) + .where( + and( + eq(schema.albums.title, scrobble.value.album), + eq(schema.albums.artist, scrobble.value.albumArtist), + ), + ) + .execute() + .then(([album]) => album), + ), + ); + + const artists = await Promise.all( + scrobbles.map((scrobble) => + ctx.db + .select() + .from(schema.artists) + .where(and(eq(schema.artists.name, scrobble.value.artist))) + .execute() + .then(([artist]) => artist), + ), + ); + + const validScrobbleData = scrobbles + .map((scrobble, index) => ({ + scrobble, + track: tracks[index], + album: albums[index], + artist: artists[index], + })) + .filter(({ track, album, artist }) => track && album && artist); + + // Process in batches to avoid stack overflow with large datasets + const BATCH_SIZE = 500; + let totalScrobblesImported = 0; + + for (let i = 0; i < validScrobbleData.length; i += BATCH_SIZE) { + const batch = validScrobbleData.slice(i, i + BATCH_SIZE); + + const result = await ctx.db + .insert(schema.scrobbles) + .values( + batch.map(({ scrobble, track, album, artist }) => ({ + id: createId(), + userId: user.id, + trackId: track.id, + albumId: album.id, + artistId: artist.id, + uri: scrobble.uri, + cid: scrobble.cid, + timestamp: new Date(scrobble.value.createdAt), + })), + ) + .onConflictDoNothing({ + target: schema.scrobbles.cid, + }) + .returning() + .execute(); + + totalScrobblesImported += result.length; + } + + logger.info`🕒 ${totalScrobblesImported} scrobbles imported`; }; const subscribeToJetstream = (_did: string) => { @@ -301,12 +570,11 @@ const getRockskyUserSongs = async (agent: Agent): Promise => { cid: string; }[] = []; let cursor: string | undefined; - let i = 1; do { const res = await agent.com.atproto.repo.listRecords({ repo: agent.assertDid, collection: "app.rocksky.song", - limit: 100, + limit: PAGE_SIZE, cursor, }); const records = res.data.records as Array<{ @@ -316,8 +584,9 @@ const getRockskyUserSongs = async (agent: Agent): Promise => { }>; results = results.concat(records); cursor = res.data.cursor; - logger.info(`${chalk.greenBright(i)} songs`); - i += 100; + logger.info( + `${chalk.cyanBright(agent.assertDid)} ${chalk.greenBright(results.length)} songs`, + ); } while (cursor); return results; @@ -330,12 +599,11 @@ const getRockskyUserAlbums = async (agent: Agent): Promise => { cid: string; }[] = []; let cursor: string | undefined; - let i = 1; do { const res = await agent.com.atproto.repo.listRecords({ repo: agent.assertDid, collection: "app.rocksky.album", - limit: 100, + limit: PAGE_SIZE, cursor, }); @@ -348,8 +616,9 @@ const getRockskyUserAlbums = async (agent: Agent): Promise => { results = results.concat(records); cursor = res.data.cursor; - logger.info(`${chalk.greenBright(i)} albums`); - i += 100; + logger.info( + `${chalk.cyanBright(agent.assertDid)} ${chalk.greenBright(results.length)} albums`, + ); } while (cursor); return results; @@ -362,12 +631,11 @@ const getRockskyUserArtists = async (agent: Agent): Promise => { cid: string; }[] = []; let cursor: string | undefined; - let i = 1; do { const res = await agent.com.atproto.repo.listRecords({ repo: agent.assertDid, collection: "app.rocksky.artist", - limit: 100, + limit: PAGE_SIZE, cursor, }); @@ -380,8 +648,9 @@ const getRockskyUserArtists = async (agent: Agent): Promise => { results = results.concat(records); cursor = res.data.cursor; - logger.info(`${chalk.greenBright(i)} artists`); - i += 100; + logger.info( + `${chalk.cyanBright(agent.assertDid)} ${chalk.greenBright(results.length)} artists`, + ); } while (cursor); return results; @@ -394,12 +663,11 @@ const getRockskyUserScrobbles = async (agent: Agent): Promise => { cid: string; }[] = []; let cursor: string | undefined; - let i = 1; do { const res = await agent.com.atproto.repo.listRecords({ repo: agent.assertDid, collection: "app.rocksky.scrobble", - limit: 100, + limit: PAGE_SIZE, cursor, }); @@ -412,8 +680,9 @@ const getRockskyUserScrobbles = async (agent: Agent): Promise => { results = results.concat(records); cursor = res.data.cursor; - logger.info(`${chalk.greenBright(i)} scrobbles`); - i += 100; + logger.info( + `${chalk.cyanBright(agent.assertDid)} ${chalk.greenBright(results.length)} scrobbles`, + ); } while (cursor); return results; diff --git a/apps/cli/src/lib/agent.ts b/apps/cli/src/lib/agent.ts index 70a81a09..a9d9d28f 100644 --- a/apps/cli/src/lib/agent.ts +++ b/apps/cli/src/lib/agent.ts @@ -25,9 +25,7 @@ export async function createAgent(did: string, handle: string): Promise { await agent.resumeSession(JSON.parse(data.session)); return agent; } catch (e) { - ctx.logger.error`resuming session ${did}`; - ctx.logger.error(e); - + ctx.logger.error`Resuming session ${did}`; await ctx.db .delete(authSessions) .where(eq(authSessions.key, did)) @@ -50,6 +48,8 @@ export async function createAgent(did: string, handle: string): Promise { }) .execute(); + ctx.logger.info`Logged in as ${handle}`; + return agent; } } diff --git a/apps/cli/src/schema/album-tracks.ts b/apps/cli/src/schema/album-tracks.ts index 8e0ee192..7ccf3e3a 100644 --- a/apps/cli/src/schema/album-tracks.ts +++ b/apps/cli/src/schema/album-tracks.ts @@ -1,23 +1,28 @@ import { type InferInsertModel, type InferSelectModel, sql } from "drizzle-orm"; -import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core"; +import { integer, sqliteTable, text, unique } from "drizzle-orm/sqlite-core"; import albums from "./albums"; import tracks from "./tracks"; -const albumTracks = sqliteTable("album_tracks", { - id: text("id").primaryKey().notNull(), - albumId: text("album_id") - .notNull() - .references(() => albums.id), - trackId: text("track_id") - .notNull() - .references(() => tracks.id), - createdAt: integer("created_at") - .notNull() - .default(sql`CURRENT_TIMESTAMP`), - updatedAt: integer("updated_at") - .notNull() - .default(sql`CURRENT_TIMESTAMP`), -}); +const albumTracks = sqliteTable( + "album_tracks", + { + id: text("id").primaryKey().notNull(), + albumId: text("album_id") + .notNull() + .references(() => albums.id), + trackId: text("track_id") + .notNull() + .references(() => tracks.id), + createdAt: integer("created_at") + .notNull() + .default(sql`CURRENT_TIMESTAMP`), + updatedAt: integer("updated_at") + .notNull() + .default(sql`CURRENT_TIMESTAMP`), + }, + + (t) => [unique("album_tracks_unique_index").on(t.albumId, t.trackId)], +); export type SelectAlbumTrack = InferSelectModel; export type InsertAlbumTrack = InferInsertModel; diff --git a/apps/cli/src/schema/artist-albums.ts b/apps/cli/src/schema/artist-albums.ts index 31b97008..78e23fc7 100644 --- a/apps/cli/src/schema/artist-albums.ts +++ b/apps/cli/src/schema/artist-albums.ts @@ -1,23 +1,27 @@ import { type InferInsertModel, type InferSelectModel, sql } from "drizzle-orm"; -import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core"; +import { integer, sqliteTable, text, unique } from "drizzle-orm/sqlite-core"; import albums from "./albums"; import artists from "./artists"; -const artistAlbums = sqliteTable("artist_albums", { - id: text("id").primaryKey().notNull(), - artistId: text("artist_id") - .notNull() - .references(() => artists.id), - albumId: text("album_id") - .notNull() - .references(() => albums.id), - createdAt: integer("created_at") - .notNull() - .default(sql`CURRENT_TIMESTAMP`), - updatedAt: integer("updated_at") - .notNull() - .default(sql`CURRENT_TIMESTAMP`), -}); +const artistAlbums = sqliteTable( + "artist_albums", + { + id: text("id").primaryKey().notNull(), + artistId: text("artist_id") + .notNull() + .references(() => artists.id), + albumId: text("album_id") + .notNull() + .references(() => albums.id), + createdAt: integer("created_at") + .notNull() + .default(sql`CURRENT_TIMESTAMP`), + updatedAt: integer("updated_at") + .notNull() + .default(sql`CURRENT_TIMESTAMP`), + }, + (t) => [unique("artist_albums_unique_index").on(t.artistId, t.albumId)], +); export type SelectArtistAlbum = InferSelectModel; export type InsertArtistAlbum = InferInsertModel; diff --git a/apps/cli/src/schema/artist-genres.ts b/apps/cli/src/schema/artist-genres.ts new file mode 100644 index 00000000..531a19df --- /dev/null +++ b/apps/cli/src/schema/artist-genres.ts @@ -0,0 +1,17 @@ +import { type InferInsertModel, type InferSelectModel } from "drizzle-orm"; +import { sqliteTable, text, unique } from "drizzle-orm/sqlite-core"; + +const artistGenres = sqliteTable( + "artist_genres ", + { + id: text("id").primaryKey().notNull(), + artistId: text("artist_id").notNull(), + genreId: text("genre_id").notNull(), + }, + (t) => [unique("artist_genre_unique_index").on(t.artistId, t.genreId)], +); + +export type SelectArtistGenre = InferSelectModel; +export type InsertArtistGenre = InferInsertModel; + +export default artistGenres; diff --git a/apps/cli/src/schema/artist-tracks.ts b/apps/cli/src/schema/artist-tracks.ts index 1614d5f7..1919c476 100644 --- a/apps/cli/src/schema/artist-tracks.ts +++ b/apps/cli/src/schema/artist-tracks.ts @@ -1,23 +1,27 @@ import { type InferInsertModel, type InferSelectModel, sql } from "drizzle-orm"; -import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core"; +import { integer, sqliteTable, text, unique } from "drizzle-orm/sqlite-core"; import artists from "./artists"; import tracks from "./tracks"; -const artistTracks = sqliteTable("artist_tracks", { - id: text("id").primaryKey().notNull(), - artistId: text("artist_id") - .notNull() - .references(() => artists.id), - trackId: text("track_id") - .notNull() - .references(() => tracks.id), - createdAt: integer("created_at", { mode: "timestamp" }) - .notNull() - .default(sql`CURRENT_TIMESTAMP`), - updatedAt: integer("updated_at", { mode: "timestamp" }) - .notNull() - .default(sql`CURRENT_TIMESTAMP`), -}); +const artistTracks = sqliteTable( + "artist_tracks", + { + id: text("id").primaryKey().notNull(), + artistId: text("artist_id") + .notNull() + .references(() => artists.id), + trackId: text("track_id") + .notNull() + .references(() => tracks.id), + createdAt: integer("created_at", { mode: "timestamp" }) + .notNull() + .default(sql`CURRENT_TIMESTAMP`), + updatedAt: integer("updated_at", { mode: "timestamp" }) + .notNull() + .default(sql`CURRENT_TIMESTAMP`), + }, + (t) => [unique("artist_tracks_unique_index").on(t.artistId, t.trackId)], +); export type SelectArtistTrack = InferSelectModel; export type InsertArtistTrack = InferInsertModel; diff --git a/apps/cli/src/schema/genres.ts b/apps/cli/src/schema/genres.ts new file mode 100644 index 00000000..60d0a8e6 --- /dev/null +++ b/apps/cli/src/schema/genres.ts @@ -0,0 +1,18 @@ +import { type InferInsertModel, type InferSelectModel, sql } from "drizzle-orm"; +import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core"; + +const genres = sqliteTable("genres", { + id: text("id").primaryKey().notNull(), + name: text("name").unique().notNull(), + createdAt: integer("created_at", { mode: "timestamp" }) + .notNull() + .default(sql`CURRENT_TIMESTAMP`), + updatedAt: integer("updated_at", { mode: "timestamp" }) + .notNull() + .default(sql`CURRENT_TIMESTAMP`), +}); + +export type SelectGenre = InferSelectModel; +export type InsertGenre = InferInsertModel; + +export default genres; diff --git a/apps/cli/src/schema/index.ts b/apps/cli/src/schema/index.ts index 91505b86..b2fcf4e7 100644 --- a/apps/cli/src/schema/index.ts +++ b/apps/cli/src/schema/index.ts @@ -1,9 +1,11 @@ import albumTracks from "./album-tracks"; import albums from "./albums"; import artistAlbums from "./artist-albums"; +import artistGenres from "./artist-genres"; import artistTracks from "./artist-tracks"; import artists from "./artists"; import authSessions from "./auth-session"; +import genres from "./genres"; import lovedTracks from "./loved-tracks"; import scrobbles from "./scrobbles"; import tracks from "./tracks"; @@ -26,4 +28,6 @@ export default { userAlbums, userArtists, userTracks, + genres, + artistGenres, }; diff --git a/apps/cli/src/schema/loved-tracks.ts b/apps/cli/src/schema/loved-tracks.ts index ea3b5851..bc958b12 100644 --- a/apps/cli/src/schema/loved-tracks.ts +++ b/apps/cli/src/schema/loved-tracks.ts @@ -1,21 +1,25 @@ import { type InferInsertModel, type InferSelectModel, sql } from "drizzle-orm"; -import { sqliteTable, integer, text } from "drizzle-orm/sqlite-core"; +import { sqliteTable, integer, text, unique } from "drizzle-orm/sqlite-core"; import tracks from "./tracks"; import users from "./users"; -const lovedTracks = sqliteTable("loved_tracks", { - id: text("id").primaryKey().notNull(), - userId: text("user_id") - .notNull() - .references(() => users.id), - trackId: text("track_id") - .notNull() - .references(() => tracks.id), - uri: text("uri").unique(), - createdAt: integer("created_at") - .notNull() - .default(sql`CURRENT_TIMESTAMP`), -}); +const lovedTracks = sqliteTable( + "loved_tracks", + { + id: text("id").primaryKey().notNull(), + userId: text("user_id") + .notNull() + .references(() => users.id), + trackId: text("track_id") + .notNull() + .references(() => tracks.id), + uri: text("uri").unique(), + createdAt: integer("created_at") + .notNull() + .default(sql`CURRENT_TIMESTAMP`), + }, + (t) => [unique("loved_tracks_unique_index").on(t.userId, t.trackId)], +); export type SelectLovedTrack = InferSelectModel; export type InsertLovedTrack = InferInsertModel; diff --git a/apps/cli/src/schema/user-albums.ts b/apps/cli/src/schema/user-albums.ts index 86b28e99..c24f1f33 100644 --- a/apps/cli/src/schema/user-albums.ts +++ b/apps/cli/src/schema/user-albums.ts @@ -1,25 +1,29 @@ import { type InferInsertModel, type InferSelectModel, sql } from "drizzle-orm"; -import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core"; +import { integer, sqliteTable, text, unique } from "drizzle-orm/sqlite-core"; import albums from "./albums"; import users from "./users"; -const userAlbums = sqliteTable("user_albums", { - id: text("id").primaryKey().notNull(), - userId: text("user_id") - .notNull() - .references(() => users.id), - albumId: text("album_id") - .notNull() - .references(() => albums.id), - createdAt: integer("created_at", { mode: "timestamp" }) - .notNull() - .default(sql`CURRENT_TIMESTAMP`), - updatedAt: integer("updated_at", { mode: "timestamp" }) - .notNull() - .default(sql`CURRENT_TIMESTAMP`), - scrobbles: integer("scrobbles"), - uri: text("uri").unique().notNull(), -}); +const userAlbums = sqliteTable( + "user_albums", + { + id: text("id").primaryKey().notNull(), + userId: text("user_id") + .notNull() + .references(() => users.id), + albumId: text("album_id") + .notNull() + .references(() => albums.id), + createdAt: integer("created_at", { mode: "timestamp" }) + .notNull() + .default(sql`CURRENT_TIMESTAMP`), + updatedAt: integer("updated_at", { mode: "timestamp" }) + .notNull() + .default(sql`CURRENT_TIMESTAMP`), + scrobbles: integer("scrobbles"), + uri: text("uri").unique().notNull(), + }, + (t) => [unique("user_albums_unique_index").on(t.userId, t.albumId)], +); export type SelectUserAlbum = InferSelectModel; export type InsertUserAlbum = InferInsertModel; diff --git a/apps/cli/src/schema/user-artists.ts b/apps/cli/src/schema/user-artists.ts index 0452d759..eafbf7fe 100644 --- a/apps/cli/src/schema/user-artists.ts +++ b/apps/cli/src/schema/user-artists.ts @@ -1,25 +1,30 @@ import { type InferInsertModel, type InferSelectModel, sql } from "drizzle-orm"; -import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core"; +import { integer, sqliteTable, text, unique } from "drizzle-orm/sqlite-core"; import artists from "./artists"; import users from "./users"; -const userArtists = sqliteTable("user_albums", { - id: text("id").primaryKey().notNull(), - userId: text("user_id") - .notNull() - .references(() => users.id), - albumId: text("artist_id") - .notNull() - .references(() => artists.id), - createdAt: integer("created_at", { mode: "timestamp" }) - .notNull() - .default(sql`CURRENT_TIMESTAMP`), - updatedAt: integer("updated_at", { mode: "timestamp" }) - .notNull() - .default(sql`CURRENT_TIMESTAMP`), - scrobbles: integer("scrobbles"), - uri: text("uri").unique().notNull(), -}); +const userArtists = sqliteTable( + "user_artists", + { + id: text("id").primaryKey().notNull(), + userId: text("user_id") + .notNull() + .references(() => users.id), + artistId: text("artist_id") + .notNull() + .references(() => artists.id), + createdAt: integer("created_at", { mode: "timestamp" }) + .notNull() + .default(sql`CURRENT_TIMESTAMP`), + updatedAt: integer("updated_at", { mode: "timestamp" }) + .notNull() + .default(sql`CURRENT_TIMESTAMP`), + scrobbles: integer("scrobbles"), + uri: text("uri").unique().notNull(), + }, + + (t) => [unique("user_artists_unique_index").on(t.userId, t.artistId)], +); export type SelectUserArtist = InferSelectModel; export type InsertUserArtist = InferInsertModel; diff --git a/apps/cli/src/schema/user-tracks.ts b/apps/cli/src/schema/user-tracks.ts index d79b85af..a69b5efc 100644 --- a/apps/cli/src/schema/user-tracks.ts +++ b/apps/cli/src/schema/user-tracks.ts @@ -1,25 +1,29 @@ import { type InferInsertModel, type InferSelectModel, sql } from "drizzle-orm"; -import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core"; +import { integer, sqliteTable, text, unique } from "drizzle-orm/sqlite-core"; import tracks from "./tracks"; import users from "./users"; -const userTracks = sqliteTable("user_tracks", { - id: text("id").primaryKey().notNull(), - userId: text("user_id") - .notNull() - .references(() => users.id), - albumId: text("track_id") - .notNull() - .references(() => tracks.id), - createdAt: integer("created_at", { mode: "timestamp" }) - .notNull() - .default(sql`CURRENT_TIMESTAMP`), - updatedAt: integer("updated_at", { mode: "timestamp" }) - .notNull() - .default(sql`CURRENT_TIMESTAMP`), - scrobbles: integer("scrobbles"), - uri: text("uri").unique().notNull(), -}); +const userTracks = sqliteTable( + "user_tracks", + { + id: text("id").primaryKey().notNull(), + userId: text("user_id") + .notNull() + .references(() => users.id), + trackId: text("track_id") + .notNull() + .references(() => tracks.id), + createdAt: integer("created_at", { mode: "timestamp" }) + .notNull() + .default(sql`CURRENT_TIMESTAMP`), + updatedAt: integer("updated_at", { mode: "timestamp" }) + .notNull() + .default(sql`CURRENT_TIMESTAMP`), + scrobbles: integer("scrobbles"), + uri: text("uri").unique().notNull(), + }, + (t) => [unique("user_tracks_unique_index").on(t.userId, t.trackId)], +); export type SelectUser = InferSelectModel; export type InsertUserTrack = InferInsertModel; -- 2.51.2