diff --git a/androidApp/build.gradle.kts b/androidApp/build.gradle.kts index ff993ed..e2b22dc 100644 --- a/androidApp/build.gradle.kts +++ b/androidApp/build.gradle.kts @@ -76,8 +76,8 @@ android { applicationId = "com.derakkuma" minSdk = 26 targetSdk = 36 - versionCode = 2 - versionName = "0.1.1" + versionCode = 3 + versionName = "0.2.0" } compileOptions { diff --git a/composeApp/src/commonMain/kotlin/com/derakkuma/data/HistoricalCrawler.kt b/composeApp/src/commonMain/kotlin/com/derakkuma/data/HistoricalCrawler.kt index 1605d5b..eaad096 100644 --- a/composeApp/src/commonMain/kotlin/com/derakkuma/data/HistoricalCrawler.kt +++ b/composeApp/src/commonMain/kotlin/com/derakkuma/data/HistoricalCrawler.kt @@ -45,46 +45,66 @@ class HistoricalCrawler( delayMs: Long = 300, ): Result = withContext(Dispatchers.Default) { val errors = mutableListOf() - val allIdxs = mutableSetOf() - - // Difficulty values for musicSort: 0=Basic, 1=Advanced, 2=Expert, 3=Master, 4=ReMaster, 10=Utage. - val difficulties = listOf( - 0 to "Basic", - 1 to "Advanced", - 2 to "Expert", - 3 to "Master", - 4 to "Re:Master", - 10 to "Utage", - ) - val utageIdxs = mutableSetOf() - // Phase 1: Fetch all search pages to collect idx values - onProgress(Progress(0, difficulties.size, "Fetching song lists...")) + val cursor = playStore.getHistoricalBackfillCursor() ?: run { + val allIdxs = mutableSetOf() + val utageIdxs = mutableSetOf() + + // Difficulty values for musicSort: 0=Basic, 1=Advanced, 2=Expert, 3=Master, 4=ReMaster, 10=Utage. + val difficulties = listOf( + 0 to "Basic", + 1 to "Advanced", + 2 to "Expert", + 3 to "Master", + 4 to "Re:Master", + 10 to "Utage", + ) + + // Phase 1: Fetch all search pages to collect idx values + onProgress(Progress(0, difficulties.size, "Fetching song lists...")) + + for ((i, diffPair) in difficulties.withIndex()) { + val (diff, diffName) = diffPair + try { + onProgress(Progress(i + 1, difficulties.size, "Fetching $diffName songs...")) + val idxs = client.fetchMusicSearchIdxs(diff = diff, sort = 5) + if (diff == 10) utageIdxs.addAll(idxs) + allIdxs.addAll(idxs) + println("[crawler] Found ${idxs.size} songs on $diffName (${allIdxs.size} unique total)") + delay(delayMs) + } catch (e: Exception) { + errors.add("Failed to fetch $diffName: ${e.message}") + println("[crawler] Error fetching $diffName: ${e.message}") + } + } - for ((i, diffPair) in difficulties.withIndex()) { - val (diff, diffName) = diffPair - try { - onProgress(Progress(i + 1, difficulties.size, "Fetching $diffName songs...")) - val idxs = client.fetchMusicSearchIdxs(diff = diff, sort = 5) - if (diff == 10) utageIdxs.addAll(idxs) - allIdxs.addAll(idxs) - println("[crawler] Found ${idxs.size} songs on $diffName (${allIdxs.size} unique total)") - delay(delayMs) - } catch (e: Exception) { - errors.add("Failed to fetch $diffName: ${e.message}") - println("[crawler] Error fetching $diffName: ${e.message}") + if (errors.isNotEmpty()) { + return@withContext Result( + totalSongs = allIdxs.size, + totalPlays = 0, + errors = errors, + ) } - } - println("[crawler] Total unique songs: ${allIdxs.size}") + println("[crawler] Total unique songs: ${allIdxs.size}") + + PlayStore.HistoricalBackfillCursor( + idxList = allIdxs.toList(), + utageIdxs = utageIdxs.toList(), + nextIndex = 0, + runStartedAt = Clock.System.now().toEpochMilliseconds(), + ).also { playStore.putHistoricalBackfillCursor(it) } + } // Phase 2: Fetch musicDetail for each unique idx - val idxList = allIdxs.toList() - val historicalPlays = mutableListOf() + val idxList = cursor.idxList + val utageIdxs = cursor.utageIdxs.toSet() + var historicalPlayCount = cursor.totalPlaysInserted - onProgress(Progress(0, idxList.size, "Fetching song details...")) + onProgress(Progress(cursor.nextIndex, idxList.size, "Fetching song details...")) - for ((i, idx) in idxList.withIndex()) { + for (i in cursor.nextIndex until idxList.size) { + val idx = idxList[i] try { onProgress(Progress(i + 1, idxList.size, "Fetching song ${i + 1}/${idxList.size}...")) val detail = client.fetchMusicDetail(idx) @@ -94,7 +114,7 @@ class HistoricalCrawler( for (score in detail.scores) { if (score.lastPlayed.isNotBlank() && score.lastPlayed != "-") { val difficulty = if (isUtageSong && !score.difficulty.startsWith("【")) "utage" else score.difficulty - historicalPlays.add( + playStore.upsertHistoricalPlay( PlayStore.HistoricalPlay( datetime = score.lastPlayed, songName = detail.name, @@ -109,26 +129,30 @@ class HistoricalCrawler( albumArtUrl = detail.albumArtUrl, detailIdx = idx, ), + syncedAt = cursor.runStartedAt, ) + historicalPlayCount += 1 } } + playStore.putHistoricalBackfillCursor(cursor.copy(nextIndex = i + 1, totalPlaysInserted = historicalPlayCount)) delay(delayMs) } catch (e: Exception) { errors.add("Failed to fetch song $i: ${e.message}") println("[crawler] Error fetching song idx=$i: ${e.message}") + break } } - // Phase 3: Store in database - onProgress(Progress(0, 1, "Storing ${historicalPlays.size} plays...")) - playStore.clearHistorical() // Clear old historical data first - playStore.upsertHistoricalPlays(historicalPlays) + if (errors.isEmpty()) { + playStore.deleteHistoricalSyncedBefore(cursor.runStartedAt) + playStore.clearHistoricalBackfillCursor() + } - println("[crawler] Stored ${historicalPlays.size} historical plays") + println("[crawler] Stored $historicalPlayCount historical plays") return@withContext Result( totalSongs = idxList.size, - totalPlays = historicalPlays.size, + totalPlays = historicalPlayCount, errors = errors, ) } diff --git a/composeApp/src/commonMain/kotlin/com/derakkuma/data/PlayStore.kt b/composeApp/src/commonMain/kotlin/com/derakkuma/data/PlayStore.kt index 84dbc10..f19abb0 100644 --- a/composeApp/src/commonMain/kotlin/com/derakkuma/data/PlayStore.kt +++ b/composeApp/src/commonMain/kotlin/com/derakkuma/data/PlayStore.kt @@ -20,6 +20,15 @@ class PlayStore(driverFactory: DriverFactory) { private val queries by lazy { db.playHistoryQueries } private val json = Json { ignoreUnknownKeys = true } + @Serializable + data class HistoricalBackfillCursor( + val idxList: List, + val utageIdxs: List, + val nextIndex: Int, + val runStartedAt: Long, + val totalPlaysInserted: Int = 0, + ) + /** * A play record extracted from musicDetail pages. * Note: musicDetail only shows the MOST RECENT play for each song+difficulty, @@ -102,6 +111,47 @@ class PlayStore(driverFactory: DriverFactory) { } } + /** + * Insert a single historical play as soon as it is fetched. + * Keeps long historical syncs from losing all progress if the DX NET session expires mid-crawl. + */ + suspend fun upsertHistoricalPlay(play: HistoricalPlay, syncedAt: Long = Clock.System.now().toEpochMilliseconds()) = withContext(Dispatchers.IO) { + val id = generateId(play.datetime, play.songName, play.difficulty) + queries.insert( + id = id, + datetime = play.datetime, + song_name = play.songName, + difficulty = play.difficulty, + level = play.level, + achievement = play.achievement, + score_rank = play.scoreRank, + fc_status = play.fcStatus, + sync_status = play.syncStatus, + is_dx = if (play.isDx) 1L else 0L, + play_count = play.playCount.toLong(), + album_art_url = play.albumArtUrl, + detail_idx = play.detailIdx, + source = "historical", + synced_at = syncedAt, + ) + } + + suspend fun getHistoricalBackfillCursor(): HistoricalBackfillCursor? = withContext(Dispatchers.IO) { + queries.getBackfillState("historical_cursor").executeAsOneOrNull()?.let { json.decodeFromString(it) } + } + + suspend fun putHistoricalBackfillCursor(cursor: HistoricalBackfillCursor) = withContext(Dispatchers.IO) { + queries.upsertBackfillState("historical_cursor", json.encodeToString(cursor)) + } + + suspend fun clearHistoricalBackfillCursor() = withContext(Dispatchers.IO) { + queries.deleteBackfillState("historical_cursor") + } + + suspend fun deleteHistoricalSyncedBefore(syncedAt: Long) = withContext(Dispatchers.IO) { + queries.deleteHistoricalSyncedBefore(syncedAt) + } + suspend fun getByDateRange(startDate: String, endDate: String): List = withContext(Dispatchers.IO) { queries.getByDateRange(startDate, endDate).executeAsList().map { it.toHistoricalPlay() } } diff --git a/composeApp/src/commonMain/sqldelight/com/derakkuma/db/PlayHistory.sq b/composeApp/src/commonMain/sqldelight/com/derakkuma/db/PlayHistory.sq index 86a77f4..ee5c22e 100644 --- a/composeApp/src/commonMain/sqldelight/com/derakkuma/db/PlayHistory.sq +++ b/composeApp/src/commonMain/sqldelight/com/derakkuma/db/PlayHistory.sq @@ -25,6 +25,12 @@ CREATE TABLE IF NOT EXISTS play_detail ( fetched_at INTEGER NOT NULL ); +-- Persistent cursor for resumable long-running historical backfills. +CREATE TABLE IF NOT EXISTS play_history_backfill_state ( + key TEXT NOT NULL PRIMARY KEY, + data TEXT NOT NULL +); + -- Index for quick lookup by date range (heatmap uses this) CREATE INDEX IF NOT EXISTS idx_play_history_datetime ON play_history(datetime); @@ -58,6 +64,9 @@ DELETE FROM play_history WHERE datetime < ?; clearHistorical: DELETE FROM play_history WHERE source = 'historical'; +deleteHistoricalSyncedBefore: +DELETE FROM play_history WHERE source = 'historical' AND synced_at < ?; + clear: DELETE FROM play_history; @@ -75,3 +84,12 @@ SELECT detail_json FROM play_detail WHERE idx = ?; clearPlayDetails: DELETE FROM play_detail; + +getBackfillState: +SELECT data FROM play_history_backfill_state WHERE key = ?; + +upsertBackfillState: +INSERT OR REPLACE INTO play_history_backfill_state (key, data) VALUES (?, ?); + +deleteBackfillState: +DELETE FROM play_history_backfill_state WHERE key = ?; diff --git a/composeApp/src/commonMain/sqldelight/com/derakkuma/db/migrations/4.sqm b/composeApp/src/commonMain/sqldelight/com/derakkuma/db/migrations/4.sqm new file mode 100644 index 0000000..2703823 --- /dev/null +++ b/composeApp/src/commonMain/sqldelight/com/derakkuma/db/migrations/4.sqm @@ -0,0 +1,4 @@ +CREATE TABLE IF NOT EXISTS play_history_backfill_state ( + key TEXT NOT NULL PRIMARY KEY, + data TEXT NOT NULL +);