From 6913c3608aa4ac9d129de4d192f3b52ef62ef1da Mon Sep 17 00:00:00 2001 From: ToBinio Date: Tue, 16 Jun 2026 11:01:22 +0200 Subject: [PATCH] move global stats to sql --- server/api/stats/global.ts | 2 +- .../migrations/0001_strong_spectrum.sql | 7 + .../migrations/meta/0001_snapshot.json | 152 ++++++++++++++++++ server/database/migrations/meta/_journal.json | 7 + server/database/schema.ts | 14 ++ server/utils/api/export/globalStats.ts | 2 +- server/utils/db/kv.ts | 24 --- server/utils/db/sql.ts | 71 +++++++- server/utils/processing/global/processing.ts | 6 +- 9 files changed, 257 insertions(+), 28 deletions(-) create mode 100644 server/database/migrations/0001_strong_spectrum.sql create mode 100644 server/database/migrations/meta/0001_snapshot.json diff --git a/server/api/stats/global.ts b/server/api/stats/global.ts index 111a61d..fb0fd29 100644 --- a/server/api/stats/global.ts +++ b/server/api/stats/global.ts @@ -3,7 +3,7 @@ import type { GlobalStats } from "~~/server/utils/processing/global/types"; export default defineCachedEventHandler( async (event): Promise => { - const globalStats = await KV.GlobalStats.getLatest(); + const globalStats = await DB.GlobalStats.getLatest(); if (globalStats instanceof Error) { consola.error(globalStats.message); diff --git a/server/database/migrations/0001_strong_spectrum.sql b/server/database/migrations/0001_strong_spectrum.sql new file mode 100644 index 0000000..446c45c --- /dev/null +++ b/server/database/migrations/0001_strong_spectrum.sql @@ -0,0 +1,7 @@ +CREATE TABLE "global_stats" ( + "projects" integer NOT NULL, + "versions" integer NOT NULL, + "files" integer NOT NULL, + "authors" integer NOT NULL, + "date" date PRIMARY KEY NOT NULL +); diff --git a/server/database/migrations/meta/0001_snapshot.json b/server/database/migrations/meta/0001_snapshot.json new file mode 100644 index 0000000..c50aeed --- /dev/null +++ b/server/database/migrations/meta/0001_snapshot.json @@ -0,0 +1,152 @@ +{ + "id": "1827b468-795b-47a1-9b65-14b3d65ba15e", + "prevId": "0eb5e94d-5e9c-4d40-9ee7-288e4331c6f7", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.dependencies": { + "name": "dependencies", + "schema": "", + "columns": { + "project_id": { + "name": "project_id", + "type": "varchar(8)", + "primaryKey": false, + "notNull": true + }, + "dependency_id": { + "name": "dependency_id", + "type": "varchar(8)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "dependencies_project_id_modpacks_id_fk": { + "name": "dependencies_project_id_modpacks_id_fk", + "tableFrom": "dependencies", + "tableTo": "modpacks", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "dependencies_project_id_dependency_id_pk": { + "name": "dependencies_project_id_dependency_id_pk", + "columns": [ + "project_id", + "dependency_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.global_stats": { + "name": "global_stats", + "schema": "", + "columns": { + "projects": { + "name": "projects", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "versions": { + "name": "versions", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "files": { + "name": "files", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "authors": { + "name": "authors", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "date": { + "name": "date", + "type": "date", + "primaryKey": true, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.modpacks": { + "name": "modpacks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(8)", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "icon_url": { + "name": "icon_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "downloads": { + "name": "downloads", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/server/database/migrations/meta/_journal.json b/server/database/migrations/meta/_journal.json index 985262b..a0b7289 100644 --- a/server/database/migrations/meta/_journal.json +++ b/server/database/migrations/meta/_journal.json @@ -8,6 +8,13 @@ "when": 1781597912217, "tag": "0000_breezy_rick_jones", "breakpoints": true + }, + { + "idx": 1, + "version": "7", + "when": 1781599684292, + "tag": "0001_strong_spectrum", + "breakpoints": true } ] } \ No newline at end of file diff --git a/server/database/schema.ts b/server/database/schema.ts index 261c20f..9a14a2f 100644 --- a/server/database/schema.ts +++ b/server/database/schema.ts @@ -1,4 +1,5 @@ import { + date, integer, pgTable, primaryKey, @@ -28,3 +29,16 @@ export const modpacks = pgTable("modpacks", { }); export type NewModpack = typeof modpacks.$inferInsert; + +export const globalStats = pgTable("global_stats", { + // Values + projects: integer("projects").notNull(), + versions: integer("versions").notNull(), + files: integer("files").notNull(), + authors: integer("authors").notNull(), + + // Metadata + date: date("date").notNull().primaryKey(), +}); + +export type NewGlobalStats = typeof globalStats.$inferInsert; diff --git a/server/utils/api/export/globalStats.ts b/server/utils/api/export/globalStats.ts index 542fad8..7fd962f 100644 --- a/server/utils/api/export/globalStats.ts +++ b/server/utils/api/export/globalStats.ts @@ -34,7 +34,7 @@ export async function exportGlobalStatsOverTime( date = new_date; } - const data = await KV.GlobalStats.getBulk(dates); + const data = await DB.GlobalStats.getBulk(dates); days_to_fetch_left -= data.length; diff --git a/server/utils/db/kv.ts b/server/utils/db/kv.ts index 07e9310..1209ea6 100644 --- a/server/utils/db/kv.ts +++ b/server/utils/db/kv.ts @@ -2,7 +2,6 @@ import type { GameVersions, VersionCategories, } from "~~/server/utils/processing/gameVersions/types"; -import type { GlobalStats } from "~~/server/utils/processing/global/types"; import type { ProjectStats } from "~~/server/utils/processing/projects/types"; import { dateToKey } from "~~/shared/utils/date"; import type { ProjectTypes } from "~~/shared/utils/project"; @@ -38,29 +37,6 @@ export const KV = { return data; }, }, - GlobalStats: { - async set(data: GlobalStats, date: Date) { - const storage = useStorage("globalStatistics"); - const dateKey = dateToKey(date); - - await storage.setItem(`globalStats${dateKey}`, data); - }, - async getBulk(dates: Date[]): Promise<(GlobalStats | Error)[]> { - const keys = dates.map((date) => { - const dateKey = dateToKey(date); - return `globalStats${dateKey}`; - }); - - return getFromStorageCachedBulk("globalStatistics", keys); - }, - async getLatest(): Promise { - const dateKey = await KV.LatestDate.get(); - if (dateKey instanceof Error) return dateKey; - - const key = `globalStats${dateKey}`; - return getFromStorageCached("globalStatistics", key); - }, - }, ProjectStats: { async set( data: ProjectStats, diff --git a/server/utils/db/sql.ts b/server/utils/db/sql.ts index 883d495..2821a1d 100644 --- a/server/utils/db/sql.ts +++ b/server/utils/db/sql.ts @@ -2,10 +2,12 @@ import { count, desc, eq } from "drizzle-orm"; import { drizzle } from "drizzle-orm/node-postgres"; import { dependencies, + globalStats, modpacks, type NewDependency, type NewModpack, } from "~~/server/database/schema"; +import { dateToKey } from "~~/shared/utils/date"; import type { ProjectData } from "../processing/connections/types"; let _drizzle: ReturnType; @@ -19,14 +21,81 @@ export function useDrizzle() { return _drizzle; } -//TODO: dont catch in functions or force error handling outside +//TODO: dont catch in functions or force error handling outside (for functions that dont return values) +//TODO: cache functions again export const DB = { + GlobalStats: { + async set(data: GlobalStats, date: Date): Promise { + const db = useDrizzle(); + + try { + await db.insert(globalStats).values({ + ...data, + date: dateToKey(date), + }); + } catch (error) { + return new Error(`Failed to add connections: ${error}`); + } + }, + async getBulk(dates: Date[]): Promise<(GlobalStats | Error)[]> { + const db = useDrizzle(); + const results: (GlobalStats | Error)[] = []; + + for (const date of dates) { + try { + const result = await db + .select() + .from(globalStats) + .where(eq(globalStats.date, dateToKey(date))) + .limit(1); + + if (result.length === 0) { + results.push(new Error(`No global stats found for date ${date}`)); + continue; + } + + results.push({ + projects: result[0].projects, + versions: result[0].versions, + files: result[0].files, + authors: result[0].authors, + }); + } catch (error) { + results.push( + new Error(`Failed to get global stats for date ${date}: ${error}`), + ); + } + } + + return results; + }, + async getLatest(): Promise { + const db = useDrizzle(); + const result = await db + .select() + .from(globalStats) + .orderBy(desc(globalStats.date)) + .limit(1); + + if (result.length === 0) { + return new Error("No global stats found"); + } + + return { + projects: result[0].projects, + versions: result[0].versions, + files: result[0].files, + authors: result[0].authors, + }; + }, + }, async clear() { const db = useDrizzle(); await db.delete(dependencies); await db.delete(modpacks); }, + async getAllForProject( dependencyId: string, offset: number, diff --git a/server/utils/processing/global/processing.ts b/server/utils/processing/global/processing.ts index 2705c7d..3d0320f 100644 --- a/server/utils/processing/global/processing.ts +++ b/server/utils/processing/global/processing.ts @@ -6,7 +6,11 @@ export async function updateGlobalStats() { LOGGER.info("updating globalStats [starting]"); const data = await $modrinthFetch("/statistics"); - await KV.GlobalStats.set(data, new Date()); + const result = await DB.GlobalStats.set(data, new Date()); + if (result instanceof Error) { + LOGGER.error("failed to update globalStats", result); + throw result; + } LOGGER.info("updating globalStats [finished]"); } -- 2.51.2