From e4be318f779cbfc45564c133e71902a0dbc251e5 Mon Sep 17 00:00:00 2001 From: ToBinio Date: Thu, 18 Jun 2026 09:51:40 +0200 Subject: [PATCH] import data task --- justfile | 6 + package.json | 4 + pnpm-lock.yaml | 41 +++ pnpm-workspace.yaml | 4 +- .../migrations/0003_boring_colleen_wing.sql | 5 + .../migrations/meta/0003_snapshot.json | 256 ++++++++++++++++++ server/database/migrations/meta/_journal.json | 9 +- server/database/schema.ts | 11 +- server/tasks/script/import.ts | 89 ++++++ server/utils/db/sql.ts | 21 +- 10 files changed, 437 insertions(+), 9 deletions(-) create mode 100644 server/database/migrations/0003_boring_colleen_wing.sql create mode 100644 server/database/migrations/meta/0003_snapshot.json create mode 100644 server/tasks/script/import.ts diff --git a/justfile b/justfile index 595edba..b67bf89 100644 --- a/justfile +++ b/justfile @@ -10,3 +10,9 @@ postgres: postgres_down: docker compose -f postgres.compose.yml down + +run-task name param: + curl -X POST \ + "http://localhost:3000/_nitro/tasks/{{ name }}" \ + -H "Content-Type: application/json" \ + -d '{ "payload": {{ param }} }' diff --git a/package.json b/package.json index 08d9833..4f2eee6 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,8 @@ "dependencies": { "@nuxt/kit": "^4.2.2", "@nuxtjs/sitemap": "7.4.9", + "@types/stream-chain": "^2.1.0", + "@types/stream-json": "^1.7.8", "@upstash/redis": "^1.35.8", "chart.js": "^4.5.1", "consola": "^3.4.2", @@ -28,6 +30,8 @@ "h3": "1.15.4", "nuxt": "^4.2.2", "pg": "^8.21.0", + "stream-chain": "^4.2.5", + "stream-json": "^3.4.0", "tailwindcss": "^4.2.2", "unstorage": "^1.17.3", "vue": "^3.5.25", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0e1e340..add1028 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14,6 +14,12 @@ importers: '@nuxtjs/sitemap': specifier: 7.4.9 version: 7.4.9(h3@1.15.4)(magicast@0.5.1)(vite@7.3.0(@types/node@25.0.3)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.44.1)(yaml@2.8.2))(vue@3.5.25(typescript@5.9.2))(zod@4.2.1) + '@types/stream-chain': + specifier: ^2.1.0 + version: 2.1.0 + '@types/stream-json': + specifier: ^1.7.8 + version: 1.7.8 '@upstash/redis': specifier: ^1.35.8 version: 1.35.8 @@ -35,6 +41,12 @@ importers: pg: specifier: ^8.21.0 version: 8.21.0 + stream-chain: + specifier: ^4.2.5 + version: 4.2.5 + stream-json: + specifier: ^3.4.0 + version: 3.4.0 tailwindcss: specifier: ^4.2.2 version: 4.2.2 @@ -1842,6 +1854,12 @@ packages: '@types/resolve@1.20.2': resolution: {integrity: sha512-60BCwRFOZCQhDncwQdxxeOEEkbc5dIMccYLwbxsS4TUNeVECQ/pBJ0j09mrHOl/JJvpRPGwO9SvE4nR2Nb/a4Q==} + '@types/stream-chain@2.1.0': + resolution: {integrity: sha512-guDyAl6s/CAzXUOWpGK2bHvdiopLIwpGu8v10+lb9hnQOyo4oj/ZUQFOvqFjKGsE3wJP1fpIesCcMvbXuWsqOg==} + + '@types/stream-json@1.7.8': + resolution: {integrity: sha512-MU1OB1eFLcYWd1LjwKXrxdoPtXSRzRmAnnxs4Js/ayB5O/NvHraWwuOaqMWIebpYwM6khFlsJOHEhI9xK/ab4Q==} + '@types/web-bluetooth@0.0.21': resolution: {integrity: sha512-oIQLCGWtcFZy2JW77j9k8nHzAOpqMHLQejDA48XXMWH6tjCQHz5RCFz1bzsmROyL6PUm+LLnUiI4BCn221inxA==} @@ -4021,6 +4039,14 @@ packages: std-env@3.10.0: resolution: {integrity: sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==} + stream-chain@4.2.5: + resolution: {integrity: sha512-Wtyq3bNE3ggLR0v2vftqvuhltym3WbZAkZpfIrkr5F/6vpeUmWmwTgXa16zD87gpahwJ/Qulq3zVfUlgIc0J2A==} + engines: {node: '>=22'} + + stream-json@3.4.0: + resolution: {integrity: sha512-7NEiOhtS0W7ZrwnqJc/fugsGQMdpNJng+v1s2xLTrmfBN2vcYDyBglCK2+eITif0VkHLXnTdALgQI9TnZMBhYA==} + engines: {node: '>=22'} + streamx@2.23.0: resolution: {integrity: sha512-kn+e44esVfn2Fa/O0CPFcex27fjIL6MkVae0Mm6q+E6f0hWv578YCERbv+4m02cjxvDsPKLnmxral/rR6lBMAg==} @@ -6229,6 +6255,15 @@ snapshots: '@types/resolve@1.20.2': {} + '@types/stream-chain@2.1.0': + dependencies: + '@types/node': 25.0.3 + + '@types/stream-json@1.7.8': + dependencies: + '@types/node': 25.0.3 + '@types/stream-chain': 2.1.0 + '@types/web-bluetooth@0.0.21': {} '@types/whatwg-mimetype@3.0.2': {} @@ -8616,6 +8651,12 @@ snapshots: std-env@3.10.0: {} + stream-chain@4.2.5: {} + + stream-json@3.4.0: + dependencies: + stream-chain: 4.2.5 + streamx@2.23.0: dependencies: events-universal: 1.0.1 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 5abead7..c186e2d 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,4 +1,4 @@ allowBuilds: - '@parcel/watcher': false - better-sqlite3: false + "@parcel/watcher": false + deasync: true esbuild: false diff --git a/server/database/migrations/0003_boring_colleen_wing.sql b/server/database/migrations/0003_boring_colleen_wing.sql new file mode 100644 index 0000000..fb1f9ff --- /dev/null +++ b/server/database/migrations/0003_boring_colleen_wing.sql @@ -0,0 +1,5 @@ +ALTER TABLE "global_stats" ALTER COLUMN "versions" SET DATA TYPE bigint;--> statement-breakpoint +ALTER TABLE "global_stats" ALTER COLUMN "files" SET DATA TYPE bigint;--> statement-breakpoint +ALTER TABLE "project_stats" ALTER COLUMN "downloads" SET DATA TYPE bigint;--> statement-breakpoint +ALTER TABLE "project_stats" ALTER COLUMN "versions" SET DATA TYPE bigint;--> statement-breakpoint +ALTER TABLE "project_stats" ALTER COLUMN "count" SET DATA TYPE bigint; \ No newline at end of file diff --git a/server/database/migrations/meta/0003_snapshot.json b/server/database/migrations/meta/0003_snapshot.json new file mode 100644 index 0000000..f316da6 --- /dev/null +++ b/server/database/migrations/meta/0003_snapshot.json @@ -0,0 +1,256 @@ +{ + "id": "2686b21f-04d0-46f4-8e4f-9d008f353773", + "prevId": "298b38ca-d53b-4543-bc52-970358a2a0f7", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.dependencies": { + "name": "dependencies", + "schema": "", + "columns": { + "project_id": { + "name": "project_id", + "type": "varchar(8)", + "primaryKey": false, + "notNull": true + }, + "dependency_id": { + "name": "dependency_id", + "type": "varchar(8)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "dependencies_project_id_modpacks_id_fk": { + "name": "dependencies_project_id_modpacks_id_fk", + "tableFrom": "dependencies", + "tableTo": "modpacks", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "dependencies_project_id_dependency_id_pk": { + "name": "dependencies_project_id_dependency_id_pk", + "columns": [ + "project_id", + "dependency_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.global_stats": { + "name": "global_stats", + "schema": "", + "columns": { + "projects": { + "name": "projects", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "versions": { + "name": "versions", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "files": { + "name": "files", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "authors": { + "name": "authors", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "date": { + "name": "date", + "type": "date", + "primaryKey": true, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.modpacks": { + "name": "modpacks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(8)", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "icon_url": { + "name": "icon_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "downloads": { + "name": "downloads", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.project_stats": { + "name": "project_stats", + "schema": "", + "columns": { + "downloads": { + "name": "downloads", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "versions": { + "name": "versions", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "count": { + "name": "count", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "loader": { + "name": "loader", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "date": { + "name": "date", + "type": "date", + "primaryKey": false, + "notNull": true + }, + "projectType": { + "name": "projectType", + "type": "project_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "versionCategory": { + "name": "versionCategory", + "type": "version_category", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "excluded": { + "name": "excluded", + "type": "boolean", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "project_stats_version_loader_date_projectType_versionCategory_excluded_pk": { + "name": "project_stats_version_loader_date_projectType_versionCategory_excluded_pk", + "columns": [ + "version", + "loader", + "date", + "projectType", + "versionCategory", + "excluded" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.project_type": { + "name": "project_type", + "schema": "public", + "values": [ + "mod", + "plugin", + "datapack", + "shader", + "resourcepack", + "modpack" + ] + }, + "public.version_category": { + "name": "version_category", + "schema": "public", + "values": [ + "all", + "major", + "minor", + "unified" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/server/database/migrations/meta/_journal.json b/server/database/migrations/meta/_journal.json index 74dbb2a..b4de9d2 100644 --- a/server/database/migrations/meta/_journal.json +++ b/server/database/migrations/meta/_journal.json @@ -22,6 +22,13 @@ "when": 1781601788874, "tag": "0002_romantic_lucky_pierre", "breakpoints": true + }, + { + "idx": 3, + "version": "7", + "when": 1781766491223, + "tag": "0003_boring_colleen_wing", + "breakpoints": true } ] -} +} \ No newline at end of file diff --git a/server/database/schema.ts b/server/database/schema.ts index b9c81ac..2046bfa 100644 --- a/server/database/schema.ts +++ b/server/database/schema.ts @@ -1,4 +1,5 @@ import { + bigint, boolean, date, integer, @@ -35,8 +36,8 @@ export type NewModpack = typeof modpacks.$inferInsert; export const globalStats = pgTable("global_stats", { // Values projects: integer("projects").notNull(), - versions: integer("versions").notNull(), - files: integer("files").notNull(), + versions: bigint("versions", { mode: "number" }).notNull(), + files: bigint("files", { mode: "number" }).notNull(), authors: integer("authors").notNull(), // Metadata @@ -65,9 +66,9 @@ export const projectStats = pgTable( "project_stats", { // Values - downloads: integer("downloads").notNull(), - versions: integer("versions").notNull(), - count: integer("count").notNull(), + downloads: bigint("downloads", { mode: "number" }).notNull(), + versions: bigint("versions", { mode: "number" }).notNull(), + count: bigint("count", { mode: "number" }).notNull(), // Version Info version: text("version").notNull(), diff --git a/server/tasks/script/import.ts b/server/tasks/script/import.ts new file mode 100644 index 0000000..91d3b4f --- /dev/null +++ b/server/tasks/script/import.ts @@ -0,0 +1,89 @@ +import fs from "node:fs"; +import { parser } from "stream-json"; +import { streamArray } from "stream-json/streamers/stream-array.js"; +import chain from "stream-chain"; +import { GlobalStats } from "~~/server/utils/processing/global/types"; +import { keyToDate } from "~~/shared/utils/date"; +import { ProjectTypes } from "~~/shared/utils/project"; + +export default defineTask({ + meta: { + name: "script:import", + description: "Import data from the local file", + }, + async run({ payload }) { + const path = payload.path as string; + if (!path) { + return { error: "No path provided" }; + } + + const pipeline = chain([ + fs.createReadStream(path), + parser(), + streamArray<{ key: string; value: string }>(), + ]); + + for await (const data of pipeline) { + const { key, value } = data.value; + + if (key.startsWith("globalStatistics:")) { + const json: GlobalStats = JSON.parse(value); + const date = keyToDate(key.replace("globalStatistics:globalStats", "")); + + const result = await DB.GlobalStats.set(json, date); + if (result instanceof Error) { + return { error: result.message }; + } + } else if (key.startsWith("projectStatistics:")) { + const json: ProjectStats = JSON.parse(value); + const keyData = parseProjectStorageKey( + key.replace("projectStatistics:", ""), + ); + + const result = await DB.ProjectStats.set_for_date( + keyToDate(keyData.dateKey), + json, + keyData.type, + keyData.versionCategory, + keyData.exclusive, + ); + if (result instanceof Error) { + return { error: result.message }; + } + } + + if (data.key % 100 === 0) { + console.log(`Imported ${data.key}`); + } + } + + return { result: "ok" }; + }, +}); + +function parseProjectStorageKey(key: string) { + const statsIndex = key.indexOf("Stats"); + + const type = key.slice(0, statsIndex) as ProjectTypes; + + let rest = key.slice(statsIndex + "Stats".length); + + const exclusive = rest.includes("Exclusive"); + + if (exclusive) { + rest = rest.replace("Exclusive", ""); + } + + const versionCategoryMatch = rest.match(/^[A-Z][a-z]*/)!; + const versionCategory = (versionCategoryMatch[0][0].toLowerCase() + + versionCategoryMatch[0].slice(1)) as VersionCategories; + + const dateKey = rest.slice(versionCategoryMatch[0].length); + + return { + type, + versionCategory, + exclusive, + dateKey, + }; +} diff --git a/server/utils/db/sql.ts b/server/utils/db/sql.ts index d74458a..7fc2344 100644 --- a/server/utils/db/sql.ts +++ b/server/utils/db/sql.ts @@ -34,10 +34,25 @@ export const DB = { type: ProjectTypes, versionCategory: VersionCategories, exclusive: boolean, + ): Promise { + return DB.ProjectStats.set_for_date( + new Date(), + data, + type, + versionCategory, + exclusive, + ); + }, + async set_for_date( + date: Date, + data: ProjectStats, + type: ProjectTypes, + versionCategory: VersionCategories, + exclusive: boolean, ): Promise { const db = useDrizzle(); - const dateKey = dateToKey(new Date()); + const dateKey = dateToKey(date); const to_insert: NewProjectStats[] = []; for (let i = 0; i < data.versions.length; i++) { @@ -80,6 +95,8 @@ export const DB = { } } + if (to_insert.length === 0) return; + try { await db .insert(projectStats) @@ -100,6 +117,7 @@ export const DB = { }, }); } catch (error) { + console.error(error); return new Error(`Failed to add project stats: ${error}`); } }, @@ -229,6 +247,7 @@ export const DB = { }, }); } catch (error) { + console.error(error); return new Error(`Failed to add global stats: ${error}`); } }, -- 2.51.2