diff --git a/app/layouts/default.vue b/app/layouts/default.vue index ac37392..9ad7ab8 100644 --- a/app/layouts/default.vue +++ b/app/layouts/default.vue @@ -18,18 +18,27 @@ - - - Charts - +
+ + + Charts + + + ModSearch + +
+ const text = ref(""); + + const result = useFetch("/api/reverseSearch", { + params: { + id: text, + }, + }); + + +> diff --git a/nuxt.config.ts b/nuxt.config.ts index 8f650c6..a36938a 100644 --- a/nuxt.config.ts +++ b/nuxt.config.ts @@ -76,6 +76,7 @@ export default defineNuxtConfig({ }, experimental: { tasks: true, + database: true, }, scheduledTasks: { "0 * * * *": ["analyze"], diff --git a/server/api/reverseSearch.ts b/server/api/reverseSearch.ts new file mode 100644 index 0000000..da41aa8 --- /dev/null +++ b/server/api/reverseSearch.ts @@ -0,0 +1,12 @@ +export default defineEventHandler(async (event): Promise => { + const query = getQuery(event); + + const projectIds = await DB.getForProject(query.id as string); + + if (projectIds instanceof Error) { + setResponseStatus(event, 500); + return []; + } + + return projectIds; +}); diff --git a/server/plugins/db.ts b/server/plugins/db.ts new file mode 100644 index 0000000..397f0ff --- /dev/null +++ b/server/plugins/db.ts @@ -0,0 +1,15 @@ +import consola from "consola"; + +export default defineNitroPlugin(async () => { + const db = useDatabase(); + + //TODO: handle via migrations + await db.sql`DROP TABLE IF EXISTS connections`; + await db.sql`CREATE TABLE IF NOT EXISTS connections ( + project_id VARCHAR(8), + dependency_id VARCHAR(8), + PRIMARY KEY (project_id, dependency_id) + )`; + + consola.info("Database initialized"); +}); diff --git a/server/tasks/analyze.ts b/server/tasks/analyze.ts index 82d00d8..b85fe14 100644 --- a/server/tasks/analyze.ts +++ b/server/tasks/analyze.ts @@ -2,6 +2,8 @@ import consola from "consola"; import { updateGameVersions } from "~~/server/utils/processing/gameVersions/processing"; import { updateGlobalStats } from "~~/server/utils/processing/global/processing"; import { updateStatistics } from "~~/server/utils/processing/projects/processing"; +import { DB } from "../utils/db/sql"; +import { updateConnections } from "../utils/processing/connections/processing"; export const LOGGER = consola.withTag("Analyze"); @@ -20,9 +22,14 @@ export default defineTask({ const start = new Date(); LOGGER.info("starting analyze - at", start.toISOString()); try { + //TODO: do something less destructive... + await DB.clear(); + await updateConnections(); + await updateGameVersions(); await updateStatistics(); await updateGlobalStats(); + await KV.LatestDate.set(new Date()); } catch (e) { LOGGER.fail(e); diff --git a/server/utils/db/sql.ts b/server/utils/db/sql.ts new file mode 100644 index 0000000..7e79199 --- /dev/null +++ b/server/utils/db/sql.ts @@ -0,0 +1,45 @@ +import type { Connection } from "../processing/connections/types"; + +export const DB = { + async clear(): Promise { + const db = useDatabase(); + + try { + await db.sql` + TRUNCATE TABLE connections; + `; + } catch (error) { + return new Error(`Failed to clear database: ${error}`); + } + }, + async getForProject(dependencyId: string): Promise { + const db = useDatabase(); + + const result = await db.sql` + SELECT project_id FROM connections WHERE dependency_id = ${dependencyId}; + `; + + if (!result.rows) { + return new Error( + `Failed to fetch connections for project ${dependencyId}: ${result.error}`, + ); + } + + return result.rows.map((row) => { + return row.project_id as string; + }); + }, + async addBulk(connections: Connection[]): Promise { + const db = useDatabase(); + + try { + //TODO: dont use becouse of sql injection risk + await db.sql` + INSERT INTO connections (project_id, dependency_id) + VALUES {${connections.map((connection) => `('${connection.project_id}', '${connection.dependency_id}')`).join(", ")}}; + `; + } catch (error) { + return new Error(`Failed to add connections: ${error}`); + } + }, +}; diff --git a/server/utils/processing/connections/fetching.ts b/server/utils/processing/connections/fetching.ts new file mode 100644 index 0000000..e4a2360 --- /dev/null +++ b/server/utils/processing/connections/fetching.ts @@ -0,0 +1,49 @@ +export async function getModpackIds( + offset: number, + limit: number, +): Promise { + return getProjectIds(offset, "modpack", limit); +} + +export async function getFirstVersionIds( + projectIds: string[], +): Promise { + type Project = { + versions: string[]; + }; + + const data = await $modrinthFetch("/projects", { + query: { + ids: `["${projectIds.join('","')}"]`, + }, + }); + + return data.map((value) => value.versions[0]); +} + +export async function getVersionDependencies( + versionIds: string[], +): Promise<{ project_id: string; dependencies: string[] }[]> { + type Version = { + project_id: string; + dependencies: { project_id: string }[]; + }; + + const data = await $modrinthFetch("/versions", { + query: { + ids: `["${versionIds.join('","')}"]`, + }, + }); + + //done to remove unused values from the ram + return data.map((value) => { + const dependencyIds = Array.from( + new Set(value.dependencies.map((d) => d.project_id).filter(Boolean)), + ); + + return { + project_id: value.project_id, + dependencies: dependencyIds, + }; + }); +} diff --git a/server/utils/processing/connections/processing.ts b/server/utils/processing/connections/processing.ts new file mode 100644 index 0000000..4832327 --- /dev/null +++ b/server/utils/processing/connections/processing.ts @@ -0,0 +1,56 @@ +import { LOGGER } from "~~/server/tasks/analyze"; +import { DB } from "../../db/sql"; +import { + getFirstVersionIds, + getModpackIds, + getVersionDependencies, +} from "./fetching"; + +export async function updateConnections() { + LOGGER.info("updating connections [starting]"); + + const BATCH_SIZE = import.meta.dev ? 1 : 10_000; + + let projectIndex = 0; + while (true) { + let done = false; + + const batchProjectIds = []; + while (batchProjectIds.length < BATCH_SIZE) { + const projectIds = await getModpackIds(projectIndex, 100); + batchProjectIds.push(...projectIds); + projectIndex += 100; + + if (projectIds.length !== 100) { + done = true; + break; + } + } + + const versionIds: string[] = []; + for (const chunk of chunkArray(batchProjectIds, 200)) { + const versions = await getFirstVersionIds(chunk); + versionIds.push(...versions); + } + + const versionDependencies = await getVersionDependencies(versionIds); + for (const pair of versionDependencies) { + const connections = pair.dependencies.map((dependency) => { + return { + project_id: pair.project_id, + dependency_id: dependency, + }; + }); + + const result = await DB.addBulk(connections); + + if (result instanceof Error) { + LOGGER.warn(result, connections); + } + } + + if (done || import.meta.dev) break; + } + + LOGGER.info("updating connections [finished]"); +} diff --git a/server/utils/processing/connections/types.ts b/server/utils/processing/connections/types.ts new file mode 100644 index 0000000..6cb9187 --- /dev/null +++ b/server/utils/processing/connections/types.ts @@ -0,0 +1,4 @@ +export type Connection = { + project_id: string; + dependency_id: string; +};