diff --git a/server/utils/fetchData.ts b/server/utils/fetchData.ts index 290aa84..46f11b7 100644 --- a/server/utils/fetchData.ts +++ b/server/utils/fetchData.ts @@ -1,12 +1,11 @@ import {$modrinthFetch} from "~/utils/modrinthFetch"; -export async function getProjectIds(offset: number, type: string): Promise { +export async function getProjectIds(offset: number, type: string, limit: number): Promise { const data = await $modrinthFetch("/search", { query: { - limit: 100, + limit: limit, offset: offset, - //todo - remove newest - index: "newest", + index: import.meta.dev ? "newest" : "downloads", facets: `[[\"project_types:${type}\"]]` } }) diff --git a/server/utils/statistics.ts b/server/utils/statistics.ts index 2f6d07a..3cb94eb 100644 --- a/server/utils/statistics.ts +++ b/server/utils/statistics.ts @@ -22,17 +22,37 @@ async function saveStats(stats: AllStats, type: string) { } async function getStatistics(type: string): Promise { - const projectIds = await getProjectIds(0, type); - console.log("project ids", projectIds.length); + const BATCH_COUNT = import.meta.dev ? 1 : 10; + let data: StatsData = new Map() - const versionIds = (await getVersionIds(projectIds)) - console.log("version ids", versionIds.length); + let index = 0; - let data: StatsData = new Map() - await analyzeVersionsFromIds(versionIds, data); + while (true) { + let done = false; - let versions = await getGameVersions() + const batchProjectIds = [] + for (let i = 0; i < BATCH_COUNT; i++) { + const projectIds = await getProjectIds(index, type, 100); + batchProjectIds.push(...projectIds); + index += 100; + + if (projectIds.length != 100) { + done = true; + break + } + } + console.log("project ids", batchProjectIds.length); + + const versionIds = (await getVersionIds(batchProjectIds)) + console.log("version ids", versionIds.length); + + await analyzeVersionsFromIds(versionIds, data); + + if (done || import.meta.dev) break + } + + let versions = await getGameVersions() let allDownloads = StatsFromData(versions, data); let {gameVersions: minorGameVersions, downloads: minorVersionDownloads} = onlyMinorVersions(versions, allDownloads); let majorVersionDownloads = onlyMajorVersions(minorGameVersions, minorVersionDownloads);