From 7a1fc23e5a86781e2e401c47dec784df8c37c787 Mon Sep 17 00:00:00 2001
From: tobinio <Tobias.frischmann1@gmail.com>
Date: Mon, 17 Jun 2024 21:53:29 +0200
Subject: [PATCH] load data from modrinth

---
 composables/useModrinthFetch.ts     |  11 +++
 layouts/default.vue                 |  19 +++-
 pages/downloads.vue                 |   3 +
 pages/index.vue                     |   3 +-
 pages/majorVersionDownloads.vue     |  52 ++++++++++
 pages/minorVersionDownloads.vue     |  52 ++++++++++
 server/api/downloads.ts             |  42 ++++----
 server/api/majorVersionDownloads.ts |  35 +++++++
 server/api/minorVersionDownloads.ts |  35 +++++++
 server/api/update.ts                | 147 ++++++++++++++++++++++++++++
 server/utils/color.ts               |   4 +-
 server/utils/fetchData.ts           |  56 +++++++++++
 server/utils/types/downloads.ts     |   7 ++
 utils/modrinthFetch.ts              |   4 +
 14 files changed, 445 insertions(+), 25 deletions(-)
 create mode 100644 composables/useModrinthFetch.ts
 create mode 100644 pages/majorVersionDownloads.vue
 create mode 100644 pages/minorVersionDownloads.vue
 create mode 100644 server/api/majorVersionDownloads.ts
 create mode 100644 server/api/minorVersionDownloads.ts
 create mode 100644 server/api/update.ts
 create mode 100644 server/utils/fetchData.ts
 create mode 100644 server/utils/types/downloads.ts
 create mode 100644 utils/modrinthFetch.ts

diff --git a/composables/useModrinthFetch.ts b/composables/useModrinthFetch.ts
new file mode 100644
index 0000000..4779ace
--- /dev/null
+++ b/composables/useModrinthFetch.ts
@@ -0,0 +1,11 @@
+import type {UseFetchOptions} from "#app";
+
+export function useModrinthFetch<T>(
+    url: string | (() => string),
+    options: UseFetchOptions<T> = {}
+) {
+    return useFetch(url, {
+        ...options,
+        $fetch: $modrinthFetch,
+    })
+}
\ No newline at end of file
diff --git a/layouts/default.vue b/layouts/default.vue
index 61351ef..9b2314a 100644
--- a/layouts/default.vue
+++ b/layouts/default.vue
@@ -1,5 +1,7 @@
 <script setup lang="ts">
-
+async function onUpdateData() {
+  await $fetch("/api/update")
+}
 </script>
 
 <template>
@@ -7,9 +9,18 @@
     <NuxtLink id="header" href="/">
       Modrinth Statistics
     </NuxtLink>
+    <button @click="onUpdateData">
+      update Data
+    </button>
     <div>
       <NuxtLink href="/downloads">
-        Downloads
+        All Downloads
+      </NuxtLink>
+      <NuxtLink href="/minorVersionDownloads">
+        minor Downloads
+      </NuxtLink>
+      <NuxtLink href="/majorVersionDownloads">
+        major Downloads
       </NuxtLink>
     </div>
   </div>
@@ -34,6 +45,10 @@
     font-size: xx-large;
   }
 
+  button {
+    color: black;
+  }
+
   a {
     font-size: large;
 
diff --git a/pages/downloads.vue b/pages/downloads.vue
index 1cce2d5..d6df085 100644
--- a/pages/downloads.vue
+++ b/pages/downloads.vue
@@ -34,6 +34,9 @@ const chartOptions = ref({
   scales: {
     x: {
       stacked: true
+    },
+    y: {
+      stacked: true
     }
   },
 })
diff --git a/pages/index.vue b/pages/index.vue
index 8e7637f..1f391b7 100644
--- a/pages/index.vue
+++ b/pages/index.vue
@@ -5,7 +5,8 @@ type statistics = {
   "files": number,
   "authors": number
 };
-const {data} = useFetch<statistics>("https://api.modrinth.com/v2/statistics");
+
+const {data} = useModrinthFetch<statistics>("/statistics");
 </script>
 
 <template>
diff --git a/pages/majorVersionDownloads.vue b/pages/majorVersionDownloads.vue
new file mode 100644
index 0000000..dbc7971
--- /dev/null
+++ b/pages/majorVersionDownloads.vue
@@ -0,0 +1,52 @@
+<script setup lang="ts">
+import {Bar} from 'vue-chartjs'
+
+const {data} = await useFetch("/api/majorVersionDownloads");
+let downloadData = computed(() => {
+  if (data.value) {
+    return data.value
+  }
+
+  return {
+    labels: [],
+    data: []
+  }
+})
+
+const chartData = computed(() => {
+  const datasets = downloadData.value.data.map((launcher) => {
+    return {
+      label: launcher.name,
+      backgroundColor: launcher.color,
+      data: launcher.downloads,
+    }
+  })
+
+  return {
+    labels: downloadData.value.labels,
+    datasets: datasets
+  }
+})
+
+const chartOptions = ref({
+  responsive: true,
+  maintainAspectRatio: false,
+  scales: {
+    x: {
+      stacked: true
+    },
+    y: {
+      stacked: true
+    }
+  },
+})
+</script>
+
+<template>
+  <Bar
+      :data="chartData"
+      :options="chartOptions"
+  />
+</template>
+<style scoped>
+</style>
\ No newline at end of file
diff --git a/pages/minorVersionDownloads.vue b/pages/minorVersionDownloads.vue
new file mode 100644
index 0000000..6bf3aa7
--- /dev/null
+++ b/pages/minorVersionDownloads.vue
@@ -0,0 +1,52 @@
+<script setup lang="ts">
+import {Bar} from 'vue-chartjs'
+
+const {data} = await useFetch("/api/minorVersionDownloads");
+let downloadData = computed(() => {
+  if (data.value) {
+    return data.value
+  }
+
+  return {
+    labels: [],
+    data: []
+  }
+})
+
+const chartData = computed(() => {
+  const datasets = downloadData.value.data.map((launcher) => {
+    return {
+      label: launcher.name,
+      backgroundColor: launcher.color,
+      data: launcher.downloads,
+    }
+  })
+
+  return {
+    labels: downloadData.value.labels,
+    datasets: datasets
+  }
+})
+
+const chartOptions = ref({
+  responsive: true,
+  maintainAspectRatio: false,
+  scales: {
+    x: {
+      stacked: true
+    },
+    y: {
+      stacked: true
+    }
+  },
+})
+</script>
+
+<template>
+  <Bar
+      :data="chartData"
+      :options="chartOptions"
+  />
+</template>
+<style scoped>
+</style>
\ No newline at end of file
diff --git a/server/api/downloads.ts b/server/api/downloads.ts
index da60e93..4212023 100644
--- a/server/api/downloads.ts
+++ b/server/api/downloads.ts
@@ -1,4 +1,4 @@
-import {getColorForLauncher} from "~/server/utils/color";
+import {Downloads} from "~/server/utils/types/downloads";
 
 type result = {
     labels: string[]
@@ -9,27 +9,27 @@ type result = {
     }[]
 }
 
-export default defineEventHandler((event): result => {
+export default defineEventHandler(async (event): Promise<result> => {
+    const storage = useStorage();
+    let downloads = await storage.getItem<Downloads>("allDownloads")
+
+    if (!downloads) {
+        // todo - error message
         return {
-            labels: ["1.16.5", "1.20.1", "1.21.0"],
-            data: [
-                {
-                    name: "forge",
-                    color: getColorForLauncher("forge"),
-                    downloads: [5, 10, 100]
-                },
-                {
-                    name: "neoforge",
-                    color: getColorForLauncher("neoforge"),
-                    downloads: [100, 10, 500]
-                },
-                {
-                    name: "fabric",
-                    color: getColorForLauncher("fabric"),
-                    downloads: [50, 100, 2000]
-                }
-            ]
+            labels: [],
+            data: []
         }
     }
-)
+
+    return {
+        labels: downloads.versions,
+        data: downloads.data.map(value => {
+            return {
+                name: value.name,
+                color: getColorForLauncher(value.name),
+                downloads: value.downloads
+            }
+        })
+    }
+})
 
diff --git a/server/api/majorVersionDownloads.ts b/server/api/majorVersionDownloads.ts
new file mode 100644
index 0000000..9aaec93
--- /dev/null
+++ b/server/api/majorVersionDownloads.ts
@@ -0,0 +1,35 @@
+import {Downloads} from "~/server/utils/types/downloads";
+
+type result = {
+    labels: string[]
+    data: {
+        name: string
+        color: string,
+        downloads: number[]
+    }[]
+}
+
+export default defineEventHandler(async (event): Promise<result> => {
+    const storage = useStorage();
+    let downloads = await storage.getItem<Downloads>("majorVersionDownloads")
+
+    if (!downloads) {
+        // todo - error message
+        return {
+            labels: [],
+            data: []
+        }
+    }
+
+    return {
+        labels: downloads.versions,
+        data: downloads.data.map(value => {
+            return {
+                name: value.name,
+                color: getColorForLauncher(value.name),
+                downloads: value.downloads
+            }
+        })
+    }
+})
+
diff --git a/server/api/minorVersionDownloads.ts b/server/api/minorVersionDownloads.ts
new file mode 100644
index 0000000..3e4d82e
--- /dev/null
+++ b/server/api/minorVersionDownloads.ts
@@ -0,0 +1,35 @@
+import {Downloads} from "~/server/utils/types/downloads";
+
+type result = {
+    labels: string[]
+    data: {
+        name: string
+        color: string,
+        downloads: number[]
+    }[]
+}
+
+export default defineEventHandler(async (event): Promise<result> => {
+    const storage = useStorage();
+    let downloads = await storage.getItem<Downloads>("minorVersionDownloads")
+
+    if (!downloads) {
+        // todo - error message
+        return {
+            labels: [],
+            data: []
+        }
+    }
+
+    return {
+        labels: downloads.versions,
+        data: downloads.data.map(value => {
+            return {
+                name: value.name,
+                color: getColorForLauncher(value.name),
+                downloads: value.downloads
+            }
+        })
+    }
+})
+
diff --git a/server/api/update.ts b/server/api/update.ts
new file mode 100644
index 0000000..5b7ed32
--- /dev/null
+++ b/server/api/update.ts
@@ -0,0 +1,147 @@
+import {Downloads} from "~/server/utils/types/downloads";
+import downloads from "~/server/api/downloads";
+import {min} from "@floating-ui/utils";
+
+export default defineEventHandler(async (event) => {
+
+    const projectIds = await getProjectIds(0);
+    console.log("project ids", projectIds.length);
+
+    const versionIds = (await getVersionIds(projectIds)).slice(0, 1000)
+    console.log("version ids", versionIds.length);
+
+    const versions = await getVersions(versionIds)
+    console.log("versions", versions.length);
+
+    let data = new Map<string, Map<string, number>>()
+    let versionsSet = new Set<string>()
+
+    for (let version of versions) {
+        for (let loader of version.loaders) {
+            if (loader == "datapack") {
+                continue
+            }
+
+            let downloads = new Map<string, number>();
+
+            if (data.has(loader)) {
+                downloads = data.get(loader)!;
+            } else {
+                data.set(loader, downloads)
+            }
+
+            for (let gameVersion of version.game_versions) {
+                versionsSet.add(gameVersion);
+
+                if (downloads.has(gameVersion)) {
+                    downloads.set(gameVersion, downloads.get(gameVersion)! + version.downloads)
+                } else {
+                    downloads.set(gameVersion, version.downloads)
+                }
+            }
+        }
+    }
+
+    let allDownloads = downloadsFromData(versionsSet, data);
+    let minorVersionDownloads = onlyMinorVersions(allDownloads);
+    let majorVersionDownloads = onlyMajorVersions(allDownloads);
+
+    const storage = useStorage();
+    await storage.setItem<Downloads>("allDownloads", allDownloads)
+    await storage.setItem<Downloads>("minorVersionDownloads", minorVersionDownloads)
+    await storage.setItem<Downloads>("majorVersionDownloads", majorVersionDownloads)
+})
+
+function onlyMinorVersions(all: Downloads): Downloads {
+
+    let data = new Map<string, Map<string, number>>()
+    let versionsSet = new Set<string>()
+
+    for (let loader of all.data) {
+        let downloads = new Map<string, number>();
+
+        for (let i = 0; i < all.versions.length; i++) {
+            let versionText = all.versions[i];
+
+            //todo - dont somehow handle snapshot versions
+            if (versionText.includes("w")) {
+                continue
+            }
+
+            let version = versionText.split("-")[0];
+
+            versionsSet.add(version);
+
+            if (downloads.has(version)) {
+                downloads.set(version, downloads.get(version)! + loader.downloads[i])
+            } else {
+                downloads.set(version, loader.downloads[i])
+            }
+        }
+
+        data.set(loader.name, downloads)
+    }
+
+    return downloadsFromData(versionsSet, data);
+}
+
+function onlyMajorVersions(all: Downloads): Downloads {
+
+    let data = new Map<string, Map<string, number>>()
+    let versionsSet = new Set<string>()
+
+    for (let loader of all.data) {
+        let downloads = new Map<string, number>();
+
+        for (let i = 0; i < all.versions.length; i++) {
+            let versionText = all.versions[i];
+
+            //todo - dont somehow handle snapshot versions
+            if (versionText.includes("w")) {
+                continue
+            }
+
+            let version = versionText.split("-")[0];
+            version = version.split(".").slice(0, 2).join(".")
+
+            versionsSet.add(version);
+
+            if (downloads.has(version)) {
+                downloads.set(version, downloads.get(version)! + loader.downloads[i])
+            } else {
+                downloads.set(version, loader.downloads[i])
+            }
+        }
+
+        data.set(loader.name, downloads)
+    }
+
+    return downloadsFromData(versionsSet, data);
+}
+
+function downloadsFromData(versionsSet: Set<string>, data: Map<string, Map<string, number>>): Downloads {
+    const versionArray = Array.from(versionsSet)
+    versionArray.sort((a, b) => a.localeCompare(b, undefined, {numeric: true}))
+
+    const loaderData = []
+
+    for (let loader of data) {
+        let downloadsMap = loader[1];
+        let downloads = []
+
+        for (let version of versionArray) {
+            if (downloadsMap.has(version)) {
+                downloads.push(downloadsMap.get(version)!);
+            } else {
+                downloads.push(0)
+            }
+        }
+
+        loaderData.push({
+            name: loader[0],
+            downloads: downloads
+        })
+    }
+
+    return {versions: versionArray, data: loaderData}
+}
\ No newline at end of file
diff --git a/server/utils/color.ts b/server/utils/color.ts
index a5dc137..f22384e 100644
--- a/server/utils/color.ts
+++ b/server/utils/color.ts
@@ -8,9 +8,11 @@ export function getColorForLauncher(name: string): string {
             return "#959eef"
         case 'neoforge':
             return "#f99e6b"
+        case 'liteloader':
+            return "#7ab0ee"
         default: {
             console.log(`unknown Launcher trying to access color - ${name}`);
-            return "#ffffff"
+            return "#6b7280"
         }
     }
 }
\ No newline at end of file
diff --git a/server/utils/fetchData.ts b/server/utils/fetchData.ts
new file mode 100644
index 0000000..09c35b4
--- /dev/null
+++ b/server/utils/fetchData.ts
@@ -0,0 +1,56 @@
+import {$modrinthFetch} from "~/utils/modrinthFetch";
+
+export async function getProjectIds(offset: number): Promise<string[]> {
+    const data = await $modrinthFetch<SimpleProjects>("/search", {
+        query: {
+            limit: 100,
+            offset: offset,
+            //todo - remove newest
+            // index: "newest",
+            facets: "[[\"project_types:mod\"]]"
+        }
+    })
+
+    return data.hits.map(value => value.project_id)
+}
+
+type SimpleProjects = {
+    hits: { project_id: string }[]
+}
+
+export async function getVersionIds(projectIds: string[]): Promise<string[]> {
+    const data = await $modrinthFetch<Project[]>("/projects", {
+        query: {
+            ids: "[\"" + projectIds.join("\",\"") + "\"]"
+        }
+    })
+
+    return data.flatMap(value => value.versions)
+}
+
+type Project = {
+    versions: string[]
+}
+
+export async function getVersions(versionIds: string[]): Promise<Version[]> {
+    const data = await $modrinthFetch<Version[]>("/versions", {
+        query: {
+            ids: "[\"" + versionIds.join("\",\"") + "\"]"
+        }
+    })
+
+    //done to remove unused values from the ram
+    return data.flatMap(value => {
+        return {
+            loaders: value.loaders,
+            game_versions: value.game_versions,
+            downloads: value.downloads
+        }
+    })
+}
+
+type Version = {
+    loaders: string[]
+    game_versions: string[]
+    downloads: number
+}
\ No newline at end of file
diff --git a/server/utils/types/downloads.ts b/server/utils/types/downloads.ts
new file mode 100644
index 0000000..24349a1
--- /dev/null
+++ b/server/utils/types/downloads.ts
@@ -0,0 +1,7 @@
+export type Downloads = {
+    versions: string[]
+    data: {
+       name: string
+       downloads: number[]
+    }[]
+}
\ No newline at end of file
diff --git a/utils/modrinthFetch.ts b/utils/modrinthFetch.ts
new file mode 100644
index 0000000..c16c449
--- /dev/null
+++ b/utils/modrinthFetch.ts
@@ -0,0 +1,4 @@
+export const $modrinthFetch = $fetch.create({
+    headers: {"User-Agent": "ToBinio/modrinth-statistics"},
+    baseURL: "https://api.modrinth.com/v2/"
+})
\ No newline at end of file
-- 
2.51.2