diff --git a/pages/index.vue b/pages/index.vue index e768e09..9b17fdb 100644 --- a/pages/index.vue +++ b/pages/index.vue @@ -1,5 +1,5 @@ + + + + diff --git a/server/api/metrics.get.ts b/server/api/metrics.get.ts index 8105f38..1efe03a 100644 --- a/server/api/metrics.get.ts +++ b/server/api/metrics.get.ts @@ -1,8 +1,5 @@ -import fs from "node:fs"; -import path from "node:path"; import { createError, defineEventHandler, getQuery } from "h3"; -import { useStorage } from "nitropack/runtime"; -import initSqlJs from "sql.js/dist/sql-asm.js"; +import { createClient } from "@libsql/client"; type ReplyRow = { parent_ts: string; @@ -44,27 +41,37 @@ const PALETTE = [ "#F25F5C" ]; -let sqlJsModulePromise: Promise | null = null; +const EXCLUDED_REPLY_USER_IDS = new Set(["U0AFC27QCF7"]); -function getSqlJsModule(): Promise { - if (!sqlJsModulePromise) { - sqlJsModulePromise = initSqlJs(); - } - return sqlJsModulePromise!; +let tursoClient: ReturnType | null = null; + +function readEnv(name: string): string | undefined { + return (globalThis as any)?.process?.env?.[name]; } -function queryAll(db: any, sql: string): T[] { - const result = db.exec(sql); - if (!result.length) return []; +function getTursoClient() { + const url = readEnv("TURSO_DATABASE_URL") || readEnv("TURSO_URL"); + if (!url) return null; + + if (!tursoClient) { + tursoClient = createClient({ + url, + authToken: readEnv("TURSO_AUTH_TOKEN") || readEnv("TURSO_TOKEN") + }); + } + + return tursoClient; +} - const first = result[0]; - const columns: string[] = first.columns ?? []; - const values: Array> = first.values ?? []; +async function queryAll(db: any, sql: string): Promise { + const result = await db.execute(sql); + const columns: string[] = result.columns ?? []; + const rows: Array> = result.rows ?? []; - return values.map((row) => { + return rows.map((row) => { const obj: Record = {}; for (let i = 0; i < columns.length; i++) { - obj[columns[i]] = row[i] ?? null; + obj[columns[i]] = row[columns[i]] ?? null; } return obj as T; }); @@ -87,29 +94,6 @@ function dayRange(fromDay: string, toDay: string): string[] { return result; } -function normalizePath(inputPath?: string): string { - const projectRoot = process.cwd(); - - if (!inputPath || !inputPath.trim()) { - return path.join(projectRoot, "slack_messages.public.db"); - } - - if (path.isAbsolute(inputPath)) { - return inputPath; - } - - return path.join(projectRoot, inputPath); -} - -function isDefaultPublicDbRequest(requestedDbPath?: string): boolean { - if (!requestedDbPath || !requestedDbPath.trim()) return true; - const trimmed = requestedDbPath.trim(); - if (path.isAbsolute(trimmed)) return false; - - const normalized = trimmed.replace(/^\.\//, "").replace(/^\//, ""); - return normalized === "slack_messages.public.db"; -} - function parseUsers(rawUsers?: string): string[] { if (!rawUsers) return []; @@ -127,389 +111,381 @@ function parseCsv(raw?: string): string[] { .filter(Boolean); } -function colorAt(index: number): string { - return PALETTE[index % PALETTE.length]; +function parseDay(raw?: string): string | null { + if (!raw) return null; + return /^\d{4}-\d{2}-\d{2}$/.test(raw) ? raw : null; } -async function resolveBundledPublicDbPath(): Promise { - const tmpPath = path.join("/tmp", "slack_messages.public.db"); - if (fs.existsSync(tmpPath)) return tmpPath; - - const storage = useStorage("assets"); - const keys = await storage.keys(); - const candidateKeys = keys.filter( - (key) => - key.endsWith(":db:slack_messages.public.db") || - key.endsWith("/db/slack_messages.public.db") || - key.endsWith("slack_messages.public.db") - ); - - for (const key of candidateKeys) { - const raw = await storage.get(key); - if (!raw) continue; - - const bytes = raw instanceof Uint8Array ? raw : Buffer.from(String(raw)); - fs.writeFileSync(tmpPath, bytes); - if (fs.existsSync(tmpPath)) return tmpPath; - } - - return null; +function colorAt(index: number): string { + return PALETTE[index % PALETTE.length]; } export default defineEventHandler(async (event) => { const query = getQuery(event); - const requestedDbPath = typeof query.dbPath === "string" ? query.dbPath : undefined; - let dbPath = normalizePath(requestedDbPath); const selectedUsers = parseUsers(typeof query.users === "string" ? query.users : undefined); const selectedThreadTypes = parseCsv(typeof query.threadTypes === "string" ? query.threadTypes : undefined); const selectedKeywords = parseCsv(typeof query.keywords === "string" ? query.keywords : undefined); const excludedUsers = new Set(parseCsv(typeof query.excludedUsers === "string" ? query.excludedUsers : undefined)); + const startDate = parseDay(typeof query.startDate === "string" ? query.startDate : undefined); + const endDate = parseDay(typeof query.endDate === "string" ? query.endDate : undefined); + const tursoDb = getTursoClient(); - // On serverless platforms, the project-root DB file may not exist; recover from bundled server assets. - if (!fs.existsSync(dbPath) && isDefaultPublicDbRequest(requestedDbPath)) { - const bundledDbPath = await resolveBundledPublicDbPath(); - if (bundledDbPath) { - dbPath = bundledDbPath; - } - } - - if (!fs.existsSync(dbPath)) { + if (!tursoDb) { throw createError({ - statusCode: 404, - statusMessage: `DB file not found: ${dbPath}` + statusCode: 500, + statusMessage: + "Turso is not configured. Set TURSO_DATABASE_URL (or TURSO_URL) and TURSO_AUTH_TOKEN (or TURSO_TOKEN)." }); } - const SQL = await getSqlJsModule(); - const db = new SQL.Database(new Uint8Array(fs.readFileSync(dbPath))); - - try { - const messageColumns = queryAll<{ name: string }>(db, "PRAGMA table_info(messages)"); - const hasTextColumn = messageColumns.some((col) => col.name === "text"); - const hasKeywordTagsColumn = messageColumns.some((col) => col.name === "keyword_tags"); - - const parents = queryAll( - db, - ` - SELECT ts, user, created_at, reply_count, thread_ts, "type" - FROM messages - WHERE is_thread_reply = 0 - ORDER BY created_at ASC - ` - ); - - const replies = queryAll( - db, - ` - SELECT parent_ts, user, - ${hasTextColumn ? "text AS text" : "NULL AS text"}, - ${hasKeywordTagsColumn ? "keyword_tags AS keyword_tags" : "NULL AS keyword_tags"}, - created_at, reply_type - FROM messages - WHERE is_thread_reply = 1 - ORDER BY created_at ASC - ` - ); - - const threadTypes = - selectedThreadTypes.length > 0 - ? selectedThreadTypes - : ["help req", "api_help_req"]; - - const helpParents = parents.filter( - (parent) => threadTypes.includes(parent.type ?? "") && (!parent.user || !excludedUsers.has(parent.user)) - ); - const helpParentIds = new Set(helpParents.map((parent) => parent.ts)); - - const filteredReplies = replies.filter( - (reply) => - reply.parent_ts && helpParentIds.has(reply.parent_ts) && (!reply.user || !excludedUsers.has(reply.user)) - ); - - const repliesByParent = new Map(); - const repliesByUserTotal = new Map(); - - for (const reply of filteredReplies) { - if (!reply.parent_ts) continue; - const arr = repliesByParent.get(reply.parent_ts) ?? []; - arr.push(reply); - repliesByParent.set(reply.parent_ts, arr); - - if (reply.user) { - repliesByUserTotal.set(reply.user, (repliesByUserTotal.get(reply.user) ?? 0) + 1); - } - } + const dbPath = readEnv("TURSO_DATABASE_URL") || readEnv("TURSO_URL") || "turso"; - const responseTimesByDay = new Map(); - const solutionTimesByDay = new Map(); - const threadCountsByDay = new Map(); - const solvedThreadCountsByDay = new Map(); + const messageColumns = await queryAll<{ name: string }>(tursoDb, "PRAGMA table_info(messages)"); + const hasTextColumn = messageColumns.some((col) => col.name === "text"); + const hasKeywordTagsColumn = messageColumns.some((col) => col.name === "keyword_tags"); - for (const parent of helpParents) { - const day = toDay(parent.created_at); + const parents = await queryAll( + tursoDb, + ` + SELECT ts, user, created_at, reply_count, thread_ts, "type" + FROM messages + WHERE is_thread_reply = 0 + ORDER BY created_at ASC + ` + ); - const isThreadStarter = parent.reply_count > 0 || parent.thread_ts === parent.ts; - if (isThreadStarter) { - threadCountsByDay.set(day, (threadCountsByDay.get(day) ?? 0) + 1); - } + const replies = await queryAll( + tursoDb, + ` + SELECT parent_ts, user, + ${hasTextColumn ? "text AS text" : "NULL AS text"}, + ${hasKeywordTagsColumn ? "keyword_tags AS keyword_tags" : "NULL AS keyword_tags"}, + created_at, reply_type + FROM messages + WHERE is_thread_reply = 1 + ORDER BY created_at ASC + ` + ); - if (parent.reply_count <= 0) continue; - const threadReplies = repliesByParent.get(parent.ts) ?? []; - const firstNonAuthorReply = threadReplies.find((reply) => reply.user && reply.user !== parent.user); - - if (!firstNonAuthorReply) continue; - const deltaSeconds = firstNonAuthorReply.created_at - parent.created_at; - if (deltaSeconds < 0) continue; - - const arr = responseTimesByDay.get(day) ?? []; - arr.push(deltaSeconds); - responseTimesByDay.set(day, arr); - - const firstSolutionReply = threadReplies.find((reply) => reply.reply_type === "final solution"); - if (firstSolutionReply) { - solvedThreadCountsByDay.set(day, (solvedThreadCountsByDay.get(day) ?? 0) + 1); - const solutionDelta = firstSolutionReply.created_at - parent.created_at; - if (solutionDelta >= 0) { - const solutionArr = solutionTimesByDay.get(day) ?? []; - solutionArr.push(solutionDelta); - solutionTimesByDay.set(day, solutionArr); - } - } - } + const filteredParents = parents.filter((parent) => { + const day = toDay(parent.created_at); + if (startDate && day < startDate) return false; + if (endDate && day > endDate) return false; + return true; + }); - const responseDays = [...responseTimesByDay.keys()].sort(); - const responseLabels = - responseDays.length > 0 ? dayRange(responseDays[0], responseDays[responseDays.length - 1]) : []; + const filteredRepliesByDate = replies.filter((reply) => { + const day = toDay(reply.created_at); + if (startDate && day < startDate) return false; + if (endDate && day > endDate) return false; + return true; + }); - const responseAvgMins = responseLabels.map((day) => { - const values = responseTimesByDay.get(day) ?? []; - if (!values.length) return 0; - const avg = values.reduce((sum, val) => sum + val, 0) / values.length; - return Number((avg / 60).toFixed(2)); - }); - const responseSampleCounts = responseLabels.map((day) => (responseTimesByDay.get(day) ?? []).length); - - const solutionDays = [...solutionTimesByDay.keys()].sort(); - const solutionLabels = - solutionDays.length > 0 ? dayRange(solutionDays[0], solutionDays[solutionDays.length - 1]) : []; - const solutionAvgMins = solutionLabels.map((day) => { - const values = solutionTimesByDay.get(day) ?? []; - if (!values.length) return 0; - const avg = values.reduce((sum, val) => sum + val, 0) / values.length; - return Number((avg / 60).toFixed(2)); - }); - const solutionSampleCounts = solutionLabels.map((day) => (solutionTimesByDay.get(day) ?? []).length); - - const threadDays = [...threadCountsByDay.keys()].sort(); - const threadLabels = - threadDays.length > 0 ? dayRange(threadDays[0], threadDays[threadDays.length - 1]) : []; - const threadSeries = threadLabels.map((day) => threadCountsByDay.get(day) ?? 0); - const solvedRateSeries = threadLabels.map((day) => { - const solved = solvedThreadCountsByDay.get(day) ?? 0; - const total = threadCountsByDay.get(day) ?? 0; - if (total <= 0) return 0; - return Number(((solved / total) * 100).toFixed(2)); - }); + const threadTypes = + selectedThreadTypes.length > 0 + ? selectedThreadTypes + : ["help req", "api_help_req", "dev disc", "other"]; - const finalUsers = - selectedUsers.length > 0 - ? selectedUsers - : [...repliesByUserTotal.entries()] - .sort((a, b) => b[1] - a[1]) - .slice(0, 5) - .map(([user]) => user); - - const repliesByUserByDay = new Map>(); - for (const user of finalUsers) { - repliesByUserByDay.set(user, new Map()); - } + const helpParents = filteredParents.filter( + (parent) => + threadTypes.includes(parent.type ?? "") && + (!parent.user || !excludedUsers.has(parent.user)) + ); + const helpParentIds = new Set(helpParents.map((parent) => parent.ts)); - for (const reply of filteredReplies) { - if (!reply.user || !finalUsers.includes(reply.user)) continue; - const day = toDay(reply.created_at); - const map = repliesByUserByDay.get(reply.user); - if (!map) continue; - map.set(day, (map.get(day) ?? 0) + 1); - } + const filteredReplies = filteredRepliesByDate.filter( + (reply) => + reply.parent_ts && + helpParentIds.has(reply.parent_ts) && + (!reply.user || (!excludedUsers.has(reply.user) && !EXCLUDED_REPLY_USER_IDS.has(reply.user))) + ); - const userDays = new Set(); - for (const map of repliesByUserByDay.values()) { - for (const day of map.keys()) userDays.add(day); - } + const repliesByParent = new Map(); + const repliesByUserTotal = new Map(); - const sortedUserDays = [...userDays].sort(); - const repliesByUserLabels = - sortedUserDays.length > 0 - ? dayRange(sortedUserDays[0], sortedUserDays[sortedUserDays.length - 1]) - : []; - - const repliesByUserSeries = finalUsers.map((user, idx) => { - const map = repliesByUserByDay.get(user) ?? new Map(); - return { - name: user, - color: colorAt(idx), - values: repliesByUserLabels.map((day) => map.get(day) ?? 0) - }; - }); + for (const reply of filteredReplies) { + if (!reply.parent_ts) continue; + const arr = repliesByParent.get(reply.parent_ts) ?? []; + arr.push(reply); + repliesByParent.set(reply.parent_ts, arr); - const solutionRepliesByUserTotal = new Map(); - for (const reply of filteredReplies) { - if (reply.reply_type !== "final solution" || !reply.user) continue; - solutionRepliesByUserTotal.set(reply.user, (solutionRepliesByUserTotal.get(reply.user) ?? 0) + 1); + if (reply.user) { + repliesByUserTotal.set(reply.user, (repliesByUserTotal.get(reply.user) ?? 0) + 1); } + } - const solutionUsers = - selectedUsers.length > 0 - ? selectedUsers - : [...solutionRepliesByUserTotal.entries()] - .sort((a, b) => b[1] - a[1]) - .slice(0, 5) - .map(([user]) => user); - - const solutionsByUserByDay = new Map>(); - for (const user of solutionUsers) { - solutionsByUserByDay.set(user, new Map()); - } + const responseTimesByDay = new Map(); + const solutionTimesByDay = new Map(); + const threadCountsByDay = new Map(); + const solvedThreadCountsByDay = new Map(); - for (const reply of filteredReplies) { - if (reply.reply_type !== "final solution" || !reply.user || !solutionUsers.includes(reply.user)) continue; - const day = toDay(reply.created_at); - const map = solutionsByUserByDay.get(reply.user); - if (!map) continue; - map.set(day, (map.get(day) ?? 0) + 1); + for (const parent of helpParents) { + const day = toDay(parent.created_at); + + const isThreadStarter = parent.reply_count > 0 || parent.thread_ts === parent.ts; + if (isThreadStarter) { + threadCountsByDay.set(day, (threadCountsByDay.get(day) ?? 0) + 1); } - const solutionUserDays = new Set(); - for (const map of solutionsByUserByDay.values()) { - for (const day of map.keys()) solutionUserDays.add(day); + if (parent.reply_count <= 0) continue; + const threadReplies = repliesByParent.get(parent.ts) ?? []; + const firstNonAuthorReply = threadReplies.find((reply) => reply.user && reply.user !== parent.user); + + if (!firstNonAuthorReply) continue; + const deltaSeconds = firstNonAuthorReply.created_at - parent.created_at; + if (deltaSeconds < 0) continue; + + const arr = responseTimesByDay.get(day) ?? []; + arr.push(deltaSeconds); + responseTimesByDay.set(day, arr); + + const firstSolutionReply = threadReplies.find((reply) => reply.reply_type === "final solution"); + if (firstSolutionReply) { + solvedThreadCountsByDay.set(day, (solvedThreadCountsByDay.get(day) ?? 0) + 1); + const solutionDelta = firstSolutionReply.created_at - parent.created_at; + if (solutionDelta >= 0) { + const solutionArr = solutionTimesByDay.get(day) ?? []; + solutionArr.push(solutionDelta); + solutionTimesByDay.set(day, solutionArr); + } } + } - const sortedSolutionUserDays = [...solutionUserDays].sort(); - const solutionsByUserLabels = - sortedSolutionUserDays.length > 0 - ? dayRange(sortedSolutionUserDays[0], sortedSolutionUserDays[sortedSolutionUserDays.length - 1]) - : []; - - const solutionsByUserSeries = solutionUsers.map((user, idx) => { - const map = solutionsByUserByDay.get(user) ?? new Map(); - return { - name: user, - color: colorAt(idx), - values: solutionsByUserLabels.map((day) => map.get(day) ?? 0) - }; - }); + const responseDays = [...responseTimesByDay.keys()].sort(); + const responseLabels = + responseDays.length > 0 ? dayRange(responseDays[0], responseDays[responseDays.length - 1]) : []; - const topRepliers = [...repliesByUserTotal.entries()] - .sort((a, b) => b[1] - a[1]) - .slice(0, 10) - .map(([user, replies], index) => ({ - rank: index + 1, - user, - replies - })); - - const activeKeywords = - selectedKeywords.length > 0 - ? KEYWORDS.filter((kw) => selectedKeywords.includes(kw.key)) - : KEYWORDS; - - const keywordMap = new Map>(); - for (const kw of activeKeywords) { - keywordMap.set(kw.key, new Map()); - } + const responseAvgMins = responseLabels.map((day) => { + const values = responseTimesByDay.get(day) ?? []; + if (!values.length) return 0; + const avg = values.reduce((sum, val) => sum + val, 0) / values.length; + return Number((avg / 60).toFixed(2)); + }); + const responseSampleCounts = responseLabels.map((day) => (responseTimesByDay.get(day) ?? []).length); + + const solutionDays = [...solutionTimesByDay.keys()].sort(); + const solutionLabels = + solutionDays.length > 0 ? dayRange(solutionDays[0], solutionDays[solutionDays.length - 1]) : []; + const solutionAvgMins = solutionLabels.map((day) => { + const values = solutionTimesByDay.get(day) ?? []; + if (!values.length) return 0; + const avg = values.reduce((sum, val) => sum + val, 0) / values.length; + return Number((avg / 60).toFixed(2)); + }); + const solutionSampleCounts = solutionLabels.map((day) => (solutionTimesByDay.get(day) ?? []).length); + + const threadDays = [...threadCountsByDay.keys()].sort(); + const threadLabels = + threadDays.length > 0 ? dayRange(threadDays[0], threadDays[threadDays.length - 1]) : []; + const threadSeries = threadLabels.map((day) => threadCountsByDay.get(day) ?? 0); + const solvedRateSeries = threadLabels.map((day) => { + const solved = solvedThreadCountsByDay.get(day) ?? 0; + const total = threadCountsByDay.get(day) ?? 0; + if (total <= 0) return 0; + return Number(((solved / total) * 100).toFixed(2)); + }); - for (const reply of filteredReplies) { - const day = toDay(reply.created_at); - - if (reply.keyword_tags) { - const present = new Set( - String(reply.keyword_tags) - .split(",") - .map((item) => item.trim()) - .filter(Boolean) - ); - - for (const kw of activeKeywords) { - if (!present.has(kw.key)) continue; - const map = keywordMap.get(kw.key); - if (!map) continue; - map.set(day, (map.get(day) ?? 0) + 1); - } - - continue; - } + const finalUsers = + selectedUsers.length > 0 + ? selectedUsers + : [...repliesByUserTotal.entries()] + .sort((a, b) => b[1] - a[1]) + .slice(0, 5) + .map(([user]) => user); + + const repliesByUserByDay = new Map>(); + for (const user of finalUsers) { + repliesByUserByDay.set(user, new Map()); + } + + for (const reply of filteredReplies) { + if (!reply.user || !finalUsers.includes(reply.user)) continue; + const day = toDay(reply.created_at); + const map = repliesByUserByDay.get(reply.user); + if (!map) continue; + map.set(day, (map.get(day) ?? 0) + 1); + } + + const userDays = new Set(); + for (const map of repliesByUserByDay.values()) { + for (const day of map.keys()) userDays.add(day); + } + + const sortedUserDays = [...userDays].sort(); + const repliesByUserLabels = + sortedUserDays.length > 0 + ? dayRange(sortedUserDays[0], sortedUserDays[sortedUserDays.length - 1]) + : []; - const text = reply.text ?? ""; - if (!text) continue; + const repliesByUserSeries = finalUsers.map((user, idx) => { + const map = repliesByUserByDay.get(user) ?? new Map(); + return { + name: user, + color: colorAt(idx), + values: repliesByUserLabels.map((day) => map.get(day) ?? 0) + }; + }); + + const solutionRepliesByUserTotal = new Map(); + for (const reply of filteredReplies) { + if (reply.reply_type !== "final solution" || !reply.user) continue; + solutionRepliesByUserTotal.set(reply.user, (solutionRepliesByUserTotal.get(reply.user) ?? 0) + 1); + } + + const solutionUsers = + selectedUsers.length > 0 + ? selectedUsers + : [...solutionRepliesByUserTotal.entries()] + .sort((a, b) => b[1] - a[1]) + .slice(0, 5) + .map(([user]) => user); + + const solutionsByUserByDay = new Map>(); + for (const user of solutionUsers) { + solutionsByUserByDay.set(user, new Map()); + } + + for (const reply of filteredReplies) { + if (reply.reply_type !== "final solution" || !reply.user || !solutionUsers.includes(reply.user)) continue; + const day = toDay(reply.created_at); + const map = solutionsByUserByDay.get(reply.user); + if (!map) continue; + map.set(day, (map.get(day) ?? 0) + 1); + } + + const solutionUserDays = new Set(); + for (const map of solutionsByUserByDay.values()) { + for (const day of map.keys()) solutionUserDays.add(day); + } + + const sortedSolutionUserDays = [...solutionUserDays].sort(); + const solutionsByUserLabels = + sortedSolutionUserDays.length > 0 + ? dayRange(sortedSolutionUserDays[0], sortedSolutionUserDays[sortedSolutionUserDays.length - 1]) + : []; + + const solutionsByUserSeries = solutionUsers.map((user, idx) => { + const map = solutionsByUserByDay.get(user) ?? new Map(); + return { + name: user, + color: colorAt(idx), + values: solutionsByUserLabels.map((day) => map.get(day) ?? 0) + }; + }); + + const topRepliers = [...repliesByUserTotal.entries()] + .sort((a, b) => b[1] - a[1]) + .slice(0, 10) + .map(([user, replies], index) => ({ + rank: index + 1, + user, + replies + })); + + const activeKeywords = + selectedKeywords.length > 0 + ? KEYWORDS.filter((kw) => selectedKeywords.includes(kw.key)) + : KEYWORDS; + + const keywordMap = new Map>(); + for (const kw of activeKeywords) { + keywordMap.set(kw.key, new Map()); + } + + for (const reply of filteredReplies) { + const day = toDay(reply.created_at); + + if (reply.keyword_tags) { + const present = new Set( + String(reply.keyword_tags) + .split(",") + .map((item) => item.trim()) + .filter(Boolean) + ); for (const kw of activeKeywords) { - if (!kw.pattern.test(text)) continue; + if (!present.has(kw.key)) continue; const map = keywordMap.get(kw.key); if (!map) continue; map.set(day, (map.get(day) ?? 0) + 1); } + + continue; } - const keywordDays = new Set(); - for (const map of keywordMap.values()) { - for (const day of map.keys()) keywordDays.add(day); + const text = reply.text ?? ""; + if (!text) continue; + + for (const kw of activeKeywords) { + if (!kw.pattern.test(text)) continue; + const map = keywordMap.get(kw.key); + if (!map) continue; + map.set(day, (map.get(day) ?? 0) + 1); } - const sortedKeywordDays = [...keywordDays].sort(); - const keywordLabels = - sortedKeywordDays.length > 0 - ? dayRange(sortedKeywordDays[0], sortedKeywordDays[sortedKeywordDays.length - 1]) - : []; - - const keywordSeries = activeKeywords.map((kw, idx) => { - const map = keywordMap.get(kw.key) ?? new Map(); - return { - name: kw.key, - color: colorAt(idx), - values: keywordLabels.map((day) => map.get(day) ?? 0) - }; - }); + } + const keywordDays = new Set(); + for (const map of keywordMap.values()) { + for (const day of map.keys()) keywordDays.add(day); + } + const sortedKeywordDays = [...keywordDays].sort(); + const keywordLabels = + sortedKeywordDays.length > 0 + ? dayRange(sortedKeywordDays[0], sortedKeywordDays[sortedKeywordDays.length - 1]) + : []; + + const keywordSeries = activeKeywords.map((kw, idx) => { + const map = keywordMap.get(kw.key) ?? new Map(); return { - meta: { - dbPath, - selectedUsers: finalUsers, - availableUsers: [...repliesByUserTotal.keys()].sort(), - topRepliers, - activeThreadTypes: threadTypes, - activeKeywords: activeKeywords.map((kw) => kw.key) - }, - responseTime: { - labels: responseLabels, - avgMinutes: responseAvgMins, - sampleCounts: responseSampleCounts - }, - timeToSolution: { - labels: solutionLabels, - avgMinutes: solutionAvgMins, - sampleCounts: solutionSampleCounts - }, - threadsOverTime: { - labels: threadLabels, - counts: threadSeries - }, - solvedRateOverTime: { - labels: threadLabels, - ratePct: solvedRateSeries - }, - repliesByUser: { - labels: repliesByUserLabels, - series: repliesByUserSeries - }, - solutionsByUser: { - labels: solutionsByUserLabels, - series: solutionsByUserSeries - }, - keywordMentions: { - labels: keywordLabels, - series: keywordSeries - } + name: kw.key, + color: colorAt(idx), + values: keywordLabels.map((day) => map.get(day) ?? 0) }; - } finally { - db.close(); - } -}); \ No newline at end of file + }); + + return { + meta: { + dbPath, + selectedUsers: finalUsers, + availableUsers: [...repliesByUserTotal.keys()].sort(), + topRepliers, + activeThreadTypes: threadTypes, + activeKeywords: activeKeywords.map((kw) => kw.key), + dateRange: { + startDate: startDate ?? "", + endDate: endDate ?? "" + } + }, + responseTime: { + labels: responseLabels, + avgMinutes: responseAvgMins, + sampleCounts: responseSampleCounts + }, + timeToSolution: { + labels: solutionLabels, + avgMinutes: solutionAvgMins, + sampleCounts: solutionSampleCounts + }, + threadsOverTime: { + labels: threadLabels, + counts: threadSeries, + solvedRate: solvedRateSeries + }, + solvedRateOverTime: { + labels: threadLabels, + ratePct: solvedRateSeries + }, + repliesByUser: { + labels: repliesByUserLabels, + series: repliesByUserSeries + }, + solutionsByUser: { + labels: solutionsByUserLabels, + series: solutionsByUserSeries + }, + keywordsOverTime: { + labels: keywordLabels, + series: keywordSeries + } + }; +}); diff --git a/server/api/ship-metrics.get.ts b/server/api/ship-metrics.get.ts new file mode 100644 index 0000000..63cc3a5 --- /dev/null +++ b/server/api/ship-metrics.get.ts @@ -0,0 +1,1063 @@ +import fs from "node:fs"; +import path from "node:path"; +import { createError, defineEventHandler } from "h3"; +import { useStorage } from "nitropack/runtime"; +import { createClient } from "@libsql/client"; + +type HistogramPayload = { + labels: string[]; + counts: number[]; + average: number | null; + sampleCount: number; +}; + +type XyPayload = { + labels: string[]; + series: Array<{ + name: string; + color: string; + values: number[]; + }>; +}; + +type ShipMetricsPayload = { + meta: { + responses: number; + withComments: number; + source: string; + }; + comments: { + total: number; + suggestions: Array<{ title: string; count: number; pct: number; examples: string[] }>; + recommendations: Array<{ title: string; count: number; pct: number; examples: string[] }>; + opinions: Array<{ title: string; count: number; pct: number; examples: string[] }>; + }; + usage: { + posting: { labels: string[]; counts: number[] }; + browsing: { labels: string[]; counts: number[] }; + }; + ratings: { + browsingQuality: HistogramPayload; + postingQuality: HistogramPayload; + importance: HistogramPayload; + correlation: Array<{ a: string; b: string; r: number | null; sampleCount: number }>; + }; + priorities: { + labels: string[]; + bordaScores: number[]; + topChoiceCounts: number[]; + averageRank: Array<{ label: string; avgRank: number | null; sampleCount: number }>; + }; + phenomena: { + labels: string[]; + counts: number[]; + pct: number[]; + }; + relationships: { + topCoWitnessPairs: Array<{ a: string; b: string; count: number; pct: number }>; + ratingDeltasByPhenomenon: Array<{ + phenomenon: string; + deltaBrowsingQuality: number | null; + deltaPostingQuality: number | null; + withCount: number; + withoutCount: number; + }>; + }; + examples: Array<{ + index: number; + breakdown: Array<{ label: string; count: number; pct: number }>; + }>; +}; + +const CSV_FILE_NAME = "ship_feedback.csv"; +const ROOT_CSV_GLOB = /^Anonymous ship Feedback Form submissions.*\.csv$/; + +function readEnv(name: string): string | undefined { + return (globalThis as any)?.process?.env?.[name]; +} + +let _tursoClient: ReturnType | null = null; +function getTursoClient() { + const url = readEnv("TURSO_DATABASE_URL") || readEnv("TURSO_URL"); + if (!url) return null; + + if (!_tursoClient) { + _tursoClient = createClient({ + url, + authToken: readEnv("TURSO_AUTH_TOKEN") || readEnv("TURSO_TOKEN") + }); + } + + return _tursoClient; +} + +async function tryLoadCsvFromTurso(): Promise<{ text: string; source: string } | null> { + const table = readEnv("SHIP_FEEDBACK_TURSO_TABLE"); + if (!table) return null; + + const key = readEnv("SHIP_FEEDBACK_TURSO_KEY") || CSV_FILE_NAME; + const client = getTursoClient(); + if (!client) return null; + + try { + const res = await client.execute(`SELECT * FROM ${table} WHERE key = '${key}' LIMIT 1`); + const rows = res.rows ?? []; + if (!rows || !rows.length) return null; + + const row = rows[0] as Record; + for (const k of Object.keys(row)) { + const v = row[k]; + if (typeof v === "string" && v.trim().length > 0) { + return { text: v, source: `turso:${table}:${key}` }; + } + if (v instanceof Uint8Array) { + return { text: Buffer.from(v).toString("utf8"), source: `turso:${table}:${key}` }; + } + } + } catch (err) { + return null; + } + + return null; +} + +function isLikelyShipFeedbackCsv(text: string): boolean { + const rows = parseCsv(text); + if (rows.length < 2) return false; + + const header = rows[0].map((c) => normalizeSpaces(String(c ?? "")).toLowerCase()); + const joined = header.join(" | "); + + // Keep this intentionally heuristic (survey exports may rename columns). + const hints = [ + "#ship", + "phenomena", + "priorit", + "browsing experience", + "posting experience", + "elaborate" + ]; + + const hintHits = hints.filter((h) => joined.includes(h)).length; + return header.length >= 8 && hintHits >= 2; +} + +function normalizeSpaces(input: string): string { + return String(input ?? "") + .replaceAll("\u00A0", " ") + .replaceAll(/\s+/g, " ") + .trim(); +} + +function normKey(input: string): string { + return normalizeSpaces(input) + .toLowerCase() + .replaceAll(/\s*[–—-]\s*/g, "-") + .replaceAll(/[“”]/g, '"') + .replaceAll(/[’]/g, "'"); +} + +function parseCsv(raw: string): string[][] { + const rows: string[][] = []; + let row: string[] = []; + let field = ""; + let inQuotes = false; + + for (let i = 0; i < raw.length; i += 1) { + const ch = raw[i]; + + if (inQuotes) { + if (ch === '"') { + if (raw[i + 1] === '"') { + field += '"'; + i += 1; + } else { + inQuotes = false; + } + } else { + field += ch; + } + continue; + } + + if (ch === '"') { + inQuotes = true; + continue; + } + + if (ch === ",") { + row.push(field); + field = ""; + continue; + } + + if (ch === "\r") { + continue; + } + + if (ch === "\n") { + row.push(field); + field = ""; + const isEmptyRow = row.every((cell) => !String(cell ?? "").trim()); + if (!isEmptyRow) rows.push(row); + row = []; + continue; + } + + field += ch; + } + + row.push(field); + const isEmptyRow = row.every((cell) => !String(cell ?? "").trim()); + if (!isEmptyRow) rows.push(row); + + return rows; +} + +async function loadCsvText(): Promise<{ text: string; source: string }> { + // Try Turso-backed CSV if configured (set SHIP_FEEDBACK_TURSO_TABLE and optional SHIP_FEEDBACK_TURSO_KEY) + const fromTurso = await tryLoadCsvFromTurso(); + if (fromTurso) return fromTurso; + + const projectRoot = process.cwd(); + const override = process.env.SHIP_FEEDBACK_CSV_PATH; + const overridePath = override ? (path.isAbsolute(override) ? override : path.join(projectRoot, override)) : null; + + const localPath = path.join(projectRoot, "server", "assets", "db", CSV_FILE_NAME); + + const rootCandidates: string[] = []; + try { + const rootFiles = fs.readdirSync(projectRoot); + for (const f of rootFiles) { + if (ROOT_CSV_GLOB.test(f)) rootCandidates.push(path.join(projectRoot, f)); + } + } catch { + // ignore + } + + const rootCandidatesNewestFirst = rootCandidates + .map((p) => { + try { + return { p, mtimeMs: fs.statSync(p).mtimeMs }; + } catch { + return null; + } + }) + .filter((v): v is { p: string; mtimeMs: number } => Boolean(v)) + .sort((a, b) => b.mtimeMs - a.mtimeMs) + .map((v) => v.p); + + const preferredPaths = [overridePath, ...rootCandidatesNewestFirst, localPath].filter( + (p): p is string => Boolean(p) + ); + + for (const p of preferredPaths) { + if (!fs.existsSync(p)) continue; + const text = fs.readFileSync(p, "utf8"); + if (!isLikelyShipFeedbackCsv(text)) continue; + return { text, source: p }; + } + + const storage = useStorage("assets"); + const keys = await storage.keys(); + const candidateKeys = keys.filter( + (key) => + key.endsWith(`:db:${CSV_FILE_NAME}`) || + key.endsWith(`/db/${CSV_FILE_NAME}`) || + key.endsWith(CSV_FILE_NAME) + ); + + for (const key of candidateKeys) { + const raw = await storage.get(key); + if (!raw) continue; + + if (raw instanceof Uint8Array) { + return { text: Buffer.from(raw).toString("utf8"), source: `bundled:${key}` }; + } + + return { text: String(raw), source: `bundled:${key}` }; + } + + throw createError({ + statusCode: 404, + statusMessage: `Could not load ship feedback CSV (not found on disk or in bundled assets)` + }); +} + +function safeNumber(value: unknown): number | null { + if (value === null || value === undefined) return null; + const trimmed = String(value).trim(); + if (!trimmed) return null; + const n = Number(trimmed); + if (!Number.isFinite(n)) return null; + return n; +} + +function mean(values: number[]): number | null { + if (!values.length) return null; + const sum = values.reduce((acc, v) => acc + v, 0); + return sum / values.length; +} + +function pearson(x: Array, y: Array): { r: number | null; sampleCount: number } { + const pairs: Array<[number, number]> = []; + for (let i = 0; i < Math.min(x.length, y.length); i += 1) { + const a = x[i]; + const b = y[i]; + if (a === null || b === null) continue; + pairs.push([a, b]); + } + + if (pairs.length < 3) return { r: null, sampleCount: pairs.length }; + + const xs = pairs.map((p) => p[0]); + const ys = pairs.map((p) => p[1]); + const mx = mean(xs)!; + const my = mean(ys)!; + + let num = 0; + let dx = 0; + let dy = 0; + + for (let i = 0; i < pairs.length; i += 1) { + const vx = xs[i] - mx; + const vy = ys[i] - my; + num += vx * vy; + dx += vx * vx; + dy += vy * vy; + } + + const denom = Math.sqrt(dx * dy); + if (!denom) return { r: null, sampleCount: pairs.length }; + + return { r: Number((num / denom).toFixed(3)), sampleCount: pairs.length }; +} + +function buildHistogram(values: Array, min: number, max: number): HistogramPayload { + const labels = Array.from({ length: max - min + 1 }, (_, i) => String(min + i)); + const counts = new Array(labels.length).fill(0); + const valid: number[] = []; + + for (const v of values) { + if (v === null) continue; + if (!Number.isFinite(v)) continue; + if (v < min || v > max) continue; + counts[Math.round(v) - min] += 1; + valid.push(v); + } + + const avg = mean(valid); + return { + labels, + counts, + average: avg === null ? null : Number(avg.toFixed(2)), + sampleCount: valid.length + }; +} + +function countByLabel(values: string[]): { labels: string[]; counts: number[] } { + const map = new Map(); + for (const v of values) { + const label = normalizeSpaces(v); + if (!label) continue; + map.set(label, (map.get(label) ?? 0) + 1); + } + + const labels = [...map.keys()].sort((a, b) => (map.get(b) ?? 0) - (map.get(a) ?? 0)); + const counts = labels.map((l) => map.get(l) ?? 0); + return { labels, counts }; +} + +function countByLabelInOrder(values: string[], desiredOrder: string[]): { labels: string[]; counts: number[] } { + const map = new Map(); + for (const v of values) { + const label = normalizeSpaces(v); + if (!label) continue; + map.set(label, (map.get(label) ?? 0) + 1); + } + + const desired = desiredOrder.map((v) => normalizeSpaces(v)).filter(Boolean); + const desiredSet = new Set(desired); + + const labels: string[] = []; + for (const label of desired) { + if (map.has(label)) labels.push(label); + } + + const remaining = [...map.keys()] + .filter((k) => !desiredSet.has(k)) + .sort((a, b) => (map.get(b) ?? 0) - (map.get(a) ?? 0)); + + labels.push(...remaining); + const counts = labels.map((l) => map.get(l) ?? 0); + return { labels, counts }; +} + +const PRIORITY_ISSUES: Array<{ key: string; label: string; patterns: string[] }> = [ + { + key: "ai_projects", + label: "AI-generated / entirely vibe-coded projects", + patterns: ["ai-generated / entirely vibe-coded projects", "ai-generated / entirely-vibecoded projects"] + }, + { + key: "ai_descriptions", + label: "AI-generated ship descriptions", + patterns: ["ai-generated ship descriptions"] + }, + { + key: "coc", + label: "Unconstructive criticism or CoC violations", + patterns: [ + "unconstructive criticism or coc violations", + "abuse, unconstructive criticism, or coc-violations", + "abuse, unconstructive criticism, or coc violations" + ] + }, + { + key: "nonship_ads", + label: "Non-ship advertising", + patterns: ["non-ship (whatever that means to you) advertising"] + }, + { + key: "low_detail", + label: "Not detailed enough ship posts", + patterns: ["not detailed enough (vague) ship posts", "not detailed enough ship posts"] + }, + { + key: "spam", + label: "Spam / repeated reposts", + patterns: ["spam or repeatedly reposted ships", "spam or un-threaded posts", "spam or unthreaded posts"] + }, + { + key: "help_threads", + label: "Help request threads", + patterns: ["help request threads"] + }, + { + key: "offtopic", + label: "Off-topic discussions", + patterns: ["off-topic discussions", "off-topic discussion"] + }, + { + key: "overly_detailed", + label: "Overly detailed ship posts", + patterns: ["overly detailed ship posts"] + } +]; + +function parseRankedIssues(rawValue: string): string[] { + const text = normKey(rawValue); + + const found: Array<{ key: string; idx: number }> = []; + for (const issue of PRIORITY_ISSUES) { + let best = -1; + for (const pattern of issue.patterns) { + const p = normKey(pattern); + const idx = text.indexOf(p); + if (idx >= 0 && (best < 0 || idx < best)) { + best = idx; + } + } + if (best >= 0) found.push({ key: issue.key, idx: best }); + } + + if (found.length) { + found.sort((a, b) => a.idx - b.idx); + return found.map((f) => f.key); + } + + // Fallback: naive comma split + normalization + return rawValue + .split(",") + .map((s) => normKey(s)) + .filter(Boolean) + .map((token) => { + for (const issue of PRIORITY_ISSUES) { + if (issue.patterns.some((p) => normKey(p) === token) || normKey(issue.label) === token) { + return issue.key; + } + } + return ""; + }) + .filter(Boolean); +} + +function parsePhenomena(rawValue: string): Set { + const parts = rawValue + .split(",") + .map((s) => normalizeSpaces(s)) + .filter(Boolean); + const result = new Set(); + + for (const display of parts) { + const part = normKey(display); + if (!part) continue; + if (part === "none of the above") continue; + + // Map known issues into the same canonical set where possible + let matchedKey: string | null = null; + for (const issue of PRIORITY_ISSUES) { + if (issue.patterns.some((p) => normKey(p) === part) || normKey(issue.label) === part) { + matchedKey = issue.key; + break; + } + } + + if (matchedKey) { + result.add(matchedKey); + } else { + // Keep unknown/extra phenomena under their display name so we don't lose signal. + result.add(display); + } + } + + return result; +} + +function labelForIssueKey(key: string): string { + const match = PRIORITY_ISSUES.find((i) => i.key === key); + if (match) return match.label; + return key; +} + +function redactMentions(text: string): string { + return text + .replaceAll(/@[A-Za-z0-9._-]+/g, "@redacted") + .replaceAll(/\bU[A-Z0-9]{8,}\b/g, "U…"); +} + +function softenProfanity(text: string): string { + const replacements: Array<[RegExp, string]> = [ + [/\bshitbox\b/gi, "s***box"], + [/\bshit\b/gi, "s***"], + [/\bfuck\b/gi, "f***"], + [/\bfucking\b/gi, "f***ing"], + [/\bbullshit\b/gi, "b******t"] + ]; + + let out = text; + for (const [re, repl] of replacements) out = out.replace(re, repl); + return out; +} + +function sanitizeExcerpt(text: string, maxLen = 220): string { + let t = normalizeSpaces(text); + t = redactMentions(t); + t = softenProfanity(t); + if (t.length > maxLen) t = `${t.slice(0, maxLen).trimEnd()}…`; + return t; +} + +type CommentThemeDef = { + key: string; + title: string; + category: "suggestion" | "recommendation" | "opinion"; + patterns: RegExp[]; +}; + +const COMMENT_THEMES: CommentThemeDef[] = [ + { + key: "require_links", + title: "Require repo/demo links in ship posts", + category: "recommendation", + patterns: [/\brepo\b/i, /\bdemo\b/i, /github\b/i, /link to the (shipped project|github|code)\b/i, /should always be a link/i] + }, + { + key: "more_technical_journey", + title: "Encourage technical journey / challenges in posts", + category: "recommendation", + patterns: [/technical journey/i, /development experience/i, /what challenges/i, /talk more about/i] + }, + { + key: "increase_interaction", + title: "More interaction/comments on ships", + category: "opinion", + patterns: [/not every ship has comments/i, /discourages shipping/i, /isn't any interaction/i] + }, + { + key: "reduce_offtopic_help", + title: "Reduce off-topic and help threads in #ship", + category: "recommendation", + patterns: [/off ?topic/i, /help request/i, /getting rid of off/i, /keep(s)? true to its purpose/i, /automod/i] + }, + { + key: "ship_help_channel", + title: "Create a #ship-help (or dedicated help) channel", + category: "suggestion", + patterns: [/#ship-help/i, /somewhere centralized.*ask for help/i] + }, + { + key: "require_image", + title: "Require or strongly encourage an image", + category: "suggestion", + patterns: [/require an image/i, /maybe require an image/i, /link and image/i, /images would be recommended/i] + }, + { + key: "don't_be_too_strict", + title: "Don’t be too strict about post detail", + category: "opinion", + patterns: [/shouldn't be too strict/i, /feels very unfair/i, /deleted because they didn't write enough/i, /write a lot/i] + }, + { + key: "ship_as_launch", + title: "Treat #ship as launches (big updates only)", + category: "recommendation", + patterns: [/ship is like a launch/i, /post only the first ship/i, /big enough update/i] + }, + { + key: "feed_follow", + title: "Make #ship easier to follow (feed/discovery)", + category: "suggestion", + patterns: [/easy to follow/i, /like a feed/i, /browse/i] + }, + { + key: "highlight_quality", + title: "Give high-quality ships a special place", + category: "suggestion", + patterns: [/special place/i, /high quality ships/i, /highlight/i] + }, + { + key: "ship_website", + title: "Make a ship website (sync/highlight top ships)", + category: "suggestion", + patterns: [/ship website/i, /hackclub-wide ship website/i, /sync to the channel/i] + }, + { + key: "happenings_motivation", + title: "Motivation: ships that reach #happenings", + category: "opinion", + patterns: [/#happenings/i] + }, + { + key: "event_quality_effect", + title: "Perceived quality changes during flagship events", + category: "opinion", + patterns: [/arcade style flagship/i, /whenever there is no .*flagship/i] + }, + { + key: "examples_obvious", + title: "Survey examples felt obvious", + category: "opinion", + patterns: [/obvious examples/i] + }, + { + key: "flavortown_standard", + title: "Use a known standard for descriptions (e.g., Flavortown-style)", + category: "recommendation", + patterns: [/flavortown/i, /level of expectation/i] + } +]; + +function analyzeComments(comments: string[]) { + const total = comments.length; + const byKey = new Map; examples: string[] }>(); + + for (let i = 0; i < comments.length; i += 1) { + const raw = comments[i]; + const text = normalizeSpaces(raw); + if (!text) continue; + + for (const def of COMMENT_THEMES) { + if (!def.patterns.some((re) => re.test(text))) continue; + + const entry = byKey.get(def.key) ?? { def, indices: new Set(), examples: [] }; + entry.indices.add(i); + if (entry.examples.length < 3) { + entry.examples.push(sanitizeExcerpt(text)); + } + byKey.set(def.key, entry); + } + } + + function summaries(category: "suggestion" | "recommendation" | "opinion") { + const items = [...byKey.values()] + .filter((e) => e.def.category === category) + .map((e) => ({ + title: e.def.title, + count: e.indices.size, + pct: total ? Number(((e.indices.size / total) * 100).toFixed(1)) : 0, + examples: e.examples + })) + .sort((a, b) => b.count - a.count) + .slice(0, 8); + + return items; + } + + return { + total, + suggestions: summaries("suggestion"), + recommendations: summaries("recommendation"), + opinions: summaries("opinion") + }; +} + +export default defineEventHandler(async () => { + const { text, source } = await loadCsvText(); + const rows = parseCsv(text); + if (!rows.length) { + throw createError({ statusCode: 500, statusMessage: "CSV appears empty" }); + } + + const rawHeaders = rows[0].map((h) => normalizeSpaces(h)); + const dataRows = rows.slice(1); + + function mode(nums: number[]): number | null { + const counts = new Map(); + for (const n of nums) counts.set(n, (counts.get(n) ?? 0) + 1); + let best: { n: number; c: number } | null = null; + for (const [n, c] of counts) { + if (!best || c > best.c) best = { n, c }; + } + return best ? best.n : null; + } + + // Some Google Sheets exports can produce a duplicated leading "ID" header while data rows + // contain only one leading ID value. That shifts every subsequent column by 1. + const sampleLens = dataRows.slice(0, 30).map((r) => r.length).filter((n) => n > 0); + const typicalRowLen = mode(sampleLens); + + let headers = rawHeaders; + if ( + typicalRowLen !== null && + typicalRowLen === rawHeaders.length - 1 && + normKey(rawHeaders[0] ?? "") === "id" && + normKey(rawHeaders[1] ?? "") === "id" + ) { + headers = rawHeaders.slice(1); + } + + function headerIndexIncludes(substr: string): number { + const needle = normKey(substr); + return headers.findIndex((h) => normKey(h).includes(needle)); + } + + const postingUsageIdx = headerIndexIncludes("used #ship to share"); + const browsingUsageIdx = headerIndexIncludes("open #ship to read"); + const browsingQualityIdx = headerIndexIncludes("quality of the browsing experience"); + const postingQualityIdx = headerIndexIncludes("quality of the posting experience"); + const importanceIdx = headerIndexIncludes("how important is posting it"); + const phenomenaIdx = headerIndexIncludes("phenomena have you witnessed"); + const prioritiesIdx = headerIndexIncludes("how should these issues be prioritized"); + const commentsIdx = headerIndexIncludes("Anything you want to elaborate"); + + function getShifted(row: string[], headerIdx: number, shift: number): string { + if (headerIdx < 0) return ""; + const idx = headerIdx + shift; + if (idx < 0 || idx >= row.length) return ""; + return String(row[idx] ?? ""); + } + + function scoreShift(shift: number, limit = 25): number { + const take = Math.min(limit, dataRows.length); + let score = 0; + + for (let i = 0; i < take; i += 1) { + const row = dataRows[i]; + + const bq = safeNumber(getShifted(row, browsingQualityIdx, shift)); + if (bq !== null && bq >= 1 && bq <= 10) score += 2; + + const pq = safeNumber(getShifted(row, postingQualityIdx, shift)); + if (pq !== null && pq >= 1 && pq <= 10) score += 2; + + const imp = safeNumber(getShifted(row, importanceIdx, shift)); + if (imp !== null && imp >= 1 && imp <= 10) score += 2; + + const ranked = parseRankedIssues(getShifted(row, prioritiesIdx, shift)); + if (ranked.length) score += 3; + + const phenomena = normalizeSpaces(getShifted(row, phenomenaIdx, shift)); + if (phenomena && phenomena.length >= 8) score += 1; + + const comment = normalizeSpaces(getShifted(row, commentsIdx, shift)); + if (comment) score += 1; + } + + return score; + } + + // Some exports have headers shifted by +1 after the first column. + // Choose the shift that best matches expected data types (ratings 1–10, parsable rankings, etc.). + const score0 = scoreShift(0); + const scoreNeg1 = scoreShift(-1); + const headerValueShift = scoreNeg1 > score0 ? -1 : 0; + + const exampleIndices = headers + .map((h, i) => ({ h, i })) + .filter(({ h }) => /^\(\d+\)\s*Do you think this post should belong in #ship/i.test(h)) + .map(({ i }) => i); + + if (phenomenaIdx < 0 || prioritiesIdx < 0) { + throw createError({ + statusCode: 500, + statusMessage: "CSV schema mismatch: required columns not found (phenomena/priorities)" + }); + } + + const postingUsageValues: string[] = []; + const browsingUsageValues: string[] = []; + const commentTexts: string[] = []; + + const browsingQualityValues: Array = []; + const postingQualityValues: Array = []; + const importanceValues: Array = []; + + const phenomenaByRespondent: Array> = []; + const rankedIssuesByRespondent: Array = []; + + const commentCountByRow: boolean[] = []; + + const exampleVotes: Array> = Array.from({ length: Math.max(0, exampleIndices.length) }, () => new Map()); + + for (const row of dataRows) { + const get = (headerIdx: number) => { + if (headerIdx < 0) return ""; + const idx = headerIdx + headerValueShift; + if (idx < 0 || idx >= row.length) return ""; + return String(row[idx] ?? ""); + }; + + if (postingUsageIdx >= 0) postingUsageValues.push(get(postingUsageIdx)); + if (browsingUsageIdx >= 0) browsingUsageValues.push(get(browsingUsageIdx)); + + browsingQualityValues.push(safeNumber(get(browsingQualityIdx))); + postingQualityValues.push(safeNumber(get(postingQualityIdx))); + importanceValues.push(safeNumber(get(importanceIdx))); + + const phenomena = parsePhenomena(get(phenomenaIdx)); + const ranked = parseRankedIssues(get(prioritiesIdx)); + + phenomenaByRespondent.push(phenomena); + rankedIssuesByRespondent.push(ranked); + + const comment = normalizeSpaces(get(commentsIdx)); + commentCountByRow.push(Boolean(comment)); + if (comment) commentTexts.push(comment); + + for (let k = 0; k < exampleIndices.length; k += 1) { + const idx = exampleIndices[k]; + const vote = normalizeSpaces(get(idx)); + if (!vote) continue; + const map = exampleVotes[k]; + map.set(vote, (map.get(vote) ?? 0) + 1); + } + } + + const responses = dataRows.length; + const withComments = commentCountByRow.filter(Boolean).length; + const commentAnalysis = analyzeComments(commentTexts); + + const postingUsage = countByLabel(postingUsageValues); + const browsingUsage = countByLabel(browsingUsageValues); + + const POSTING_ORDER_LEAST_TO_MOST = [ + "I've never posted in #ship", + "Not at all", + "Not last year", + "1 - 2 times", + "3 - 6 times", + "6 - 10 times", + "10+ times" + ]; + + const BROWSING_ORDER_LEAST_TO_MOST = [ + "I've never browsed #ship", + "Never", + "Every couple of months", + "Every so often (less than once per month)", + "1 - 2 times per month", + "3 - 6 times per month", + "6 - 10 times per month", + "More than 10 times per month" + ]; + + const postingUsageOrdered = countByLabelInOrder(postingUsageValues, POSTING_ORDER_LEAST_TO_MOST); + const browsingUsageOrdered = countByLabelInOrder(browsingUsageValues, BROWSING_ORDER_LEAST_TO_MOST); + + const browsingHist = buildHistogram(browsingQualityValues, 1, 10); + const postingHist = buildHistogram(postingQualityValues, 1, 10); + const importanceHist = buildHistogram(importanceValues, 1, 10); + + const corr = [ + { a: "Browsing quality", b: "Posting quality", x: browsingQualityValues, y: postingQualityValues }, + { a: "Browsing quality", b: "Importance", x: browsingQualityValues, y: importanceValues }, + { a: "Posting quality", b: "Importance", x: postingQualityValues, y: importanceValues } + ].map(({ a, b, x, y }) => { + const { r, sampleCount } = pearson(x, y); + return { a, b, r, sampleCount }; + }); + + // Priorities: Borda score + avg rank + top choice counts + const borda = new Map(); + const top1 = new Map(); + const rankSums = new Map(); + + for (const ranked of rankedIssuesByRespondent) { + const keys = ranked.filter(Boolean); + if (!keys.length) continue; + + if (keys[0]) top1.set(keys[0], (top1.get(keys[0]) ?? 0) + 1); + + const k = keys.length; + for (let i = 0; i < k; i += 1) { + const key = keys[i]; + const points = k - i; + borda.set(key, (borda.get(key) ?? 0) + points); + const current = rankSums.get(key) ?? { sum: 0, n: 0 }; + current.sum += i + 1; + current.n += 1; + rankSums.set(key, current); + } + } + + const allIssueKeys = PRIORITY_ISSUES.map((i) => i.key); + const priorityLabels = allIssueKeys.map(labelForIssueKey); + const bordaScores = allIssueKeys.map((k) => borda.get(k) ?? 0); + const topChoiceCounts = allIssueKeys.map((k) => top1.get(k) ?? 0); + + const averageRank = allIssueKeys.map((k) => { + const rs = rankSums.get(k); + if (!rs || !rs.n) { + return { label: labelForIssueKey(k), avgRank: null, sampleCount: 0 }; + } + return { + label: labelForIssueKey(k), + avgRank: Number((rs.sum / rs.n).toFixed(2)), + sampleCount: rs.n + }; + }); + + // Phenomena counts + const phenomenaCounts = new Map(); + for (const set of phenomenaByRespondent) { + for (const item of set) { + phenomenaCounts.set(item, (phenomenaCounts.get(item) ?? 0) + 1); + } + } + + const phenomenaKeysSorted = [...phenomenaCounts.keys()].sort( + (a, b) => (phenomenaCounts.get(b) ?? 0) - (phenomenaCounts.get(a) ?? 0) + ); + const phenomenaLabels = phenomenaKeysSorted.map((k) => labelForIssueKey(k)); + const phenomenaCountArr = phenomenaKeysSorted.map((k) => phenomenaCounts.get(k) ?? 0); + const phenomenaPctArr = phenomenaCountArr.map((c) => Number(((c / Math.max(1, responses)) * 100).toFixed(1))); + + // Relationships: co-witness pairs (canonical + extras) + const pairCounts = new Map(); + for (const set of phenomenaByRespondent) { + const items = [...set].sort(); + for (let i = 0; i < items.length; i += 1) { + for (let j = i + 1; j < items.length; j += 1) { + const key = `${items[i]}||${items[j]}`; + pairCounts.set(key, (pairCounts.get(key) ?? 0) + 1); + } + } + } + + const topCoWitnessPairs = [...pairCounts.entries()] + .sort((a, b) => b[1] - a[1]) + .slice(0, 12) + .map(([key, count]) => { + const [a, b] = key.split("||"); + return { + a: labelForIssueKey(a), + b: labelForIssueKey(b), + count, + pct: Number(((count / Math.max(1, responses)) * 100).toFixed(1)) + }; + }); + + // Relationships: rating deltas for phenomena that map to canonical issues + const ratingDeltasByPhenomenon = PRIORITY_ISSUES.map((issue) => { + const withBrowsing: number[] = []; + const withoutBrowsing: number[] = []; + const withPosting: number[] = []; + const withoutPosting: number[] = []; + + for (let i = 0; i < phenomenaByRespondent.length; i += 1) { + const has = phenomenaByRespondent[i].has(issue.key); + const bq = browsingQualityValues[i]; + const pq = postingQualityValues[i]; + + if (bq !== null) { + (has ? withBrowsing : withoutBrowsing).push(bq); + } + + if (pq !== null) { + (has ? withPosting : withoutPosting).push(pq); + } + } + + const withB = mean(withBrowsing); + const withoutB = mean(withoutBrowsing); + const withP = mean(withPosting); + const withoutP = mean(withoutPosting); + + const deltaB = withB === null || withoutB === null ? null : Number((withB - withoutB).toFixed(2)); + const deltaP = withP === null || withoutP === null ? null : Number((withP - withoutP).toFixed(2)); + + return { + phenomenon: issue.label, + deltaBrowsingQuality: deltaB, + deltaPostingQuality: deltaP, + withCount: withBrowsing.length, + withoutCount: withoutBrowsing.length + }; + }).sort((a, b) => { + const da = Math.abs(a.deltaBrowsingQuality ?? 0); + const db = Math.abs(b.deltaBrowsingQuality ?? 0); + return db - da; + }); + + // Example post votes breakdowns + const examples = exampleVotes.map((map, idx) => { + const total = [...map.values()].reduce((acc, v) => acc + v, 0); + const breakdown = [...map.entries()] + .sort((a, b) => b[1] - a[1]) + .map(([label, count]) => ({ + label, + count, + pct: Number(((count / Math.max(1, total)) * 100).toFixed(1)) + })); + + return { index: idx + 1, breakdown }; + }); + + const payload: ShipMetricsPayload = { + meta: { + responses, + withComments, + source + }, + comments: commentAnalysis, + usage: { + posting: postingUsageOrdered, + browsing: browsingUsageOrdered + }, + ratings: { + browsingQuality: browsingHist, + postingQuality: postingHist, + importance: importanceHist, + correlation: corr + }, + priorities: { + labels: priorityLabels, + bordaScores, + topChoiceCounts, + averageRank + }, + phenomena: { + labels: phenomenaLabels, + counts: phenomenaCountArr, + pct: phenomenaPctArr + }, + relationships: { + topCoWitnessPairs, + ratingDeltasByPhenomenon + }, + examples + }; + + return payload; +}); diff --git a/server/api/ship-upload.post.ts b/server/api/ship-upload.post.ts new file mode 100644 index 0000000..cfc7783 --- /dev/null +++ b/server/api/ship-upload.post.ts @@ -0,0 +1,48 @@ +import { createError, defineEventHandler, readBody } from "h3"; +import { createClient } from "@libsql/client"; + +function readEnv(name: string): string | undefined { + return (globalThis as any)?.process?.env?.[name]; +} + +function getTursoClient() { + const url = readEnv("TURSO_DATABASE_URL") || readEnv("TURSO_URL"); + if (!url) return null; + + return createClient({ + url, + authToken: readEnv("TURSO_AUTH_TOKEN") || readEnv("TURSO_TOKEN") + }); +} + +export default defineEventHandler(async (event) => { + const client = getTursoClient(); + if (!client) { + throw createError({ statusCode: 500, statusMessage: "Turso not configured" }); + } + + const table = readEnv("SHIP_FEEDBACK_TURSO_TABLE"); + if (!table) { + throw createError({ statusCode: 500, statusMessage: "SHIP_FEEDBACK_TURSO_TABLE not set" }); + } + + const body = await readBody(event); + // Accept either raw text or { csv: '...' } + const csvText = typeof body === "string" ? body : (body && (body.csv || body.text)) || null; + if (!csvText || typeof csvText !== "string") { + throw createError({ statusCode: 400, statusMessage: "Missing CSV text in request body (send raw text or JSON {csv: '...'})" }); + } + + const key = readEnv("SHIP_FEEDBACK_TURSO_KEY") || "ship_feedback.csv"; + + try { + // Ensure table exists with simple (key,value) storage + await client.execute(`CREATE TABLE IF NOT EXISTS ${table} (key TEXT PRIMARY KEY, value TEXT)`); + // Use parameterized query to avoid injection + await client.execute(`INSERT OR REPLACE INTO ${table} (key, value) VALUES (?, ?);`, [key, csvText]); + + return { ok: true, storedAt: `turso:${table}:${key}` }; + } catch (err: any) { + throw createError({ statusCode: 500, statusMessage: `Failed to store CSV to Turso: ${String(err?.message ?? err)}` }); + } +});