From 481df1784a6c7199ea20ae12c60604de1fd81cef Mon Sep 17 00:00:00 2001 From: theMackabu Date: Sat, 25 Apr 2026 18:12:13 -0700 Subject: [PATCH] add deduplication by hash/hitcount --- .../migrations/0001_fluffy_lockheed.sql | 1 + .../migrations/meta/0001_snapshot.json | 245 ++++++++++++++++++ docs/reports/migrations/meta/_journal.json | 7 + docs/reports/src/reports.ts | 74 ++++-- docs/reports/src/schema.ts | 44 ++-- 5 files changed, 329 insertions(+), 42 deletions(-) create mode 100644 docs/reports/migrations/0001_fluffy_lockheed.sql create mode 100644 docs/reports/migrations/meta/0001_snapshot.json diff --git a/docs/reports/migrations/0001_fluffy_lockheed.sql b/docs/reports/migrations/0001_fluffy_lockheed.sql new file mode 100644 index 0000000..dff32a0 --- /dev/null +++ b/docs/reports/migrations/0001_fluffy_lockheed.sql @@ -0,0 +1 @@ +CREATE UNIQUE INDEX `reports_trace_unique` ON `reports` (`trace`); \ No newline at end of file diff --git a/docs/reports/migrations/meta/0001_snapshot.json b/docs/reports/migrations/meta/0001_snapshot.json new file mode 100644 index 0000000..36a18e0 --- /dev/null +++ b/docs/reports/migrations/meta/0001_snapshot.json @@ -0,0 +1,245 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "bbaeff3b-328a-4c14-a851-5a252a334c26", + "prevId": "f888823f-8746-428a-894d-7a6e8578f8d2", + "tables": { + "frames": { + "name": "frames", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "frame": { + "name": "frame", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "report_frames": { + "name": "report_frames", + "columns": { + "report_id": { + "name": "report_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "frame_index": { + "name": "frame_index", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "frame_hash": { + "name": "frame_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "report_frames_report_id_reports_id_fk": { + "name": "report_frames_report_id_reports_id_fk", + "tableFrom": "report_frames", + "tableTo": "reports", + "columnsFrom": [ + "report_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "report_frames_frame_hash_frames_hash_fk": { + "name": "report_frames_frame_hash_frames_hash_fk", + "tableFrom": "report_frames", + "tableTo": "frames", + "columnsFrom": [ + "frame_hash" + ], + "columnsTo": [ + "hash" + ], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "report_frames_report_id_frame_index_pk": { + "columns": [ + "report_id", + "frame_index" + ], + "name": "report_frames_report_id_frame_index_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "reports": { + "name": "reports", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "runtime": { + "name": "runtime", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "trace": { + "name": "trace", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "code": { + "name": "code", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target": { + "name": "target", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "os": { + "name": "os", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "arch": { + "name": "arch", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fault_address": { + "name": "fault_address", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "elapsed_ms": { + "name": "elapsed_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "peak_rss": { + "name": "peak_rss", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "first_seen_at": { + "name": "first_seen_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "hit_count": { + "name": "hit_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "reports_trace_unique": { + "name": "reports_trace_unique", + "columns": [ + "trace" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/docs/reports/migrations/meta/_journal.json b/docs/reports/migrations/meta/_journal.json index 14b57ac..fe60e3f 100644 --- a/docs/reports/migrations/meta/_journal.json +++ b/docs/reports/migrations/meta/_journal.json @@ -8,6 +8,13 @@ "when": 1777160084343, "tag": "0000_melodic_maginty", "breakpoints": true + }, + { + "idx": 1, + "version": "6", + "when": 1777165898584, + "tag": "0001_fluffy_lockheed", + "breakpoints": true } ] } \ No newline at end of file diff --git a/docs/reports/src/reports.ts b/docs/reports/src/reports.ts index 0b22671..9f802ab 100644 --- a/docs/reports/src/reports.ts +++ b/docs/reports/src/reports.ts @@ -1,6 +1,6 @@ import { sha256 } from './helpers'; -import { asc, eq } from 'drizzle-orm'; import { drizzle } from 'drizzle-orm/d1'; +import { asc, eq, sql } from 'drizzle-orm'; import { generate as generateShortUuid } from 'short-uuid'; import { frames as frameTable, reportFrames, reports, type CrashReport } from './schema'; @@ -127,29 +127,59 @@ export async function getRawReport(db: ReportDb, id: string): Promise { const id = generateShortUuid(); const trace = await reportTrace(report); + const now = new Date(); const expires = new Date(now.getTime() + 30 * 24 * 60 * 60 * 1000); - await db.insert(reports).values({ - id, - trace, - hitCount: 1, - runtime: report.runtime, - version: report.version, - kind: report.kind, - reason: report.reason, - code: report.code, - target: report.target, - os: report.os, - arch: report.arch, - faultAddress: report.addr, - elapsedMs: report.elapsedMs, - peakRss: report.peakRss, - firstSeenAt: now.toISOString(), - lastSeenAt: now.toISOString(), - expiresAt: expires.toISOString(), - }); + const nowIso = now.toISOString(); + const expiresIso = expires.toISOString(); + + await db + .insert(reports) + .values({ + id, + trace, + hitCount: 1, + runtime: report.runtime, + version: report.version, + kind: report.kind, + reason: report.reason, + code: report.code, + target: report.target, + os: report.os, + arch: report.arch, + faultAddress: report.addr, + elapsedMs: report.elapsedMs, + peakRss: report.peakRss, + firstSeenAt: nowIso, + lastSeenAt: nowIso, + expiresAt: expiresIso, + }) + .onConflictDoNothing({ target: reports.trace }); + + const [row] = await db + .select({ + id: reports.id, + hitCount: reports.hitCount, + }) + .from(reports) + .where(eq(reports.trace, trace)) + .limit(1); + + if (!row) return id; + if (row.id === id) { + await insertReportFrames(db, id, report.frames); + return id; + } + + await db + .update(reports) + .set({ + hitCount: sql`${reports.hitCount} + 1`, + lastSeenAt: nowIso, + expiresAt: expiresIso, + }) + .where(eq(reports.id, row.id)); - await insertReportFrames(db, id, report.frames); - return id; + return row.id; } diff --git a/docs/reports/src/schema.ts b/docs/reports/src/schema.ts index 631da5c..5d0f42f 100644 --- a/docs/reports/src/schema.ts +++ b/docs/reports/src/schema.ts @@ -1,5 +1,5 @@ import { z } from 'zod'; -import { integer, primaryKey, sqliteTable, text } from 'drizzle-orm/sqlite-core'; +import { integer, primaryKey, sqliteTable, text, uniqueIndex } from 'drizzle-orm/sqlite-core'; const boundedString = z .string() @@ -27,25 +27,29 @@ export const CrashReportSchema = z }) .strict(); -export const reports = sqliteTable('reports', { - id: text('id').primaryKey(), - runtime: text('runtime').notNull(), - version: text('version').notNull(), - trace: text('trace').notNull(), - kind: text('kind').notNull(), - reason: text('reason').notNull(), - code: text('code').notNull(), - target: text('target').notNull(), - os: text('os').notNull(), - arch: text('arch').notNull(), - faultAddress: text('fault_address').notNull(), - elapsedMs: integer('elapsed_ms'), - peakRss: integer('peak_rss'), - firstSeenAt: text('first_seen_at').notNull(), - lastSeenAt: text('last_seen_at').notNull(), - hitCount: integer('hit_count').notNull(), - expiresAt: text('expires_at').notNull(), -}); +export const reports = sqliteTable( + 'reports', + { + id: text('id').primaryKey(), + runtime: text('runtime').notNull(), + version: text('version').notNull(), + trace: text('trace').notNull(), + kind: text('kind').notNull(), + reason: text('reason').notNull(), + code: text('code').notNull(), + target: text('target').notNull(), + os: text('os').notNull(), + arch: text('arch').notNull(), + faultAddress: text('fault_address').notNull(), + elapsedMs: integer('elapsed_ms'), + peakRss: integer('peak_rss'), + firstSeenAt: text('first_seen_at').notNull(), + lastSeenAt: text('last_seen_at').notNull(), + hitCount: integer('hit_count').notNull(), + expiresAt: text('expires_at').notNull(), + }, + table => [uniqueIndex('reports_trace_unique').on(table.trace)], +); export const frames = sqliteTable('frames', { hash: text('hash').primaryKey(), -- 2.51.2